feature: add unstructured formatter operator for doc/docx (#17)

* feature: add UnstructuredFormatter

* feature: add UnstructuredFormatter in db

* feature: add unstructured[docx]==0.18.15

* feature: support doc

---------

Co-authored-by: Startalker <438747480@qq.com>

runtime/ops/formatter/unstructured_formatter/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from datamate.core.base_op import OPERATORS
+
+OPERATORS.register_module(module_name='UnstructuredFormatter',
+                          module_path="ops.formatter.unstructured_formatter.process")

runtime/ops/formatter/unstructured_formatter/metadata.yml

@@ -0,0 +1,16 @@
+name: '非结构化文本抽取'
+name_en: 'Unstructured Text Extraction'
+description: '抽取非结构化文件的文本，目前支持word文档'
+description_en: 'Extracts text from Unstructured files, currently supporting Word documents.'
+language: 'python'
+vendor: 'huawei'
+raw_id: 'UnstructuredFormatter'
+version: '1.0.0'
+types:
+  - 'collect'
+modal: 'text'
+effect:
+  before: ''
+  after: ''
+inputs: 'text'
+outputs: 'text'

runtime/ops/formatter/unstructured_formatter/process.py

@@ -0,0 +1,35 @@
+
+#!/user/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+Description: 非结构化文本抽取
+Create: 2025/10/22 15:15
+"""
+import time
+from typing import Dict, Any
+
+from loguru import logger
+from unstructured.partition.auto import partition
+
+from datamate.core.base_op import Mapper
+
+
+class UnstructuredFormatter(Mapper):
+    """把输入的非结构化文本抽取为txt"""
+
+    def __init__(self, *args, **kwargs):
+        super(UnstructuredFormatter, self).__init__(*args, **kwargs)
+
+    def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:
+        start = time.time()
+        filepath = sample.get(self.filepath_key)
+        filename = sample.get(self.filename_key)
+        try:
+            elements = partition(filename=filepath)
+            sample[self.text_key] = "\n\n".join([str(el) for el in elements])
+            logger.info(f"fileName: {filename}, method: UnstructuredFormatter costs {(time.time() - start):6f} s")
+        except UnicodeDecodeError as err:
+            logger.exception(f"fileName: {filename}, method: UnstructuredFormatter causes decode error: {err}")
+            raise
+        return sample