feature: 修改算子开发指南 (#127)

runtime/ops/README.md

@@ -7,83 +7,83 @@
 每个自定义算子都需要包含一个 `metadata.yml` 文件：
 
 ```yaml
-name: '落盘算子'
+name: '测试算子'
-name_en: 'save file operator'
+description: '这是一个测试算子。'
-description: '将文件内容保存为文件。'
+language: 'python'
-description_en: 'Save the file data as a file.'
+vendor: 'huawei'
-language: 'Python'
+raw_id: 'TestMapper'
-vendor: 'Huawei'
-raw_id: 'FileExporter'
 version: '1.0.0'
-types:
+modal: 'text'
-  - 'collect'
+inputs: 'text'
-modal: 'others'
+outputs: 'text'
-effect:
-  before: ''
-  after: ''
-inputs: 'all'
-outputs: 'all'
 ```
 
 ### 算子实现
 
-创建 `process.py` 文件：
+#### process.py
 
 ```python
 # -*- coding: utf-8 -*-
 
-"""
+# 导入所需数据结构，可以通过以下方式直接导入使用
-Description: Json文本抽取
+# 提供两种算子类：
-Create: 2024/06/06 15:43
+# Mapper用于映射和转换数据，使用时直接修改数据内容
-"""
-import time
-from loguru import logger
-from typing import Dict, Any
-
 from datamate.core.base_op import Mapper
 
+class TestMapper(Mapper):
+    def execute(self, sample):
+        sample[self.text_key] += "\n新增的数据"
+        return sample
 
-class TextFormatter(Mapper):
-    """把输入的json文件流抽取为txt"""
     
-    def __init__(self, *args, **kwargs):
+# Filter用于过滤和选择性保留数据，使用时将需要过滤的数据的text或data置为空值
-        super(TextFormatter, self).__init__(*args, **kwargs)
+from datamate.core.base_op import Filter
 
-    @staticmethod
+class TestFilter(Filter):
-    def _extract_json(byte_io):
+    def execute(self, sample):
-        """将默认使用utf-8编码的Json文件流解码，抽取为txt"""
+        if len(sample[self.text_key]) > 100:
-        # 用utf-8-sig的格式进行抽取，可以避免uft-8 BOM编码格式的文件在抽取后产生隐藏字符作为前缀。
+            sample[self.text_key] += ""
-        return byte_io.decode("utf-8-sig").replace("\r\n", "\n")
-
-    def byte_read(self, sample: Dict[str, Any]):
-        filepath = sample[self.filepath_key]
-        with open(filepath, "rb") as file:
-            byte_data = file.read()
-        sample[self.data_key] = byte_data
-
-    def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:
-        start = time.time()
-        try:
-            self.byte_read(sample)
-            sample[self.text_key] = self._extract_json(sample[self.data_key])
-            sample[self.data_key] = b""  # 将sample[self.data_key]置空
-            logger.info(
-                f"fileName: {sample[self.filename_key]}, method: TextFormatter costs {(time.time() - start):6f} s")
-        except UnicodeDecodeError as err:
-            logger.exception(f"fileName: {sample[self.filename_key]}, method: TextFormatter causes decode error: {err}")
-            raise
         return sample
 
 ```
 
-创建 `__init__.py` 文件：
+其中，sample的数据结构如下所示:
+```json lines
+// 数据结构
+{
+  "text": "数据文件的文本内容",
+  "data": "多模态文件的内容",
+  "fileName": "文件名称",
+  "fileType": "文件类型",
+  "filePath": "文件路径",
+  "fileSize": "文件大小",
+  "export_path": "保存的文件路径",
+  "extraFileType": "导出的文件类型"
+}
+
+// 数据示例
+{
+  "text": "text",
+  "data": "data",
+  "fileName": "test",
+  "fileType": "pdf",
+  "filePath": "/dataset/test.pdf",
+  "fileSize": "100B",
+  "export_path": "/dataset/test.txt",
+  "extraFileType": "txt"
+}
+```
+
+####  \_\_init__.py
 
 ```python
 # -*- coding: utf-8 -*-
 
+# 导入OPERATORS用于进行模块注册，可以通过以下方式直接导入使用
 from datamate.core.base_op import OPERATORS
 
-OPERATORS.register_module(module_name='TextFormatter',
+# module_name必须填写算子类名称；module_path中须替换模块的算子压缩包名称：python_operator.user.压缩包名.process
-                          module_path="ops.formatter.text_formatter.process")
+OPERATORS.register_module(module_name='TestMapper',
+                          module_path="ops.user.test_operator.process")
 
 ```

runtime/ops/examples/test_operator/metadata.yml

@@ -5,9 +5,6 @@ vendor: 'huawei'
 raw_id: 'TestMapper'
 version: '1.0.0'
 modal: 'text'
-effect:
-  before: '使用方式很简单，只需要将代码放入Markdown文本中即可，富文本格式可直接复制表情😀使用。'
-  after: '使用方式很简单，只需要将代码放入Markdown文本中即可，富文本格式可直接复制表情使用。'
 inputs: 'text'
 outputs: 'text'
 settings:

runtime/python-executor/README.md

@@ -1,89 +0,0 @@
-# 自定义算子开发指南
-
-## 算子规范
-
-### 算子元数据格式
-
-每个自定义算子都需要包含一个 `metadata.yml` 文件：
-
-```yaml
-name: '落盘算子'
-name_en: 'save file operator'
-description: '将文件内容保存为文件。'
-description_en: 'Save the file data as a file.'
-language: 'Python'
-vendor: 'Huawei'
-raw_id: 'FileExporter'
-version: '1.0.0'
-types:
-  - 'collect'
-modal: 'others'
-effect:
-  before: ''
-  after: ''
-inputs: 'all'
-outputs: 'all'
-```
-
-### 算子实现
-
-创建 `process.py` 文件：
-
-```python
-# -*- coding: utf-8 -*-
-
-"""
-Description: Json文本抽取
-Create: 2024/06/06 15:43
-"""
-import time
-from loguru import logger
-from typing import Dict, Any
-
-from datamate.core.base_op import Mapper
-
-
-class TextFormatter(Mapper):
-    """把输入的json文件流抽取为txt"""
-
-    def __init__(self, *args, **kwargs):
-        super(TextFormatter, self).__init__(*args, **kwargs)
-
-    @staticmethod
-    def _extract_json(byte_io):
-        """将默认使用utf-8编码的Json文件流解码，抽取为txt"""
-        # 用utf-8-sig的格式进行抽取，可以避免uft-8 BOM编码格式的文件在抽取后产生隐藏字符作为前缀。
-        return byte_io.decode("utf-8-sig").replace("\r\n", "\n")
-
-    def byte_read(self, sample: Dict[str, Any]):
-        filepath = sample[self.filepath_key]
-        with open(filepath, "rb") as file:
-            byte_data = file.read()
-        sample[self.data_key] = byte_data
-
-    def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:
-        start = time.time()
-        try:
-            self.byte_read(sample)
-            sample[self.text_key] = self._extract_json(sample[self.data_key])
-            sample[self.data_key] = b""  # 将sample[self.data_key]置空
-            logger.info(
-                f"fileName: {sample[self.filename_key]}, method: TextFormatter costs {(time.time() - start):6f} s")
-        except UnicodeDecodeError as err:
-            logger.exception(f"fileName: {sample[self.filename_key]}, method: TextFormatter causes decode error: {err}")
-            raise
-        return sample
-
-```
-
-创建 `__init__.py` 文件：
-
-```python
-# -*- coding: utf-8 -*-
-
-from datamate.core.base_op import OPERATORS
-
-OPERATORS.register_module(module_name='TextFormatter',
-                          module_path="ops.formatter.text_formatter.process")
-
-```

runtime/python-executor/pyproject.toml

@@ -5,7 +5,6 @@ description = "Data Processing for and with Foundation Models."
 authors = [
     { name = "Huawei datamate team" }
 ]
-readme = "README.md"
 license = { text = "Apache-2.0" }
 requires-python = ">=3.10"
 urls = { repository = "https://github.com/ModelEngine-Group/datamate" }