diff --git a/runtime/ops/README.md b/runtime/ops/README.md index dc4678e..6d0bf3d 100644 --- a/runtime/ops/README.md +++ b/runtime/ops/README.md @@ -7,83 +7,83 @@ 每个自定义算子都需要包含一个 `metadata.yml` 文件： ```yaml -name: '落盘算子' -name_en: 'save file operator' -description: '将文件内容保存为文件。' -description_en: 'Save the file data as a file.' -language: 'Python' -vendor: 'Huawei' -raw_id: 'FileExporter' +name: '测试算子' +description: '这是一个测试算子。' +language: 'python' +vendor: 'huawei' +raw_id: 'TestMapper' version: '1.0.0' -types: - - 'collect' -modal: 'others' -effect: - before: '' - after: '' -inputs: 'all' -outputs: 'all' +modal: 'text' +inputs: 'text' +outputs: 'text' ``` ### 算子实现 -创建 `process.py` 文件： +#### process.py ```python # -*- coding: utf-8 -*- -""" -Description: Json文本抽取 -Create: 2024/06/06 15:43 -""" -import time -from loguru import logger -from typing import Dict, Any - +# 导入所需数据结构，可以通过以下方式直接导入使用 +# 提供两种算子类： +# Mapper用于映射和转换数据，使用时直接修改数据内容 from datamate.core.base_op import Mapper +class TestMapper(Mapper): + def execute(self, sample): + sample[self.text_key] += "\n新增的数据" + return sample -class TextFormatter(Mapper): - """把输入的json文件流抽取为txt""" + +# Filter用于过滤和选择性保留数据，使用时将需要过滤的数据的text或data置为空值 +from datamate.core.base_op import Filter - def __init__(self, *args, **kwargs): - super(TextFormatter, self).__init__(*args, **kwargs) - - @staticmethod - def _extract_json(byte_io): - """将默认使用utf-8编码的Json文件流解码，抽取为txt""" - # 用utf-8-sig的格式进行抽取，可以避免uft-8 BOM编码格式的文件在抽取后产生隐藏字符作为前缀。 - return byte_io.decode("utf-8-sig").replace("\r\n", "\n") - - def byte_read(self, sample: Dict[str, Any]): - filepath = sample[self.filepath_key] - with open(filepath, "rb") as file: - byte_data = file.read() - sample[self.data_key] = byte_data - - def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]: - start = time.time() - try: - self.byte_read(sample) - sample[self.text_key] = self._extract_json(sample[self.data_key]) - sample[self.data_key] = b"" # 将sample[self.data_key]置空 - logger.info( - f"fileName: {sample[self.filename_key]}, method: TextFormatter costs {(time.time() - start):6f} s") - except UnicodeDecodeError as err: - logger.exception(f"fileName: {sample[self.filename_key]}, method: TextFormatter causes decode error: {err}") - raise +class TestFilter(Filter): + def execute(self, sample): + if len(sample[self.text_key]) > 100: + sample[self.text_key] += "" return sample ``` -创建 `__init__.py` 文件： +其中，sample的数据结构如下所示: +```json lines +// 数据结构 +{ + "text": "数据文件的文本内容", + "data": "多模态文件的内容", + "fileName": "文件名称", + "fileType": "文件类型", + "filePath": "文件路径", + "fileSize": "文件大小", + "export_path": "保存的文件路径", + "extraFileType": "导出的文件类型" +} + +// 数据示例 +{ + "text": "text", + "data": "data", + "fileName": "test", + "fileType": "pdf", + "filePath": "/dataset/test.pdf", + "fileSize": "100B", + "export_path": "/dataset/test.txt", + "extraFileType": "txt" +} +``` + +#### \_\_init__.py ```python # -*- coding: utf-8 -*- +# 导入OPERATORS用于进行模块注册，可以通过以下方式直接导入使用 from datamate.core.base_op import OPERATORS -OPERATORS.register_module(module_name='TextFormatter', - module_path="ops.formatter.text_formatter.process") +# module_name必须填写算子类名称；module_path中须替换模块的算子压缩包名称：python_operator.user.压缩包名.process +OPERATORS.register_module(module_name='TestMapper', + module_path="ops.user.test_operator.process") ``` \ No newline at end of file diff --git a/runtime/ops/examples/test_operator/metadata.yml b/runtime/ops/examples/test_operator/metadata.yml index 0d54fe2..d4f80fd 100644 --- a/runtime/ops/examples/test_operator/metadata.yml +++ b/runtime/ops/examples/test_operator/metadata.yml @@ -5,9 +5,6 @@ vendor: 'huawei' raw_id: 'TestMapper' version: '1.0.0' modal: 'text' -effect: - before: '使用方式很简单，只需要将代码放入Markdown文本中即可，富文本格式可直接复制表情😀使用。' - after: '使用方式很简单，只需要将代码放入Markdown文本中即可，富文本格式可直接复制表情使用。' inputs: 'text' outputs: 'text' settings: diff --git a/runtime/python-executor/README.md b/runtime/python-executor/README.md deleted file mode 100644 index dc4678e..0000000 --- a/runtime/python-executor/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# 自定义算子开发指南 - -## 算子规范 - -### 算子元数据格式 - -每个自定义算子都需要包含一个 `metadata.yml` 文件： - -```yaml -name: '落盘算子' -name_en: 'save file operator' -description: '将文件内容保存为文件。' -description_en: 'Save the file data as a file.' -language: 'Python' -vendor: 'Huawei' -raw_id: 'FileExporter' -version: '1.0.0' -types: - - 'collect' -modal: 'others' -effect: - before: '' - after: '' -inputs: 'all' -outputs: 'all' -``` - -### 算子实现 - -创建 `process.py` 文件： - -```python -# -*- coding: utf-8 -*- - -""" -Description: Json文本抽取 -Create: 2024/06/06 15:43 -""" -import time -from loguru import logger -from typing import Dict, Any - -from datamate.core.base_op import Mapper - - -class TextFormatter(Mapper): - """把输入的json文件流抽取为txt""" - - def __init__(self, *args, **kwargs): - super(TextFormatter, self).__init__(*args, **kwargs) - - @staticmethod - def _extract_json(byte_io): - """将默认使用utf-8编码的Json文件流解码，抽取为txt""" - # 用utf-8-sig的格式进行抽取，可以避免uft-8 BOM编码格式的文件在抽取后产生隐藏字符作为前缀。 - return byte_io.decode("utf-8-sig").replace("\r\n", "\n") - - def byte_read(self, sample: Dict[str, Any]): - filepath = sample[self.filepath_key] - with open(filepath, "rb") as file: - byte_data = file.read() - sample[self.data_key] = byte_data - - def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]: - start = time.time() - try: - self.byte_read(sample) - sample[self.text_key] = self._extract_json(sample[self.data_key]) - sample[self.data_key] = b"" # 将sample[self.data_key]置空 - logger.info( - f"fileName: {sample[self.filename_key]}, method: TextFormatter costs {(time.time() - start):6f} s") - except UnicodeDecodeError as err: - logger.exception(f"fileName: {sample[self.filename_key]}, method: TextFormatter causes decode error: {err}") - raise - return sample - -``` - -创建 `__init__.py` 文件： - -```python -# -*- coding: utf-8 -*- - -from datamate.core.base_op import OPERATORS - -OPERATORS.register_module(module_name='TextFormatter', - module_path="ops.formatter.text_formatter.process") - -``` \ No newline at end of file diff --git a/runtime/python-executor/pyproject.toml b/runtime/python-executor/pyproject.toml index 5914565..2a28280 100644 --- a/runtime/python-executor/pyproject.toml +++ b/runtime/python-executor/pyproject.toml @@ -5,7 +5,6 @@ description = "Data Processing for and with Foundation Models." authors = [ { name = "Huawei datamate team" } ] -readme = "README.md" license = { text = "Apache-2.0" } requires-python = ">=3.10" urls = { repository = "https://github.com/ModelEngine-Group/datamate" }