feature: 更新算子名称；增加创建任务、模板校验 (#57)

* feature: 更新算子名称；增加创建任务、模板校验

* feature: 镜像构建增加缓存

runtime/ops/formatter/__init__.py

@@ -21,7 +21,7 @@ def _import_operators():
     from . import file_exporter
     from . import slide_formatter
     from . import unstructured_formatter
-    from . import external_pdf_formatter
+    from . import mineru_formatter
 
 
 _import_operators()

runtime/ops/formatter/mineru_formatter/__init__.py

@@ -2,5 +2,5 @@
 
 from datamate.core.base_op import OPERATORS
 
-OPERATORS.register_module(module_name='ExternalPDFFormatter',
+OPERATORS.register_module(module_name='MineruFormatter',
                           module_path="ops.formatter.external_pdf_formatter.process")

runtime/ops/formatter/mineru_formatter/metadata.yml

@@ -1,10 +1,10 @@
 name: 'MinerU PDF文本抽取'
-name_en: 'External PDF Text Extraction'
+name_en: 'MinerU PDF Text Extraction'
 description: '基于MinerU API，抽取PDF中的文本。'
 description_en: 'Extracts text from PDF files based on MinerU API.'
 language: 'python'
 vendor: 'huawei'
-raw_id: 'ExternalPDFFormatter'
+raw_id: 'MineruFormatter'
 version: '1.0.0'
 types:
   - 'collect'

runtime/ops/formatter/mineru_formatter/process.py

@@ -15,11 +15,11 @@ from datamate.core.base_op import Mapper
 from datamate.common.utils.rest_client import http_request
 
 
-class ExternalPDFFormatter(Mapper):
+class MineruFormatter(Mapper):
     """基于外部API，抽取PDF中的文本"""
 
     def __init__(self, *args, **kwargs):
-        super(ExternalPDFFormatter, self).__init__(*args, **kwargs)
+        super(MineruFormatter, self).__init__(*args, **kwargs)
         self.base_url = os.getenv("EXTERNAL_PDF_BASE_URL", "http://datamate-mineru:9001")
         self.pdf_extract_url = f"{self.base_url}/api/pdf-extract"
 
@@ -31,8 +31,8 @@ class ExternalPDFFormatter(Mapper):
             response = http_request(method="POST", url=self.pdf_extract_url, data=data)
             sample[self.text_key] = json.loads(response.text).get("result")
             logger.info(
-                f"fileName: {filename}, method: ExternalPDFFormatter costs {(time.time() - start):6f} s")
+                f"fileName: {filename}, method: MineruFormatter costs {(time.time() - start):6f} s")
         except UnicodeDecodeError as err:
-            logger.exception(f"fileName: {filename}, method: ExternalPDFFormatter causes decode error: {err}")
+            logger.exception(f"fileName: {filename}, method: MineruFormatter causes decode error: {err}")
             raise
         return sample

runtime/ops/formatter/unstructured_formatter/metadata.yml

@@ -1,4 +1,4 @@
-name: '非结构化文本抽取'
+name: 'Unstructured文本抽取'
 name_en: 'Unstructured Text Extraction'
 description: '抽取非结构化文件的文本，目前支持PowerPoint演示文稿、Word文档以及Excel工作簿。'
 description_en: 'Extracts text from Unstructured files, currently supporting PowerPoint presentations, Word documents and Excel spreadsheets files.'