feature: 更新算子名称；增加创建任务、模板校验 (#57)

* feature: 更新算子名称；增加创建任务、模板校验

* feature: 镜像构建增加缓存

runtime/ops/formatter/__init__.py

@@ -21,7 +21,7 @@ def _import_operators():
     from . import file_exporter
     from . import slide_formatter
     from . import unstructured_formatter
-    from . import external_pdf_formatter
+    from . import mineru_formatter
 
 
 _import_operators()

runtime/ops/formatter/mineru_formatter/__init__.py

@@ -2,5 +2,5 @@
 
 from datamate.core.base_op import OPERATORS
 
-OPERATORS.register_module(module_name='ExternalPDFFormatter',
+OPERATORS.register_module(module_name='MineruFormatter',
                           module_path="ops.formatter.external_pdf_formatter.process")

runtime/ops/formatter/mineru_formatter/metadata.yml

@@ -1,10 +1,10 @@
 name: 'MinerU PDF文本抽取'
-name_en: 'External PDF Text Extraction'
+name_en: 'MinerU PDF Text Extraction'
 description: '基于MinerU API，抽取PDF中的文本。'
 description_en: 'Extracts text from PDF files based on MinerU API.'
 language: 'python'
 vendor: 'huawei'
-raw_id: 'ExternalPDFFormatter'
+raw_id: 'MineruFormatter'
 version: '1.0.0'
 types:
   - 'collect'

runtime/ops/formatter/mineru_formatter/process.py

@@ -15,11 +15,11 @@ from datamate.core.base_op import Mapper
 from datamate.common.utils.rest_client import http_request
 
 
-class ExternalPDFFormatter(Mapper):
+class MineruFormatter(Mapper):
     """基于外部API，抽取PDF中的文本"""
 
     def __init__(self, *args, **kwargs):
-        super(ExternalPDFFormatter, self).__init__(*args, **kwargs)
+        super(MineruFormatter, self).__init__(*args, **kwargs)
         self.base_url = os.getenv("EXTERNAL_PDF_BASE_URL", "http://datamate-mineru:9001")
         self.pdf_extract_url = f"{self.base_url}/api/pdf-extract"
 
@@ -31,8 +31,8 @@ class ExternalPDFFormatter(Mapper):
             response = http_request(method="POST", url=self.pdf_extract_url, data=data)
             sample[self.text_key] = json.loads(response.text).get("result")
             logger.info(
-                f"fileName: {filename}, method: ExternalPDFFormatter costs {(time.time() - start):6f} s")
+                f"fileName: {filename}, method: MineruFormatter costs {(time.time() - start):6f} s")
         except UnicodeDecodeError as err:
-            logger.exception(f"fileName: {filename}, method: ExternalPDFFormatter causes decode error: {err}")
+            logger.exception(f"fileName: {filename}, method: MineruFormatter causes decode error: {err}")
             raise
         return sample

runtime/ops/formatter/unstructured_formatter/metadata.yml

@@ -1,4 +1,4 @@
-name: '非结构化文本抽取'
+name: 'Unstructured文本抽取'
 name_en: 'Unstructured Text Extraction'
 description: '抽取非结构化文件的文本，目前支持PowerPoint演示文稿、Word文档以及Excel工作簿。'
 description_en: 'Extracts text from Unstructured files, currently supporting PowerPoint presentations, Word documents and Excel spreadsheets files.'

runtime/python-executor/datamate/common/utils/lazy_loader.py

@@ -51,7 +51,7 @@ class LazyLoader(ModuleType):
     def __init__(self,
                  package_name,
                  module_name=None,
-                 whl_path="/dataset/ops_whl",
+                 whl_path=None,
                  exact_version=None,
                  force_reinstall=False
                  ):
@@ -72,7 +72,7 @@ class LazyLoader(ModuleType):
         self._module_name = module_name if module_name else package_name
         self._package_name = package_name
 
-        self.whl_path = Path(whl_path).resolve()
+        self.whl_path = whl_path
         self.exact_version = exact_version
 
         self.force_reinstall = force_reinstall
@@ -126,7 +126,10 @@ class LazyLoader(ModuleType):
                 need_install = True
 
         if need_install:
-            self._pip_install_package(package_name)
+            if self.whl_path is None:
+                self._pip_install_package_pypi(package_name)
+            else:
+                self._pip_install_package_local(package_name)
             module = importlib.import_module(module_name)
             self._cached_module = module
             self._register_alias(module)
@@ -168,13 +171,26 @@ class LazyLoader(ModuleType):
                 return line.split()[-1]
         raise PackageNotFoundError()
 
-    def _pip_install_package(self, package_name: str):
+    def _pip_install_package_pypi(self, package_name: str):
+        if self.exact_version:
+            package_name += f"=={self.exact_version}"
+        try:
+            subprocess.check_call([
+                sys.executable, "-m", "pip", "install", str(package_name)
+            ], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
+            logger.info(f"Successfully installed {package_name}")
+        except subprocess.CalledProcessError as e:
+            logger.error(f"Installation failed: {e}")
+            raise RuntimeError(f"Installation failed: {e}") from e
+
+    def _pip_install_package_local(self, package_name: str):
         """安装逻辑 """
 
-        if not self.whl_path.exists():
+        whl_path = Path(self.whl_path).resolve()
+        if not whl_path.exists():
             raise FileNotFoundError(f"WHL directory not found: {self.whl_path}")
 
-        whl_files = list(self.whl_path.glob(f"{package_name}*.whl"))
+        whl_files = list(whl_path.glob(f"{package_name}*.whl"))
         if not whl_files:
             raise RuntimeError(f"No WHL files found for {package_name}")