支持mineru npu处理 (#174)

* feature: unstructured支持简单pdf处理

* feature: update values.yaml to enhance ray-cluster configuration with security context, environment variables, and resource limits

* feature: update deploy.yaml and process.py for mineru server configuration and PDF processing enhancements

* feature: update deploy.yaml and process.py for mineru server configuration and PDF processing enhancements

* feature: improve PDF processing logic and update dependencies in process.py and pyproject.toml

* feature: improve PDF processing logic and update dependencies in process.py and pyproject.toml

* feature: update Dockerfile for improved package source mirrors and add mineru-npu to build targets

deployment/docker/datamate/docker-compose.yml

@@ -99,19 +99,28 @@ services:
     restart: on-failure
     environment:
       MINERU_MODEL_SOURCE: local
-      MINERU_DEVICE_MODE: cpu  # cpu|cuda|npu|mps
-      MINERU_BACKEND_MODE: pipeline
+      MINERU_DEVICE_MODE: npu  # cpu|cuda|npu|mps
+      VLLM_WORKER_MULTIPROC_METHOD: spawn
     privileged: true
     command:
-      - python
-      - /opt/runtime/datamate/mineru/mineru_api.py
-      - --port
-      - "9001"
+      - mineru-openai-server
+      - --engine vllm
+      - --host 0.0.0.0
+      - --port "8000"
     volumes:
       - dataset_volume:/dataset
       - mineru_log_volume:/var/log/datamate/mineru
+      - /var/log/npu/:/usr/slog
+      - /usr/local/dcmi:/usr/local/dcmi
+      - /usr/local/bin/npu-smi:/usr/local/bin/npu-smi
+      - /usr/local/Ascend/driver:/usr/local/Ascend/driver
     networks: [ datamate ]
     profiles: [ mineru ]
+    devices:
+      - /dev/davinci0
+      - /dev/davinci_manager
+      - /dev/devmm_svm
+      - /dev/hisi_hdc
 
   # 5) redis
   datamate-redis:

deployment/helm/datamate/values.yaml

@@ -175,7 +175,6 @@ ray-cluster:
   enabled: true
   head:
     rayStartParams:
-      object-store-memory: '78643200'
       num-cpus: '0'
     containerEnv:
       - name: RAY_DEDUP_LOGS
@@ -194,8 +193,8 @@ ray-cluster:
         value: "datamate"
     resources:
       limits:
-        cpu: "2"
-        memory: "8G"
+        cpu: "4"
+        memory: "16G"
       requests:
         cpu: "1"
         memory: "2G"

deployment/kubernetes/mineru/deploy.yaml

@@ -22,26 +22,32 @@ spec:
           image: datamate-mineru
           imagePullPolicy: IfNotPresent
           command:
-            - python
-            - /opt/runtime/datamate/mineru/mineru_api.py
+            - mineru-openai-server
+          args:
+            - --engine
+            - vllm
+            - --host
+            - 0.0.0.0
             - --port
-            - "9001"
+            - "8000"
           env:
             - name: MINERU_MODEL_SOURCE
               value: local
             - name: MINERU_DEVICE_MODE
-              value: cpu
-            - name: MINERU_BACKEND_MODE
-              value: pipeline
+              value: npu
+            - name: VLLM_WORKER_MULTIPROC_METHOD
+              value: spawn
           ports:
-            - containerPort: 9001
+            - containerPort: 8000
           resources:
             limits:
-              cpu: 16
+              cpu: 8
               memory: 32Gi
+              huawei.com/Ascend910: 1
             requests:
               cpu: 100m
               memory: 100Mi
+              huawei.com/Ascend910: 1
           volumeMounts:
             - name: dataset-volume
               mountPath: /dataset
@@ -67,8 +73,8 @@ metadata:
 spec:
   type: ClusterIP
   ports:
-    - port: 9001
-      targetPort: 9001
+    - port: 8000
+      targetPort: 8000
       protocol: TCP
   selector:
     app: datamate