Develop op (#35)

* refactor: enhance CleaningTaskService and related components with validation and repository updates
* feature: 支持算子上传创建

deployment/helm/datamate/charts/frontend/templates/configmap.yaml

@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: datamate-nginx-conf
+data:
+  backend.conf: |
+    server {
+        listen       80;
+        server_name  0.0.0.0;
+
+        access_log /var/log/datamate/frontend/access.log main;
+        error_log /var/log/datamate/frontend/error.log notice;
+
+        client_max_body_size 1024M;
+
+        location /api/ {
+            proxy_pass http://datamate-backend:8080/api/;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+
+        location / {
+            root   /opt/frontend;
+            try_files $uri $uri/ /index.html;
+        }
+    }
+

deployment/helm/datamate/charts/ray-cluster/values.yaml

@@ -113,63 +113,11 @@ head:
   securityContext: {}
   # Optional: The following volumes/volumeMounts configurations are optional but recommended because
   # Ray writes logs to /tmp/ray/session_latests/logs instead of stdout/stderr.
-  volumes:
-    - name: log-volume
-      hostPath:
-        path: /opt/datamate/data/log
-        type: DirectoryOrCreate
-    - name: dataset-volume
-      hostPath:
-        path: /opt/datamate/data/dataset
-        type: DirectoryOrCreate
-    - name: flow-volume
-      hostPath:
-        path: /opt/datamate/data/flow
-        type: DirectoryOrCreate
-  volumeMounts:
-    - mountPath: /tmp/ray
-      name: log-volume
-      subPath: ray/head
-    - mountPath: /dataset
-      name: dataset-volume
-    - mountPath: /flow
-      name: flow-volume
+  volumes: []
+  volumeMounts: []
   # sidecarContainers specifies additional containers to attach to the Ray pod.
   # Follows standard K8s container spec.
-  sidecarContainers:
-    - name: runtime
-      image: datamate-runtime
-      imagePullPolicy: IfNotPresent
-      command:
-        - python
-        - /opt/runtime/datamate/operator_runtime.py
-        - --port
-        - "8081"
-      env:
-        - name: MYSQL_HOST
-          value: "datamate-database"
-        - name: MYSQL_PORT
-          value: "3306"
-        - name: MYSQL_USER
-          value: "root"
-        - name: MYSQL_PASSWORD
-          value: "password"
-        - name: MYSQL_DATABASE
-          value: "datamate"
-        - name: PDF_FORMATTER_BASE_URL
-          value: "http://datamate-mineru:9001"
-      ports:
-        - containerPort: 8081
-      volumeMounts:
-        - mountPath: /tmp/ray
-          name: log-volume
-          subPath: ray/head
-        - mountPath: /var/log/datamate
-          name: log-volume
-        - mountPath: /dataset
-          name: dataset-volume
-        - mountPath: /flow
-          name: flow-volume
+  sidecarContainers: []
   # See docs/guidance/pod-command.md for more details about how to specify
   # container command for head Pod.
   command: []
@@ -260,27 +208,8 @@ worker:
   securityContext: {}
   # Optional: The following volumes/volumeMounts configurations are optional but recommended because
   # Ray writes logs to /tmp/ray/session_latests/logs instead of stdout/stderr.
-  volumes:
-    - name: log-volume
-      hostPath:
-        path: /opt/datamate/data/log
-        type: DirectoryOrCreate
-    - name: dataset-volume
-      hostPath:
-        path: /opt/datamate/data/dataset
-        type: DirectoryOrCreate
-    - name: flow-volume
-      hostPath:
-        path: /opt/datamate/data/flow
-        type: DirectoryOrCreate
-  volumeMounts:
-    - mountPath: /tmp/ray
-      name: log-volume
-      subPath: ray/worker
-    - mountPath: /dataset
-      name: dataset-volume
-    - mountPath: /flow
-      name: flow-volume
+  volumes: []
+  volumeMounts: []
   # sidecarContainers specifies additional containers to attach to the Ray pod.
   # Follows standard K8s container spec.
   sidecarContainers: []

deployment/helm/datamate/values.yaml

@@ -40,11 +40,17 @@ dataVolume: &dataVolume
     path: /opt/datamate/data/mysql
     type: DirectoryOrCreate
 
+operatorVolume: &operatorVolume
+  name: operator-volume
+  hostPath:
+    path: /opt/datamate/data/operator
+
 backend:
   volumes:
     - *datasetVolume
     - *flowVolume
     - *logVolume
+    - *operatorVolume
   volumeMounts:
     - name: dataset-volume
       mountPath: /dataset
@@ -52,14 +58,22 @@ backend:
       mountPath: /flow
     - name: log-volume
       mountPath: /var/log/datamate
+    - name: operator-volume
+      mountPath: /operators
 
 frontend:
   volumes:
     - *logVolume
+    - name: datamate-nginx-conf
+      configMap:
+        name: datamate-nginx-conf
   volumeMounts:
     - name: log-volume
       mountPath: /var/log/datamate/frontend
       subPath: frontend
+    - mountPath: /etc/nginx/conf.d/backend.conf
+      name: datamate-nginx-conf
+      subPath: backend.conf
 
 database:
   volumes:
@@ -81,3 +95,76 @@ database:
       mountPath: /docker-entrypoint-initdb.d
     - name: mysql-utf8-config
       mountPath: /etc/mysql/conf.d
+
+ray-cluster:
+  head:
+    volumes:
+      - *datasetVolume
+      - *flowVolume
+      - *logVolume
+      - *operatorVolume
+    volumeMounts:
+      - mountPath: /tmp/ray
+        name: log-volume
+        subPath: ray/head
+      - mountPath: /dataset
+        name: dataset-volume
+      - mountPath: /flow
+        name: flow-volume
+      - mountPath: /opt/runtime/datamate/ops/user
+        name: operator-volume
+        subPath: extract
+    sidecarContainers:
+      - name: runtime
+        image: datamate-runtime
+        imagePullPolicy: IfNotPresent
+        args:
+          - python
+          - /opt/runtime/datamate/operator_runtime.py
+          - --port
+          - "8081"
+        env:
+          - name: MYSQL_HOST
+            value: "datamate-database"
+          - name: MYSQL_PORT
+            value: "3306"
+          - name: MYSQL_USER
+            value: "root"
+          - name: MYSQL_PASSWORD
+            value: "password"
+          - name: MYSQL_DATABASE
+            value: "datamate"
+          - name: PDF_FORMATTER_BASE_URL
+            value: "http://datamate-mineru:9001"
+        ports:
+          - containerPort: 8081
+        volumeMounts:
+          - mountPath: /tmp/ray
+            name: log-volume
+            subPath: ray/head
+          - mountPath: /var/log/datamate
+            name: log-volume
+          - mountPath: /dataset
+            name: dataset-volume
+          - mountPath: /flow
+            name: flow-volume
+          - mountPath: /opt/runtime/datamate/ops/user
+            name: operator-volume
+            subPath: extract
+  worker:
+    volumes:
+      - *datasetVolume
+      - *flowVolume
+      - *logVolume
+      - *operatorVolume
+    volumeMounts:
+      - mountPath: /tmp/ray
+        name: log-volume
+        subPath: ray/worker
+      - mountPath: /dataset
+        name: dataset-volume
+      - mountPath: /flow
+        name: flow-volume
+      - mountPath: /opt/runtime/datamate/ops/user
+        name: operator-volume
+        subPath: extract