[Feature] Refactor project to use 'datamate' naming convention for services and configurations (#14)

* Enhance CleaningTaskService to track cleaning process progress and update ExecutorType to DATAMATE * Refactor project to use 'datamate' naming convention for services and configurations
2025-10-22 17:53:16 +08:00
parent 175d9ded93
commit 31ef8bc265
39 changed files with 312 additions and 737 deletions
--- a/deployment/docker/datamate/docker-compose.yml
+++ b/deployment/docker/datamate/docker-compose.yml
@@ -1,63 +1,70 @@
 services:
  # 1) backend
-  backend:
-    container_name: backend
-    image: backend
+  datamate-backend:
+    container_name: datamate-backend
+    image: datamate-backend
    restart: on-failure
    privileged: true
    ports:
-      - "8080:8080"
+      - "8080"
    volumes:
      - dataset_volume:/dataset
      - flow_volume:/flow
-      - log_volume:/var/log/data-mate
-    networks: [ edatamate ]
+      - log_volume:/var/log/datamate
+    networks: [ datamate ]
    depends_on:
-      - mysql
+      - datamate-database

  # 2) frontend（NodePort 30000）
-  frontend:
-    container_name: frontend
-    image: frontend
+  datamate-frontend:
+    container_name: datamate-frontend
+    image: datamate-frontend
    restart: on-failure
    ports:
      - "30000:80"            # nodePort → hostPort
    volumes:
-      - log_volume:/var/log/data-mate
-    networks: [ edatamate ]
+      - frontend_log_volume:/var/log/datamate/frontend
+    networks: [ datamate ]
    depends_on:
-      - backend
+      - datamate-backend

-  # 3) mysql
-  mysql:
-    container_name: mysql
+  # 3) database
+  datamate-database:
+    container_name: datamate-database
    image: mysql:8
    restart: on-failure
    environment:
      MYSQL_ROOT_PASSWORD: Huawei@123
    ports:
-      - "3306:3306"
+      - "3306"
+    command: |
+      sh -c "
+        chown mysql:mysql /var/log/datamate/database &&
+        chmod 755 /var/log/datamate/database &&
+        exec docker-entrypoint.sh mysqld
+      "
    volumes:
      - mysql_volume:/var/lib/mysql
      - ../../../scripts/db:/docker-entrypoint-initdb.d
-      - ./utf8.cnf:/etc/mysql/conf.d/utf8.cnf
-      - log_volume:/var/log/data-mate
-    networks: [ edatamate ]
+      - ./utf8.cnf:/etc/mysql/conf.d/utf8.cnf:ro
+      - database_log_volume:/var/log/datamate/database
+    networks: [ datamate ]

-  runtime:
-    container_name: runtime
-    image: runtime
+  # 3) runtime
+  datamate-runtime:
+    container_name: datamate-runtime
+    image: datamate-runtime
    restart: on-failure
    environment:
      RAY_DEDUP_LOGS: "0"
      RAY_TQDM_PATCH_PRINT: "0"
-      MYSQL_HOST: "mysql"
+      MYSQL_HOST: "datamate-database"
      MYSQL_PORT: "3306"
      MYSQL_USER: "root"
      MYSQL_PASSWORD: "Huawei@123"
      MYSQL_DATABASE: "datamate"
    ports:
-      - "8081:8081"
+      - "8081"
    command:
      - python
      - /opt/runtime/datamate/operator_runtime.py
@@ -65,22 +72,27 @@ services:
      - "8081"
    volumes:
      - ray_log_volume:/tmp/ray
-      - log_volume:/var/log/data-mate
+      - log_volume:/var/log/datamate
      - dataset_volume:/dataset
      - flow_volume:/flow
+    networks: [ datamate ]

 volumes:
  dataset_volume:
-    name: data-mate-dataset-volume
+    name: datamate-dataset-volume
  flow_volume:
-    name: data-mate-flow-volume
+    name: datamate-flow-volume
  log_volume:
-    name: data-mate-log-volume
+    name: datamate-log-volume
  mysql_volume:
-    name: data-mate-mysql-volume
+    name: datamate-mysql-volume
  ray_log_volume:
-    name: data-mate-ray-log-volume
+    name: datamate-ray-log-volume
+  frontend_log_volume:
+    name: datamate-frontend-log-volume
+  database_log_volume:
+    name: datamate-database-log-volume

 networks:
-  edatamate:
+  datamate:
    driver: bridge
--- a/deployment/docker/datamate/utf8.cnf
+++ b/deployment/docker/datamate/utf8.cnf
@@ -5,6 +5,7 @@ character-set-server = utf8mb4
 collation-server = utf8mb4_unicode_ci
 # 或者使用 utf8_general_ci (性能稍好，但排序规则稍宽松)
 default-time-zone = 'Asia/Shanghai'
+log_error=/var/log/datamate/database/error.log

 [client]
 # 设置客户端连接默认字符集
@@ -12,4 +13,4 @@ default-character-set = utf8mb4

 [mysql]
 # 设置 mysql 命令行客户端默认字符集
-default-character-set = utf8mb4
+default-character-set = utf8mb4
--- a/deployment/helm/ray/kuberay-operator/values.yaml
+++ b/deployment/helm/ray/kuberay-operator/values.yaml
@@ -3,13 +3,13 @@
 # Declare variables to be passed into your templates.

 # -- String to partially override release name.
-nameOverride: kuberay-operator
+nameOverride: datamate-kuberay-operator

 # -- String to fully override release name.
-fullnameOverride: kuberay-operator
+fullnameOverride: datamate-kuberay-operator

 # -- String to override component name.
-componentOverride: kuberay-operator
+componentOverride: datamate-kuberay-operator

 image:
  # -- Image repository.
@@ -32,7 +32,7 @@ serviceAccount:
  create: true
  # -- The name of the service account to use.
  # If not set and create is true, a name is generated using the fullname template.
-  name: kuberay-operator
+  name: datamate-kuberay-operator

 logging:
  # -- Log encoder to use for stdout (one of `json` or `console`).
@@ -88,7 +88,7 @@ featureGates:
 # Configurations for KubeRay operator metrics.
 metrics:
  # -- Whether KubeRay operator should emit control plane metrics.
-  enabled: true
+  enabled: false
  serviceMonitor:
    # -- Enable a prometheus ServiceMonitor
    enabled: false
--- a/deployment/helm/ray/ray-cluster/values.yaml
+++ b/deployment/helm/ray/ray-cluster/values.yaml
@@ -6,12 +6,12 @@
 # in this Helm chart.

 image:
-  repository: runtime
+  repository: datamate-runtime
  tag: latest
  pullPolicy: IfNotPresent

 nameOverride: "kuberay"
-fullnameOverride: ""
+fullnameOverride: "datamate-raycluster"

 imagePullSecrets: []
 # - name: an-existing-secret
@@ -68,7 +68,7 @@ head:
    - name: RAY_TQDM_PATCH_PRINT
      value: "0"
    - name: MYSQL_HOST
-      value: "mysql"
+      value: "datamate-database"
    - name: MYSQL_PORT
      value: "3306"
    - name: MYSQL_USER
@@ -114,15 +114,15 @@ head:
  volumes:
    - name: log-volume
      hostPath:
-        path: /opt/data-mate/data/log
+        path: /opt/datamate/data/log
        type: DirectoryOrCreate
    - name: dataset-volume
      hostPath:
-        path: /opt/data-mate/data/dataset
+        path: /opt/datamate/data/dataset
        type: DirectoryOrCreate
    - name: flow-volume
      hostPath:
-        path: /opt/data-mate/data/flow
+        path: /opt/datamate/data/flow
        type: DirectoryOrCreate
  volumeMounts:
    - mountPath: /tmp/ray
@@ -136,7 +136,7 @@ head:
  # Follows standard K8s container spec.
  sidecarContainers:
    - name: runtime
-      image: runtime
+      image: datamate-runtime
      imagePullPolicy: IfNotPresent
      command:
        - python
@@ -145,7 +145,7 @@ head:
        - "8081"
      env:
        - name: MYSQL_HOST
-          value: "mysql"
+          value: "datamate-database"
        - name: MYSQL_PORT
          value: "3306"
        - name: MYSQL_USER
@@ -160,7 +160,7 @@ head:
        - mountPath: /tmp/ray
          name: log-volume
          subPath: ray/head
-        - mountPath: /var/log/data-mate
+        - mountPath: /var/log/datamate
          name: log-volume
        - mountPath: /dataset
          name: dataset-volume
@@ -212,7 +212,7 @@ worker:
    - name: RAY_TQDM_PATCH_PRINT
      value: "0"
    - name: MYSQL_HOST
-      value: "mysql"
+      value: "datamate-database"
    - name: MYSQL_PORT
      value: "3306"
    - name: MYSQL_USER
@@ -257,15 +257,15 @@ worker:
  volumes:
    - name: log-volume
      hostPath:
-        path: /opt/data-mate/data/log
+        path: /opt/datamate/data/log
        type: DirectoryOrCreate
    - name: dataset-volume
      hostPath:
-        path: /opt/data-mate/data/dataset
+        path: /opt/datamate/data/dataset
        type: DirectoryOrCreate
    - name: flow-volume
      hostPath:
-        path: /opt/data-mate/data/flow
+        path: /opt/datamate/data/flow
        type: DirectoryOrCreate
  volumeMounts:
    - mountPath: /tmp/ray
@@ -350,15 +350,15 @@ additionalWorkerGroups:
    volumes:
      - name: log-volume
        hostPath:
-          path: /opt/data-mate/data/log
+          path: /opt/datamate/data/log
          type: DirectoryOrCreate
      - name: dataset-volume
        hostPath:
-          path: /opt/data-mate/data/dataset
+          path: /opt/datamate/data/dataset
          type: DirectoryOrCreate
      - name: flow-volume
        hostPath:
-          path: /opt/data-mate/data/flow
+          path: /opt/datamate/data/flow
          type: DirectoryOrCreate
    volumeMounts:
      - mountPath: /tmp/ray
--- a/deployment/helm/ray/service.yaml
+++ b/deployment/helm/ray/service.yaml
@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: runtime
+  name: datamate-runtime
  labels:
    ray.io/node-type: head
 spec:
--- a/deployment/kubernetes/backend/deploy.yaml
+++ b/deployment/kubernetes/backend/deploy.yaml
@@ -2,8 +2,9 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
  labels:
-    app: backend
-  name: backend
+    app: datamate
+    tier: backend
+  name: datamate-backend
 rules:
  - verbs:
      - create
@@ -33,83 +34,89 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
  labels:
-    app: backend
-  name: backend
+    app: datamate
+    tier: backend
+  name: datamate-backend

 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
  labels:
-    app: backend
-  name: backend
+    app: datamate
+    tier: backend
+  name: datamate-backend
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
-  name: backend
+  name: datamate-backend
 subjects:
  - kind: ServiceAccount
-    name: backend
+    name: datamate-backend

 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: backend
+  name: datamate-backend
  labels:
-    app: backend
+    app: datamate
+    tier: backend
 spec:
  replicas: 1
  selector:
    matchLabels:
-      app: backend
+      app: datamate
+      tier: backend
  template:
    metadata:
      labels:
-        app: backend
+        app: datamate
+        tier: backend
    spec:
-      serviceAccountName: backend
+      serviceAccountName: datamate-backend
      containers:
-      - name: backend
-        image: backend
-        imagePullPolicy: IfNotPresent
-        env:
-          - name: namespace
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.namespace
-          - name: SPRING_CONFIG_LOCATION
-            value: file:/opt/backend/application.yml
-        ports:
-        - containerPort: 8080
-        volumeMounts:
-        - name: dataset-volume
-          mountPath: /dataset
-        - name: flow-volume
-          mountPath: /flow
-        - name: log-volume
-          mountPath: /var/log/data-mate
+        - name: backend
+          image: datamate-backend
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: namespace
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: SPRING_CONFIG_LOCATION
+              value: file:/opt/backend/application.yml
+          ports:
+            - containerPort: 8080
+          volumeMounts:
+            - name: dataset-volume
+              mountPath: /dataset
+            - name: flow-volume
+              mountPath: /flow
+            - name: log-volume
+              mountPath: /var/log/datamate
      volumes:
-      - name: dataset-volume
-        hostPath:
-          path: /opt/data-mate/data/dataset
-          type: DirectoryOrCreate
-      - name: flow-volume
-        hostPath:
-          path: /opt/data-mate/data/flow
-          type: DirectoryOrCreate
-      - name: log-volume
-        hostPath:
-          path: /opt/data-mate/data/log
-          type: DirectoryOrCreate
+        - name: dataset-volume
+          hostPath:
+            path: /opt/datamate/data/dataset
+            type: DirectoryOrCreate
+        - name: flow-volume
+          hostPath:
+            path: /opt/datamate/data/flow
+            type: DirectoryOrCreate
+        - name: log-volume
+          hostPath:
+            path: /opt/datamate/data/log
+            type: DirectoryOrCreate

 ---
 apiVersion: v1
 kind: Service
 metadata:
-  name: backend
+  name: datamate-backend
  labels:
-    app: backend
+    app: datamate
+    tier: backend
 spec:
  type: ClusterIP
  ports:
@@ -117,4 +124,5 @@ spec:
      targetPort: 8080
      protocol: TCP
  selector:
-    app: backend
+    app: datamate
+    tier: backend
--- a/deployment/kubernetes/datax/deploy.yaml
+++ b/deployment/kubernetes/datax/deploy.yaml
@@ -1,54 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: datax
-  labels:
-    app: datax
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: datax
-  template:
-    metadata:
-      labels:
-        app: datax
-    spec:
-      containers:
-      - name: datax
-        image: datax
-        imagePullPolicy: IfNotPresent
-        securityContext:
-          capabilities:
-            add:
-              - SYS_ADMIN
-        command:
-          - bash
-          - -c
-          - rpcbind && python3 /opt/datax/bin/app.py
-        ports:
-        - containerPort: 8000
-        volumeMounts:
-        - name: dataset
-          mountPath: /dataset
-          subPath: dataset
-      volumes:
-      - name: dataset
-        hostPath:
-          path: /tmp/data-mate
-
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: datax
-  labels:
-    app: datax
-spec:
-  type: ClusterIP
-  ports:
-    - port: 8000
-      targetPort: 8000
-      protocol: TCP
-  selector:
-    app: datax
--- a/deployment/kubernetes/frontend/deploy.yaml
+++ b/deployment/kubernetes/frontend/deploy.yaml
@@ -1,32 +1,45 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: frontend
+  name: datamate-frontend
  labels:
-    app: frontend
+    app: datamate
+    tier: frontend
 spec:
  replicas: 1
  selector:
    matchLabels:
-      app: frontend
+      app: datamate
+      tier: frontend
  template:
    metadata:
      labels:
-        app: frontend
+        app: datamate
+        tier: frontend
    spec:
      containers:
-      - name: frontend
-        image: frontend
-        imagePullPolicy: IfNotPresent
-        ports:
-        - containerPort: 80
+        - name: frontend
+          image: datamate-frontend
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 80
+          volumeMounts:
+            - name: log-volume
+              mountPath: /var/log/datamate/frontend
+              subPath: frontend
+      volumes:
+        - name: log-volume
+          hostPath:
+            path: /opt/datamate/data/log
+            type: DirectoryOrCreate
 ---
 apiVersion: v1
 kind: Service
 metadata:
-  name: frontend
+  name: datamate-frontend
  labels:
-    app: frontend
+    app: datamate
+    tier: frontend
 spec:
  type: NodePort
  ports:
@@ -35,4 +48,5 @@ spec:
      nodePort: 30000
      protocol: TCP
  selector:
-    app: frontend
+    app: datamate
+    tier: frontend
--- a/deployment/kubernetes/mineru/deploy.yaml
+++ b/deployment/kubernetes/mineru/deploy.yaml
@@ -1,52 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: mineru
-  labels:
-    app: mineru
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: mineru
-  template:
-    metadata:
-      labels:
-        app: mineru
-    spec:
-      containers:
-      - name: mineru
-        image: mineru
-        imagePullPolicy: IfNotPresent
-        command:
-          - mineru-api
-        args:
-          - --host
-          - "0.0.0.0"
-          - --port
-          - "8000"
-        ports:
-        - containerPort: 8000
-        volumeMounts:
-        - name: tmp
-          mountPath: /tmp/data-mate
-      volumes:
-      - name: tmp
-        hostPath:
-          path: /tmp/data-mate
-
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: mineru
-  labels:
-    app: mineru
-spec:
-  type: ClusterIP
-  ports:
-    - port: 8000
-      targetPort: 8000
-      protocol: TCP
-  selector:
-    app: mineru
--- a/deployment/kubernetes/mysql/configmap.yaml
+++ b/deployment/kubernetes/mysql/configmap.yaml
@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: mysql-utf8-config
+  name: datamate-mysql-utf8-config
 data:
  utf8.cnf: |
    [mysqld]
@@ -11,6 +11,7 @@ data:
    collation-server = utf8mb4_unicode_ci
    # 或者使用 utf8_general_ci (性能稍好，但排序规则稍宽松)
    default-time-zone = 'Asia/Shanghai'
+    log_error=/var/log/datamate/database/error.log

    [client]
    # 设置客户端连接默认字符集
@@ -18,4 +19,4 @@ data:

    [mysql]
    # 设置 mysql 命令行客户端默认字符集
-    default-character-set = utf8mb4
+    default-character-set = utf8mb4
--- a/deployment/kubernetes/mysql/deploy.yaml
+++ b/deployment/kubernetes/mysql/deploy.yaml
@@ -1,21 +1,39 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: mysql
+  name: datamate-database
  labels:
-    app: mysql
+    app: datamate
+    tier: database
 spec:
  replicas: 1
  selector:
    matchLabels:
-      app: mysql
+      app: datamate
+      tier: database
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
-        app: mysql
+        app: datamate
+        tier: database
    spec:
+      initContainers:
+        - name: init-log
+          image: mysql:8
+          imagePullPolicy: IfNotPresent
+          command:
+            - sh
+            - -c
+          args:
+            - |
+              chown mysql:mysql /var/log/datamate/database
+              chmod 755 /var/log/datamate/database
+          volumeMounts:
+            - name: log-volume
+              mountPath: /var/log/datamate/database
+              subPath: database
      containers:
        - name: mysql
          image: mysql:8
@@ -28,6 +46,9 @@ spec:
          volumeMounts:
            - name: data-volume
              mountPath: /var/lib/mysql
+            - name: log-volume
+              mountPath: /var/log/datamate/database
+              subPath: database
            - name: init-sql
              mountPath: /docker-entrypoint-initdb.d
            - name: mysql-utf8-config
@@ -35,22 +56,27 @@ spec:
      volumes:
        - name: data-volume
          hostPath:
-            path: /opt/data-mate/data/mysql
+            path: /opt/datamate/data/mysql
+            type: DirectoryOrCreate
+        - name: log-volume
+          hostPath:
+            path: /opt/datamate/data/log
            type: DirectoryOrCreate
        - name: init-sql
          configMap:
-            name: init-sql
+            name: datamate-init-sql
        - name: mysql-utf8-config
          configMap:
-            name: mysql-utf8-config
+            name: datamate-mysql-utf8-config

 ---
 apiVersion: v1
 kind: Service
 metadata:
-  name: mysql
+  name: datamate-database
  labels:
-    app: mysql
+    app: datamate
+    tier: database
 spec:
  type: ClusterIP
  ports:
@@ -58,4 +84,5 @@ spec:
      targetPort: 3306
      protocol: TCP
  selector:
-    app: mysql
+    app: datamate
+    tier: database
--- a/deployment/kubernetes/unstructured/deploy.yaml
+++ b/deployment/kubernetes/unstructured/deploy.yaml
@@ -1,49 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: unstructured
-  labels:
-    app: unstructured
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: unstructured
-  template:
-    metadata:
-      labels:
-        app: unstructured
-    spec:
-      containers:
-      - name: unstructured
-        image: unstructured
-        imagePullPolicy: IfNotPresent
-        command:
-          - python
-        args:
-          - app.py
-        ports:
-        - containerPort: 8000
-        volumeMounts:
-        - name: tmp
-          mountPath: /tmp/data-mate
-      volumes:
-      - name: tmp
-        hostPath:
-          path: /tmp/data-mate
-
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: unstructured
-  labels:
-    app: unstructured
-spec:
-  type: ClusterIP
-  ports:
-    - port: 8000
-      targetPort: 8000
-      protocol: TCP
-  selector:
-    app: unstructured