DataMate/deployment/helm/datamate/values.yaml

# Default values for datamate.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.

global:
  namespace: datamate
  image:
    repository: "ghcr.io/modelengine-group/"
    pullPolicy: "IfNotPresent"
    backend:
      name: "datamate-backend"
      tag: "latest"
    backendPython:
      name: "datamate-backend-python"
      tag: "latest"
    gateway:
      name: "datamate-gateway"
      tag: "latest"
    frontend:
      name: "datamate-frontend"
      tag: "latest"
    runtime:
      name: "datamate-runtime"
      tag: "latest"
    database:
      name: "datamate-database"
      tag: "latest"

public:
  persistentVolumeClaim:
    storageClass: ""
    storagePath: ""
    storageNode: ""
    size:
      dataset: 10Gi
      flow: 1Gi
      log: 1Gi
      database: 1Gi
      operator: 1Gi

datasetVolume: &datasetVolume
  name: dataset-volume
  persistentVolumeClaim:
    claimName: datamate-dataset-pvc

flowVolume: &flowVolume
  name: flow-volume
  persistentVolumeClaim:
    claimName: datamate-flow-pvc

logVolume: &logVolume
  name: log-volume
  persistentVolumeClaim:
    claimName: datamate-log-pvc

dataVolume: &dataVolume
  name: data-volume
  persistentVolumeClaim:
    claimName: datamate-database-pvc

operatorVolume: &operatorVolume
  name: operator-volume
  persistentVolumeClaim:
    claimName: datamate-operator-pvc

database:
  env:
    - name: MYSQL_ROOT_PASSWORD
      value: &dbPass "password"
  volumes:
    - *dataVolume
    - *logVolume
  volumeMounts:
    - name: data-volume
      mountPath: /var/lib/mysql
    - name: log-volume
      mountPath: /var/log/datamate/database
      subPath: database

backend:
  securityContext:
    capabilities:
      add:
        - SYS_ADMIN
  env:
    - name: DB_PASSWORD
      value: *dbPass
    - name: datamate.rag.milvus-uri
      value: "http://milvus:19530"
  volumes:
    - *datasetVolume
    - *flowVolume
    - *logVolume
    - *operatorVolume
  volumeMounts:
    - name: dataset-volume
      mountPath: /dataset
    - name: flow-volume
      mountPath: /flow
    - name: log-volume
      mountPath: /var/log/datamate
    - name: operator-volume
      mountPath: /operators

backend-python:
  env:
    - name: DB_PASSWORD
      value: *dbPass
  volumes:
    - *datasetVolume
    - *flowVolume
    - *logVolume
  volumeMounts:
    - name: dataset-volume
      mountPath: /dataset
    - name: flow-volume
      mountPath: /flow
    - name: log-volume
      mountPath: /var/log/datamate

gateway:
  service:
    type: NodePort
    nodePort: 30000
  volumes:
    - *logVolume
  volumeMounts:
    - name: log-volume
      mountPath: /var/log/datamate/gateway
      subPath: gateway

frontend:
  service:
    type: NodePort
    nodePort: 30000
  volumes:
    - *logVolume
    - name: datamate-nginx-conf
      configMap:
        name: datamate-nginx-conf
  volumeMounts:
    - name: log-volume
      mountPath: /var/log/datamate/frontend
      subPath: frontend
    - mountPath: /etc/nginx/conf.d/backend.conf
      name: datamate-nginx-conf
      subPath: backend.conf

runtime:
  enabled: false
  args: &runtimeArgs
    - python
    - /opt/runtime/datamate/operator_runtime.py
    - --port
    - "8081"
  env: &runtimeEnv
    - name: MYSQL_HOST
      value: "datamate-database"
    - name: MYSQL_PORT
      value: "3306"
    - name: MYSQL_USER
      value: "root"
    - name: MYSQL_PASSWORD
      value: *dbPass
    - name: MYSQL_DATABASE
      value: "datamate"
  volumes:
    - *datasetVolume
    - *flowVolume
    - *logVolume
    - *operatorVolume
  volumeMounts: &runtimeVolumeMounts
    - mountPath: /tmp/ray
      name: log-volume
      subPath: ray/head
    - mountPath: /var/log/datamate
      name: log-volume
    - mountPath: /dataset
      name: dataset-volume
    - mountPath: /flow
      name: flow-volume
    - mountPath: /opt/runtime/datamate/ops/user
      name: operator-volume
      subPath: extract
    - mountPath: /usr/local/lib/ops/site-packages
      name: operator-volume
      subPath: site-packages

ray-cluster:
  enabled: true
  head:
    enableInTreeAutoscaling: true
    autoscalerOptions:
      upscalingMode: Default
      idleTimeoutSeconds: 60
      imagePullPolicy: IfNotPresent
      resources:
        limits:
          cpu: "500m"
          memory: "512Mi"
        requests:
          cpu: "500m"
          memory: "512Mi"
    rayStartParams:
      num-cpus: "0"
    containerEnv:
      - name: RAY_DEDUP_LOGS
        value: "0"
      - name: RAY_TQDM_PATCH_PRINT
        value: "0"
      - name: MYSQL_HOST
        value: "datamate-database"
      - name: MYSQL_PORT
        value: "3306"
      - name: MYSQL_USER
        value: "root"
      - name: MYSQL_PASSWORD
        value: *dbPass
      - name: MYSQL_DATABASE
        value: "datamate"
      - name: RAY_enable_autoscaler_v2
        value: "1"
    resources:
      limits:
        cpu: "4"
        memory: "16G"
      requests:
        cpu: "1"
        memory: "2G"
    volumes:
      - *datasetVolume
      - *flowVolume
      - *logVolume
      - *operatorVolume
    volumeMounts:
      - mountPath: /tmp/ray
        name: log-volume
        subPath: ray/head
      - mountPath: /dataset
        name: dataset-volume
      - mountPath: /flow
        name: flow-volume
      - mountPath: /opt/runtime/datamate/ops/user
        name: operator-volume
        subPath: extract
      - mountPath: /usr/local/lib/ops/site-packages
        name: operator-volume
        subPath: site-packages
    sidecarContainers:
      - name: runtime
        imagePullPolicy: IfNotPresent
        args: *runtimeArgs
        env: *runtimeEnv
        ports:
          - containerPort: 8081
        volumeMounts: *runtimeVolumeMounts
  worker:
    containerEnv:
      - name: RAY_DEDUP_LOGS
        value: "0"
      - name: RAY_TQDM_PATCH_PRINT
        value: "0"
      - name: MYSQL_HOST
        value: "datamate-database"
      - name: MYSQL_PORT
        value: "3306"
      - name: MYSQL_USER
        value: "root"
      - name: MYSQL_PASSWORD
        value: *dbPass
      - name: MYSQL_DATABASE
        value: "datamate"
    resources:
      limits:
        cpu: "8"
        memory: "64G"
      requests:
        cpu: "1"
        memory: "2G"
    volumes:
      - *datasetVolume
      - *flowVolume
      - *logVolume
      - *operatorVolume
    volumeMounts:
      - mountPath: /tmp/ray
        name: log-volume
        subPath: ray/worker
      - mountPath: /dataset
        name: dataset-volume
      - mountPath: /flow
        name: flow-volume
      - mountPath: /opt/runtime/datamate/ops/user
        name: operator-volume
        subPath: extract
      - mountPath: /usr/local/lib/ops/site-packages
        name: operator-volume
        subPath: site-packages
  additionalWorkerGroups:
    npuGroup:
      disabled: false
      replicas: 0
      minReplicas: 0
      maxReplicas: 8
      rayStartParams:
        resources: '"{\"npu\": 1}"'
      containerEnv:
        - name: RAY_DEDUP_LOGS
          value: "0"
        - name: RAY_TQDM_PATCH_PRINT
          value: "0"
        - name: MYSQL_HOST
          value: "datamate-database"
        - name: MYSQL_PORT
          value: "3306"
        - name: MYSQL_USER
          value: "root"
        - name: MYSQL_PASSWORD
          value: *dbPass
        - name: MYSQL_DATABASE
          value: "datamate"
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
      resources:
        limits:
          cpu: "8"
          memory: "64G"
          huawei.com/Ascend910: 1
        requests:
          cpu: "1"
          memory: "2G"
          huawei.com/Ascend910: 1
      volumes:
        - *datasetVolume
        - *flowVolume
        - *logVolume
        - *operatorVolume
        - name: ascend
          hostPath:
            path: /usr/local/Ascend
      volumeMounts:
        - mountPath: /tmp/ray
          name: log-volume
          subPathExpr: ray/$(POD_NAME)
        - mountPath: /dataset
          name: dataset-volume
        - mountPath: /flow
          name: flow-volume
        - mountPath: /opt/runtime/datamate/ops/user
          name: operator-volume
          subPath: extract
        - mountPath: /usr/local/lib/ops/site-packages
          name: operator-volume
          subPath: site-packages
        - mountPath: /usr/local/Ascend
          name: ascend