feature: 数据库镜像制作 (#70)

* feature: 数据库镜像制作

* feature: 增加归档包流水线

.github/workflows/docker-image-backend-python.yml

@@ -14,6 +14,7 @@ on:
       - 'scripts/images/datamate-python/**'
       - '.github/workflows/docker-image-backend-python.yml'
   workflow_dispatch:
+  workflow_call:
 
 jobs:
   call-docker-build:

.github/workflows/docker-image-backend.yml

@@ -14,6 +14,7 @@ on:
       - 'scripts/images/backend/**'
       - '.github/workflows/docker-image-backend.yml'
   workflow_dispatch:
+  workflow_call:
 
 jobs:
   call-docker-build:

.github/workflows/docker-image-database.yml

@@ -0,0 +1,28 @@
+name: Database Docker Image CI
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'scripts/db/**'
+      - 'scripts/images/database/**'
+      - '.github/workflows/docker-image-database.yml'
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - 'scripts/db/**'
+      - 'scripts/images/database/**'
+      - '.github/workflows/docker-image-database.yml'
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  call-docker-build:
+    name: Build and Push Database Docker Image
+    uses: ./.github/workflows/docker-images-reusable.yml
+    permissions:
+      contents: read
+      packages: write
+    with:
+      service_name: database
+      build_dir: .

.github/workflows/docker-image-deer-flow.yml

@@ -16,6 +16,7 @@ on:
       - 'scripts/images/deer-flow-frontend/**'
       - '.github/workflows/docker-image-deer-flow.yml'
   workflow_dispatch:
+  workflow_call:
 
 jobs:
   build-and-push:

.github/workflows/docker-image-frontend.yml

@@ -14,6 +14,7 @@ on:
       - 'scripts/images/frontend/**'
       - '.github/workflows/docker-image-frontend.yml'
   workflow_dispatch:
+  workflow_call:
 
 jobs:
   call-docker-build:

.github/workflows/docker-image-runtime.yml

@@ -4,16 +4,19 @@ on:
   push:
     branches: [ "main" ]
     paths:
-      - 'runtime/**'
+      - 'runtime/ops/**'
+      - 'runtime/python-executor/**'
       - 'scripts/images/runtime/**'
       - '.github/workflows/docker-image-runtime.yml'
   pull_request:
     branches: [ "main" ]
     paths:
-      - 'runtime/**'
+      - 'runtime/ops/**'
+      - 'runtime/python-executor/**'
       - 'scripts/images/runtime/**'
       - '.github/workflows/docker-image-runtime.yml'
   workflow_dispatch:
+  workflow_call:
 
 jobs:
   call-docker-build:

.github/workflows/docker-images-reusable.yml

@@ -51,7 +51,18 @@ jobs:
           docker tag datamate-${{ inputs.service_name }}:latest ${{ steps.set-tag.outputs.TAGS }}
 
       - name: Push Docker Image
-        if: github.event_name != 'pull_request'
+        if: github.event_name != 'pull_request' && !startsWith(github.workflow, 'Package')
         run: |
           docker push ${{ steps.set-tag.outputs.TAGS }}
 
+      - name: Save Docker Image
+        if: startsWith(github.workflow, 'Package')
+        run: |
+          docker save -o datamate-${{ inputs.service_name }}.tar ${{ steps.set-tag.outputs.TAGS }}
+
+      - name: Upload Docker Image
+        if: startsWith(github.workflow, 'Package')
+        uses: actions/upload-artifact@v4
+        with:
+          name: datamate-${{ inputs.service_name }}
+          path: datamate-${{ inputs.service_name }}.tar

.github/workflows/package.yml

@@ -0,0 +1,58 @@
+name: Package All
+
+on:
+  workflow_dispatch:
+
+jobs:
+  backend-docker-build:
+    name: Build and Push Backend Docker Image
+    uses: ./.github/workflows/docker-image-backend.yml
+
+  frontend-docker-build:
+    name: Build and Push Frontend Docker Image
+    uses: ./.github/workflows/docker-image-frontend.yml
+
+  database-docker-build:
+    name: Build and Push Database Docker Image
+    uses: ./.github/workflows/docker-image-database.yml
+
+  runtime-docker-build:
+    name: Build and Push Runtime Docker Image
+    uses: ./.github/workflows/docker-image-runtime.yml
+
+  backend-python-docker-build:
+    name: Build and Push Backend Python Docker Image
+    uses: ./.github/workflows/docker-image-backend-python.yml
+
+  package-all:
+    needs:
+      - backend-docker-build
+      - frontend-docker-build
+      - database-docker-build
+      - backend-python-docker-build
+      - runtime-docker-build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Save Image
+        uses: actions/download-artifact@v5
+        with:
+          path: images
+          pattern: datamate-*
+          merge-multiple: true
+
+      - name: Package
+        run: |
+          cp runtime/deer-flow/.env.example deployment/docker/deer-flow/.env
+          cp runtime/deer-flow/conf.yaml.example deployment/docker/deer-flow/conf.yaml
+          cp runtime/deer-flow/.env.example deployment/helm/deer-flow/charts/public/.env
+          cp runtime/deer-flow/conf.yaml.example deployment/helm/deer-flow/charts/public/conf.yaml
+          tar -cvf datamate.tar deployment/ images
+
+      - name: Upload Package
+        uses: actions/upload-artifact@v4
+        with:
+          name: package
+          path: datamate.tar

Makefile

@@ -18,7 +18,7 @@ build-%:
 	$(MAKE) $*-docker-build
 
 .PHONY: build
-build: backend-docker-build frontend-docker-build runtime-docker-build
+build: database-docker-build backend-docker-build frontend-docker-build runtime-docker-build
 
 .PHONY: create-namespace
 create-namespace:
@@ -105,6 +105,10 @@ endif
 backend-docker-build:
 	docker build -t datamate-backend:$(VERSION) . -f scripts/images/backend/Dockerfile
 
+.PHONY: database-docker-build
+database-docker-build:
+	docker build -t datamate-database:$(VERSION) . -f scripts/images/database/Dockerfile
+
 .PHONY: frontend-docker-build
 frontend-docker-build:
 	docker build -t datamate-frontend:$(VERSION) . -f scripts/images/frontend/Dockerfile
@@ -206,20 +210,18 @@ milvus-docker-uninstall:
 
 .PHONY: datamate-k8s-install
 datamate-k8s-install: create-namespace
-	kubectl create configmap datamate-init-sql --from-file=scripts/db/ --dry-run=client -o yaml | kubectl apply -f - -n $(NAMESPACE)
-	helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.image.repository=$(REPOSITORY)
+	helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY)
 
 .PHONY: datamate-k8s-uninstall
 datamate-k8s-uninstall:
 	helm uninstall datamate -n $(NAMESPACE) --ignore-not-found
-	kubectl delete configmap datamate-init-sql -n $(NAMESPACE) --ignore-not-found
 
 .PHONY: deer-flow-k8s-install
 deer-flow-k8s-install:
-	helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.deerFlow.enable=true --set global.image.repository=$(REPOSITORY)
+	helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.deerFlow.enable=true --set global.image.repository=$(REGISTRY)
 	cp runtime/deer-flow/.env deployment/helm/deer-flow/charts/public/.env
 	cp runtime/deer-flow/conf.yaml deployment/helm/deer-flow/charts/public/conf.yaml
-	helm upgrade deer-flow deployment/helm/deer-flow -n $(NAMESPACE) --install --set global.image.repository=$(REPOSITORY)
+	helm upgrade deer-flow deployment/helm/deer-flow -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY)
 
 .PHONY: deer-flow-k8s-uninstall
 deer-flow-k8s-uninstall:

deployment/docker/datamate/docker-compose.yml

@@ -49,7 +49,7 @@ services:
   # 3) database
   datamate-database:
     container_name: datamate-database
-    image: mysql:8
+    image: ${REGISTRY:-}datamate-database
     restart: on-failure
     environment:
       MYSQL_ROOT_PASSWORD: password
@@ -61,8 +61,6 @@ services:
       "
     volumes:
       - mysql_volume:/var/lib/mysql
-      - ../../../scripts/db:/docker-entrypoint-initdb.d
-      - ./utf8.cnf:/etc/mysql/conf.d/utf8.cnf:ro
       - database_log_volume:/var/log/datamate/database
     ports:
       - "3306:3306"

deployment/helm/datamate/charts/database/templates/_helpers.tpl

@@ -67,5 +67,9 @@ Name of image
 {{- define "database.image" -}}
 {{- $name := default .Values.image.repository .Values.global.image.database.name }}
 {{- $tag := default .Values.image.tag .Values.global.image.database.tag }}
+{{- if .Values.global.image.repository }}
+{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}:{{ $tag }}
+{{- else }}
 {{- $name }}:{{ $tag }}
 {{- end }}
+{{- end }}

deployment/helm/datamate/charts/database/templates/configmap.yaml

@@ -1,22 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: datamate-mysql-utf8-config
-data:
-  utf8.cnf: |
-    [mysqld]
-    # 设置服务器默认字符集为 utf8mb4 (推荐，支持完整的 UTF-8，包括 emoji)
-    character-set-server = utf8mb4
-    # 设置默认排序规则
-    collation-server = utf8mb4_unicode_ci
-    # 或者使用 utf8_general_ci (性能稍好，但排序规则稍宽松)
-    default-time-zone = 'Asia/Shanghai'
-    log_error=/var/log/datamate/database/error.log
-
-    [client]
-    # 设置客户端连接默认字符集
-    default-character-set = utf8mb4
-
-    [mysql]
-    # 设置 mysql 命令行客户端默认字符集
-    default-character-set = utf8mb4

deployment/helm/datamate/charts/database/values.yaml

@@ -7,11 +7,11 @@ replicaCount: 1
 
 # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
 image:
-  repository: "mysql"
+  repository: "datamate-database"
   # This sets the pull policy for images.
   pullPolicy: "IfNotPresent"
   # Overrides the image tag whose default is the chart appVersion.
-  tag: "8"
+  tag: "latest"
 
 # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
 imagePullSecrets: []
@@ -19,9 +19,7 @@ imagePullSecrets: []
 nameOverride: "datamate-database"
 fullnameOverride: "datamate-database"
 
-env:
-  - name: MYSQL_ROOT_PASSWORD
-    value: "password"
+env: []
 
 initContainers:
   - name: init-log

deployment/helm/datamate/charts/public/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deployment/helm/datamate/charts/public/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: public
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.0.1
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "0.0.1"

deployment/helm/datamate/charts/public/templates/_helpers.tpl

@@ -0,0 +1,51 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "public.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "public.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "public.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "public.labels" -}}
+helm.sh/chart: {{ include "public.chart" . }}
+{{ include "public.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "public.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "public.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

deployment/helm/datamate/charts/public/templates/database-pvc.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: datamate-database-pvc
+{{- with .Values.annotations  }}
+  annotations:
+    {{- toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "public.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  {{- if .Values.persistentVolumeClaim.storageClass }}
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.persistentVolumeClaim.size.database }}

deployment/helm/datamate/charts/public/templates/dataset-pvc.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: datamate-dataset-pvc
+{{- with .Values.annotations  }}
+  annotations:
+    {{- toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "public.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  {{- if .Values.persistentVolumeClaim.storageClass }}
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.persistentVolumeClaim.size.dataset }}

deployment/helm/datamate/charts/public/templates/flow-pvc.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: datamate-flow-pvc
+{{- with .Values.annotations  }}
+  annotations:
+    {{- toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "public.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  {{- if .Values.persistentVolumeClaim.storageClass }}
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.persistentVolumeClaim.size.flow }}

deployment/helm/datamate/charts/public/templates/log-pvc.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: datamate-log-pvc
+{{- with .Values.annotations  }}
+  annotations:
+    {{- toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "public.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  {{- if .Values.persistentVolumeClaim.storageClass }}
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.persistentVolumeClaim.size.log }}

deployment/helm/datamate/charts/public/templates/operator-pvc.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: datamate-operator-pvc
+{{- with .Values.annotations  }}
+  annotations:
+    {{- toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "public.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  {{- if .Values.persistentVolumeClaim.storageClass }}
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.persistentVolumeClaim.size.operator }}

deployment/helm/datamate/charts/public/values.yaml

@@ -0,0 +1,25 @@
+# Default values for datamate.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# This is to override the chart name.
+nameOverride: "datamate-public"
+fullnameOverride: "datamate-public"
+
+# This is for setting Kubernetes Annotations to a Pvc.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+annotations: {}
+
+persistentVolumeClaim:
+  ## If defined, storageClassName: <storageClass>
+  ## If undefined (the default) or set to null, no storageClassName spec is
+  ##   set, choosing the default provisioner.
+  ##
+  storageClass: ""
+  accessModes: ReadWriteMany
+  size:
+    dataset: 10Gi
+    flow: 1Gi
+    log: 1Gi
+    database: 1Gi
+    operator: 1Gi

deployment/helm/datamate/values.yaml

@@ -21,39 +21,52 @@ global:
       name: "datamate-runtime"
       tag: "latest"
     database:
-      name: "mysql"
-      tag: "8"
+      name: "datamate-database"
+      tag: "latest"
 
 datasetVolume: &datasetVolume
   name: dataset-volume
-  hostPath:
-    path: /opt/datamate/data/dataset
-    type: DirectoryOrCreate
+  persistentVolumeClaim:
+    claimName: datamate-dataset-pvc
 
 flowVolume: &flowVolume
   name: flow-volume
-  hostPath:
-    path: /opt/datamate/data/flow
-    type: DirectoryOrCreate
+  persistentVolumeClaim:
+    claimName: datamate-flow-pvc
 
 logVolume: &logVolume
   name: log-volume
-  hostPath:
-    path: /opt/datamate/data/log
-    type: DirectoryOrCreate
+  persistentVolumeClaim:
+    claimName: datamate-log-pvc
 
 dataVolume: &dataVolume
   name: data-volume
-  hostPath:
-    path: /opt/datamate/data/mysql
-    type: DirectoryOrCreate
+  persistentVolumeClaim:
+    claimName: datamate-database-pvc
 
 operatorVolume: &operatorVolume
   name: operator-volume
-  hostPath:
-    path: /opt/datamate/data/operator
+  persistentVolumeClaim:
+    claimName: datamate-operator-pvc
+
+database:
+  env:
+    - name: MYSQL_ROOT_PASSWORD
+      value: &dbPass "password"
+  volumes:
+    - *dataVolume
+    - *logVolume
+  volumeMounts:
+    - name: data-volume
+      mountPath: /var/lib/mysql
+    - name: log-volume
+      mountPath: /var/log/datamate/database
+      subPath: database
 
 backend:
+  env:
+    - name: DB_PASSWORD
+      value: *dbPass
   volumes:
     - *datasetVolume
     - *flowVolume
@@ -83,27 +96,6 @@ frontend:
       name: datamate-nginx-conf
       subPath: backend.conf
 
-database:
-  volumes:
-    - *dataVolume
-    - *logVolume
-    - name: init-sql
-      configMap:
-        name: datamate-init-sql
-    - name: mysql-utf8-config
-      configMap:
-        name: datamate-mysql-utf8-config
-  volumeMounts:
-    - name: data-volume
-      mountPath: /var/lib/mysql
-    - name: log-volume
-      mountPath: /var/log/datamate/database
-      subPath: database
-    - name: init-sql
-      mountPath: /docker-entrypoint-initdb.d
-    - name: mysql-utf8-config
-      mountPath: /etc/mysql/conf.d
-
 ray-cluster:
   head:
     volumes:

frontend/src/pages/OperatorMarket/Create/components/ConfigureStep.tsx

@@ -80,7 +80,7 @@ export default function ConfigureStep({
             <Input value={parsedInfo.outputs}/>
           </Form.Item>
 
-          {parsedInfo.configs && (
+          {parsedInfo.configs && Object.keys(parsedInfo.configs).length > 0 && (
             <>
               <h3 className="text-lg font-semibold text-gray-900 mt-10 mb-2">
                 高级配置

runtime/deer-flow/.env.example

@@ -54,7 +54,7 @@ TAVILY_API_KEY=tvly-xxx
 # MOI_LIST_LIMIT=10
 
 # RAG_PROVIDER: milvus  (using free milvus instance on zilliz cloud: https://docs.zilliz.com/docs/quick-start )
-# RAG_PROVIDER=milvus
+RAG_PROVIDER=milvus
 MILVUS_URI=http://milvus-standalone:19530
 MILVUS_USER=root
 MILVUS_PASSWORD=Milvus

scripts/db/data-operator-init.sql

@@ -123,7 +123,7 @@ AND o.id IN ('TextFormatter', 'FileWithShortOrLongLengthFilter', 'FileWithHighRe
             'AnonymizedIpAddress', 'AnonymizedPhoneNumber', 'AnonymizedUrlCleaner', 'HtmlTagCleaner', 'XMLTagCleaner',
             'ContentCleaner', 'EmailNumberCleaner', 'EmojiCleaner', 'ExtraSpaceCleaner', 'FullWidthCharacterCleaner',
             'GrableCharactersCleaner', 'InvisibleCharactersCleaner', 'LegendCleaner', 'PoliticalWordCleaner',
-            'SexualAndViolentWordCleaner', 'TraditionalChineseCleaner', 'UnicodeSpaceCleaner');
+            'SexualAndViolentWordCleaner', 'TraditionalChineseCleaner', 'UnicodeSpaceCleaner', 'MineruFormatter');
 
 INSERT IGNORE INTO t_operator_category_relation(category_id, operator_id)
 SELECT c.id, o.id

scripts/images/database/Dockerfile

@@ -0,0 +1,6 @@
+FROM mysql:8
+
+COPY scripts/images/database/utf8.cnf /etc/mysql/conf.d/utf8.cnf
+COPY scripts/db/ /docker-entrypoint-initdb.d/
+
+RUN chmod 644 /etc/mysql/conf.d/utf8.cnf