feature: 数据库镜像制作 (#70)

* feature: 数据库镜像制作 * feature: 增加归档包流水线
2025-11-10 19:06:53 +08:00
parent 8a0228b20e
commit 9dd26d622f
28 changed files with 379 additions and 79 deletions
--- a/deployment/docker/datamate/docker-compose.yml
+++ b/deployment/docker/datamate/docker-compose.yml
@@ -49,7 +49,7 @@ services:
  # 3) database
  datamate-database:
    container_name: datamate-database
-    image: mysql:8
+    image: ${REGISTRY:-}datamate-database
    restart: on-failure
    environment:
      MYSQL_ROOT_PASSWORD: password
@@ -61,8 +61,6 @@ services:
      "
    volumes:
      - mysql_volume:/var/lib/mysql
-      - ../../../scripts/db:/docker-entrypoint-initdb.d
-      - ./utf8.cnf:/etc/mysql/conf.d/utf8.cnf:ro
      - database_log_volume:/var/log/datamate/database
    ports:
      - "3306:3306"
--- a/deployment/docker/datamate/utf8.cnf
+++ b/deployment/docker/datamate/utf8.cnf
@@ -1,16 +0,0 @@
-[mysqld]
-# 设置服务器默认字符集为 utf8mb4 (推荐，支持完整的 UTF-8，包括 emoji)
-character-set-server = utf8mb4
-# 设置默认排序规则
-collation-server = utf8mb4_unicode_ci
-# 或者使用 utf8_general_ci (性能稍好，但排序规则稍宽松)
-default-time-zone = 'Asia/Shanghai'
-log_error=/var/log/datamate/database/error.log
-
-[client]
-# 设置客户端连接默认字符集
-default-character-set = utf8mb4
-
-[mysql]
-# 设置 mysql 命令行客户端默认字符集
-default-character-set = utf8mb4
--- a/deployment/helm/datamate/charts/database/templates/_helpers.tpl
+++ b/deployment/helm/datamate/charts/database/templates/_helpers.tpl
@@ -67,5 +67,9 @@ Name of image
 {{- define "database.image" -}}
 {{- $name := default .Values.image.repository .Values.global.image.database.name }}
 {{- $tag := default .Values.image.tag .Values.global.image.database.tag }}
+{{- if .Values.global.image.repository }}
+{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}:{{ $tag }}
+{{- else }}
 {{- $name }}:{{ $tag }}
 {{- end }}
+{{- end }}
--- a/deployment/helm/datamate/charts/database/templates/configmap.yaml
+++ b/deployment/helm/datamate/charts/database/templates/configmap.yaml
@@ -1,22 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: datamate-mysql-utf8-config
-data:
-  utf8.cnf: |
-    [mysqld]
-    # 设置服务器默认字符集为 utf8mb4 (推荐，支持完整的 UTF-8，包括 emoji)
-    character-set-server = utf8mb4
-    # 设置默认排序规则
-    collation-server = utf8mb4_unicode_ci
-    # 或者使用 utf8_general_ci (性能稍好，但排序规则稍宽松)
-    default-time-zone = 'Asia/Shanghai'
-    log_error=/var/log/datamate/database/error.log
-
-    [client]
-    # 设置客户端连接默认字符集
-    default-character-set = utf8mb4
-
-    [mysql]
-    # 设置 mysql 命令行客户端默认字符集
-    default-character-set = utf8mb4
--- a/deployment/helm/datamate/charts/database/values.yaml
+++ b/deployment/helm/datamate/charts/database/values.yaml
@@ -7,11 +7,11 @@ replicaCount: 1

 # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
 image:
-  repository: "mysql"
+  repository: "datamate-database"
  # This sets the pull policy for images.
  pullPolicy: "IfNotPresent"
  # Overrides the image tag whose default is the chart appVersion.
-  tag: "8"
+  tag: "latest"

 # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
 imagePullSecrets: []
@@ -19,9 +19,7 @@ imagePullSecrets: []
 nameOverride: "datamate-database"
 fullnameOverride: "datamate-database"

-env:
-  - name: MYSQL_ROOT_PASSWORD
-    value: "password"
+env: []

 initContainers:
  - name: init-log
--- a/deployment/helm/datamate/charts/public/.helmignore
+++ b/deployment/helm/datamate/charts/public/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
--- a/deployment/helm/datamate/charts/public/Chart.yaml
+++ b/deployment/helm/datamate/charts/public/Chart.yaml
@@ -0,0 +1,24 @@
+apiVersion: v2
+name: public
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.0.1
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "0.0.1"
--- a/deployment/helm/datamate/charts/public/templates/_helpers.tpl
+++ b/deployment/helm/datamate/charts/public/templates/_helpers.tpl
@@ -0,0 +1,51 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "public.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "public.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "public.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "public.labels" -}}
+helm.sh/chart: {{ include "public.chart" . }}
+{{ include "public.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "public.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "public.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
--- a/deployment/helm/datamate/charts/public/templates/database-pvc.yaml
+++ b/deployment/helm/datamate/charts/public/templates/database-pvc.yaml
@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: datamate-database-pvc
+{{- with .Values.annotations  }}
+  annotations:
+    {{- toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "public.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  {{- if .Values.persistentVolumeClaim.storageClass }}
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.persistentVolumeClaim.size.database }}
--- a/deployment/helm/datamate/charts/public/templates/dataset-pvc.yaml
+++ b/deployment/helm/datamate/charts/public/templates/dataset-pvc.yaml
@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: datamate-dataset-pvc
+{{- with .Values.annotations  }}
+  annotations:
+    {{- toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "public.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  {{- if .Values.persistentVolumeClaim.storageClass }}
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.persistentVolumeClaim.size.dataset }}
--- a/deployment/helm/datamate/charts/public/templates/flow-pvc.yaml
+++ b/deployment/helm/datamate/charts/public/templates/flow-pvc.yaml
@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: datamate-flow-pvc
+{{- with .Values.annotations  }}
+  annotations:
+    {{- toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "public.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  {{- if .Values.persistentVolumeClaim.storageClass }}
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.persistentVolumeClaim.size.flow }}
--- a/deployment/helm/datamate/charts/public/templates/log-pvc.yaml
+++ b/deployment/helm/datamate/charts/public/templates/log-pvc.yaml
@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: datamate-log-pvc
+{{- with .Values.annotations  }}
+  annotations:
+    {{- toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "public.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  {{- if .Values.persistentVolumeClaim.storageClass }}
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.persistentVolumeClaim.size.log }}
--- a/deployment/helm/datamate/charts/public/templates/operator-pvc.yaml
+++ b/deployment/helm/datamate/charts/public/templates/operator-pvc.yaml
@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: datamate-operator-pvc
+{{- with .Values.annotations  }}
+  annotations:
+    {{- toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "public.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistentVolumeClaim.accessModes }}
+  {{- if .Values.persistentVolumeClaim.storageClass }}
+  storageClassName: {{ .Values.persistentVolumeClaim.storageClass }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.persistentVolumeClaim.size.operator }}
--- a/deployment/helm/datamate/charts/public/values.yaml
+++ b/deployment/helm/datamate/charts/public/values.yaml
@@ -0,0 +1,25 @@
+# Default values for datamate.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# This is to override the chart name.
+nameOverride: "datamate-public"
+fullnameOverride: "datamate-public"
+
+# This is for setting Kubernetes Annotations to a Pvc.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+annotations: {}
+
+persistentVolumeClaim:
+  ## If defined, storageClassName: <storageClass>
+  ## If undefined (the default) or set to null, no storageClassName spec is
+  ##   set, choosing the default provisioner.
+  ##
+  storageClass: ""
+  accessModes: ReadWriteMany
+  size:
+    dataset: 10Gi
+    flow: 1Gi
+    log: 1Gi
+    database: 1Gi
+    operator: 1Gi
--- a/deployment/helm/datamate/values.yaml
+++ b/deployment/helm/datamate/values.yaml
@@ -21,39 +21,52 @@ global:
      name: "datamate-runtime"
      tag: "latest"
    database:
-      name: "mysql"
-      tag: "8"
+      name: "datamate-database"
+      tag: "latest"

 datasetVolume: &datasetVolume
  name: dataset-volume
-  hostPath:
-    path: /opt/datamate/data/dataset
-    type: DirectoryOrCreate
+  persistentVolumeClaim:
+    claimName: datamate-dataset-pvc

 flowVolume: &flowVolume
  name: flow-volume
-  hostPath:
-    path: /opt/datamate/data/flow
-    type: DirectoryOrCreate
+  persistentVolumeClaim:
+    claimName: datamate-flow-pvc

 logVolume: &logVolume
  name: log-volume
-  hostPath:
-    path: /opt/datamate/data/log
-    type: DirectoryOrCreate
+  persistentVolumeClaim:
+    claimName: datamate-log-pvc

 dataVolume: &dataVolume
  name: data-volume
-  hostPath:
-    path: /opt/datamate/data/mysql
-    type: DirectoryOrCreate
+  persistentVolumeClaim:
+    claimName: datamate-database-pvc

 operatorVolume: &operatorVolume
  name: operator-volume
-  hostPath:
-    path: /opt/datamate/data/operator
+  persistentVolumeClaim:
+    claimName: datamate-operator-pvc
+
+database:
+  env:
+    - name: MYSQL_ROOT_PASSWORD
+      value: &dbPass "password"
+  volumes:
+    - *dataVolume
+    - *logVolume
+  volumeMounts:
+    - name: data-volume
+      mountPath: /var/lib/mysql
+    - name: log-volume
+      mountPath: /var/log/datamate/database
+      subPath: database

 backend:
+  env:
+    - name: DB_PASSWORD
+      value: *dbPass
  volumes:
    - *datasetVolume
    - *flowVolume
@@ -83,27 +96,6 @@ frontend:
      name: datamate-nginx-conf
      subPath: backend.conf

-database:
-  volumes:
-    - *dataVolume
-    - *logVolume
-    - name: init-sql
-      configMap:
-        name: datamate-init-sql
-    - name: mysql-utf8-config
-      configMap:
-        name: datamate-mysql-utf8-config
-  volumeMounts:
-    - name: data-volume
-      mountPath: /var/lib/mysql
-    - name: log-volume
-      mountPath: /var/log/datamate/database
-      subPath: database
-    - name: init-sql
-      mountPath: /docker-entrypoint-initdb.d
-    - name: mysql-utf8-config
-      mountPath: /etc/mysql/conf.d
-
 ray-cluster:
  head:
    volumes: