q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Develop hsc (#58)

Browse Source 
feature: 优化镜像构建/部署
This commit is contained in:
hhhhsc701
2025-11-06 17:14:54 +08:00
committed by GitHub
parent d84152b45f
commit f78475e29f
27 changed files with 540 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
.github/workflows/docker-image-backend-python.yml vendored Normal file
View File

@@ -0,0 +1,27 @@
name: Python Backend Docker Image CI
on:
push:
branches: [ "main" ]
paths:
- 'backend/**'
- 'scripts/images/datamate-python/**'
- '.github/workflows/docker-image-backend-python.yml'
pull_request:
branches: [ "main" ]
paths:
- 'backend/**'
- 'scripts/images/datamate-python/**'
- '.github/workflows/docker-image-backend-python.yml'
workflow_dispatch:
jobs:
call-docker-build:
name: Build and Push Backend Python Docker Image
uses: ./.github/workflows/docker-images-reusable.yml
permissions:
contents: read
packages: write
with:
service_name: backend-python
build_dir: .

78
Makefile
View File

@@ -3,6 +3,7 @@ MAKEFLAGS += --no-print-directory
WITH_MINERU ?= false # 默认不构建mineru
VERSION ?= latest
NAMESPACE ?= datamate
REGISTRY ?= "ghcr.io/modelengine-group/"
ifdef COMSPEC
# Windows 环境
@@ -17,7 +18,7 @@ build-%:
$(MAKE) $*-docker-build
.PHONY: build
build: backend-docker-build frontend-docker-build runtime-docker-build $(if $(WITH_MINERU),mineru-docker-build)
build: backend-docker-build frontend-docker-build runtime-docker-build
.PHONY: create-namespace
create-namespace:
@@ -42,7 +43,24 @@ else
endif
.PHONY: install
install: install-datamate
install:
ifeq ($(origin INSTALLER), undefined)
@echo "Choose a deployment method:"
@echo "1. Docker/Docker-Compose"
@echo "2. Kubernetes/Helm"
@echo -n "Enter choice: "
@read choice; \
case $$choice in \
1) INSTALLER=docker ;; \
2) INSTALLER=k8s ;; \
*) echo "Invalid choice" && exit 1 ;; \
esac; \
$(MAKE) datamate-$$INSTALLER-install; \
$(MAKE) milvus-$$INSTALLER-install
else
$(MAKE) datamate-$(INSTALLER)-install; \
$(MAKE) milvus-$(INSTALLER)-install
endif
.PHONY: uninstall-%
uninstall-%:
@@ -63,7 +81,24 @@ else
endif
.PHONY: uninstall
uninstall: uninstall-datamate
uninstall:
ifeq ($(origin INSTALLER), undefined)
@echo "Choose a deployment method:"
@echo "1. Docker/Docker-Compose"
@echo "2. Kubernetes/Helm"
@echo -n "Enter choice: "
@read choice; \
case $$choice in \
1) INSTALLER=docker ;; \
2) INSTALLER=k8s ;; \
*) echo "Invalid choice" && exit 1 ;; \
esac; \
$(MAKE) milvus-$$INSTALLER-uninstall; \
$(MAKE) datamate-$$INSTALLER-uninstall
else
$(MAKE) milvus-$(INSTALLER)-uninstall; \
$(MAKE) datamate-$(INSTALLER)-uninstall
endif
# build
.PHONY: backend-docker-build
@@ -92,6 +127,11 @@ deer-flow-docker-build:
.PHONY: mineru-docker-build
mineru-docker-build:
docker build -t datamate-mineru:$(VERSION) . -f scripts/images/mineru/Dockerfile
.PHONY: backend-python-docker-build
backend-python-docker-build:
docker build -t datamate-backend-python:$(VERSION) . -f scripts/images/datamate-python/Dockerfile
.PHONY: backend-docker-install
backend-docker-install:
cd deployment/docker/datamate && docker compose up -d backend
@@ -118,7 +158,7 @@ runtime-docker-uninstall:
.PHONY: mineru-docker-install
mineru-docker-install:
cd deployment/docker/datamate && cp .env.example .env && docker compose up -d datamate-mineru
cd deployment/docker/datamate && export REGISTRY=$(REGISTRY) && docker compose up -d datamate-mineru
.PHONY: mineru-docker-uninstall
mineru-docker-uninstall:
@@ -134,31 +174,40 @@ mineru-k8s-uninstall:
.PHONY: datamate-docker-install
datamate-docker-install:
cd deployment/docker/datamate && cp .env.example .env && docker compose -f docker-compose.yml up -d
cd deployment/docker/datamate && export REGISTRY=$(REGISTRY) && docker compose -f docker-compose.yml up -d
.PHONY: datamate-docker-uninstall
datamate-docker-uninstall:
cd deployment/docker/datamate && docker compose -f docker-compose.yml down -v
cd deployment/docker/datamate && docker compose -f docker-compose.yml --profile mineru down -v
.PHONY: deer-flow-docker-install
deer-flow-docker-install:
cd deployment/docker/datamate && cp .env.deer-flow.example .env && docker compose -f docker-compose.yml up -d
cd deployment/docker/datamate && export NGINX_CONF="./backend-with-deer-flow.conf" && export REGISTRY=$(REGISTRY) && docker compose -f docker-compose.yml up -d
cp -n runtime/deer-flow/.env.example runtime/deer-flow/.env
cp -n runtime/deer-flow/conf.yaml.example runtime/deer-flow/conf.yaml
cp runtime/deer-flow/.env deployment/docker/deer-flow/.env && cp runtime/deer-flow/conf.yaml deployment/docker/deer-flow/conf.yaml
cd deployment/docker/deer-flow && docker compose -f docker-compose.yml up -d
cp runtime/deer-flow/.env deployment/docker/deer-flow/.env
cp runtime/deer-flow/conf.yaml deployment/docker/deer-flow/conf.yaml
cd deployment/docker/deer-flow && export REGISTRY=$(REGISTRY) && docker compose -f docker-compose.yml up -d
.PHONY: deer-flow-docker-uninstall
deer-flow-docker-uninstall:
@if docker compose ls --filter name=datamate | grep -q datamate; then \
cd deployment/docker/datamate && docker compose -f docker-compose.yml up -d; \
cd deployment/docker/datamate && export REGISTRY=$(REGISTRY) && docker compose -f docker-compose.yml up -d; \
fi
cd deployment/docker/deer-flow && docker compose -f docker-compose.yml down
.PHONY: milvus-docker-install
milvus-docker-install:
cd deployment/docker/milvus && docker compose -f docker-compose.yml up -d
.PHONY: milvus-docker-uninstall
milvus-docker-uninstall:
cd deployment/docker/milvus && docker compose -f docker-compose.yml down
.PHONY: datamate-k8s-install
datamate-k8s-install: create-namespace
kubectl create configmap datamate-init-sql --from-file=scripts/db/ --dry-run=client -o yaml | kubectl apply -f - -n $(NAMESPACE)
helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install
helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.image.repository=$(REPOSITORY)
.PHONY: datamate-k8s-uninstall
datamate-k8s-uninstall:
@@ -167,14 +216,17 @@ datamate-k8s-uninstall:
.PHONY: deer-flow-k8s-install
deer-flow-k8s-install:
helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.deerFlow.enable=true
helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.deerFlow.enable=true --set global.image.repository=$(REPOSITORY)
cp runtime/deer-flow/.env deployment/helm/deer-flow/charts/public/.env
cp runtime/deer-flow/conf.yaml deployment/helm/deer-flow/charts/public/conf.yaml
helm upgrade deer-flow deployment/helm/deer-flow -n $(NAMESPACE) --install
helm upgrade deer-flow deployment/helm/deer-flow -n $(NAMESPACE) --install --set global.image.repository=$(REPOSITORY)
.PHONY: deer-flow-k8s-uninstall
deer-flow-k8s-uninstall:
helm uninstall deer-flow -n $(NAMESPACE) --ignore-not-found
@if helm ls -n $(NAMESPACE) --filter datamate | grep -q datamate; then \
helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --set global.deerFlow.enable=false; \
fi
.PHONY: milvus-k8s-install
milvus-k8s-install:

1
deployment/docker/datamate/.env.deer-flow.example
View File

@@ -1 +0,0 @@
NGINX_CONF=./backend-with-deer-flow.conf

1
deployment/docker/datamate/.env.example
View File

@@ -1 +0,0 @@
NGINX_CONF=./backend.conf

14
deployment/docker/datamate/backend-with-deer-flow.conf
View File

@@ -9,6 +9,20 @@ server {
add_header Set-Cookie "NEXT_LOCALE=zh";
location /api/synthesis/ {
proxy_pass http://datamate-backend-python:18000/api/synthesis/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /api/annotation/ {
proxy_pass http://datamate-backend-python:18000/api/annotation/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /api/ {
proxy_pass http://datamate-backend:8080/api/;
proxy_set_header Host $host;

13
deployment/docker/datamate/docker-compose.yml
View File

@@ -2,7 +2,7 @@ services:
# 1) backend
datamate-backend:
container_name: datamate-backend
image: datamate-backend
image: ${REGISTRY:-}datamate-backend
restart: on-failure
privileged: true
volumes:
@@ -18,7 +18,7 @@ services:
# 1) backend (Python)
datamate-backend-python:
container_name: datamate-backend-python
image: datamate-backend-python
image: ${REGISTRY:-}datamate-backend-python
restart: on-failure
privileged: true
environment:
@@ -34,13 +34,13 @@ services:
# 2) frontend（NodePort 30000）
datamate-frontend:
container_name: datamate-frontend
image: datamate-frontend
image: ${REGISTRY:-}datamate-frontend
restart: on-failure
ports:
- "30000:80" # nodePort → hostPort
volumes:
- frontend_log_volume:/var/log/datamate/frontend
- $NGINX_CONF:/etc/nginx/conf.d/backend.conf
- ${NGINX_CONF:-./backend.conf}:/etc/nginx/conf.d/backend.conf
networks: [ datamate ]
depends_on:
- datamate-backend
@@ -71,7 +71,7 @@ services:
# 3) runtime
datamate-runtime:
container_name: datamate-runtime
image: datamate-runtime
image: ${REGISTRY:-}datamate-runtime
restart: on-failure
environment:
RAY_DEDUP_LOGS: "0"
@@ -81,7 +81,6 @@ services:
MYSQL_USER: "root"
MYSQL_PASSWORD: "password"
MYSQL_DATABASE: "datamate"
PDF_FORMATTER_BASE_URL: "http://datamate-mineru:9001"
command:
- python
- /opt/runtime/datamate/operator_runtime.py
@@ -135,6 +134,8 @@ volumes:
name: datamate-operator-upload-volume
operator-runtime-volume:
name: datamate-operator-runtime-volume
mineru_log_volume:
name: datamate-mineru_log_volume
networks:
datamate:

4
deployment/docker/deer-flow/docker-compose.yml
View File

@@ -1,6 +1,6 @@
services:
deer-flow-backend:
image: deer-flow-backend
image: ${REGISTRY:-}deer-flow-backend
container_name: deer-flow-backend
env_file:
- .env
@@ -11,7 +11,7 @@ services:
- datamate
deer-flow-frontend:
image: deer-flow-frontend
image: ${REGISTRY:-}deer-flow-frontend
container_name: deer-flow-frontend
env_file:
- .env

2
deployment/docker/milvus/docker-compose.yml
View File

@@ -1,5 +1,3 @@
version: '3.5'
services:
etcd:
container_name: milvus-etcd

23
deployment/helm/datamate/charts/backend-python/.helmignore Normal file
View File

@@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/

29
deployment/helm/datamate/charts/backend-python/Chart.yaml Normal file
View File

@@ -0,0 +1,29 @@
apiVersion: v2
name: backend-python
description: A Helm chart for Kubernetes
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.0.1
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "0.0.1"
dependencies:
- name: database
repository: file://../database
version: 0.0.1

75
deployment/helm/datamate/charts/backend-python/templates/_helpers.tpl Normal file
View File

@@ -0,0 +1,75 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "backend-python.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "backend-python.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "backend-python.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end }}
{{/*
Common labels
*/}}
{{- define "backend-python.labels" -}}
helm.sh/chart: {{ include "backend-python.chart" . }}
{{ include "backend-python.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "backend-python.selectorLabels" -}}
app.kubernetes.io/name: {{ include "backend-python.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "backend-python.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "backend-python.fullname" .) .Values.serviceAccount.name -}}
{{- else }}
{{- default "default" .Values.serviceAccount.name -}}
{{- end }}
{{- end }}
{{/*
Name of image
*/}}
{{- define "backend-python.image" -}}
{{- $name := default .Values.image.repository .Values.global.image.backendPython.name }}
{{- $tag := default .Values.image.tag .Values.global.image.backendPython.tag }}
{{- if .Values.global.image.repository }}
{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}:{{ $tag }}
{{- else }}
{{- $name }}:{{ $tag }}
{{- end }}
{{- end }}

82
deployment/helm/datamate/charts/backend-python/templates/deployment.yaml Normal file
View File

@@ -0,0 +1,82 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "backend-python.fullname" . }}
labels:
{{- include "backend-python.labels" . | nindent 4 }}
spec:
{{- if not .Values.autoscaling.enabled }}
replicas: {{ .Values.replicaCount }}
{{- end }}
selector:
matchLabels:
{{- include "backend-python.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "backend-python.labels" . | nindent 8 }}
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "backend-python.serviceAccountName" . }}
{{- with .Values.podSecurityContext }}
securityContext:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: {{ .Chart.Name }}
{{- with .Values.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
image: "{{ include "backend-python.image" . }}"
imagePullPolicy: {{ default .Values.image.pullPolicy .Values.global.image.pullPolicy }}
ports:
- name: http
containerPort: {{ .Values.service.port }}
protocol: TCP
{{- with .Values.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.readinessProbe }}
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.env }}
env:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.volumeMounts }}
volumeMounts:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.volumes }}
volumes:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}

15
deployment/helm/datamate/charts/backend-python/templates/service.yaml Normal file
View File

@@ -0,0 +1,15 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "backend-python.fullname" . }}
labels:
{{- include "backend-python.labels" . | nindent 4 }}
spec:
type: {{ .Values.service.type }}
ports:
- port: {{ .Values.service.port }}
targetPort: {{ .Values.service.port }}
protocol: TCP
name: {{ .Chart.Name }}
selector:
{{- include "backend.selectorLabels" . | nindent 4 }}

13
deployment/helm/datamate/charts/backend-python/templates/serviceaccount.yaml Normal file
View File

@@ -0,0 +1,13 @@
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "backend-python.serviceAccountName" . }}
labels:
{{- include "backend-python.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
{{- end }}

108
deployment/helm/datamate/charts/backend-python/values.yaml Normal file
View File

@@ -0,0 +1,108 @@
# Default values for datamate.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
replicaCount: 1
# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
image:
repository: "datamate-backend-python"
# This sets the pull policy for images.
pullPolicy: "IfNotPresent"
# Overrides the image tag whose default is the chart appVersion.
tag: "latest"
# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
imagePullSecrets: []
# This is to override the chart name.
nameOverride: "datamate-backend-python"
fullnameOverride: "datamate-backend-python"
env: []
# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
serviceAccount:
# Specifies whether a service account should be created
create: true
# Automatically mount a ServiceAccount's API credentials?
automount: true
# Annotations to add to the service account
annotations: {}
# The name of the service account to use.
# If not set and create is true, a name is generated using the fullname template
name: ""
# This is for setting Kubernetes Annotations to a Pod.
# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
podAnnotations: {}
# This is for setting Kubernetes Labels to a Pod.
# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
podLabels: {}
podSecurityContext: {}
# fsGroup: 2000
securityContext: {}
# capabilities:
# drop:
# - ALL
# readOnlyRootFilesystem: true
# runAsNonRoot: true
# runAsUser: 1000
# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/
service:
# This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types
type: ClusterIP
# This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports
port: 18000
resources: {}
# We usually recommend not to specify default resources and to leave this as a conscious
# choice for the user. This also increases chances charts run on environments with little
# resources, such as Minikube. If you do want to specify resources, uncomment the following
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
# livenessProbe:
# httpGet:
# path: /
# port: http
# readinessProbe:
# httpGet:
# path: /
# port: http
# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/
autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 100
targetCPUUtilizationPercentage: 80
# targetMemoryUtilizationPercentage: 80
# Additional volumes on the output Deployment definition.
volumes: []
# - name: foo
# secret:
# secretName: mysecret
# optional: false
# Additional volumeMounts on the output Deployment definition.
volumeMounts: []
# - name: foo
# mountPath: "/etc/foo"
# readOnly: true
nodeSelector: {}
tolerations: []
affinity: {}

28
deployment/helm/datamate/charts/frontend/templates/configmap.yaml
View File

@@ -16,6 +16,20 @@ data:
add_header Set-Cookie "NEXT_LOCALE=zh";
location /api/synthesis/ {
proxy_pass http://datamate-backend-python:18000/api/synthesis/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /api/annotation/ {
proxy_pass http://datamate-backend-python:18000/api/annotation/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /api/ {
proxy_pass http://datamate-backend:8080/api/;
proxy_set_header Host $host;
@@ -70,6 +84,20 @@ data:
client_max_body_size 1024M;
location /api/synthesis/ {
proxy_pass http://datamate-backend-python:18000/api/synthesis/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /api/annotation/ {
proxy_pass http://datamate-backend-python:18000/api/annotation/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /api/ {
proxy_pass http://datamate-backend:8080/api/;
proxy_set_header Host $host;

27
deployment/helm/datamate/charts/ray-cluster/templates/_helpers.tpl
View File

@@ -52,4 +52,29 @@ Create the name of the service account to use
{{- else -}}
{{ default "default" .Values.serviceAccount.name }}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Name of image
*/}}
{{- define "ray-cluster.image" -}}
{{- $name := default .Values.image.repository .Values.global.image.runtime.name }}
{{- $tag := default .Values.image.tag .Values.global.image.runtime.tag }}
{{- if .Values.global.image.repository }}
{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}:{{ $tag }}
{{- else }}
{{- $name }}:{{ $tag }}
{{- end }}
{{- end }}
{{/*
Name of sidecar image
*/}}
{{- define "ray-cluster-sidecar.image" -}}
{{- $name := default (printf "%s:%s" .Values.image.repository .Values.image.tag) .Values.head.sidecarContainers.image }}
{{- if .Values.global.image.repository }}
{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}
{{- else }}
{{- $name }}
{{- end }}
{{- end }}

28
deployment/helm/datamate/charts/ray-cluster/templates/raycluster-cluster.yaml
View File

@@ -58,13 +58,8 @@ spec:
{{- end }}
containers:
- name: ray-head
{{- if .Values.head.image }}
image: {{ .Values.head.image.repository }}:{{ .Values.head.image.tag }}
imagePullPolicy: {{ .Values.head.image.pullPolicy }}
{{- else }}
image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
{{- end }}
image: {{ include "ray-cluster.image" . }}
imagePullPolicy: {{ default .Values.image.pullPolicy .Values.global.image.pullPolicy }}
{{- with .Values.head.command }}
command:
{{- toYaml . | nindent 10 }}
@@ -101,8 +96,16 @@ spec:
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- $defult := printf "%s:%s" .Values.image.repository .Values.image.tag }}
{{- with .Values.head.sidecarContainers }}
{{- toYaml . | nindent 8 }}
{{- range $index, $container := . }}
{{- $image := default $defult $container.image -}}
{{- with $.Values.global.image.repository -}}
{{- $image = printf "%s/%s" (trimSuffix "/" .) (default $defult $container.image) -}}
{{- end -}}
{{- $mergedObj := mergeOverwrite $container (dict "image" $image) }}
- {{- toYaml $mergedObj | nindent 10 }}
{{- end }}
{{- end }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
@@ -185,13 +188,8 @@ spec:
{{- end }}
containers:
- name: ray-worker
{{- if .Values.worker.image }}
image: {{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag }}
imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
{{- else }}
image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
{{- end }}
image: {{ include "ray-cluster.image" . }}
imagePullPolicy: {{ default .Values.image.pullPolicy .Values.global.image.pullPolicy }}
{{- with .Values.worker.command }}
command:
{{- toYaml . | nindent 10 }}

4
deployment/helm/datamate/charts/ray-cluster/values.yaml
View File

@@ -77,8 +77,6 @@ head:
value: "password"
- name: MYSQL_DATABASE
value: "datamate"
- name: PDF_FORMATTER_BASE_URL
value: "http://datamate-mineru:9001"
# - name: EXAMPLE_ENV
# value: "1"
envFrom: []
@@ -173,8 +171,6 @@ worker:
value: "password"
- name: MYSQL_DATABASE
value: "datamate"
- name: PDF_FORMATTER_BASE_URL
value: "http://datamate-mineru:9001"
# - name: EXAMPLE_ENV
# value: "1"
envFrom: []

10
deployment/helm/datamate/values.yaml
View File

@@ -6,14 +6,20 @@ global:
deerFlow:
enable: false
image:
repository: ""
repository: "ghcr.io/modelengine-group/"
pullPolicy: "IfNotPresent"
backend:
name: "datamate-backend"
tag: "latest"
backendPython:
name: "datamate-backend-python"
tag: "latest"
frontend:
name: "datamate-frontend"
tag: "latest"
runtime:
name: "datamate-runtime"
tag: "latest"
database:
name: "mysql"
tag: "8"
@@ -136,8 +142,6 @@ ray-cluster:
value: "password"
- name: MYSQL_DATABASE
value: "datamate"
- name: PDF_FORMATTER_BASE_URL
value: "http://datamate-mineru:9001"
ports:
- containerPort: 8081
volumeMounts:

2
deployment/helm/deer-flow/values.yaml
View File

@@ -4,7 +4,7 @@
global:
image:
repository: ""
repository: "ghcr.io/modelengine-group/"
pullPolicy: "IfNotPresent"
backend:
name: "deer-flow-backend"

86
deployment/kubernetes/backend/deploy.yaml
View File

@@ -1,86 +0,0 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
app: datamate
tier: backend
name: datamate-backend
---
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: datamate-backend
labels:
app: datamate
tier: backend
spec:
replicas: 1
selector:
matchLabels:
app: datamate
tier: backend
template:
metadata:
labels:
app: datamate
tier: backend
spec:
serviceAccountName: datamate-backend
containers:
- name: backend
image: datamate-backend
imagePullPolicy: IfNotPresent
env:
- name: namespace
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: SPRING_CONFIG_LOCATION
value: file:/opt/backend/application.yml
ports:
- containerPort: 8080
volumeMounts:
- name: dataset-volume
mountPath: /dataset
- name: flow-volume
mountPath: /flow
- name: log-volume
mountPath: /var/log/datamate
volumes:
- name: dataset-volume
hostPath:
path: /opt/datamate/data/dataset
type: DirectoryOrCreate
- name: flow-volume
hostPath:
path: /opt/datamate/data/flow
type: DirectoryOrCreate
- name: log-volume
hostPath:
path: /opt/datamate/data/log
type: DirectoryOrCreate
---
apiVersion: v1
kind: Service
metadata:
name: datamate-backend
labels:
app: datamate
tier: backend
spec:
type: ClusterIP
ports:
- port: 8080
targetPort: 8080
protocol: TCP
selector:
app: datamate
tier: backend

52
deployment/kubernetes/frontend/deploy.yaml
View File

@@ -1,52 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: datamate-frontend
labels:
app: datamate
tier: frontend
spec:
replicas: 1
selector:
matchLabels:
app: datamate
tier: frontend
template:
metadata:
labels:
app: datamate
tier: frontend
spec:
containers:
- name: frontend
image: datamate-frontend
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
volumeMounts:
- name: log-volume
mountPath: /var/log/datamate/frontend
subPath: frontend
volumes:
- name: log-volume
hostPath:
path: /opt/datamate/data/log
type: DirectoryOrCreate
---
apiVersion: v1
kind: Service
metadata:
name: datamate-frontend
labels:
app: datamate
tier: frontend
spec:
type: NodePort
ports:
- port: 80
targetPort: 80
nodePort: 30000
protocol: TCP
selector:
app: datamate
tier: frontend

22
deployment/kubernetes/mysql/configmap.yaml
View File

@@ -1,22 +0,0 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: datamate-mysql-utf8-config
data:
utf8.cnf: |
[mysqld]
# 设置服务器默认字符集为 utf8mb4 (推荐，支持完整的 UTF-8，包括 emoji)
character-set-server = utf8mb4
# 设置默认排序规则
collation-server = utf8mb4_unicode_ci
# 或者使用 utf8_general_ci (性能稍好，但排序规则稍宽松)
default-time-zone = 'Asia/Shanghai'
log_error=/var/log/datamate/database/error.log
[client]
# 设置客户端连接默认字符集
default-character-set = utf8mb4
[mysql]
# 设置 mysql 命令行客户端默认字符集
default-character-set = utf8mb4

88
deployment/kubernetes/mysql/deploy.yaml
View File

@@ -1,88 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: datamate-database
labels:
app: datamate
tier: database
spec:
replicas: 1
selector:
matchLabels:
app: datamate
tier: database
strategy:
type: Recreate
template:
metadata:
labels:
app: datamate
tier: database
spec:
initContainers:
- name: init-log
image: mysql:8
imagePullPolicy: IfNotPresent
command:
- sh
- -c
args:
- |
chown mysql:mysql /var/log/datamate/database
chmod 755 /var/log/datamate/database
volumeMounts:
- name: log-volume
mountPath: /var/log/datamate/database
subPath: database
containers:
- name: mysql
image: mysql:8
imagePullPolicy: IfNotPresent
env:
- name: MYSQL_ROOT_PASSWORD
value: "password"
ports:
- containerPort: 3306
volumeMounts:
- name: data-volume
mountPath: /var/lib/mysql
- name: log-volume
mountPath: /var/log/datamate/database
subPath: database
- name: init-sql
mountPath: /docker-entrypoint-initdb.d
- name: mysql-utf8-config
mountPath: /etc/mysql/conf.d
volumes:
- name: data-volume
hostPath:
path: /opt/datamate/data/mysql
type: DirectoryOrCreate
- name: log-volume
hostPath:
path: /opt/datamate/data/log
type: DirectoryOrCreate
- name: init-sql
configMap:
name: datamate-init-sql
- name: mysql-utf8-config
configMap:
name: datamate-mysql-utf8-config
---
apiVersion: v1
kind: Service
metadata:
name: datamate-database
labels:
app: datamate
tier: database
spec:
type: ClusterIP
ports:
- port: 3306
targetPort: 3306
protocol: TCP
selector:
app: datamate
tier: database

9
runtime/deer-flow/.env.example
View File

@@ -53,18 +53,17 @@ TAVILY_API_KEY=tvly-xxx
# MOI_RETRIEVAL_SIZE=10
# MOI_LIST_LIMIT=10
# RAG_PROVIDER: milvus (using free milvus instance on zilliz cloud: https://docs.zilliz.com/docs/quick-start )
# RAG_PROVIDER=milvus
# MILVUS_URI=<endpoint_of_self_hosted_milvus_or_zilliz_cloud>
# MILVUS_USER=<username_of_self_hosted_milvus_or_zilliz_cloud>
# MILVUS_PASSWORD=<password_of_self_hosted_milvus_or_zilliz_cloud>
MILVUS_URI=http://milvus-standalone:19530
MILVUS_USER=root
MILVUS_PASSWORD=Milvus
MILVUS_AUTO_LOAD_EXAMPLES=false
# MILVUS_COLLECTION=documents
# MILVUS_EMBEDDING_PROVIDER=openai # support openai,dashscope
# MILVUS_EMBEDDING_BASE_URL=
# MILVUS_EMBEDDING_MODEL=
# MILVUS_EMBEDDING_API_KEY=
# MILVUS_AUTO_LOAD_EXAMPLES=true
# RAG_PROVIDER: milvus (using milvus lite on Mac or Linux)
# RAG_PROVIDER=milvus

3
runtime/ops/formatter/mineru_formatter/process.py
View File

@@ -6,7 +6,6 @@ Description: MinerU PDF文本抽取
Create: 2025/10/29 17:24
"""
import json
import os
import time
from loguru import logger
from typing import Dict, Any
@@ -20,7 +19,7 @@ class MineruFormatter(Mapper):
def __init__(self, *args, **kwargs):
super(MineruFormatter, self).__init__(*args, **kwargs)
self.base_url = os.getenv("EXTERNAL_PDF_BASE_URL", "http://datamate-mineru:9001")
self.base_url = "http://datamate-mineru:9001"
self.pdf_extract_url = f"{self.base_url}/api/pdf-extract"
def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]: