diff --git a/.github/workflows/docker-image-backend-python.yml b/.github/workflows/docker-image-backend-python.yml new file mode 100644 index 0000000..1b8f6d4 --- /dev/null +++ b/.github/workflows/docker-image-backend-python.yml @@ -0,0 +1,27 @@ +name: Python Backend Docker Image CI + +on: + push: + branches: [ "main" ] + paths: + - 'backend/**' + - 'scripts/images/datamate-python/**' + - '.github/workflows/docker-image-backend-python.yml' + pull_request: + branches: [ "main" ] + paths: + - 'backend/**' + - 'scripts/images/datamate-python/**' + - '.github/workflows/docker-image-backend-python.yml' + workflow_dispatch: + +jobs: + call-docker-build: + name: Build and Push Backend Python Docker Image + uses: ./.github/workflows/docker-images-reusable.yml + permissions: + contents: read + packages: write + with: + service_name: backend-python + build_dir: . diff --git a/Makefile b/Makefile index ae87539..9468bce 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ MAKEFLAGS += --no-print-directory WITH_MINERU ?= false # 默认不构建mineru VERSION ?= latest NAMESPACE ?= datamate +REGISTRY ?= "ghcr.io/modelengine-group/" ifdef COMSPEC # Windows 环境 @@ -17,7 +18,7 @@ build-%: $(MAKE) $*-docker-build .PHONY: build -build: backend-docker-build frontend-docker-build runtime-docker-build $(if $(WITH_MINERU),mineru-docker-build) +build: backend-docker-build frontend-docker-build runtime-docker-build .PHONY: create-namespace create-namespace: @@ -42,7 +43,24 @@ else endif .PHONY: install -install: install-datamate +install: +ifeq ($(origin INSTALLER), undefined) + @echo "Choose a deployment method:" + @echo "1. Docker/Docker-Compose" + @echo "2. Kubernetes/Helm" + @echo -n "Enter choice: " + @read choice; \ + case $$choice in \ + 1) INSTALLER=docker ;; \ + 2) INSTALLER=k8s ;; \ + *) echo "Invalid choice" && exit 1 ;; \ + esac; \ + $(MAKE) datamate-$$INSTALLER-install; \ + $(MAKE) milvus-$$INSTALLER-install +else + $(MAKE) datamate-$(INSTALLER)-install; \ + $(MAKE) milvus-$(INSTALLER)-install +endif .PHONY: uninstall-% uninstall-%: @@ -63,7 +81,24 @@ else endif .PHONY: uninstall -uninstall: uninstall-datamate +uninstall: +ifeq ($(origin INSTALLER), undefined) + @echo "Choose a deployment method:" + @echo "1. Docker/Docker-Compose" + @echo "2. Kubernetes/Helm" + @echo -n "Enter choice: " + @read choice; \ + case $$choice in \ + 1) INSTALLER=docker ;; \ + 2) INSTALLER=k8s ;; \ + *) echo "Invalid choice" && exit 1 ;; \ + esac; \ + $(MAKE) milvus-$$INSTALLER-uninstall; \ + $(MAKE) datamate-$$INSTALLER-uninstall +else + $(MAKE) milvus-$(INSTALLER)-uninstall; \ + $(MAKE) datamate-$(INSTALLER)-uninstall +endif # build .PHONY: backend-docker-build @@ -92,6 +127,11 @@ deer-flow-docker-build: .PHONY: mineru-docker-build mineru-docker-build: docker build -t datamate-mineru:$(VERSION) . -f scripts/images/mineru/Dockerfile + +.PHONY: backend-python-docker-build +backend-python-docker-build: + docker build -t datamate-backend-python:$(VERSION) . -f scripts/images/datamate-python/Dockerfile + .PHONY: backend-docker-install backend-docker-install: cd deployment/docker/datamate && docker compose up -d backend @@ -118,7 +158,7 @@ runtime-docker-uninstall: .PHONY: mineru-docker-install mineru-docker-install: - cd deployment/docker/datamate && cp .env.example .env && docker compose up -d datamate-mineru + cd deployment/docker/datamate && export REGISTRY=$(REGISTRY) && docker compose up -d datamate-mineru .PHONY: mineru-docker-uninstall mineru-docker-uninstall: @@ -134,31 +174,40 @@ mineru-k8s-uninstall: .PHONY: datamate-docker-install datamate-docker-install: - cd deployment/docker/datamate && cp .env.example .env && docker compose -f docker-compose.yml up -d + cd deployment/docker/datamate && export REGISTRY=$(REGISTRY) && docker compose -f docker-compose.yml up -d .PHONY: datamate-docker-uninstall datamate-docker-uninstall: - cd deployment/docker/datamate && docker compose -f docker-compose.yml down -v + cd deployment/docker/datamate && docker compose -f docker-compose.yml --profile mineru down -v .PHONY: deer-flow-docker-install deer-flow-docker-install: - cd deployment/docker/datamate && cp .env.deer-flow.example .env && docker compose -f docker-compose.yml up -d + cd deployment/docker/datamate && export NGINX_CONF="./backend-with-deer-flow.conf" && export REGISTRY=$(REGISTRY) && docker compose -f docker-compose.yml up -d cp -n runtime/deer-flow/.env.example runtime/deer-flow/.env cp -n runtime/deer-flow/conf.yaml.example runtime/deer-flow/conf.yaml - cp runtime/deer-flow/.env deployment/docker/deer-flow/.env && cp runtime/deer-flow/conf.yaml deployment/docker/deer-flow/conf.yaml - cd deployment/docker/deer-flow && docker compose -f docker-compose.yml up -d + cp runtime/deer-flow/.env deployment/docker/deer-flow/.env + cp runtime/deer-flow/conf.yaml deployment/docker/deer-flow/conf.yaml + cd deployment/docker/deer-flow && export REGISTRY=$(REGISTRY) && docker compose -f docker-compose.yml up -d .PHONY: deer-flow-docker-uninstall deer-flow-docker-uninstall: @if docker compose ls --filter name=datamate | grep -q datamate; then \ - cd deployment/docker/datamate && docker compose -f docker-compose.yml up -d; \ + cd deployment/docker/datamate && export REGISTRY=$(REGISTRY) && docker compose -f docker-compose.yml up -d; \ fi cd deployment/docker/deer-flow && docker compose -f docker-compose.yml down +.PHONY: milvus-docker-install +milvus-docker-install: + cd deployment/docker/milvus && docker compose -f docker-compose.yml up -d + +.PHONY: milvus-docker-uninstall +milvus-docker-uninstall: + cd deployment/docker/milvus && docker compose -f docker-compose.yml down + .PHONY: datamate-k8s-install datamate-k8s-install: create-namespace kubectl create configmap datamate-init-sql --from-file=scripts/db/ --dry-run=client -o yaml | kubectl apply -f - -n $(NAMESPACE) - helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install + helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.image.repository=$(REPOSITORY) .PHONY: datamate-k8s-uninstall datamate-k8s-uninstall: @@ -167,14 +216,17 @@ datamate-k8s-uninstall: .PHONY: deer-flow-k8s-install deer-flow-k8s-install: - helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.deerFlow.enable=true + helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.deerFlow.enable=true --set global.image.repository=$(REPOSITORY) cp runtime/deer-flow/.env deployment/helm/deer-flow/charts/public/.env cp runtime/deer-flow/conf.yaml deployment/helm/deer-flow/charts/public/conf.yaml - helm upgrade deer-flow deployment/helm/deer-flow -n $(NAMESPACE) --install + helm upgrade deer-flow deployment/helm/deer-flow -n $(NAMESPACE) --install --set global.image.repository=$(REPOSITORY) .PHONY: deer-flow-k8s-uninstall deer-flow-k8s-uninstall: helm uninstall deer-flow -n $(NAMESPACE) --ignore-not-found + @if helm ls -n $(NAMESPACE) --filter datamate | grep -q datamate; then \ + helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --set global.deerFlow.enable=false; \ + fi .PHONY: milvus-k8s-install milvus-k8s-install: diff --git a/deployment/docker/datamate/.env.deer-flow.example b/deployment/docker/datamate/.env.deer-flow.example deleted file mode 100644 index 40b3afe..0000000 --- a/deployment/docker/datamate/.env.deer-flow.example +++ /dev/null @@ -1 +0,0 @@ -NGINX_CONF=./backend-with-deer-flow.conf diff --git a/deployment/docker/datamate/.env.example b/deployment/docker/datamate/.env.example deleted file mode 100644 index 5009927..0000000 --- a/deployment/docker/datamate/.env.example +++ /dev/null @@ -1 +0,0 @@ -NGINX_CONF=./backend.conf diff --git a/deployment/docker/datamate/backend-with-deer-flow.conf b/deployment/docker/datamate/backend-with-deer-flow.conf index eff6d5c..66e26f9 100644 --- a/deployment/docker/datamate/backend-with-deer-flow.conf +++ b/deployment/docker/datamate/backend-with-deer-flow.conf @@ -9,6 +9,20 @@ server { add_header Set-Cookie "NEXT_LOCALE=zh"; + location /api/synthesis/ { + proxy_pass http://datamate-backend-python:18000/api/synthesis/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /api/annotation/ { + proxy_pass http://datamate-backend-python:18000/api/annotation/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location /api/ { proxy_pass http://datamate-backend:8080/api/; proxy_set_header Host $host; diff --git a/deployment/docker/datamate/docker-compose.yml b/deployment/docker/datamate/docker-compose.yml index 20de073..ffd3d59 100644 --- a/deployment/docker/datamate/docker-compose.yml +++ b/deployment/docker/datamate/docker-compose.yml @@ -2,7 +2,7 @@ services: # 1) backend datamate-backend: container_name: datamate-backend - image: datamate-backend + image: ${REGISTRY:-}datamate-backend restart: on-failure privileged: true volumes: @@ -18,7 +18,7 @@ services: # 1) backend (Python) datamate-backend-python: container_name: datamate-backend-python - image: datamate-backend-python + image: ${REGISTRY:-}datamate-backend-python restart: on-failure privileged: true environment: @@ -34,13 +34,13 @@ services: # 2) frontend(NodePort 30000) datamate-frontend: container_name: datamate-frontend - image: datamate-frontend + image: ${REGISTRY:-}datamate-frontend restart: on-failure ports: - "30000:80" # nodePort → hostPort volumes: - frontend_log_volume:/var/log/datamate/frontend - - $NGINX_CONF:/etc/nginx/conf.d/backend.conf + - ${NGINX_CONF:-./backend.conf}:/etc/nginx/conf.d/backend.conf networks: [ datamate ] depends_on: - datamate-backend @@ -71,7 +71,7 @@ services: # 3) runtime datamate-runtime: container_name: datamate-runtime - image: datamate-runtime + image: ${REGISTRY:-}datamate-runtime restart: on-failure environment: RAY_DEDUP_LOGS: "0" @@ -81,7 +81,6 @@ services: MYSQL_USER: "root" MYSQL_PASSWORD: "password" MYSQL_DATABASE: "datamate" - PDF_FORMATTER_BASE_URL: "http://datamate-mineru:9001" command: - python - /opt/runtime/datamate/operator_runtime.py @@ -135,6 +134,8 @@ volumes: name: datamate-operator-upload-volume operator-runtime-volume: name: datamate-operator-runtime-volume + mineru_log_volume: + name: datamate-mineru_log_volume networks: datamate: diff --git a/deployment/docker/deer-flow/docker-compose.yml b/deployment/docker/deer-flow/docker-compose.yml index 03853e3..c491b37 100644 --- a/deployment/docker/deer-flow/docker-compose.yml +++ b/deployment/docker/deer-flow/docker-compose.yml @@ -1,6 +1,6 @@ services: deer-flow-backend: - image: deer-flow-backend + image: ${REGISTRY:-}deer-flow-backend container_name: deer-flow-backend env_file: - .env @@ -11,7 +11,7 @@ services: - datamate deer-flow-frontend: - image: deer-flow-frontend + image: ${REGISTRY:-}deer-flow-frontend container_name: deer-flow-frontend env_file: - .env diff --git a/deployment/docker/milvus/docker-compose.yml b/deployment/docker/milvus/docker-compose.yml index 6396068..240934f 100644 --- a/deployment/docker/milvus/docker-compose.yml +++ b/deployment/docker/milvus/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.5' - services: etcd: container_name: milvus-etcd diff --git a/deployment/helm/datamate/charts/backend-python/.helmignore b/deployment/helm/datamate/charts/backend-python/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/deployment/helm/datamate/charts/backend-python/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deployment/helm/datamate/charts/backend-python/Chart.yaml b/deployment/helm/datamate/charts/backend-python/Chart.yaml new file mode 100644 index 0000000..36571e1 --- /dev/null +++ b/deployment/helm/datamate/charts/backend-python/Chart.yaml @@ -0,0 +1,29 @@ +apiVersion: v2 +name: backend-python +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.0.1 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.0.1" + +dependencies: + - name: database + repository: file://../database + version: 0.0.1 diff --git a/deployment/helm/datamate/charts/backend-python/templates/_helpers.tpl b/deployment/helm/datamate/charts/backend-python/templates/_helpers.tpl new file mode 100644 index 0000000..a65a78d --- /dev/null +++ b/deployment/helm/datamate/charts/backend-python/templates/_helpers.tpl @@ -0,0 +1,75 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "backend-python.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "backend-python.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "backend-python.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "backend-python.labels" -}} +helm.sh/chart: {{ include "backend-python.chart" . }} +{{ include "backend-python.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "backend-python.selectorLabels" -}} +app.kubernetes.io/name: {{ include "backend-python.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "backend-python.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "backend-python.fullname" .) .Values.serviceAccount.name -}} +{{- else }} +{{- default "default" .Values.serviceAccount.name -}} +{{- end }} +{{- end }} + +{{/* +Name of image +*/}} +{{- define "backend-python.image" -}} +{{- $name := default .Values.image.repository .Values.global.image.backendPython.name }} +{{- $tag := default .Values.image.tag .Values.global.image.backendPython.tag }} +{{- if .Values.global.image.repository }} +{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}:{{ $tag }} +{{- else }} +{{- $name }}:{{ $tag }} +{{- end }} +{{- end }} diff --git a/deployment/helm/datamate/charts/backend-python/templates/deployment.yaml b/deployment/helm/datamate/charts/backend-python/templates/deployment.yaml new file mode 100644 index 0000000..818b233 --- /dev/null +++ b/deployment/helm/datamate/charts/backend-python/templates/deployment.yaml @@ -0,0 +1,82 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "backend-python.fullname" . }} + labels: + {{- include "backend-python.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "backend-python.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "backend-python.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "backend-python.serviceAccountName" . }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: "{{ include "backend-python.image" . }}" + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.global.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deployment/helm/datamate/charts/backend-python/templates/service.yaml b/deployment/helm/datamate/charts/backend-python/templates/service.yaml new file mode 100644 index 0000000..e58e28c --- /dev/null +++ b/deployment/helm/datamate/charts/backend-python/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "backend-python.fullname" . }} + labels: + {{- include "backend-python.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.service.port }} + protocol: TCP + name: {{ .Chart.Name }} + selector: + {{- include "backend.selectorLabels" . | nindent 4 }} diff --git a/deployment/helm/datamate/charts/backend-python/templates/serviceaccount.yaml b/deployment/helm/datamate/charts/backend-python/templates/serviceaccount.yaml new file mode 100644 index 0000000..d737b93 --- /dev/null +++ b/deployment/helm/datamate/charts/backend-python/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "backend-python.serviceAccountName" . }} + labels: + {{- include "backend-python.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/deployment/helm/datamate/charts/backend-python/values.yaml b/deployment/helm/datamate/charts/backend-python/values.yaml new file mode 100644 index 0000000..49737ce --- /dev/null +++ b/deployment/helm/datamate/charts/backend-python/values.yaml @@ -0,0 +1,108 @@ +# Default values for datamate. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ +replicaCount: 1 + +# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ +image: + repository: "datamate-backend-python" + # This sets the pull policy for images. + pullPolicy: "IfNotPresent" + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +imagePullSecrets: [] +# This is to override the chart name. +nameOverride: "datamate-backend-python" +fullnameOverride: "datamate-backend-python" + +env: [] + +# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/ +serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +# This is for setting Kubernetes Annotations to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} +# This is for setting Kubernetes Labels to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +podLabels: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/ +service: + # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: ClusterIP + # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports + port: 18000 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ +# livenessProbe: +# httpGet: +# path: / +# port: http +# readinessProbe: +# httpGet: +# path: / +# port: http + +# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/ +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +# Additional volumes on the output Deployment definition. +volumes: [] +# - name: foo +# secret: +# secretName: mysecret +# optional: false + +# Additional volumeMounts on the output Deployment definition. +volumeMounts: [] +# - name: foo +# mountPath: "/etc/foo" +# readOnly: true + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/deployment/helm/datamate/charts/frontend/templates/configmap.yaml b/deployment/helm/datamate/charts/frontend/templates/configmap.yaml index a1d81a6..8a53376 100644 --- a/deployment/helm/datamate/charts/frontend/templates/configmap.yaml +++ b/deployment/helm/datamate/charts/frontend/templates/configmap.yaml @@ -16,6 +16,20 @@ data: add_header Set-Cookie "NEXT_LOCALE=zh"; + location /api/synthesis/ { + proxy_pass http://datamate-backend-python:18000/api/synthesis/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /api/annotation/ { + proxy_pass http://datamate-backend-python:18000/api/annotation/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location /api/ { proxy_pass http://datamate-backend:8080/api/; proxy_set_header Host $host; @@ -70,6 +84,20 @@ data: client_max_body_size 1024M; + location /api/synthesis/ { + proxy_pass http://datamate-backend-python:18000/api/synthesis/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /api/annotation/ { + proxy_pass http://datamate-backend-python:18000/api/annotation/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location /api/ { proxy_pass http://datamate-backend:8080/api/; proxy_set_header Host $host; diff --git a/deployment/helm/datamate/charts/ray-cluster/templates/_helpers.tpl b/deployment/helm/datamate/charts/ray-cluster/templates/_helpers.tpl index 50da5d1..019ba06 100644 --- a/deployment/helm/datamate/charts/ray-cluster/templates/_helpers.tpl +++ b/deployment/helm/datamate/charts/ray-cluster/templates/_helpers.tpl @@ -52,4 +52,29 @@ Create the name of the service account to use {{- else -}} {{ default "default" .Values.serviceAccount.name }} {{- end -}} -{{- end -}} \ No newline at end of file +{{- end -}} + +{{/* +Name of image +*/}} +{{- define "ray-cluster.image" -}} +{{- $name := default .Values.image.repository .Values.global.image.runtime.name }} +{{- $tag := default .Values.image.tag .Values.global.image.runtime.tag }} +{{- if .Values.global.image.repository }} +{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}:{{ $tag }} +{{- else }} +{{- $name }}:{{ $tag }} +{{- end }} +{{- end }} + +{{/* +Name of sidecar image +*/}} +{{- define "ray-cluster-sidecar.image" -}} +{{- $name := default (printf "%s:%s" .Values.image.repository .Values.image.tag) .Values.head.sidecarContainers.image }} +{{- if .Values.global.image.repository }} +{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }} +{{- else }} +{{- $name }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/deployment/helm/datamate/charts/ray-cluster/templates/raycluster-cluster.yaml b/deployment/helm/datamate/charts/ray-cluster/templates/raycluster-cluster.yaml index 5e17cf1..5f26211 100644 --- a/deployment/helm/datamate/charts/ray-cluster/templates/raycluster-cluster.yaml +++ b/deployment/helm/datamate/charts/ray-cluster/templates/raycluster-cluster.yaml @@ -58,13 +58,8 @@ spec: {{- end }} containers: - name: ray-head - {{- if .Values.head.image }} - image: {{ .Values.head.image.repository }}:{{ .Values.head.image.tag }} - imagePullPolicy: {{ .Values.head.image.pullPolicy }} - {{- else }} - image: {{ .Values.image.repository }}:{{ .Values.image.tag }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- end }} + image: {{ include "ray-cluster.image" . }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.global.image.pullPolicy }} {{- with .Values.head.command }} command: {{- toYaml . | nindent 10 }} @@ -101,8 +96,16 @@ spec: securityContext: {{- toYaml . | nindent 12 }} {{- end }} + {{- $defult := printf "%s:%s" .Values.image.repository .Values.image.tag }} {{- with .Values.head.sidecarContainers }} - {{- toYaml . | nindent 8 }} + {{- range $index, $container := . }} + {{- $image := default $defult $container.image -}} + {{- with $.Values.global.image.repository -}} + {{- $image = printf "%s/%s" (trimSuffix "/" .) (default $defult $container.image) -}} + {{- end -}} + {{- $mergedObj := mergeOverwrite $container (dict "image" $image) }} + - {{- toYaml $mergedObj | nindent 10 }} + {{- end }} {{- end }} {{- with .Values.imagePullSecrets }} imagePullSecrets: @@ -185,13 +188,8 @@ spec: {{- end }} containers: - name: ray-worker - {{- if .Values.worker.image }} - image: {{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag }} - imagePullPolicy: {{ .Values.worker.image.pullPolicy }} - {{- else }} - image: {{ .Values.image.repository }}:{{ .Values.image.tag }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- end }} + image: {{ include "ray-cluster.image" . }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.global.image.pullPolicy }} {{- with .Values.worker.command }} command: {{- toYaml . | nindent 10 }} diff --git a/deployment/helm/datamate/charts/ray-cluster/values.yaml b/deployment/helm/datamate/charts/ray-cluster/values.yaml index d316aa4..7627f0f 100644 --- a/deployment/helm/datamate/charts/ray-cluster/values.yaml +++ b/deployment/helm/datamate/charts/ray-cluster/values.yaml @@ -77,8 +77,6 @@ head: value: "password" - name: MYSQL_DATABASE value: "datamate" - - name: PDF_FORMATTER_BASE_URL - value: "http://datamate-mineru:9001" # - name: EXAMPLE_ENV # value: "1" envFrom: [] @@ -173,8 +171,6 @@ worker: value: "password" - name: MYSQL_DATABASE value: "datamate" - - name: PDF_FORMATTER_BASE_URL - value: "http://datamate-mineru:9001" # - name: EXAMPLE_ENV # value: "1" envFrom: [] diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index 2ecd0ba..9ab822c 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -6,14 +6,20 @@ global: deerFlow: enable: false image: - repository: "" + repository: "ghcr.io/modelengine-group/" pullPolicy: "IfNotPresent" backend: name: "datamate-backend" tag: "latest" + backendPython: + name: "datamate-backend-python" + tag: "latest" frontend: name: "datamate-frontend" tag: "latest" + runtime: + name: "datamate-runtime" + tag: "latest" database: name: "mysql" tag: "8" @@ -136,8 +142,6 @@ ray-cluster: value: "password" - name: MYSQL_DATABASE value: "datamate" - - name: PDF_FORMATTER_BASE_URL - value: "http://datamate-mineru:9001" ports: - containerPort: 8081 volumeMounts: diff --git a/deployment/helm/deer-flow/values.yaml b/deployment/helm/deer-flow/values.yaml index c070d25..c671558 100644 --- a/deployment/helm/deer-flow/values.yaml +++ b/deployment/helm/deer-flow/values.yaml @@ -4,7 +4,7 @@ global: image: - repository: "" + repository: "ghcr.io/modelengine-group/" pullPolicy: "IfNotPresent" backend: name: "deer-flow-backend" diff --git a/deployment/kubernetes/backend/deploy.yaml b/deployment/kubernetes/backend/deploy.yaml deleted file mode 100644 index c14aa37..0000000 --- a/deployment/kubernetes/backend/deploy.yaml +++ /dev/null @@ -1,86 +0,0 @@ - - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app: datamate - tier: backend - name: datamate-backend - ---- - - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: datamate-backend - labels: - app: datamate - tier: backend -spec: - replicas: 1 - selector: - matchLabels: - app: datamate - tier: backend - template: - metadata: - labels: - app: datamate - tier: backend - spec: - serviceAccountName: datamate-backend - containers: - - name: backend - image: datamate-backend - imagePullPolicy: IfNotPresent - env: - - name: namespace - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: SPRING_CONFIG_LOCATION - value: file:/opt/backend/application.yml - ports: - - containerPort: 8080 - volumeMounts: - - name: dataset-volume - mountPath: /dataset - - name: flow-volume - mountPath: /flow - - name: log-volume - mountPath: /var/log/datamate - volumes: - - name: dataset-volume - hostPath: - path: /opt/datamate/data/dataset - type: DirectoryOrCreate - - name: flow-volume - hostPath: - path: /opt/datamate/data/flow - type: DirectoryOrCreate - - name: log-volume - hostPath: - path: /opt/datamate/data/log - type: DirectoryOrCreate - ---- -apiVersion: v1 -kind: Service -metadata: - name: datamate-backend - labels: - app: datamate - tier: backend -spec: - type: ClusterIP - ports: - - port: 8080 - targetPort: 8080 - protocol: TCP - selector: - app: datamate - tier: backend diff --git a/deployment/kubernetes/frontend/deploy.yaml b/deployment/kubernetes/frontend/deploy.yaml deleted file mode 100644 index a7a5d6f..0000000 --- a/deployment/kubernetes/frontend/deploy.yaml +++ /dev/null @@ -1,52 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: datamate-frontend - labels: - app: datamate - tier: frontend -spec: - replicas: 1 - selector: - matchLabels: - app: datamate - tier: frontend - template: - metadata: - labels: - app: datamate - tier: frontend - spec: - containers: - - name: frontend - image: datamate-frontend - imagePullPolicy: IfNotPresent - ports: - - containerPort: 80 - volumeMounts: - - name: log-volume - mountPath: /var/log/datamate/frontend - subPath: frontend - volumes: - - name: log-volume - hostPath: - path: /opt/datamate/data/log - type: DirectoryOrCreate ---- -apiVersion: v1 -kind: Service -metadata: - name: datamate-frontend - labels: - app: datamate - tier: frontend -spec: - type: NodePort - ports: - - port: 80 - targetPort: 80 - nodePort: 30000 - protocol: TCP - selector: - app: datamate - tier: frontend diff --git a/deployment/kubernetes/mysql/configmap.yaml b/deployment/kubernetes/mysql/configmap.yaml deleted file mode 100644 index c3ee49b..0000000 --- a/deployment/kubernetes/mysql/configmap.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: datamate-mysql-utf8-config -data: - utf8.cnf: | - [mysqld] - # 设置服务器默认字符集为 utf8mb4 (推荐,支持完整的 UTF-8,包括 emoji) - character-set-server = utf8mb4 - # 设置默认排序规则 - collation-server = utf8mb4_unicode_ci - # 或者使用 utf8_general_ci (性能稍好,但排序规则稍宽松) - default-time-zone = 'Asia/Shanghai' - log_error=/var/log/datamate/database/error.log - - [client] - # 设置客户端连接默认字符集 - default-character-set = utf8mb4 - - [mysql] - # 设置 mysql 命令行客户端默认字符集 - default-character-set = utf8mb4 diff --git a/deployment/kubernetes/mysql/deploy.yaml b/deployment/kubernetes/mysql/deploy.yaml deleted file mode 100644 index b83da1f..0000000 --- a/deployment/kubernetes/mysql/deploy.yaml +++ /dev/null @@ -1,88 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: datamate-database - labels: - app: datamate - tier: database -spec: - replicas: 1 - selector: - matchLabels: - app: datamate - tier: database - strategy: - type: Recreate - template: - metadata: - labels: - app: datamate - tier: database - spec: - initContainers: - - name: init-log - image: mysql:8 - imagePullPolicy: IfNotPresent - command: - - sh - - -c - args: - - | - chown mysql:mysql /var/log/datamate/database - chmod 755 /var/log/datamate/database - volumeMounts: - - name: log-volume - mountPath: /var/log/datamate/database - subPath: database - containers: - - name: mysql - image: mysql:8 - imagePullPolicy: IfNotPresent - env: - - name: MYSQL_ROOT_PASSWORD - value: "password" - ports: - - containerPort: 3306 - volumeMounts: - - name: data-volume - mountPath: /var/lib/mysql - - name: log-volume - mountPath: /var/log/datamate/database - subPath: database - - name: init-sql - mountPath: /docker-entrypoint-initdb.d - - name: mysql-utf8-config - mountPath: /etc/mysql/conf.d - volumes: - - name: data-volume - hostPath: - path: /opt/datamate/data/mysql - type: DirectoryOrCreate - - name: log-volume - hostPath: - path: /opt/datamate/data/log - type: DirectoryOrCreate - - name: init-sql - configMap: - name: datamate-init-sql - - name: mysql-utf8-config - configMap: - name: datamate-mysql-utf8-config - ---- -apiVersion: v1 -kind: Service -metadata: - name: datamate-database - labels: - app: datamate - tier: database -spec: - type: ClusterIP - ports: - - port: 3306 - targetPort: 3306 - protocol: TCP - selector: - app: datamate - tier: database diff --git a/runtime/deer-flow/.env.example b/runtime/deer-flow/.env.example index d71bf6d..48ebed2 100644 --- a/runtime/deer-flow/.env.example +++ b/runtime/deer-flow/.env.example @@ -53,18 +53,17 @@ TAVILY_API_KEY=tvly-xxx # MOI_RETRIEVAL_SIZE=10 # MOI_LIST_LIMIT=10 - # RAG_PROVIDER: milvus (using free milvus instance on zilliz cloud: https://docs.zilliz.com/docs/quick-start ) # RAG_PROVIDER=milvus -# MILVUS_URI= -# MILVUS_USER= -# MILVUS_PASSWORD= +MILVUS_URI=http://milvus-standalone:19530 +MILVUS_USER=root +MILVUS_PASSWORD=Milvus +MILVUS_AUTO_LOAD_EXAMPLES=false # MILVUS_COLLECTION=documents # MILVUS_EMBEDDING_PROVIDER=openai # support openai,dashscope # MILVUS_EMBEDDING_BASE_URL= # MILVUS_EMBEDDING_MODEL= # MILVUS_EMBEDDING_API_KEY= -# MILVUS_AUTO_LOAD_EXAMPLES=true # RAG_PROVIDER: milvus (using milvus lite on Mac or Linux) # RAG_PROVIDER=milvus diff --git a/runtime/ops/formatter/mineru_formatter/process.py b/runtime/ops/formatter/mineru_formatter/process.py index f1cff86..afd7dd7 100644 --- a/runtime/ops/formatter/mineru_formatter/process.py +++ b/runtime/ops/formatter/mineru_formatter/process.py @@ -6,7 +6,6 @@ Description: MinerU PDF文本抽取 Create: 2025/10/29 17:24 """ import json -import os import time from loguru import logger from typing import Dict, Any @@ -20,7 +19,7 @@ class MineruFormatter(Mapper): def __init__(self, *args, **kwargs): super(MineruFormatter, self).__init__(*args, **kwargs) - self.base_url = os.getenv("EXTERNAL_PDF_BASE_URL", "http://datamate-mineru:9001") + self.base_url = "http://datamate-mineru:9001" self.pdf_extract_url = f"{self.base_url}/api/pdf-extract" def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]: