diff --git a/Makefile b/Makefile index c89d6b1..c83aea0 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ create-namespace: install-%: ifeq ($(origin INSTALLER), undefined) @echo "Choose a deployment method:" - @echo "1. Docker" + @echo "1. Docker/Docker-Compose" @echo "2. Kubernetes/Helm" @echo -n "Enter choice: " @read choice; \ @@ -39,7 +39,7 @@ install: install-datamate uninstall-%: ifeq ($(origin INSTALLER), undefined) @echo "Choose a deployment method:" - @echo "1. Docker" + @echo "1. Docker/Docker-Compose" @echo "2. Kubernetes/Helm" @echo -n "Enter choice: " @read choice; \ @@ -97,52 +97,6 @@ runtime-docker-install: runtime-docker-uninstall: cd deployment/docker/datamate && docker-compose down runtime -.PHONY: runtime-k8s-install -runtime-k8s-install: create-namespace - helm upgrade datamate-kuberay-operator deployment/helm/ray/kuberay-operator --install -n $(NAMESPACE) - helm upgrade datamate-raycluster deployment/helm/ray/ray-cluster/ --install -n $(NAMESPACE) - kubectl apply -f deployment/helm/ray/service.yaml -n $(NAMESPACE) - -.PHONY: runtime-k8s-uninstall -runtime-k8s-uninstall: - helm uninstall datamate-raycluster -n $(NAMESPACE) - helm uninstall datamate-kuberay-operator -n $(NAMESPACE) - kubectl delete -f deployment/helm/ray/service.yaml -n $(NAMESPACE) - -.PHONY: mysql-k8s-install -mysql-k8s-install: create-namespace - kubectl create configmap datamate-init-sql --from-file=scripts/db/ --dry-run=client -o yaml | kubectl apply -f - -n $(NAMESPACE) - kubectl apply -f deployment/kubernetes/mysql/configmap.yaml -n $(NAMESPACE) - kubectl apply -f deployment/kubernetes/mysql/deploy.yaml -n $(NAMESPACE) - -.PHONY: mysql-k8s-uninstall -mysql-k8s-uninstall: - kubectl delete configmap datamate-init-sql -n $(NAMESPACE) --ignore-not-found - kubectl delete -f deployment/kubernetes/mysql/configmap.yaml -n $(NAMESPACE) --ignore-not-found - kubectl delete -f deployment/kubernetes/mysql/deploy.yaml -n $(NAMESPACE) --ignore-not-found - -.PHONY: database-k8s-install -database-k8s-install: mysql-k8s-install - -.PHONY: database-k8s-uninstall -database-k8s-uninstall: mysql-k8s-uninstall - -.PHONY: backend-k8s-install -backend-k8s-install: create-namespace - kubectl apply -f deployment/kubernetes/backend/deploy.yaml -n $(NAMESPACE) - -.PHONY: backend-k8s-uninstall -backend-k8s-uninstall: - kubectl delete -f deployment/kubernetes/backend/deploy.yaml -n $(NAMESPACE) --ignore-not-found - -.PHONY: frontend-k8s-install -frontend-k8s-install: create-namespace - kubectl apply -f deployment/kubernetes/frontend/deploy.yaml -n $(NAMESPACE) - -.PHONY: frontend-k8s-uninstall -frontend-k8s-uninstall: - kubectl delete -f deployment/kubernetes/frontend/deploy.yaml -n $(NAMESPACE) --ignore-not-found - .PHONY: datamate-docker-install datamate-docker-install: cd deployment/docker/datamate && docker-compose up -d @@ -152,7 +106,11 @@ datamate-docker-uninstall: cd deployment/docker/datamate && docker-compose down .PHONY: datamate-k8s-install -datamate-k8s-install: create-namespace database-k8s-install backend-k8s-install frontend-k8s-install runtime-k8s-install +datamate-k8s-install: create-namespace + kubectl create configmap datamate-init-sql --from-file=scripts/db/ --dry-run=client -o yaml | kubectl apply -f - -n $(NAMESPACE) + helm install datamate deployment/helm/datamate/ -n $(NAMESPACE) .PHONY: datamate-k8s-uninstall -datamate-k8s-uninstall: database-k8s-uninstall backend-k8s-uninstall frontend-k8s-uninstall runtime-k8s-uninstall +datamate-k8s-uninstall: + helm uninstall datamate -n $(NAMESPACE) --ignore-not-found + kubectl delete configmap datamate-init-sql -n $(NAMESPACE) --ignore-not-found diff --git a/README-zh.md b/README-zh.md index 4e492a0..bc71cdf 100644 --- a/README-zh.md +++ b/README-zh.md @@ -38,6 +38,7 @@ ```bash git clone git@github.com:ModelEngine-Group/DataMate.git +cd DataMate ``` ### 镜像构建 diff --git a/README.md b/README.md index b80be8d..087285d 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ If you like this project, please give it a Star⭐️! ```bash git clone git@github.com:ModelEngine-Group/DataMate.git +cd DataMate ``` ### Build Images diff --git a/backend/openapi/specs/data-cleaning.yaml b/backend/openapi/specs/data-cleaning.yaml index ff49274..1485235 100644 --- a/backend/openapi/specs/data-cleaning.yaml +++ b/backend/openapi/specs/data-cleaning.yaml @@ -1,491 +1,1286 @@ -openapi: 3.0.3 +openapi: "3.0.1" info: - title: Data Cleaning Service API - description: 数据清洗服务API - 策略/规则、流程编排对接 - version: 1.0.0 - contact: - name: Data Mate Platform Team - -servers: - - url: http://localhost:8084 - description: Development server - -tags: - - name: CleaningTask - description: 数据清洗任务管理 - - name: CleaningTemplate - description: 数据清洗模板管理 - + title: "API Documentation" + version: "1.0" paths: - /ray/log: - get: - summary: 获取ray日志文件 - deprecated: false - description: '' - tags: [ ] - parameters: [ ] - responses: - '200': - description: '' - content: - application/json: - schema: - type: object - properties: { } - headers: { } - security: [ ] - /cleaning/tasks: - get: - summary: 查询数据清洗任务列表 - deprecated: false - description: 获取所有数据清洗任务或根据查询参数筛选任务。 - tags: - - CleaningTask - parameters: - - name: status - in: query - description: 根据任务状态筛选 (e.g., pending, running, completed, failed) - required: false - schema: - type: string - - name: keywords - in: query - description: 关键字 - required: false - schema: - type: string - - name: page - in: query - description: 分页数 - required: true - schema: - type: integer - - name: size - in: query - description: 分页单页数 - required: true - schema: - type: integer - responses: - '200': - description: 成功获取任务列表 - content: - application/json: - schema: - type: array - items: &ref_1 - $ref: '#/components/schemas/CleaningTask' - headers: { } - security: [ ] - post: - summary: 创建新的数据清洗任务 - deprecated: false - description: 可以直接创建任务或基于现有模板创建任务。 - tags: - - CleaningTask - parameters: [ ] - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/CreateCleaningTaskRequest' - examples: { } - responses: - '201': - description: 任务创建成功 - content: - application/json: - schema: *ref_1 - headers: { } - security: [ ] - /cleaning/tasks/{taskId}: - get: - summary: 获取单个数据清洗任务详情 - deprecated: false - description: 根据任务ID获取任务的详细信息。 - tags: - - CleaningTask - parameters: - - name: taskId - in: path - description: 任务的唯一标识符 - required: true - example: '' - schema: - type: string - responses: - '200': - description: 成功获取任务详情 - content: - application/json: - schema: *ref_1 - headers: { } - security: [ ] - delete: - summary: 删除数据清洗任务 - deprecated: false - description: 根据任务ID删除指定的任务。 - tags: - - CleaningTask - parameters: - - name: taskId - in: path - description: 任务的唯一标识符 - required: true - example: '' - schema: - type: string - responses: - '204': - description: 任务删除成功 - headers: { } - security: [ ] /cleaning/templates: get: - summary: 查询数据清洗模板列表 - deprecated: false - description: 获取所有可用的数据清洗模板。 - tags: - - CleaningTemplate - parameters: [ ] + summary: "cleaningTemplatesGet" + description: "cleaningTemplatesGet" + parameters: + - name: "page" + in: "query" + description: "" + required: false + schema: + type: "integer" + nullable: false + - name: "size" + in: "query" + description: "" + required: false + schema: + type: "integer" + nullable: false + - name: "keywords" + in: "query" + description: "" + required: false + schema: + type: "string" + nullable: false responses: - '200': - description: 成功获取模板列表 + "200": + description: "" content: application/json: schema: - type: array - items: &ref_2 - $ref: '#/components/schemas/CleaningTemplate' - headers: { } - security: [ ] + type: "object" + properties: + code: + type: "string" + description: "状态码" + message: + type: "string" + description: "消息" + data: + type: "object" + properties: + page: + type: "integer" + description: "" + format: "int64" + size: + type: "integer" + description: "" + format: "int64" + totalElements: + type: "integer" + description: "" + format: "int64" + totalPages: + type: "integer" + description: "" + format: "int64" + content: + type: "array" + description: "" + items: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + instance: + type: "array" + description: "" + items: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + version: + type: "string" + description: "" + inputs: + type: "string" + description: "" + outputs: + type: "string" + description: "" + runtime: + type: "string" + description: "" + settings: + type: "string" + description: "" + isStar: + type: "boolean" + description: "" + createdAt: + type: "string" + description: "" + updatedAt: + type: "string" + description: "" + description: "OperatorResponse" + default: "new ArrayList<>()" + createdAt: + type: "string" + description: "" + updatedAt: + type: "string" + description: "" + description: "CleaningTemplate" + description: "数据" post: - summary: 创建新的数据清洗模板 - deprecated: false - description: 定义一个新的数据清洗模板。 - tags: - - CleaningTemplate - parameters: [ ] + summary: "cleaningTemplatesPost" + description: "cleaningTemplatesPost" requestBody: content: application/json: schema: - $ref: '#/components/schemas/CreateCleaningTemplateRequest' + type: "object" + properties: + name: + type: "string" + description: "" + description: + type: "string" + description: "" + instance: + type: "array" + description: "" + items: + type: "object" + properties: + id: + type: "string" + description: "" + overrides: + type: "object" + properties: + key: + type: "object" + properties: {} + description: "" + default: "new HashMap<>()" + description: "OperatorInstance" + default: "new ArrayList<>()" + description: "" responses: - '201': - description: 模板创建成功 + "200": + description: "" content: application/json: - schema: *ref_2 - headers: { } - security: [ ] + schema: + type: "object" + properties: + code: + type: "string" + description: "状态码" + message: + type: "string" + description: "消息" + data: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + instance: + type: "array" + description: "" + items: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + version: + type: "string" + description: "" + inputs: + type: "string" + description: "" + outputs: + type: "string" + description: "" + runtime: + type: "string" + description: "" + settings: + type: "string" + description: "" + isStar: + type: "boolean" + description: "" + createdAt: + type: "string" + description: "" + updatedAt: + type: "string" + description: "" + description: "OperatorResponse" + default: "new ArrayList<>()" + createdAt: + type: "string" + description: "" + updatedAt: + type: "string" + description: "" + description: "数据" /cleaning/templates/{templateId}: get: - summary: 获取单个数据清洗模板详情 - deprecated: false - description: 根据模板ID获取模板的详细信息。 - tags: - - CleaningTemplate + summary: "cleaningTemplatesTemplateIdGet" + description: "cleaningTemplatesTemplateIdGet" parameters: - - name: templateId - in: path - description: 模板的唯一标识符 - required: true - example: '' - schema: - type: string + - name: "templateId" + in: "path" + description: "" + required: true + schema: + type: "string" responses: - '200': - description: 成功获取模板详情 + "200": + description: "" content: application/json: - schema: *ref_2 - headers: { } - security: [ ] + schema: + type: "object" + properties: + code: + type: "string" + description: "状态码" + message: + type: "string" + description: "消息" + data: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + instance: + type: "array" + description: "" + items: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + version: + type: "string" + description: "" + inputs: + type: "string" + description: "" + outputs: + type: "string" + description: "" + runtime: + type: "string" + description: "" + settings: + type: "string" + description: "" + isStar: + type: "boolean" + description: "" + createdAt: + type: "string" + description: "" + updatedAt: + type: "string" + description: "" + description: "OperatorResponse" + default: "new ArrayList<>()" + createdAt: + type: "string" + description: "" + updatedAt: + type: "string" + description: "" + description: "数据" put: - summary: 更新数据清洗模板 - deprecated: false - description: 根据模板ID更新模板的全部信息。 - tags: - - CleaningTemplate + summary: "cleaningTemplatesTemplateIdPut" + description: "cleaningTemplatesTemplateIdPut" parameters: - - name: templateId - in: path - description: 模板的唯一标识符 - required: true - example: '' - schema: - type: string + - name: "templateId" + in: "path" + description: "" + required: true + schema: + type: "string" requestBody: content: application/json: schema: - $ref: '#/components/schemas/UpdateCleaningTemplateRequest' + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + instance: + type: "array" + description: "" + items: + type: "object" + properties: + id: + type: "string" + description: "" + overrides: + type: "object" + properties: + key: + type: "object" + properties: {} + description: "" + default: "new HashMap<>()" + description: "OperatorInstance" + default: "new ArrayList<>()" + description: "" responses: - '200': - description: 模板更新成功 + "200": + description: "" content: application/json: - schema: *ref_2 - headers: { } - security: [ ] + schema: + type: "object" + properties: + code: + type: "string" + description: "状态码" + message: + type: "string" + description: "消息" + data: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + instance: + type: "array" + description: "" + items: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + version: + type: "string" + description: "" + inputs: + type: "string" + description: "" + outputs: + type: "string" + description: "" + runtime: + type: "string" + description: "" + settings: + type: "string" + description: "" + isStar: + type: "boolean" + description: "" + createdAt: + type: "string" + description: "" + updatedAt: + type: "string" + description: "" + description: "OperatorResponse" + default: "new ArrayList<>()" + createdAt: + type: "string" + description: "" + updatedAt: + type: "string" + description: "" + description: "数据" delete: - summary: 删除数据清洗模板 - deprecated: false - description: 根据模板ID删除指定的模板。 - tags: - - CleaningTemplate + summary: "cleaningTemplatesTemplateIdDelete" + description: "cleaningTemplatesTemplateIdDelete" parameters: - - name: templateId - in: path - description: 模板的唯一标识符 - required: true - example: '' - schema: - type: string + - name: "templateId" + in: "path" + description: "" + required: true + schema: + type: "string" responses: - '204': - description: 模板删除成功 - headers: { } - security: [ ] - + "200": + description: "" + content: + application/json: + schema: + type: "object" + properties: + code: + type: "string" + description: "状态码" + message: + type: "string" + description: "消息" + data: + type: "object" + properties: {} + description: "数据" + /cleaning/tasks: + get: + summary: "cleaningTasksGet" + description: "cleaningTasksGet" + parameters: + - name: "page" + in: "query" + description: "" + required: true + schema: + type: "integer" + nullable: false + - name: "size" + in: "query" + description: "" + required: true + schema: + type: "integer" + nullable: false + - name: "status" + in: "query" + description: "" + required: false + schema: + type: "string" + nullable: false + - name: "keywords" + in: "query" + description: "" + required: false + schema: + type: "string" + nullable: false + responses: + "200": + description: "" + content: + application/json: + schema: + type: "object" + properties: + code: + type: "string" + description: "状态码" + message: + type: "string" + description: "消息" + data: + type: "object" + properties: + page: + type: "integer" + description: "" + format: "int64" + size: + type: "integer" + description: "" + format: "int64" + totalElements: + type: "integer" + description: "" + format: "int64" + totalPages: + type: "integer" + description: "" + format: "int64" + content: + type: "array" + description: "" + items: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + srcDatasetId: + type: "string" + description: "" + srcDatasetName: + type: "string" + description: "" + destDatasetId: + type: "string" + description: "" + destDatasetName: + type: "string" + description: "" + beforeSize: + type: "integer" + description: "" + format: "int64" + afterSize: + type: "integer" + description: "" + format: "int64" + fileCount: + type: "integer" + description: "" + status: + type: "string" + description: "{PENDING=PENDING, RUNNING=RUNNING, COMPLETED=COMPLETED,\ + \ STOPPED=STOPPED, FAILED=FAILED}" + enum: + - "PENDING" + - "RUNNING" + - "COMPLETED" + - "STOPPED" + - "FAILED" + templateId: + type: "string" + description: "" + instance: + type: "array" + description: "" + items: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + version: + type: "string" + description: "" + inputs: + type: "string" + description: "" + outputs: + type: "string" + description: "" + runtime: + type: "string" + description: "" + settings: + type: "string" + description: "" + isStar: + type: "boolean" + description: "" + createdAt: + type: "string" + description: "" + updatedAt: + type: "string" + description: "" + description: "OperatorResponse" + progress: + type: "object" + properties: + process: + type: "number" + description: "" + totalFileNum: + type: "integer" + description: "" + finishedFileNum: + type: "integer" + description: "" + description: "" + createdAt: + type: "string" + description: "" + startedAt: + type: "string" + description: "" + finishedAt: + type: "string" + description: "" + description: "CleaningTask" + description: "数据" + post: + summary: "cleaningTasksPost" + description: "cleaningTasksPost" + requestBody: + content: + application/json: + schema: + type: "object" + properties: + name: + type: "string" + description: "" + description: + type: "string" + description: "" + srcDatasetId: + type: "string" + description: "" + srcDatasetName: + type: "string" + description: "" + destDatasetName: + type: "string" + description: "" + destDatasetType: + type: "string" + description: "" + instance: + type: "array" + description: "" + items: + type: "object" + properties: + id: + type: "string" + description: "" + overrides: + type: "object" + properties: + key: + type: "object" + properties: {} + description: "" + default: "new HashMap<>()" + description: "OperatorInstance" + default: "new ArrayList<>()" + description: "" + responses: + "200": + description: "" + content: + application/json: + schema: + type: "object" + properties: + code: + type: "string" + description: "状态码" + message: + type: "string" + description: "消息" + data: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + srcDatasetId: + type: "string" + description: "" + srcDatasetName: + type: "string" + description: "" + destDatasetId: + type: "string" + description: "" + destDatasetName: + type: "string" + description: "" + beforeSize: + type: "integer" + description: "" + format: "int64" + afterSize: + type: "integer" + description: "" + format: "int64" + fileCount: + type: "integer" + description: "" + status: + type: "string" + description: "{PENDING=PENDING, RUNNING=RUNNING, COMPLETED=COMPLETED,\ + \ STOPPED=STOPPED, FAILED=FAILED}" + enum: + - "PENDING" + - "RUNNING" + - "COMPLETED" + - "STOPPED" + - "FAILED" + templateId: + type: "string" + description: "" + instance: + type: "array" + description: "" + items: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + version: + type: "string" + description: "" + inputs: + type: "string" + description: "" + outputs: + type: "string" + description: "" + runtime: + type: "string" + description: "" + settings: + type: "string" + description: "" + isStar: + type: "boolean" + description: "" + createdAt: + type: "string" + description: "" + updatedAt: + type: "string" + description: "" + description: "OperatorResponse" + progress: + type: "object" + properties: + process: + type: "number" + description: "" + totalFileNum: + type: "integer" + description: "" + finishedFileNum: + type: "integer" + description: "" + description: "" + createdAt: + type: "string" + description: "" + startedAt: + type: "string" + description: "" + finishedAt: + type: "string" + description: "" + description: "数据" + /cleaning/tasks/{taskId}/stop: + post: + summary: "cleaningTasksStop" + description: "cleaningTasksStop" + parameters: + - name: "taskId" + in: "path" + description: "" + required: true + schema: + type: "string" + responses: + "200": + description: "" + content: + application/json: + schema: + type: "object" + properties: + code: + type: "string" + description: "状态码" + message: + type: "string" + description: "消息" + data: + type: "object" + properties: {} + description: "数据" + /cleaning/tasks/{taskId}/execute: + post: + summary: "cleaningTasksStart" + description: "cleaningTasksStart" + parameters: + - name: "taskId" + in: "path" + description: "" + required: true + schema: + type: "string" + responses: + "200": + description: "" + content: + application/json: + schema: + type: "object" + properties: + code: + type: "string" + description: "状态码" + message: + type: "string" + description: "消息" + data: + type: "object" + properties: {} + description: "数据" + /cleaning/tasks/{taskId}: + get: + summary: "cleaningTasksTaskIdGet" + description: "cleaningTasksTaskIdGet" + parameters: + - name: "taskId" + in: "path" + description: "" + required: true + schema: + type: "string" + responses: + "200": + description: "" + content: + application/json: + schema: + type: "object" + properties: + code: + type: "string" + description: "状态码" + message: + type: "string" + description: "消息" + data: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + srcDatasetId: + type: "string" + description: "" + srcDatasetName: + type: "string" + description: "" + destDatasetId: + type: "string" + description: "" + destDatasetName: + type: "string" + description: "" + beforeSize: + type: "integer" + description: "" + format: "int64" + afterSize: + type: "integer" + description: "" + format: "int64" + fileCount: + type: "integer" + description: "" + status: + type: "string" + description: "{PENDING=PENDING, RUNNING=RUNNING, COMPLETED=COMPLETED,\ + \ STOPPED=STOPPED, FAILED=FAILED}" + enum: + - "PENDING" + - "RUNNING" + - "COMPLETED" + - "STOPPED" + - "FAILED" + templateId: + type: "string" + description: "" + instance: + type: "array" + description: "" + items: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + version: + type: "string" + description: "" + inputs: + type: "string" + description: "" + outputs: + type: "string" + description: "" + runtime: + type: "string" + description: "" + settings: + type: "string" + description: "" + isStar: + type: "boolean" + description: "" + createdAt: + type: "string" + description: "" + updatedAt: + type: "string" + description: "" + description: "OperatorResponse" + progress: + type: "object" + properties: + process: + type: "number" + description: "" + totalFileNum: + type: "integer" + description: "" + finishedFileNum: + type: "integer" + description: "" + description: "" + createdAt: + type: "string" + description: "" + startedAt: + type: "string" + description: "" + finishedAt: + type: "string" + description: "" + description: "数据" + delete: + summary: "cleaningTasksTaskIdDelete" + description: "cleaningTasksTaskIdDelete" + parameters: + - name: "taskId" + in: "path" + description: "" + required: true + schema: + type: "string" + responses: + "200": + description: "" + content: + application/json: + schema: + type: "object" + properties: + code: + type: "string" + description: "状态码" + message: + type: "string" + description: "消息" + data: + type: "object" + properties: {} + description: "数据" components: schemas: - OperatorInstance: - type: object + OperatorResponse: + type: "object" properties: id: - type: string + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + version: + type: "string" + description: "" + inputs: + type: "string" + description: "" + outputs: + type: "string" + description: "" + runtime: + type: "string" + description: "" + settings: + type: "string" + description: "" + isStar: + type: "boolean" + description: "" + createdAt: + type: "string" + description: "" + updatedAt: + type: "string" + description: "" + description: "OperatorResponse" + CleaningTemplate: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + instance: + type: "array" + description: "" + items: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + version: + type: "string" + description: "" + inputs: + type: "string" + description: "" + outputs: + type: "string" + description: "" + runtime: + type: "string" + description: "" + settings: + type: "string" + description: "" + isStar: + type: "boolean" + description: "" + createdAt: + type: "string" + description: "" + updatedAt: + type: "string" + description: "" + description: "OperatorResponse" + default: "new ArrayList<>()" + createdAt: + type: "string" + description: "" + updatedAt: + type: "string" + description: "" + description: "数据" + OperatorInstance: + type: "object" + properties: + id: + type: "string" + description: "" overrides: - type: object - properties: { } - additionalProperties: - type: object - properties: { } - required: - - id - - overrides + type: "object" + properties: + key: + type: "object" + properties: {} + description: "" + default: "new HashMap<>()" + description: "OperatorInstance" + java.lang.Object: + type: "object" + properties: {} + description: "数据" CleaningProcess: - type: object + type: "object" properties: process: - type: number - format: float - description: 进度百分比 + type: "number" + description: "" totalFileNum: - type: integer - description: 总文件数量 + type: "integer" + description: "" finishedFileNum: - type: integer - description: 已完成文件数量 - required: - - process - - totalFileNum - - finishedFileNum - OperatorResponse: - type: object - properties: - id: - type: string - description: 算子ID - name: - type: string - description: 算子名称 - description: - type: string - description: 算子描述 - version: - type: string - description: 算子版本 - inputs: - type: string - description: 输入类型 - outputs: - type: string - description: 输入类型 - runtime: - type: string - description: 运行时设置 - settings: - type: string - description: 算子参数 - isStar: - type: boolean - description: 是否收藏 - createdAt: - type: string - format: date-time - description: 创建时间 - updatedAt: - type: string - format: date-time - description: 更新时间 - required: - - inputs - - outputs - - runtime - - settings - - isStar - UpdateCleaningTemplateRequest: - type: object - required: - - name - - instance - - id - properties: - id: - type: string - name: - type: string - description: 模板名称 - description: - type: string - description: 模板描述 - instance: - type: array - items: &ref_3 - $ref: '#/components/schemas/OperatorInstance' - description: 模板定义的清洗规则和配置 - CreateCleaningTemplateRequest: - type: object - required: - - name - - instance - properties: - name: - type: string - description: 模板名称 - description: - type: string - description: 模板描述 - instance: - type: array - items: *ref_3 - description: 任务的具体配置（如果非模板创建，则直接定义）' - CleaningTemplate: - type: object - required: - - id - - name - - instance - - createdAt - properties: - id: - type: string - description: 模板唯一标识符 - name: - type: string - description: 模板名称 - description: - type: string - description: 模板描述 - instance: - type: array - items: &ref_4 - $ref: '#/components/schemas/OperatorResponse' - description: 模板定义的清洗规则和配置 - createdAt: - type: string - format: date-time - description: 模板创建时间 - updatedAt: - type: string - format: date-time - description: 模板最后更新时间 - CreateCleaningTaskRequest: - type: object - required: - - name - - instance - - srcDatasetId - - srcDatasetName - - destDatasetName - - destDatasetType - properties: - name: - type: string - description: 任务名称 - description: - type: string - description: 任务描述 - srcDatasetId: - type: string - srcDatasetName: - type: string - destDatasetName: - type: string - destDatasetType: - type: string - instance: - type: array - items: *ref_3 - description: 任务的具体配置（如果非模板创建，则直接定义） - ErrorResponse: - type: object - properties: - error: - type: string - description: 错误类型 - message: - type: string - description: 错误详细信息 + type: "integer" + description: "" + description: "" CleaningTask: - type: object - required: - - id - - name - - status - - createdAt - - startedAt + type: "object" properties: id: - type: string - description: 任务唯一标识符 + type: "string" + description: "" name: - type: string - description: 任务名称 + type: "string" + description: "" description: - type: string - description: 任务描述 + type: "string" + description: "" srcDatasetId: - type: string - description: 源数据集id + type: "string" + description: "" srcDatasetName: - type: string - description: 源数据集名称 + type: "string" + description: "" destDatasetId: - type: string - description: 目标数据集id + type: "string" + description: "" destDatasetName: - type: string - description: 目标数据集名称 + type: "string" + description: "" + beforeSize: + type: "integer" + description: "" + format: "int64" + afterSize: + type: "integer" + description: "" + format: "int64" + fileCount: + type: "integer" + description: "" status: - type: string - description: 任务当前状态 + type: "string" + description: "{PENDING=PENDING, RUNNING=RUNNING, COMPLETED=COMPLETED, STOPPED=STOPPED,\ + \ FAILED=FAILED}" enum: - - pending - - running - - completed - - failed + - "PENDING" + - "RUNNING" + - "COMPLETED" + - "STOPPED" + - "FAILED" templateId: - type: string - description: 关联的模板ID（如果基于模板创建） + type: "string" + description: "" instance: - type: array - items: *ref_4 - description: 任务的具体配置（如果非模板创建，则直接定义） + type: "array" + description: "" + items: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + version: + type: "string" + description: "" + inputs: + type: "string" + description: "" + outputs: + type: "string" + description: "" + runtime: + type: "string" + description: "" + settings: + type: "string" + description: "" + isStar: + type: "boolean" + description: "" + createdAt: + type: "string" + description: "" + updatedAt: + type: "string" + description: "" + description: "OperatorResponse" progress: - $ref: '#/components/schemas/CleaningProcess' + type: "object" + properties: + process: + type: "number" + description: "" + totalFileNum: + type: "integer" + description: "" + finishedFileNum: + type: "integer" + description: "" + description: "" createdAt: - type: string - description: 任务创建时间 - format: date-time + type: "string" + description: "" startedAt: - type: string - format: date-time - description: 任务开始时间 + type: "string" + description: "" finishedAt: - type: string - format: date-time - description: 任务最后更新时间 - securitySchemes: { } + type: "string" + description: "" + description: "数据" diff --git a/backend/openapi/specs/operator-market.yaml b/backend/openapi/specs/operator-market.yaml index 8a6c172..44d353e 100644 --- a/backend/openapi/specs/operator-market.yaml +++ b/backend/openapi/specs/operator-market.yaml @@ -1,547 +1,812 @@ -openapi: 3.0.1 +openapi: "3.0.1" info: - title: Operator Market Service API - description: | - 算子市场服务API，提供算子的发布、管理和订阅功能。 - - 主要功能： - - 算子发布和管理 - - 算子版本控制 - - 算子评分和评论 - - 算子分类和标签 - - 算子下载和安装 - version: 1.0.0 -tags: - - name: Operator - - name: Category - - name: Label + title: "API Documentation" + version: "1.0" paths: /operators/list: post: - summary: 获取算子列表 - deprecated: false - description: 分页查询算子列表，支持按分类、标签等条件筛选 - tags: - - Operator - parameters: [] + summary: "operatorsListPost" + description: "operatorsListPost" requestBody: content: application/json: schema: - type: object + type: "object" properties: page: - type: integer - description: 页数 + type: "integer" + description: "" size: - type: integer - description: 单页数量 + type: "integer" + description: "" categories: - type: array + type: "array" + description: "" items: - type: integer - description: 分类id列表 + type: "integer" + default: "new ArrayList<>()" operatorName: - type: string - description: 算子名称 + type: "string" + description: "" labelName: - type: string - description: 标签名称 + type: "string" + description: "" isStar: - type: boolean - description: 是否收藏 - required: - - page - - size - - categories - examples: {} + type: "boolean" + description: "" + description: "" responses: - '200': - description: 成功返回算子列表 + "200": + description: "" content: application/json: schema: - type: array - items: - $ref: '#/components/schemas/OperatorResponse' - headers: {} - security: [] - /operators/create: - post: - summary: 创建新算子 - deprecated: false - description: 创建并发布一个新的算子 - tags: - - Operator - parameters: [] + type: "object" + properties: + code: + type: "string" + description: "状态码" + message: + type: "string" + description: "消息" + data: + type: "object" + properties: + page: + type: "integer" + description: "" + format: "int64" + size: + type: "integer" + description: "" + format: "int64" + totalElements: + type: "integer" + description: "" + format: "int64" + totalPages: + type: "integer" + description: "" + format: "int64" + content: + type: "array" + description: "" + items: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + version: + type: "string" + description: "" + inputs: + type: "string" + description: "" + outputs: + type: "string" + description: "" + categories: + type: "array" + description: "" + items: + type: "integer" + runtime: + type: "string" + description: "" + settings: + type: "string" + description: "" + isStar: + type: "boolean" + description: "" + createdAt: + type: "string" + description: "" + updatedAt: + type: "string" + description: "" + description: "OperatorResponse" + description: "数据" + /operators/{id}: + get: + summary: "operatorsIdGet" + description: "operatorsIdGet" + parameters: + - name: "id" + in: "path" + description: "" + required: true + schema: + type: "string" + responses: + "200": + description: "" + content: + application/json: + schema: + type: "object" + properties: + code: + type: "string" + description: "状态码" + message: + type: "string" + description: "消息" + data: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + version: + type: "string" + description: "" + inputs: + type: "string" + description: "" + outputs: + type: "string" + description: "" + categories: + type: "array" + description: "" + items: + type: "integer" + runtime: + type: "string" + description: "" + settings: + type: "string" + description: "" + isStar: + type: "boolean" + description: "" + createdAt: + type: "string" + description: "" + updatedAt: + type: "string" + description: "" + description: "数据" + put: + summary: "operatorsIdPut" + description: "operatorsIdPut" + parameters: + - name: "id" + in: "path" + description: "" + required: true + schema: + type: "string" requestBody: content: application/json: schema: - $ref: '#/components/schemas/CreateOperatorRequest' - example: null + type: "object" + properties: + name: + type: "string" + description: "" + description: + type: "string" + description: "" + version: + type: "string" + description: "" + inputs: + type: "string" + description: "" + outputs: + type: "string" + description: "" + categories: + type: "array" + description: "" + items: + type: "integer" + runtime: + type: "string" + description: "" + settings: + type: "string" + description: "" + description: "" responses: - '201': - description: 算子创建成功 + "200": + description: "" content: application/json: - schema: &ref_0 - $ref: '#/components/schemas/OperatorResponse' - headers: {} - security: [] + schema: + type: "object" + properties: + code: + type: "string" + description: "状态码" + message: + type: "string" + description: "消息" + data: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + version: + type: "string" + description: "" + inputs: + type: "string" + description: "" + outputs: + type: "string" + description: "" + categories: + type: "array" + description: "" + items: + type: "integer" + runtime: + type: "string" + description: "" + settings: + type: "string" + description: "" + isStar: + type: "boolean" + description: "" + createdAt: + type: "string" + description: "" + updatedAt: + type: "string" + description: "" + description: "数据" + /operators/create: + post: + summary: "operatorsCreatePost" + description: "operatorsCreatePost" + requestBody: + content: + application/json: + schema: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + version: + type: "string" + description: "" + inputs: + type: "string" + description: "" + outputs: + type: "string" + description: "" + categories: + type: "array" + description: "" + items: + type: "integer" + runtime: + type: "string" + description: "" + settings: + type: "string" + description: "" + fileName: + type: "string" + description: "" + description: "" + responses: + "200": + description: "" + content: + application/json: + schema: + type: "object" + properties: + code: + type: "string" + description: "状态码" + message: + type: "string" + description: "消息" + data: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + version: + type: "string" + description: "" + inputs: + type: "string" + description: "" + outputs: + type: "string" + description: "" + categories: + type: "array" + description: "" + items: + type: "integer" + runtime: + type: "string" + description: "" + settings: + type: "string" + description: "" + isStar: + type: "boolean" + description: "" + createdAt: + type: "string" + description: "" + updatedAt: + type: "string" + description: "" + description: "数据" /operators/upload: post: - summary: 上传新算子 - deprecated: false - description: 创建并发布一个新的算子 - tags: - - Operator - parameters: [] + summary: "operatorsUploadPost" + description: "operatorsUploadPost" + parameters: + - name: "description" + in: "query" + description: "" + required: true + schema: + type: "string" + nullable: false requestBody: content: multipart/form-data: schema: - type: object + required: + - "file" + type: "object" properties: file: - type: string - format: binary - example: '' - description: - type: string - example: '' - examples: {} + type: "string" + description: "" + format: "binary" + required: true responses: - '201': - description: 算子创建成功 - content: - application/json: - schema: *ref_0 - headers: {} - security: [] - /operators/{id}: - get: - summary: 获取算子详情 - deprecated: false - description: 根据ID获取算子的详细信息 - tags: - - Operator - parameters: - - name: id - in: path - description: 算子ID - required: true - example: '' - schema: - type: string - responses: - '200': - description: 成功返回算子详情 - content: - application/json: - schema: *ref_0 - headers: {} - '404': - description: 算子不存在 + "200": + description: "" content: application/json: schema: - $ref: '#/components/schemas/ErrorResponse' - headers: {} - security: [] - put: - summary: 更新算子信息 - deprecated: false - description: 根据ID更新算子信息 - tags: - - Operator - parameters: - - name: id - in: path - description: 算子ID - required: true - example: '' - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/UpdateOperatorRequest' - example: null - responses: - '200': - description: 算子更新成功 - content: - application/json: - schema: *ref_0 - headers: {} - security: [] - /category: - post: - summary: 创建算子分类 - deprecated: false - description: '' - tags: - - Category - parameters: [] - requestBody: - content: - application/json: - schema: - type: object - properties: - name: - type: string - description: 名称 - parentId: - type: integer - description: 父分类id - required: - - name - - parentId - responses: - '201': - description: '' - headers: {} - security: [] - delete: - summary: 删除算子分类 - deprecated: false - description: '' - tags: - - Category - parameters: [] - requestBody: - content: - application/json: - schema: - type: object - properties: - id: - type: integer - description: ID 编号 - required: - - id - responses: - '204': - description: '' - headers: {} - security: [] - /categories/tree: - get: - summary: 获取算子分类列表 - deprecated: false - description: 获取所有可用的算子分类 - tags: - - Category - parameters: [] - responses: - '200': - description: 成功返回分类列表 - content: - application/json: - schema: - type: array - items: - type: object - properties: - id: - type: integer - name: - type: string - count: - type: integer - categories: - $ref: '#/components/schemas/CategoryResponse' - required: - - id - - name - - count - - categories - headers: {} - security: [] + type: "object" + properties: + code: + type: "string" + description: "状态码" + message: + type: "string" + description: "消息" + data: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: + type: "string" + description: "" + version: + type: "string" + description: "" + inputs: + type: "string" + description: "" + outputs: + type: "string" + description: "" + categories: + type: "array" + description: "" + items: + type: "integer" + runtime: + type: "string" + description: "" + settings: + type: "string" + description: "" + isStar: + type: "boolean" + description: "" + createdAt: + type: "string" + description: "" + updatedAt: + type: "string" + description: "" + description: "数据" /labels: get: - summary: 获取算子标签列表 - deprecated: false - description: 获取所有算子的标签 - tags: - - Label + summary: "labelsGet" + description: "labelsGet" parameters: - - name: page - in: query - description: 页码，从0开始 - required: false + - name: "page" + in: "query" + description: "" + required: true schema: - type: integer - default: 0 - - name: size - in: query - description: 每页大小 - required: false + type: "integer" + nullable: false + - name: "size" + in: "query" + description: "" + required: true schema: - type: integer - default: 20 - - name: keyword - in: query - description: 关键词搜索 - required: false + type: "integer" + nullable: false + - name: "keyword" + in: "query" + description: "" + required: true schema: - type: string + type: "string" + nullable: false responses: - '200': - description: 成功返回标签列表 + "200": + description: "" content: application/json: schema: - type: array - items: - $ref: '#/components/schemas/LabelResponse' - headers: {} - security: [] + type: "object" + properties: + code: + type: "string" + description: "状态码" + message: + type: "string" + description: "消息" + data: + type: "object" + properties: + page: + type: "integer" + description: "" + format: "int64" + size: + type: "integer" + description: "" + format: "int64" + totalElements: + type: "integer" + description: "" + format: "int64" + totalPages: + type: "integer" + description: "" + format: "int64" + content: + type: "array" + description: "" + items: + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" + description: "com.datamate.operator.interfaces.dto.Label" + description: "数据" post: - summary: 创建标签 - deprecated: false - description: 批量创建标签 - tags: - - Label - parameters: [] + summary: "labelsPost" + description: "labelsPost" requestBody: content: application/json: schema: - type: object + type: "object" properties: + id: + type: "string" + description: "" name: - type: string - description: 名称 - required: - - name - example: veniam + type: "string" + description: "" + description: "" responses: - '201': - description: 创建成功 - headers: {} - security: [] - delete: - summary: 删除标签 - deprecated: false - description: 批量删除标签 - tags: - - Label - parameters: [] - requestBody: - content: - application/json: - schema: - type: array - items: - type: integer - format: int64 - description: 标签id列表 - example: null - responses: - '204': - description: 删除成功 - headers: {} - security: [] + "200": + description: "" + content: + application/json: + schema: + type: "object" + properties: + code: + type: "string" + description: "状态码" + message: + type: "string" + description: "消息" + data: + type: "object" + properties: {} + description: "数据" /labels/{id}: put: - summary: 更新标签 - deprecated: false - description: 更新标签 - tags: - - Label + summary: "labelsIdPut" + description: "labelsIdPut" parameters: - - name: id - in: path - description: 标签ID + - name: "id" + in: "path" + description: "" required: true - example: '' schema: - type: string + type: "string" requestBody: content: application/json: schema: - type: array + type: "array" + description: "" items: - $ref: '#/components/schemas/UpdateLabelRequest' - example: null + type: "object" + properties: + id: + type: "string" + description: "" + name: + type: "string" + description: "" responses: - '200': - description: 更新成功 - headers: {} - security: [] + "200": + description: "" + content: + application/json: + schema: + type: "object" + properties: + code: + type: "string" + description: "状态码" + message: + type: "string" + description: "消息" + data: + type: "object" + properties: {} + description: "数据" + /categories/tree: + get: + summary: "categoryTreeGet" + description: "categoryTreeGet" + responses: + "200": + description: "" + content: + application/json: + schema: + type: "object" + properties: + code: + type: "string" + description: "状态码" + message: + type: "string" + description: "消息" + data: + type: "object" + properties: + page: + type: "integer" + description: "" + format: "int64" + size: + type: "integer" + description: "" + format: "int64" + totalElements: + type: "integer" + description: "" + format: "int64" + totalPages: + type: "integer" + description: "" + format: "int64" + content: + type: "array" + description: "" + items: + type: "object" + properties: + id: + type: "integer" + description: "" + name: + type: "string" + description: "" + count: + type: "integer" + description: "" + categories: + type: "array" + description: "" + items: + type: "object" + properties: + id: + type: "integer" + description: "" + format: "int64" + name: + type: "string" + description: "" + count: + type: "integer" + description: "" + format: "int64" + type: + type: "string" + description: "" + parentId: + type: "integer" + description: "" + format: "int64" + description: "com.datamate.operator.interfaces.dto.SubCategory" + default: "new ArrayList<>()" + description: "com.datamate.operator.interfaces.dto.CategoryTreeResponse" + description: "数据" components: schemas: - UpdateLabelRequest: - type: object - required: - - id - - name - properties: - id: - type: integer - description: 标签id - name: - type: string - description: 标签名称 - Response: - type: object - properties: - code: - type: string - message: - type: string - data: - type: object - properties: {} - required: - - code - - message - - data - LabelResponse: - type: object - properties: - id: - type: string - description: 标签ID - name: - type: string - description: 标签名称 - SubCategory: - type: object - properties: - id: - type: integer - description: 分类id - name: - type: string - description: 分类名称 - count: - type: integer - type: - type: string - description: 分类类型（0：预置，1：自定义） - parentId: - type: integer - description: 父分类id - required: - - id - - name - - type - - parentId - - count - CategoryResponse: - type: array - items: - $ref: '#/components/schemas/SubCategory' - UpdateOperatorRequest: - type: object - properties: - name: - type: string - description: 算子名称 - description: - type: string - description: 算子描述 - version: - type: string - description: 算子版本 - category: - type: string - description: 算子分类 - documentation: - type: string - description: 文档内容 - ErrorResponse: - type: object - properties: - error: - type: string - description: 错误代码 - message: - type: string - description: 错误信息 - timestamp: - type: string - format: date-time - description: 错误时间 OperatorResponse: - type: object + type: "object" properties: id: - type: string - description: 算子ID + type: "string" + description: "" name: - type: string - description: 算子名称 + type: "string" + description: "" description: - type: string - description: 算子描述 + type: "string" + description: "" version: - type: string - description: 算子版本 + type: "string" + description: "" inputs: - type: string - description: 输入类型 + type: "string" + description: "" outputs: - type: string - description: 输入类型 + type: "string" + description: "" categories: - type: array - description: 算子分类列表 + type: "array" + description: "" items: - type: integer + type: "integer" runtime: - type: string - description: 运行时设置 + type: "string" + description: "" settings: - type: string - description: 算子参数 + type: "string" + description: "" isStar: - type: boolean - description: 是否收藏 + type: "boolean" + description: "" createdAt: - type: string - format: date-time - description: 创建时间 + type: "string" + description: "" updatedAt: - type: string - format: date-time - description: 更新时间 - required: - - language - - modal - - inputs - - outputs - - runtime - - settings - - isStar - CreateOperatorRequest: - type: object - required: - - name - - description - - version - - category + type: "string" + description: "" + description: "数据" + com.datamate.operator.interfaces.dto.Label: + type: "object" properties: + id: + type: "string" + description: "" name: - type: string - description: 算子名称 - description: - type: string - description: 算子描述 - version: - type: string - description: 算子版本 - category: - type: string - description: 算子分类 - documentation: - type: string - description: 文档内容 - securitySchemes: {} -servers: [] + type: "string" + description: "" + description: "com.datamate.operator.interfaces.dto.Label" + java.lang.Object: + type: "object" + properties: {} + description: "数据" + com.datamate.operator.interfaces.dto.SubCategory: + type: "object" + properties: + id: + type: "integer" + description: "" + format: "int64" + name: + type: "string" + description: "" + count: + type: "integer" + description: "" + format: "int64" + type: + type: "string" + description: "" + parentId: + type: "integer" + description: "" + format: "int64" + description: "com.datamate.operator.interfaces.dto.SubCategory" + com.datamate.operator.interfaces.dto.CategoryTreeResponse: + type: "object" + properties: + id: + type: "integer" + description: "" + name: + type: "string" + description: "" + count: + type: "integer" + description: "" + categories: + type: "array" + description: "" + items: + type: "object" + properties: + id: + type: "integer" + description: "" + format: "int64" + name: + type: "string" + description: "" + count: + type: "integer" + description: "" + format: "int64" + type: + type: "string" + description: "" + parentId: + type: "integer" + description: "" + format: "int64" + description: "com.datamate.operator.interfaces.dto.SubCategory" + default: "new ArrayList<>()" + description: "com.datamate.operator.interfaces.dto.CategoryTreeResponse" diff --git a/backend/services/data-cleaning-service/img.png b/backend/services/data-cleaning-service/img.png deleted file mode 100644 index cee23b9..0000000 Binary files a/backend/services/data-cleaning-service/img.png and /dev/null differ diff --git a/backend/services/data-cleaning-service/img1.png b/backend/services/data-cleaning-service/img1.png deleted file mode 100644 index 8a388c3..0000000 Binary files a/backend/services/data-cleaning-service/img1.png and /dev/null differ diff --git a/backend/services/data-cleaning-service/img2.png b/backend/services/data-cleaning-service/img2.png deleted file mode 100644 index 1f8aeb1..0000000 Binary files a/backend/services/data-cleaning-service/img2.png and /dev/null differ diff --git a/backend/services/data-collection-service/pom.xml b/backend/services/data-collection-service/pom.xml index 1ea9d3c..6600bf1 100644 --- a/backend/services/data-collection-service/pom.xml +++ b/backend/services/data-collection-service/pom.xml @@ -127,35 +127,6 @@ - - - org.openapitools - openapi-generator-maven-plugin - 6.6.0 - - - - generate - - - ${project.basedir}/../../openapi/specs/data-collection.yaml - spring - ${project.build.directory}/generated-sources/openapi - com.datamate.collection.interfaces.api - com.datamate.collection.interfaces.dto - - true - true - true - springdoc - java8-localdatetime - true - - - - - - org.springframework.boot spring-boot-maven-plugin diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/application/CollectionTaskService.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/application/CollectionTaskService.java new file mode 100644 index 0000000..80bb8e7 --- /dev/null +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/application/CollectionTaskService.java @@ -0,0 +1,79 @@ +package com.datamate.collection.application; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.datamate.collection.domain.model.entity.CollectionTask; +import com.datamate.collection.domain.model.entity.TaskExecution; +import com.datamate.collection.common.enums.TaskStatus; +import com.datamate.collection.domain.repository.CollectionTaskRepository; +import com.datamate.collection.interfaces.dto.CollectionTaskPagingQuery; +import com.datamate.collection.common.enums.SyncMode; +import com.datamate.common.domain.utils.ChunksSaver; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Objects; + +@Slf4j +@Service +@RequiredArgsConstructor +public class CollectionTaskService { + private final TaskExecutionService taskExecutionService; + private final CollectionTaskRepository collectionTaskRepository; + + @Transactional + public CollectionTask create(CollectionTask task) { + task.setStatus(TaskStatus.READY); + task.setCreatedAt(LocalDateTime.now()); + task.setUpdatedAt(LocalDateTime.now()); + collectionTaskRepository.save(task); + executeTaskNow(task); + return task; + } + + private void executeTaskNow(CollectionTask task) { + if (Objects.equals(task.getSyncMode(), SyncMode.ONCE)) { + TaskExecution exec = taskExecutionService.createExecution(task); + int timeout = task.getTimeoutSeconds() == null ? 3600 : task.getTimeoutSeconds(); + taskExecutionService.runAsync(task, exec.getId(), timeout); + log.info("Triggered DataX execution for task {} at {}, execId={}", task.getId(), LocalDateTime.now(), exec.getId()); + } + } + + @Transactional + public CollectionTask update(CollectionTask task) { + task.setUpdatedAt(LocalDateTime.now()); + collectionTaskRepository.updateById(task); + return task; + } + + @Transactional + public void delete(String id) { + CollectionTask task = collectionTaskRepository.getById(id); + if (task != null) { + ChunksSaver.deleteFolder("/dataset/local/" + task.getId()); + } + collectionTaskRepository.removeById(id); + } + + public CollectionTask get(String id) { + return collectionTaskRepository.getById(id); + } + + public IPage getTasks(CollectionTaskPagingQuery query) { + LambdaQueryWrapper wrapper = new LambdaQueryWrapper() + .eq(query.getStatus() != null, CollectionTask::getStatus, query.getStatus()) + .like(StringUtils.isNotBlank(query.getName()), CollectionTask::getName, query.getName()); + return collectionTaskRepository.page(new Page<>(query.getPage(), query.getSize()), wrapper); + } + + public List selectActiveTasks() { + return collectionTaskRepository.selectActiveTasks(); + } +} diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/application/TaskExecutionService.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/application/TaskExecutionService.java new file mode 100644 index 0000000..5845515 --- /dev/null +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/application/TaskExecutionService.java @@ -0,0 +1,57 @@ +package com.datamate.collection.application; + +import com.datamate.collection.domain.model.entity.CollectionTask; +import com.datamate.collection.domain.model.entity.TaskExecution; +import com.datamate.collection.common.enums.TaskStatus; +import com.datamate.collection.domain.process.ProcessRunner; +import com.datamate.collection.domain.repository.CollectionTaskRepository; +import com.datamate.collection.domain.repository.TaskExecutionRepository; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.LocalDateTime; + +@Slf4j +@Service +@RequiredArgsConstructor +public class TaskExecutionService { + private final ProcessRunner processRunner; + private final TaskExecutionRepository executionRepository; + private final CollectionTaskRepository collectionTaskRepository; + + + @Transactional + public TaskExecution createExecution(CollectionTask task) { + TaskExecution exec = TaskExecution.initTaskExecution(); + exec.setTaskId(task.getId()); + exec.setTaskName(task.getName()); + executionRepository.save(exec); + collectionTaskRepository.updateLastExecution(task.getId(), exec.getId()); + collectionTaskRepository.updateStatus(task.getId(), TaskStatus.RUNNING.name()); + return exec; + } + + public TaskExecution selectLatestByTaskId(String taskId) { + return executionRepository.selectLatestByTaskId(taskId); + } + + @Async + public void runAsync(CollectionTask task, String executionId, int timeoutSeconds) { + try { + int code = processRunner.runJob(task, executionId, timeoutSeconds); + log.info("DataX finished with code {} for execution {}", code, executionId); + // 简化：成功即完成 + executionRepository.completeExecution(executionId, TaskStatus.SUCCESS.name(), LocalDateTime.now(), + 0, 0L, 0L, 0L, null); + collectionTaskRepository.updateStatus(task.getId(), TaskStatus.SUCCESS.name()); + } catch (Exception e) { + log.error("DataX execution failed", e); + executionRepository.completeExecution(executionId, TaskStatus.FAILED.name(), LocalDateTime.now(), + 0, 0L, 0L, 0L, e.getMessage()); + collectionTaskRepository.updateStatus(task.getId(), TaskStatus.FAILED.name()); + } + } +} diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/application/service/CollectionTaskService.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/application/service/CollectionTaskService.java deleted file mode 100644 index d2b6e21..0000000 --- a/backend/services/data-collection-service/src/main/java/com/datamate/collection/application/service/CollectionTaskService.java +++ /dev/null @@ -1,85 +0,0 @@ -package com.datamate.collection.application.service; - -import com.datamate.collection.domain.model.CollectionTask; -import com.datamate.collection.domain.model.TaskExecution; -import com.datamate.collection.domain.model.TaskStatus; -import com.datamate.collection.domain.model.DataxTemplate; -import com.datamate.collection.infrastructure.persistence.mapper.CollectionTaskMapper; -import com.datamate.collection.infrastructure.persistence.mapper.TaskExecutionMapper; -import com.datamate.collection.interfaces.dto.SyncMode; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import java.time.LocalDateTime; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -@Slf4j -@Service -@RequiredArgsConstructor -public class CollectionTaskService { - private final CollectionTaskMapper taskMapper; - private final TaskExecutionMapper executionMapper; - private final DataxExecutionService dataxExecutionService; - - @Transactional - public CollectionTask create(CollectionTask task) { - task.setStatus(TaskStatus.READY); - task.setCreatedAt(LocalDateTime.now()); - task.setUpdatedAt(LocalDateTime.now()); - taskMapper.insert(task); - executeTaskNow(task); - return task; - } - - private void executeTaskNow(CollectionTask task) { - if (Objects.equals(task.getSyncMode(), SyncMode.ONCE.getValue())) { - TaskExecution exec = dataxExecutionService.createExecution(task); - int timeout = task.getTimeoutSeconds() == null ? 3600 : task.getTimeoutSeconds(); - dataxExecutionService.runAsync(task, exec.getId(), timeout); - log.info("Triggered DataX execution for task {} at {}, execId={}", task.getId(), LocalDateTime.now(), exec.getId()); - } - } - - @Transactional - public CollectionTask update(CollectionTask task) { - task.setUpdatedAt(LocalDateTime.now()); - taskMapper.update(task); - return task; - } - - @Transactional - public void delete(String id) { taskMapper.deleteById(id); } - - public CollectionTask get(String id) { return taskMapper.selectById(id); } - - public List list(Integer page, Integer size, String status, String name) { - Map p = new HashMap<>(); - p.put("status", status); - p.put("name", name); - if (page != null && size != null) { - p.put("offset", page * size); - p.put("limit", size); - } - return taskMapper.selectAll(p); - } - - @Transactional - public TaskExecution startExecution(CollectionTask task) { - return dataxExecutionService.createExecution(task); - } - - // ---- Template related merged methods ---- - public List listTemplates(String sourceType, String targetType, int page, int size) { - int offset = page * size; - return taskMapper.selectList(sourceType, targetType, offset, size); - } - - public int countTemplates(String sourceType, String targetType) { - return taskMapper.countTemplates(sourceType, targetType); - } -} diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/application/service/DataxExecutionService.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/application/service/DataxExecutionService.java deleted file mode 100644 index fc2eb9e..0000000 --- a/backend/services/data-collection-service/src/main/java/com/datamate/collection/application/service/DataxExecutionService.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.datamate.collection.application.service; - -import com.datamate.collection.domain.model.CollectionTask; -import com.datamate.collection.domain.model.TaskExecution; -import com.datamate.collection.domain.model.TaskStatus; -import com.datamate.collection.infrastructure.persistence.mapper.CollectionTaskMapper; -import com.datamate.collection.infrastructure.persistence.mapper.TaskExecutionMapper; -import com.datamate.collection.infrastructure.runtime.datax.DataxJobBuilder; -import com.datamate.collection.infrastructure.runtime.datax.DataxProcessRunner; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.scheduling.annotation.Async; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import java.nio.file.Path; -import java.time.Duration; -import java.time.LocalDateTime; - -@Slf4j -@Service -@RequiredArgsConstructor -public class DataxExecutionService { - - private final DataxJobBuilder jobBuilder; - private final DataxProcessRunner processRunner; - private final TaskExecutionMapper executionMapper; - private final CollectionTaskMapper taskMapper; - - - @Transactional - public TaskExecution createExecution(CollectionTask task) { - TaskExecution exec = TaskExecution.initTaskExecution(); - exec.setTaskId(task.getId()); - exec.setTaskName(task.getName()); - executionMapper.insert(exec); - taskMapper.updateLastExecution(task.getId(), exec.getId()); - taskMapper.updateStatus(task.getId(), TaskStatus.RUNNING.name()); - return exec; - } - - @Async - public void runAsync(CollectionTask task, String executionId, int timeoutSeconds) { - try { - Path job = jobBuilder.buildJobFile(task); - - int code = processRunner.runJob(job.toFile(), executionId, Duration.ofSeconds(timeoutSeconds)); - log.info("DataX finished with code {} for execution {}", code, executionId); - // 简化：成功即完成 - executionMapper.completeExecution(executionId, TaskStatus.SUCCESS.name(), LocalDateTime.now(), - 0, 0L, 0L, 0L, null, null); - taskMapper.updateStatus(task.getId(), TaskStatus.SUCCESS.name()); - } catch (Exception e) { - log.error("DataX execution failed", e); - executionMapper.completeExecution(executionId, TaskStatus.FAILED.name(), LocalDateTime.now(), - 0, 0L, 0L, 0L, e.getMessage(), null); - taskMapper.updateStatus(task.getId(), TaskStatus.FAILED.name()); - } - } -} diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/application/service/TaskExecutionService.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/application/service/TaskExecutionService.java deleted file mode 100644 index 68f9e3b..0000000 --- a/backend/services/data-collection-service/src/main/java/com/datamate/collection/application/service/TaskExecutionService.java +++ /dev/null @@ -1,83 +0,0 @@ -package com.datamate.collection.application.service; - -import com.datamate.collection.domain.model.CollectionTask; -import com.datamate.collection.domain.model.TaskExecution; -import com.datamate.collection.domain.model.TaskStatus; -import com.datamate.collection.infrastructure.persistence.mapper.CollectionTaskMapper; -import com.datamate.collection.infrastructure.persistence.mapper.TaskExecutionMapper; -import lombok.RequiredArgsConstructor; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import java.time.Duration; -import java.time.LocalDateTime; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Service -@RequiredArgsConstructor -public class TaskExecutionService { - private final TaskExecutionMapper executionMapper; - private final CollectionTaskMapper taskMapper; - - public List list(String taskId, String status, LocalDateTime startDate, - LocalDateTime endDate, Integer page, Integer size) { - Map p = new HashMap<>(); - p.put("taskId", taskId); - p.put("status", status); - p.put("startDate", startDate); - p.put("endDate", endDate); - if (page != null && size != null) { - p.put("offset", page * size); - p.put("limit", size); - } - return executionMapper.selectAll(p); - } - - public long count(String taskId, String status, LocalDateTime startDate, LocalDateTime endDate) { - Map p = new HashMap<>(); - p.put("taskId", taskId); - p.put("status", status); - p.put("startDate", startDate); - p.put("endDate", endDate); - return executionMapper.count(p); - } - - // --- Added convenience methods --- - public TaskExecution get(String id) { return executionMapper.selectById(id); } - public TaskExecution getLatestByTaskId(String taskId) { return executionMapper.selectLatestByTaskId(taskId); } - - @Transactional - public void complete(String executionId, boolean success, long successCount, long failedCount, - long dataSizeBytes, String errorMessage, String resultJson) { - LocalDateTime now = LocalDateTime.now(); - TaskExecution exec = executionMapper.selectById(executionId); - if (exec == null) { return; } - int duration = (int) Duration.between(exec.getStartedAt(), now).getSeconds(); - executionMapper.completeExecution(executionId, success ? TaskStatus.SUCCESS.name() : TaskStatus.FAILED.name(), - now, duration, successCount, failedCount, dataSizeBytes, errorMessage, resultJson); - CollectionTask task = taskMapper.selectById(exec.getTaskId()); - if (task != null) { - taskMapper.updateStatus(task.getId(), success ? TaskStatus.SUCCESS.name() : TaskStatus.FAILED.name()); - } - } - - @Transactional - public void stop(String executionId) { - TaskExecution exec = executionMapper.selectById(executionId); - if (exec == null || exec.getStatus() != TaskStatus.RUNNING) { return; } - LocalDateTime now = LocalDateTime.now(); - int duration = (int) Duration.between(exec.getStartedAt(), now).getSeconds(); - // Reuse completeExecution to persist STOPPED status and timing info - executionMapper.completeExecution(exec.getId(), TaskStatus.STOPPED.name(), now, duration, - exec.getRecordsSuccess(), exec.getRecordsFailed(), exec.getDataSizeBytes(), null, exec.getResult()); - taskMapper.updateStatus(exec.getTaskId(), TaskStatus.STOPPED.name()); - } - - @Transactional - public void stopLatestByTaskId(String taskId) { - TaskExecution latest = executionMapper.selectLatestByTaskId(taskId); - if (latest != null) { stop(latest.getId()); } - } -} diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/common/enums/SyncMode.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/common/enums/SyncMode.java new file mode 100644 index 0000000..28399a2 --- /dev/null +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/common/enums/SyncMode.java @@ -0,0 +1,12 @@ +package com.datamate.collection.common.enums; + +/** + * 同步方式：一次性(ONCE) 或 定时(SCHEDULED) + */ +public enum SyncMode { + /** 一次性(ONCE) */ + ONCE, + /// 定时(SCHEDULED) + SCHEDULED; +} + diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/TaskStatus.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/common/enums/TaskStatus.java similarity index 59% rename from backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/TaskStatus.java rename to backend/services/data-collection-service/src/main/java/com/datamate/collection/common/enums/TaskStatus.java index c5273d2..ed9585a 100644 --- a/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/TaskStatus.java +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/common/enums/TaskStatus.java @@ -1,7 +1,8 @@ -package com.datamate.collection.domain.model; +package com.datamate.collection.common.enums; /** * 统一的任务和执行状态枚举 + * 任务和执行状态枚举: - DRAFT: 草稿状态 - READY: 就绪状态 - RUNNING: 运行中 - SUCCESS: 执行成功 (对应原来的COMPLETED/SUCCESS) - FAILED: 执行失败 - STOPPED: 已停止 * * @author Data Mate Platform Team */ diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/CollectionTask.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/entity/CollectionTask.java similarity index 66% rename from backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/CollectionTask.java rename to backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/entity/CollectionTask.java index f40afa7..5bbdb2a 100644 --- a/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/CollectionTask.java +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/entity/CollectionTask.java @@ -1,32 +1,36 @@ -package com.datamate.collection.domain.model; +package com.datamate.collection.domain.model.entity; +import com.baomidou.mybatisplus.annotation.TableName; +import com.datamate.collection.common.enums.SyncMode; +import com.datamate.collection.common.enums.TaskStatus; +import com.datamate.common.domain.model.base.BaseEntity; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.Data; +import lombok.Getter; +import lombok.Setter; -import java.time.LocalDateTime; import java.util.Collections; import java.util.Map; -@Data -public class CollectionTask { - private String id; +/** + * 数据采集任务实体（与数据库表 t_dc_collection_tasks 对齐） + */ +@Getter +@Setter +@TableName(value = "t_dc_collection_tasks", autoResultMap = true) +public class CollectionTask extends BaseEntity { private String name; private String description; private String config; // DataX JSON 配置，包含源端和目标端配置信息 private TaskStatus status; - private String syncMode; // ONCE / SCHEDULED + private SyncMode syncMode; // ONCE / SCHEDULED private String scheduleExpression; private Integer retryCount; private Integer timeoutSeconds; private Long maxRecords; private String sortField; private String lastExecutionId; - private LocalDateTime createdAt; - private LocalDateTime updatedAt; - private String createdBy; - private String updatedBy; public void addPath() { try { diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/DataxTemplate.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/entity/DataxTemplate.java similarity index 95% rename from backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/DataxTemplate.java rename to backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/entity/DataxTemplate.java index c537a67..54792fd 100644 --- a/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/DataxTemplate.java +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/entity/DataxTemplate.java @@ -1,4 +1,4 @@ -package com.datamate.collection.domain.model; +package com.datamate.collection.domain.model.entity; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/TaskExecution.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/entity/TaskExecution.java similarity index 70% rename from backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/TaskExecution.java rename to backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/entity/TaskExecution.java index 05f1fe8..6009957 100644 --- a/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/TaskExecution.java +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/entity/TaskExecution.java @@ -1,13 +1,19 @@ -package com.datamate.collection.domain.model; +package com.datamate.collection.domain.model.entity; +import com.baomidou.mybatisplus.annotation.TableName; +import com.datamate.collection.common.enums.TaskStatus; +import com.datamate.common.domain.model.base.BaseEntity; import lombok.Data; +import lombok.Getter; +import lombok.Setter; import java.time.LocalDateTime; import java.util.UUID; -@Data -public class TaskExecution { - private String id; +@Getter +@Setter +@TableName(value = "t_dc_task_executions", autoResultMap = true) +public class TaskExecution extends BaseEntity { private String taskId; private String taskName; private TaskStatus status; @@ -25,7 +31,6 @@ public class TaskExecution { private String dataxJobId; private String config; private String result; - private LocalDateTime createdAt; public static TaskExecution initTaskExecution() { TaskExecution exec = new TaskExecution(); diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/process/ProcessRunner.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/process/ProcessRunner.java new file mode 100644 index 0000000..73d904e --- /dev/null +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/process/ProcessRunner.java @@ -0,0 +1,21 @@ +package com.datamate.collection.domain.process; + +import com.datamate.collection.domain.model.entity.CollectionTask; + +/** + * 归集执行器接口 + * + * @since 2025/10/23 + */ +public interface ProcessRunner { + /** + * 执行归集任务 + * + * @param task 任务 + * @param executionId 执行ID + * @param timeoutSeconds 超时时间（秒） + * @return 执行结果 + * @throws Exception 执行异常 + */ + int runJob(CollectionTask task, String executionId, int timeoutSeconds) throws Exception; +} diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/repository/CollectionTaskRepository.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/repository/CollectionTaskRepository.java new file mode 100644 index 0000000..21b0d16 --- /dev/null +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/repository/CollectionTaskRepository.java @@ -0,0 +1,19 @@ +package com.datamate.collection.domain.repository; + +import com.baomidou.mybatisplus.extension.repository.IRepository; +import com.datamate.collection.domain.model.entity.CollectionTask; + +import java.util.List; + +/** + * 归集任务仓储层 + * + * @since 2025/10/23 + */ +public interface CollectionTaskRepository extends IRepository { + List selectActiveTasks(); + + void updateStatus(String id, String status); + + void updateLastExecution(String id, String lastExecutionId); +} diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/repository/TaskExecutionRepository.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/repository/TaskExecutionRepository.java new file mode 100644 index 0000000..9b0c059 --- /dev/null +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/repository/TaskExecutionRepository.java @@ -0,0 +1,19 @@ +package com.datamate.collection.domain.repository; + +import com.baomidou.mybatisplus.extension.service.IService; +import com.datamate.collection.domain.model.entity.TaskExecution; + +import java.time.LocalDateTime; + +/** + * TaskExecutionRepository + * + * @since 2025/10/23 + */ +public interface TaskExecutionRepository extends IService { + TaskExecution selectLatestByTaskId(String taskId); + + void completeExecution(String executionId, String status, LocalDateTime completedAt, + Integer recordsProcessed, Long recordsTotal, + Long recordsSuccess, Long recordsFailed, String errorMessage); +} diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/datax/DataxProcessRunner.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/datax/DataxProcessRunner.java new file mode 100644 index 0000000..29ffcd2 --- /dev/null +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/datax/DataxProcessRunner.java @@ -0,0 +1,124 @@ +package com.datamate.collection.infrastructure.datax; + +import com.datamate.collection.domain.model.entity.CollectionTask; +import com.datamate.collection.domain.process.ProcessRunner; +import com.datamate.common.infrastructure.exception.BusinessException; +import com.datamate.common.infrastructure.exception.SystemErrorCode; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.exec.*; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Slf4j +@Component +@RequiredArgsConstructor +public class DataxProcessRunner implements ProcessRunner { + + private final DataxProperties props; + + @Override + public int runJob(CollectionTask task, String executionId, int timeoutSeconds) throws Exception { + Path job = buildJobFile(task); + return runJob(job.toFile(), executionId, Duration.ofSeconds(timeoutSeconds)); + } + + private int runJob(File jobFile, String executionId, Duration timeout) throws Exception { + File logFile = new File(props.getLogPath(), String.format("datax-%s.log", executionId)); + String python = props.getPythonPath(); + String dataxPy = props.getHomePath() + File.separator + "bin" + File.separator + "datax.py"; + String cmd = String.format("%s %s %s", python, dataxPy, jobFile.getAbsolutePath()); + + log.info("Execute DataX: {}", cmd); + + CommandLine cl = CommandLine.parse(cmd); + DefaultExecutor executor = getExecutor(timeout, logFile); + + return executor.execute(cl); + } + + private static DefaultExecutor getExecutor(Duration timeout, File logFile) throws FileNotFoundException { + DefaultExecutor executor = new DefaultExecutor(); + + // 将日志追加输出到文件 + File parent = logFile.getParentFile(); + if (!parent.exists()) { + parent.mkdirs(); + } + + ExecuteStreamHandler streamHandler = new PumpStreamHandler( + new org.apache.commons.io.output.TeeOutputStream( + new java.io.FileOutputStream(logFile, true), System.out), + new org.apache.commons.io.output.TeeOutputStream( + new java.io.FileOutputStream(logFile, true), System.err) + ); + executor.setStreamHandler(streamHandler); + + ExecuteWatchdog watchdog = new ExecuteWatchdog(timeout.toMillis()); + executor.setWatchdog(watchdog); + return executor; + } + + private Path buildJobFile(CollectionTask task) throws IOException { + Files.createDirectories(Paths.get(props.getJobConfigPath())); + String fileName = String.format("datax-job-%s.json", task.getId()); + Path path = Paths.get(props.getJobConfigPath(), fileName); + // 简化：直接将任务中的 config 字段作为 DataX 作业 JSON + try (FileWriter fw = new FileWriter(path.toFile())) { + if (StringUtils.isBlank(task.getConfig())) { + throw BusinessException.of(SystemErrorCode.UNKNOWN_ERROR); + } + String json = getJobConfig(task); + log.info("Job config: {}", json); + fw.write(json); + } + return path; + } + + private String getJobConfig(CollectionTask task) { + try { + ObjectMapper objectMapper = new ObjectMapper(); + Map parameter = objectMapper.readValue( + task.getConfig(), + new TypeReference<>() { + } + ); + Map job = new HashMap<>(); + Map content = new HashMap<>(); + Map reader = new HashMap<>(); + reader.put("name", "nfsreader"); + reader.put("parameter", parameter); + content.put("reader", reader); + Map writer = new HashMap<>(); + writer.put("name", "nfswriter"); + writer.put("parameter", parameter); + content.put("writer", writer); + job.put("content", List.of(content)); + Map setting = new HashMap<>(); + Map channel = new HashMap<>(); + channel.put("channel", 2); + setting.put("speed", channel); + job.put("setting", setting); + Map jobConfig = new HashMap<>(); + jobConfig.put("job", job); + return objectMapper.writeValueAsString(jobConfig); + } catch (Exception e) { + log.error("Failed to parse task config", e); + throw new RuntimeException("Failed to parse task config", e); + } + } +} diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/runtime/datax/DataxProperties.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/datax/DataxProperties.java similarity index 90% rename from backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/runtime/datax/DataxProperties.java rename to backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/datax/DataxProperties.java index e194444..8a3644c 100644 --- a/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/runtime/datax/DataxProperties.java +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/datax/DataxProperties.java @@ -1,4 +1,4 @@ -package com.datamate.collection.infrastructure.runtime.datax; +package com.datamate.collection.infrastructure.datax; import lombok.Data; import org.springframework.boot.context.properties.ConfigurationProperties; diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/persistence/mapper/CollectionTaskMapper.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/persistence/mapper/CollectionTaskMapper.java index 9359cc9..7c65b47 100644 --- a/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/persistence/mapper/CollectionTaskMapper.java +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/persistence/mapper/CollectionTaskMapper.java @@ -1,47 +1,15 @@ package com.datamate.collection.infrastructure.persistence.mapper; -import com.datamate.collection.domain.model.CollectionTask; -import com.datamate.collection.domain.model.DataxTemplate; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.datamate.collection.domain.model.entity.CollectionTask; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; import java.util.List; -import java.util.Map; @Mapper -public interface CollectionTaskMapper { - int insert(CollectionTask entity); - int update(CollectionTask entity); - int deleteById(@Param("id") String id); - CollectionTask selectById(@Param("id") String id); - CollectionTask selectByName(@Param("name") String name); - List selectByStatus(@Param("status") String status); - List selectAll(Map params); +public interface CollectionTaskMapper extends BaseMapper { int updateStatus(@Param("id") String id, @Param("status") String status); int updateLastExecution(@Param("id") String id, @Param("lastExecutionId") String lastExecutionId); List selectActiveTasks(); - - /** - * 查询模板列表 - * - * @param sourceType 源数据源类型（可选） - * @param targetType 目标数据源类型（可选） - * @param offset 偏移量 - * @param limit 限制数量 - * @return 模板列表 - */ - List selectList(@Param("sourceType") String sourceType, - @Param("targetType") String targetType, - @Param("offset") int offset, - @Param("limit") int limit); - - /** - * 统计模板数量 - * - * @param sourceType 源数据源类型（可选） - * @param targetType 目标数据源类型（可选） - * @return 模板总数 - */ - int countTemplates(@Param("sourceType") String sourceType, - @Param("targetType") String targetType); } diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/persistence/mapper/TaskExecutionMapper.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/persistence/mapper/TaskExecutionMapper.java index 645d885..2c22426 100644 --- a/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/persistence/mapper/TaskExecutionMapper.java +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/persistence/mapper/TaskExecutionMapper.java @@ -1,38 +1,22 @@ package com.datamate.collection.infrastructure.persistence.mapper; -import com.datamate.collection.domain.model.TaskExecution; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.datamate.collection.domain.model.entity.TaskExecution; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; import java.time.LocalDateTime; -import java.util.List; -import java.util.Map; @Mapper -public interface TaskExecutionMapper { - int insert(TaskExecution entity); - int update(TaskExecution entity); - int deleteById(@Param("id") String id); - TaskExecution selectById(@Param("id") String id); - List selectByTaskId(@Param("taskId") String taskId, @Param("limit") Integer limit); - List selectByStatus(@Param("status") String status); - List selectAll(Map params); - long count(Map params); - int updateProgress(@Param("id") String id, - @Param("status") String status, - @Param("progress") Double progress, - @Param("recordsProcessed") Long recordsProcessed, - @Param("throughput") Double throughput); - int completeExecution(@Param("id") String id, - @Param("status") String status, - @Param("completedAt") LocalDateTime completedAt, - @Param("durationSeconds") Integer durationSeconds, - @Param("recordsSuccess") Long recordsSuccess, - @Param("recordsFailed") Long recordsFailed, - @Param("dataSizeBytes") Long dataSizeBytes, - @Param("errorMessage") String errorMessage, - @Param("result") String result); - List selectRunningExecutions(); +public interface TaskExecutionMapper extends BaseMapper { TaskExecution selectLatestByTaskId(@Param("taskId") String taskId); - int deleteOldExecutions(@Param("beforeDate") LocalDateTime beforeDate); + + void completeExecution(@Param("executionId") String executionId, + @Param("status") String status, + @Param("completedAt") LocalDateTime completedAt, + @Param("recordsProcessed") Integer recordsProcessed, + @Param("recordsTotal") Long recordsTotal, + @Param("recordsSuccess") Long recordsSuccess, + @Param("recordsFailed") Long recordsFailed, + @Param("errorMessage") String errorMessage); } diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/persistence/repository/CollectionTaskRepositoryImpl.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/persistence/repository/CollectionTaskRepositoryImpl.java new file mode 100644 index 0000000..c17e185 --- /dev/null +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/persistence/repository/CollectionTaskRepositoryImpl.java @@ -0,0 +1,36 @@ +package com.datamate.collection.infrastructure.persistence.repository; + +import com.baomidou.mybatisplus.extension.repository.CrudRepository; +import com.datamate.collection.domain.model.entity.CollectionTask; +import com.datamate.collection.domain.repository.CollectionTaskRepository; +import com.datamate.collection.infrastructure.persistence.mapper.CollectionTaskMapper; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Repository; + +import java.util.List; + +/** + * CollectionTaskRepositoryImpl + * + * @since 2025/10/23 + */ +@Repository +@RequiredArgsConstructor +public class CollectionTaskRepositoryImpl extends CrudRepository implements CollectionTaskRepository { + private final CollectionTaskMapper collectionTaskMapper; + + @Override + public List selectActiveTasks() { + return collectionTaskMapper.selectActiveTasks(); + } + + @Override + public void updateStatus(String id, String status) { + collectionTaskMapper.updateStatus(id, status); + } + + @Override + public void updateLastExecution(String id, String lastExecutionId) { + collectionTaskMapper.updateLastExecution(id, lastExecutionId); + } +} diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/persistence/repository/TaskExecutionRepositoryImpl.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/persistence/repository/TaskExecutionRepositoryImpl.java new file mode 100644 index 0000000..2263fc9 --- /dev/null +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/persistence/repository/TaskExecutionRepositoryImpl.java @@ -0,0 +1,37 @@ +package com.datamate.collection.infrastructure.persistence.repository; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.datamate.collection.domain.model.entity.TaskExecution; +import com.datamate.collection.domain.repository.TaskExecutionRepository; +import com.datamate.collection.infrastructure.persistence.mapper.TaskExecutionMapper; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Repository; + +import java.time.LocalDateTime; + +/** + * TaskExecutionRepositoryImpl + * + * @since 2025/10/23 + */ +@Repository +@RequiredArgsConstructor +public class TaskExecutionRepositoryImpl extends ServiceImpl + implements TaskExecutionRepository { + + private final TaskExecutionMapper taskExecutionMapper; + + @Override + public TaskExecution selectLatestByTaskId(String taskId) { + return taskExecutionMapper.selectLatestByTaskId(taskId); + } + + @Override + public void completeExecution(String executionId, String status, LocalDateTime completedAt, + Integer recordsProcessed, Long recordsTotal, + Long recordsSuccess, Long recordsFailed, String errorMessage) { + taskExecutionMapper.completeExecution(executionId, status, completedAt, + recordsProcessed, recordsTotal, + recordsSuccess, recordsFailed, errorMessage); + } +} diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/persistence/typehandler/TaskStatusTypeHandler.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/persistence/typehandler/TaskStatusTypeHandler.java deleted file mode 100644 index e69de29..0000000 diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/runtime/datax/DataxJobBuilder.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/runtime/datax/DataxJobBuilder.java deleted file mode 100644 index db57d2a..0000000 --- a/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/runtime/datax/DataxJobBuilder.java +++ /dev/null @@ -1,83 +0,0 @@ -package com.datamate.collection.infrastructure.runtime.datax; - -import com.datamate.collection.domain.model.CollectionTask; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Component; - -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * 根据任务配置拼装 DataX 作业 JSON 文件 - */ -@Slf4j -@Component -@RequiredArgsConstructor -public class DataxJobBuilder { - - private final DataxProperties props; - - public Path buildJobFile(CollectionTask task) throws IOException { - Files.createDirectories(Paths.get(props.getJobConfigPath())); - String fileName = String.format("datax-job-%s.json", task.getId()); - Path path = Paths.get(props.getJobConfigPath(), fileName); - // 简化：直接将任务中的 config 字段作为 DataX 作业 JSON - try (FileWriter fw = new FileWriter(path.toFile())) { - String json = task.getConfig() == null || task.getConfig().isEmpty() ? - defaultJobJson() : task.getConfig(); - if (StringUtils.isNotBlank(task.getConfig())) { - json = getJobConfig(task); - } - log.info("Job config: {}", json); - fw.write(json); - } - return path; - } - - private String getJobConfig(CollectionTask task) { - try { - ObjectMapper objectMapper = new ObjectMapper(); - Map parameter = objectMapper.readValue( - task.getConfig(), - new TypeReference<>() {} - ); - Map job = new HashMap<>(); - Map content = new HashMap<>(); - Map reader = new HashMap<>(); - reader.put("name", "nfsreader"); - reader.put("parameter", parameter); - content.put("reader", reader); - Map writer = new HashMap<>(); - writer.put("name", "nfswriter"); - writer.put("parameter", parameter); - content.put("writer", writer); - job.put("content", List.of(content)); - Map setting = new HashMap<>(); - Map channel = new HashMap<>(); - channel.put("channel", 2); - setting.put("speed", channel); - job.put("setting", setting); - Map jobConfig = new HashMap<>(); - jobConfig.put("job", job); - return objectMapper.writeValueAsString(jobConfig); - } catch (Exception e) { - log.error("Failed to parse task config", e); - throw new RuntimeException("Failed to parse task config", e); - } - } - - private String defaultJobJson() { - // 提供一个最小可运行的空 job，实际会被具体任务覆盖 - return "{\n \"job\": {\n \"setting\": {\n \"speed\": {\n \"channel\": 1\n }\n },\n \"content\": []\n }\n}"; - } -} diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/runtime/datax/DataxProcessRunner.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/runtime/datax/DataxProcessRunner.java deleted file mode 100644 index fda00a8..0000000 --- a/backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/runtime/datax/DataxProcessRunner.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.datamate.collection.infrastructure.runtime.datax; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.exec.*; -import org.springframework.stereotype.Component; - -import java.io.File; -import java.time.Duration; - -@Slf4j -@Component -@RequiredArgsConstructor -public class DataxProcessRunner { - - private final DataxProperties props; - - public int runJob(File jobFile, String executionId, Duration timeout) throws Exception { - File logFile = new File(props.getLogPath(), String.format("datax-%s.log", executionId)); - String python = props.getPythonPath(); - String dataxPy = props.getHomePath() + File.separator + "bin" + File.separator + "datax.py"; - String cmd = String.format("%s %s %s", python, dataxPy, jobFile.getAbsolutePath()); - - log.info("Execute DataX: {}", cmd); - - CommandLine cl = CommandLine.parse(cmd); - DefaultExecutor executor = new DefaultExecutor(); - - // 将日志追加输出到文件 - File parent = logFile.getParentFile(); - if (!parent.exists()) parent.mkdirs(); - - ExecuteStreamHandler streamHandler = new PumpStreamHandler( - new org.apache.commons.io.output.TeeOutputStream( - new java.io.FileOutputStream(logFile, true), System.out), - new org.apache.commons.io.output.TeeOutputStream( - new java.io.FileOutputStream(logFile, true), System.err) - ); - executor.setStreamHandler(streamHandler); - - ExecuteWatchdog watchdog = new ExecuteWatchdog(timeout.toMillis()); - executor.setWatchdog(watchdog); - - return executor.execute(cl); - } -} diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/converter/CollectionTaskConverter.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/converter/CollectionTaskConverter.java index 1cbdde3..2b45383 100644 --- a/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/converter/CollectionTaskConverter.java +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/converter/CollectionTaskConverter.java @@ -1,16 +1,18 @@ package com.datamate.collection.interfaces.converter; -import com.datamate.collection.domain.model.CollectionTask; -import com.datamate.collection.domain.model.DataxTemplate; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.datamate.collection.domain.model.entity.CollectionTask; import com.datamate.collection.interfaces.dto.*; import com.datamate.common.infrastructure.exception.BusinessException; import com.datamate.common.infrastructure.exception.SystemErrorCode; +import com.datamate.common.interfaces.PagedResponse; import com.fasterxml.jackson.databind.ObjectMapper; import org.mapstruct.Mapper; import org.mapstruct.Mapping; import org.mapstruct.Named; import org.mapstruct.factory.Mappers; +import java.util.List; import java.util.Map; @Mapper @@ -20,9 +22,7 @@ public interface CollectionTaskConverter { @Mapping(source = "config", target = "config", qualifiedByName = "parseJsonToMap") CollectionTaskResponse toResponse(CollectionTask task); - CollectionTaskSummary toSummary(CollectionTask task); - - DataxTemplateSummary toTemplateSummary(DataxTemplate template); + List toResponse(List tasks); @Mapping(source = "config", target = "config", qualifiedByName = "mapToJsonString") CollectionTask toCollectionTask(CreateCollectionTaskRequest request); @@ -30,11 +30,19 @@ public interface CollectionTaskConverter { @Mapping(source = "config", target = "config", qualifiedByName = "mapToJsonString") CollectionTask toCollectionTask(UpdateCollectionTaskRequest request); + @Mapping(source = "current", target = "page") + @Mapping(source = "size", target = "size") + @Mapping(source = "total", target = "totalElements") + @Mapping(source = "pages", target = "totalPages") + @Mapping(source = "records", target = "content") + PagedResponse toResponse(IPage tasks); + @Named("parseJsonToMap") default Map parseJsonToMap(String json) { try { ObjectMapper objectMapper = new ObjectMapper(); - return objectMapper.readValue(json, Map.class); + return + objectMapper.readValue(json, Map.class); } catch (Exception e) { throw BusinessException.of(SystemErrorCode.INVALID_PARAMETER); } diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/dto/CollectionTaskPagingQuery.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/dto/CollectionTaskPagingQuery.java new file mode 100644 index 0000000..024d96d --- /dev/null +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/dto/CollectionTaskPagingQuery.java @@ -0,0 +1,25 @@ +package com.datamate.collection.interfaces.dto; + +import com.datamate.collection.common.enums.TaskStatus; +import com.datamate.common.interfaces.PagingQuery; +import lombok.Getter; +import lombok.Setter; + +/** + * 归集任务分页查询参数 + * + * @since 2025/10/23 + */ +@Getter +@Setter +public class CollectionTaskPagingQuery extends PagingQuery { + /** + * 任务状态 + */ + private TaskStatus status; + + /** + * 任务名称 + */ + private String name; +} diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/dto/CollectionTaskResponse.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/dto/CollectionTaskResponse.java new file mode 100644 index 0000000..08f6103 --- /dev/null +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/dto/CollectionTaskResponse.java @@ -0,0 +1,48 @@ +package com.datamate.collection.interfaces.dto; + +import java.time.LocalDateTime; +import java.util.HashMap; +import java.util.Map; + +import com.datamate.collection.common.enums.TaskStatus; +import com.datamate.collection.common.enums.SyncMode; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.springframework.format.annotation.DateTimeFormat; +import jakarta.validation.Valid; + +/** + * CollectionTaskResponse + */ +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class CollectionTaskResponse { + + private String id; + + private String name; + + private String description; + + @Valid + private Map config = new HashMap<>(); + + private TaskStatus status; + + private SyncMode syncMode; + + private String scheduleExpression; + + private String lastExecutionId; + + @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) + private LocalDateTime createdAt; + + @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) + private LocalDateTime updatedAt; +} + diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/dto/CreateCollectionTaskRequest.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/dto/CreateCollectionTaskRequest.java new file mode 100644 index 0000000..f2f029f --- /dev/null +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/dto/CreateCollectionTaskRequest.java @@ -0,0 +1,53 @@ +package com.datamate.collection.interfaces.dto; + +import com.datamate.collection.common.enums.SyncMode; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.HashMap; +import java.util.Map; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import jakarta.validation.Valid; +import jakarta.validation.constraints.*; +import io.swagger.v3.oas.annotations.media.Schema; + +/** + * CreateCollectionTaskRequest + */ +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class CreateCollectionTaskRequest { + @NotNull + @Size(min = 1, max = 100) + @Schema(name = "name", description = "任务名称", requiredMode = Schema.RequiredMode.REQUIRED) + @JsonProperty("name") + private String name; + + @Size(max = 500) + @Schema(name = "description", description = "任务描述", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("description") + private String description; + + @Valid + @NotNull + @Schema(name = "config", description = "归集配置，包含源端和目标端配置信息", requiredMode = Schema.RequiredMode.REQUIRED) + @JsonProperty("config") + private Map config = new HashMap<>(); + + @NotNull + @Valid + @Schema(name = "syncMode", requiredMode = Schema.RequiredMode.REQUIRED) + @JsonProperty("syncMode") + private SyncMode syncMode; + + @Schema(name = "scheduleExpression", description = "Cron调度表达式 (syncMode=SCHEDULED 时必填)", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("scheduleExpression") + private String scheduleExpression; +} + diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/dto/UpdateCollectionTaskRequest.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/dto/UpdateCollectionTaskRequest.java new file mode 100644 index 0000000..9321be8 --- /dev/null +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/dto/UpdateCollectionTaskRequest.java @@ -0,0 +1,50 @@ +package com.datamate.collection.interfaces.dto; + +import com.datamate.collection.common.enums.SyncMode; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.HashMap; +import java.util.Map; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import jakarta.validation.Valid; +import jakarta.validation.constraints.*; +import io.swagger.v3.oas.annotations.media.Schema; + +/** + * UpdateCollectionTaskRequest + */ +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class UpdateCollectionTaskRequest { + @Size(min = 1, max = 100) + @Schema(name = "name", description = "任务名称", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("name") + private String name; + + @Size(max = 500) + @Schema(name = "description", description = "任务描述", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("description") + private String description; + + @Valid + @Schema(name = "config", description = "归集配置，包含源端和目标端配置信息", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("config") + private Map config = new HashMap<>(); + + @Valid + @Schema(name = "syncMode", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("syncMode") + private SyncMode syncMode; + + @Schema(name = "scheduleExpression", description = "Cron调度表达式 (syncMode=SCHEDULED 时必填)", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("scheduleExpression") + private String scheduleExpression; +} + diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/rest/CollectionTaskController.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/rest/CollectionTaskController.java index d9aa353..b893a73 100644 --- a/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/rest/CollectionTaskController.java +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/rest/CollectionTaskController.java @@ -1,38 +1,36 @@ package com.datamate.collection.interfaces.rest; -import com.datamate.collection.application.service.CollectionTaskService; -import com.datamate.collection.domain.model.CollectionTask; -import com.datamate.collection.domain.model.DataxTemplate; -import com.datamate.collection.interfaces.api.CollectionTaskApi; +import com.datamate.collection.application.CollectionTaskService; +import com.datamate.collection.domain.model.entity.CollectionTask; import com.datamate.collection.interfaces.converter.CollectionTaskConverter; import com.datamate.collection.interfaces.dto.*; +import com.datamate.common.interfaces.PagedResponse; +import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.http.ResponseEntity; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import java.util.*; -import java.util.stream.Collectors; @Slf4j @RestController +@RequestMapping("/data-collection/tasks") @RequiredArgsConstructor -@Validated -public class CollectionTaskController implements CollectionTaskApi { +public class CollectionTaskController{ private final CollectionTaskService taskService; - @Override - public ResponseEntity createTask(CreateCollectionTaskRequest request) { + @PostMapping + public ResponseEntity createTask(@Valid @RequestBody CreateCollectionTaskRequest request) { CollectionTask task = CollectionTaskConverter.INSTANCE.toCollectionTask(request); task.setId(UUID.randomUUID().toString()); task.addPath(); return ResponseEntity.ok().body(CollectionTaskConverter.INSTANCE.toResponse(taskService.create(task))); } - @Override - public ResponseEntity updateTask(String id, UpdateCollectionTaskRequest request) { + @PutMapping("/{id}") + public ResponseEntity updateTask(@PathVariable("id") String id, @Valid @RequestBody UpdateCollectionTaskRequest request) { if (taskService.get(id) == null) { return ResponseEntity.notFound().build(); } @@ -41,43 +39,20 @@ public class CollectionTaskController implements CollectionTaskApi { return ResponseEntity.ok(CollectionTaskConverter.INSTANCE.toResponse(taskService.update(task))); } - @Override - public ResponseEntity deleteTask(String id) { + @DeleteMapping("/{id}") + public ResponseEntity deleteTask(@PathVariable("id") String id) { taskService.delete(id); return ResponseEntity.ok().build(); } - @Override - public ResponseEntity getTaskDetail(String id) { + @GetMapping("/{id}") + public ResponseEntity getTaskDetail(@PathVariable("id") String id) { CollectionTask task = taskService.get(id); return task == null ? ResponseEntity.notFound().build() : ResponseEntity.ok(CollectionTaskConverter.INSTANCE.toResponse(task)); } - @Override - public ResponseEntity getTasks(Integer page, Integer size, TaskStatus status, String name) { - var list = taskService.list(page, size, status == null ? null : status.getValue(), name); - PagedCollectionTaskSummary response = new PagedCollectionTaskSummary(); - response.setContent(list.stream().map(CollectionTaskConverter.INSTANCE::toSummary).collect(Collectors.toList())); - response.setNumber(page); - response.setSize(size); - response.setTotalElements(list.size()); // 简化处理，实际项目中应该有单独的count查询 - response.setTotalPages(size == null || size == 0 ? 1 : (int) Math.ceil(list.size() * 1.0 / size)); - return ResponseEntity.ok(response); - } - - @Override - public ResponseEntity templatesGet(String sourceType, String targetType, - Integer page, Integer size) { - int pageNum = page != null ? page : 0; - int pageSize = size != null ? size : 20; - List templates = taskService.listTemplates(sourceType, targetType, pageNum, pageSize); - int totalElements = taskService.countTemplates(sourceType, targetType); - PagedDataxTemplates response = new PagedDataxTemplates(); - response.setContent(templates.stream().map(CollectionTaskConverter.INSTANCE::toTemplateSummary).collect(Collectors.toList())); - response.setNumber(pageNum); - response.setSize(pageSize); - response.setTotalElements(totalElements); - response.setTotalPages(pageSize > 0 ? (int) Math.ceil(totalElements * 1.0 / pageSize) : 1); - return ResponseEntity.ok(response); + @GetMapping + public ResponseEntity> getTasks(@Valid CollectionTaskPagingQuery query) { + return ResponseEntity.ok(CollectionTaskConverter.INSTANCE.toResponse(taskService.getTasks(query))); } } diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/rest/TaskExecutionController.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/rest/TaskExecutionController.java deleted file mode 100644 index 6d392bb..0000000 --- a/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/rest/TaskExecutionController.java +++ /dev/null @@ -1,101 +0,0 @@ -package com.datamate.collection.interfaces.rest; - -import com.datamate.collection.application.service.CollectionTaskService; -import com.datamate.collection.application.service.TaskExecutionService; -import com.datamate.collection.domain.model.TaskExecution; -import com.datamate.collection.interfaces.api.TaskExecutionApi; -import com.datamate.collection.interfaces.dto.PagedTaskExecutions; -import com.datamate.collection.interfaces.dto.TaskExecutionDetail; -import com.datamate.collection.interfaces.dto.TaskExecutionResponse; -import com.datamate.collection.interfaces.dto.TaskStatus; // DTO enum -import lombok.RequiredArgsConstructor; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.RestController; - -import java.util.stream.Collectors; - -@RestController -@RequiredArgsConstructor -@Validated -public class TaskExecutionController implements TaskExecutionApi { - - private final TaskExecutionService executionService; - private final CollectionTaskService taskService; - - private TaskExecutionDetail toDetail(TaskExecution e) { - TaskExecutionDetail d = new TaskExecutionDetail(); - d.setId(e.getId()); - d.setTaskId(e.getTaskId()); - d.setTaskName(e.getTaskName()); - if (e.getStatus() != null) { d.setStatus(TaskStatus.fromValue(e.getStatus().name())); } - d.setProgress(e.getProgress()); - d.setRecordsTotal(e.getRecordsTotal() != null ? e.getRecordsTotal().intValue() : null); - d.setRecordsProcessed(e.getRecordsProcessed() != null ? e.getRecordsProcessed().intValue() : null); - d.setRecordsSuccess(e.getRecordsSuccess() != null ? e.getRecordsSuccess().intValue() : null); - d.setRecordsFailed(e.getRecordsFailed() != null ? e.getRecordsFailed().intValue() : null); - d.setThroughput(e.getThroughput()); - d.setDataSizeBytes(e.getDataSizeBytes() != null ? e.getDataSizeBytes().intValue() : null); - d.setStartedAt(e.getStartedAt()); - d.setCompletedAt(e.getCompletedAt()); - d.setDurationSeconds(e.getDurationSeconds()); - d.setErrorMessage(e.getErrorMessage()); - return d; - } - - // GET /executions/{id} - @Override - public ResponseEntity executionsIdGet(String id) { - var exec = executionService.get(id); - return exec == null ? ResponseEntity.notFound().build() : ResponseEntity.ok(toDetail(exec)); - } - - // DELETE /executions/{id} - @Override - public ResponseEntity executionsIdDelete(String id) { - executionService.stop(id); // 幂等处理，在service内部判断状态 - return ResponseEntity.noContent().build(); - } - - // POST /tasks/{id}/execute -> 201 - @Override - public ResponseEntity tasksIdExecutePost(String id) { - var task = taskService.get(id); - if (task == null) { return ResponseEntity.notFound().build(); } - var latestExec = executionService.getLatestByTaskId(id); - if (latestExec != null && latestExec.getStatus() == com.datamate.collection.domain.model.TaskStatus.RUNNING) { - TaskExecutionResponse r = new TaskExecutionResponse(); - r.setId(latestExec.getId()); - r.setTaskId(latestExec.getTaskId()); - r.setTaskName(latestExec.getTaskName()); - r.setStatus(TaskStatus.fromValue(latestExec.getStatus().name())); - r.setStartedAt(latestExec.getStartedAt()); - return ResponseEntity.status(HttpStatus.CREATED).body(r); // 返回已有运行实例 - } - var exec = taskService.startExecution(task); - TaskExecutionResponse r = new TaskExecutionResponse(); - r.setId(exec.getId()); - r.setTaskId(exec.getTaskId()); - r.setTaskName(exec.getTaskName()); - r.setStatus(TaskStatus.fromValue(exec.getStatus().name())); - r.setStartedAt(exec.getStartedAt()); - return ResponseEntity.status(HttpStatus.CREATED).body(r); - } - - // GET /tasks/{id}/executions -> 分页 - @Override - public ResponseEntity tasksIdExecutionsGet(String id, Integer page, Integer size) { - if (page == null || page < 0) { page = 0; } - if (size == null || size <= 0) { size = 20; } - var list = executionService.list(id, null, null, null, page, size); - long total = executionService.count(id, null, null, null); - PagedTaskExecutions p = new PagedTaskExecutions(); - p.setContent(list.stream().map(this::toDetail).collect(Collectors.toList())); - p.setNumber(page); - p.setSize(size); - p.setTotalElements((int) total); - p.setTotalPages(size == 0 ? 1 : (int) Math.ceil(total * 1.0 / size)); - return ResponseEntity.ok(p); - } -} diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/application/scheduler/TaskSchedulerInitializer.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/scheduler/TaskSchedulerInitializer.java similarity index 68% rename from backend/services/data-collection-service/src/main/java/com/datamate/collection/application/scheduler/TaskSchedulerInitializer.java rename to backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/scheduler/TaskSchedulerInitializer.java index 0203cab..d0d0f26 100644 --- a/backend/services/data-collection-service/src/main/java/com/datamate/collection/application/scheduler/TaskSchedulerInitializer.java +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/scheduler/TaskSchedulerInitializer.java @@ -1,11 +1,10 @@ -package com.datamate.collection.application.scheduler; +package com.datamate.collection.interfaces.scheduler; -import com.datamate.collection.application.service.DataxExecutionService; -import com.datamate.collection.domain.model.CollectionTask; -import com.datamate.collection.domain.model.TaskStatus; -import com.datamate.collection.domain.model.TaskExecution; -import com.datamate.collection.infrastructure.persistence.mapper.CollectionTaskMapper; -import com.datamate.collection.infrastructure.persistence.mapper.TaskExecutionMapper; +import com.datamate.collection.application.CollectionTaskService; +import com.datamate.collection.application.TaskExecutionService; +import com.datamate.collection.common.enums.TaskStatus; +import com.datamate.collection.domain.model.entity.CollectionTask; +import com.datamate.collection.domain.model.entity.TaskExecution; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.scheduling.annotation.Scheduled; @@ -21,14 +20,13 @@ import java.util.List; @RequiredArgsConstructor public class TaskSchedulerInitializer { - private final CollectionTaskMapper taskMapper; - private final TaskExecutionMapper executionMapper; - private final DataxExecutionService dataxExecutionService; + private final CollectionTaskService collectionTaskService; + private final TaskExecutionService taskExecutionService; // 定期扫描激活的采集任务，根据 Cron 判断是否到期执行 @Scheduled(fixedDelayString = "${datamate.data-collection.scheduler.scan-interval-ms:10000}") public void scanAndTrigger() { - List tasks = taskMapper.selectActiveTasks(); + List tasks = collectionTaskService.selectActiveTasks(); if (tasks == null || tasks.isEmpty()) { return; } @@ -40,7 +38,7 @@ public class TaskSchedulerInitializer { } try { // 如果最近一次执行仍在运行，则跳过 - TaskExecution latest = executionMapper.selectLatestByTaskId(task.getId()); + TaskExecution latest = taskExecutionService.selectLatestByTaskId(task.getId()); if (latest != null && latest.getStatus() == TaskStatus.RUNNING) { continue; } @@ -53,9 +51,9 @@ public class TaskSchedulerInitializer { if (nextTime != null && !nextTime.isAfter(now)) { // 到期，触发一次执行 - TaskExecution exec = dataxExecutionService.createExecution(task); + TaskExecution exec = taskExecutionService.createExecution(task); int timeout = task.getTimeoutSeconds() == null ? 3600 : task.getTimeoutSeconds(); - dataxExecutionService.runAsync(task, exec.getId(), timeout); + taskExecutionService.runAsync(task, exec.getId(), timeout); log.info("Triggered DataX execution for task {} at {}, execId={}", task.getId(), now, exec.getId()); } } catch (Exception ex) { diff --git a/backend/services/data-collection-service/src/main/resources/mappers/CollectionTaskMapper.xml b/backend/services/data-collection-service/src/main/resources/mappers/CollectionTaskMapper.xml index 3a195a2..6d3033b 100644 --- a/backend/services/data-collection-service/src/main/resources/mappers/CollectionTaskMapper.xml +++ b/backend/services/data-collection-service/src/main/resources/mappers/CollectionTaskMapper.xml @@ -1,11 +1,10 @@ - - + @@ -24,21 +23,6 @@ - - - - - - - - - - - - - - - id, @@ -47,96 +31,6 @@ last_execution_id, created_at, updated_at, created_by, updated_by - - - id, name, source_type, target_type, template_content, description, version, is_system, created_at, updated_at, created_by - - - - - INSERT INTO t_dc_collection_tasks (id, name, description, config, status, sync_mode, - schedule_expression, retry_count, timeout_seconds, max_records, sort_field, - last_execution_id, created_at, updated_at, created_by, updated_by) - VALUES (#{id}, #{name}, #{description}, #{config}, #{status}, #{syncMode}, - #{scheduleExpression}, #{retryCount}, #{timeoutSeconds}, #{maxRecords}, #{sortField}, - #{lastExecutionId}, #{createdAt}, #{updatedAt}, #{createdBy}, #{updatedBy}) - - - - - UPDATE t_dc_collection_tasks - SET name = #{name}, - description = #{description}, - config = #{config}, - status = #{status}, - sync_mode = #{syncMode}, - schedule_expression = #{scheduleExpression}, - retry_count = #{retryCount}, - timeout_seconds = #{timeoutSeconds}, - max_records = #{maxRecords}, - sort_field = #{sortField}, - last_execution_id = #{lastExecutionId}, - updated_at = #{updatedAt}, - updated_by = #{updatedBy} - WHERE id = #{id} - - - - - DELETE FROM t_dc_collection_tasks WHERE id = #{id} - - - - - - - - - - - - - - - - - UPDATE t_dc_collection_tasks SET status = #{status}, updated_at = NOW() WHERE id = #{id} @@ -154,35 +48,4 @@ AND schedule_expression IS NOT NULL ORDER BY created_at DESC - - - - - - - diff --git a/backend/services/data-collection-service/src/main/resources/mappers/TaskExecutionMapper.xml b/backend/services/data-collection-service/src/main/resources/mappers/TaskExecutionMapper.xml index 6b6d0a0..e9e9c86 100644 --- a/backend/services/data-collection-service/src/main/resources/mappers/TaskExecutionMapper.xml +++ b/backend/services/data-collection-service/src/main/resources/mappers/TaskExecutionMapper.xml @@ -1,191 +1,28 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - id, task_id, task_name, status, progress, records_total, records_processed, - records_success, records_failed, throughput, data_size_bytes, started_at, - completed_at, duration_seconds, error_message, datax_job_id, config, result, created_at - - - - - INSERT INTO t_dc_task_executions ( - id, task_id, task_name, status, progress, records_total, records_processed, - records_success, records_failed, throughput, data_size_bytes, started_at, - completed_at, duration_seconds, error_message, datax_job_id, config, result, created_at - ) VALUES ( - #{id}, #{taskId}, #{taskName}, #{status}, #{progress}, #{recordsTotal}, #{recordsProcessed}, - #{recordsSuccess}, #{recordsFailed}, #{throughput}, #{dataSizeBytes}, #{startedAt}, - #{completedAt}, #{durationSeconds}, #{errorMessage}, #{dataxJobId}, #{config}, #{result}, #{createdAt} - ) - - - - - UPDATE t_dc_task_executions - SET status = #{status}, - progress = #{progress}, - records_total = #{recordsTotal}, - records_processed = #{recordsProcessed}, - records_success = #{recordsSuccess}, - records_failed = #{recordsFailed}, - throughput = #{throughput}, - data_size_bytes = #{dataSizeBytes}, - completed_at = #{completedAt}, - duration_seconds = #{durationSeconds}, - error_message = #{errorMessage}, - result = #{result} - WHERE id = #{id} - - - - - DELETE FROM t_dc_task_executions WHERE id = #{id} - - - - - - - - - - - - - - - - - - - - UPDATE t_dc_task_executions - SET status = #{status}, - progress = #{progress}, - records_processed = #{recordsProcessed}, - throughput = #{throughput} - WHERE id = #{id} - - - - - UPDATE t_dc_task_executions - SET status = #{status}, - progress = 100.00, - completed_at = #{completedAt}, - duration_seconds = #{durationSeconds}, - records_success = #{recordsSuccess}, - records_failed = #{recordsFailed}, - data_size_bytes = #{dataSizeBytes}, - error_message = #{errorMessage}, - result = #{result} - WHERE id = #{id} - - - - - - + SELECT * FROM t_dc_task_executions WHERE task_id = #{taskId} ORDER BY started_at DESC LIMIT 1 - - - DELETE FROM t_dc_task_executions - WHERE started_at < #{beforeDate} - + + + UPDATE t_dc_task_executions + SET status = #{status}, + completed_at = #{completedAt}, + records_processed = #{recordsProcessed}, + records_total = #{recordsTotal}, + records_success = #{recordsSuccess}, + records_failed = #{recordsFailed}, + error_message = #{errorMessage}, + updated_at = NOW() + WHERE id = #{executionId} + diff --git a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java index fa6b07a..b14db92 100644 --- a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java +++ b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java @@ -2,6 +2,7 @@ package com.datamate.datamanagement.application; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.datamate.common.domain.utils.ChunksSaver; import com.datamate.datamanagement.interfaces.dto.*; import com.datamate.common.infrastructure.exception.BusinessAssert; import com.datamate.common.interfaces.PagedResponse; @@ -100,8 +101,13 @@ public class DatasetApplicationService { /** * 删除数据集 */ + @Transactional public void deleteDataset(String datasetId) { + Dataset dataset = datasetRepository.getById(datasetId); datasetRepository.removeById(datasetId); + if (dataset != null) { + ChunksSaver.deleteFolder(dataset.getPath()); + } } /** diff --git a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/mapper/DatasetFileMapper.java b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/mapper/DatasetFileMapper.java index 6b0429c..f67121c 100644 --- a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/mapper/DatasetFileMapper.java +++ b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/mapper/DatasetFileMapper.java @@ -24,7 +24,6 @@ public interface DatasetFileMapper extends BaseMapper { @Param("status") String status, RowBounds rowBounds); - int insert(DatasetFile file); int update(DatasetFile file); int deleteById(@Param("id") String id); } diff --git a/backend/services/operator-market-service/img.png b/backend/services/operator-market-service/img.png deleted file mode 100644 index a8ec1f2..0000000 Binary files a/backend/services/operator-market-service/img.png and /dev/null differ diff --git a/backend/services/operator-market-service/img_1.png b/backend/services/operator-market-service/img_1.png deleted file mode 100644 index ed40d49..0000000 Binary files a/backend/services/operator-market-service/img_1.png and /dev/null differ diff --git a/backend/shared/domain-common/src/main/java/com/datamate/common/domain/service/FileService.java b/backend/shared/domain-common/src/main/java/com/datamate/common/domain/service/FileService.java index 8aa059e..1c8250b 100644 --- a/backend/shared/domain-common/src/main/java/com/datamate/common/domain/service/FileService.java +++ b/backend/shared/domain-common/src/main/java/com/datamate/common/domain/service/FileService.java @@ -60,7 +60,7 @@ public class FileService { boolean isFinish = Objects.equals(preRequest.getUploadedFileNum(), preRequest.getTotalFileNum()); if (isFinish) { // 删除存分片的临时路径 - ChunksSaver.deleteFiles(new File(preRequest.getUploadPath(), + ChunksSaver.deleteFolder(new File(preRequest.getUploadPath(), String.format(ChunksSaver.TEMP_DIR_NAME_FORMAT, preRequest.getId())).getPath()); chunkUploadRequestMapper.deleteById(preRequest.getId()); } diff --git a/backend/shared/domain-common/src/main/java/com/datamate/common/domain/utils/ChunksSaver.java b/backend/shared/domain-common/src/main/java/com/datamate/common/domain/utils/ChunksSaver.java index 539cbb6..079b2cd 100644 --- a/backend/shared/domain-common/src/main/java/com/datamate/common/domain/utils/ChunksSaver.java +++ b/backend/shared/domain-common/src/main/java/com/datamate/common/domain/utils/ChunksSaver.java @@ -2,6 +2,8 @@ package com.datamate.common.domain.utils; import com.datamate.common.domain.model.ChunkUploadPreRequest; import com.datamate.common.domain.model.ChunkUploadRequest; +import com.datamate.common.infrastructure.exception.BusinessException; +import com.datamate.common.infrastructure.exception.SystemErrorCode; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.springframework.web.multipart.MultipartFile; @@ -106,29 +108,17 @@ public class ChunksSaver { * * @param uploadPath 文件路径 */ - public static void deleteFiles(String uploadPath) { - File dic = new File(uploadPath); - if (!dic.exists()) { - return; - } - File[] files = dic.listFiles(); - if (files == null || files.length == 0) { - dic.delete(); + public static void deleteFolder(String uploadPath) { + File folder = new File(uploadPath); + + if (!folder.exists()) { + log.info("folder {} does not exist", uploadPath); return; } try { - for (File file : files) { - if (file.isDirectory()) { - deleteFiles(file.getPath()); - } else { - file.delete(); - } - } - if (dic.exists()) { - dic.delete(); - } - } catch (SecurityException e) { - log.warn("Fail to delete file", e); + FileUtils.deleteDirectory(folder); + } catch (IOException e) { + throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR); } } } diff --git a/deployment/helm/datamate/.helmignore b/deployment/helm/datamate/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/deployment/helm/datamate/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deployment/helm/datamate/Chart.yaml b/deployment/helm/datamate/Chart.yaml new file mode 100644 index 0000000..63455f2 --- /dev/null +++ b/deployment/helm/datamate/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: datamate +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.0.1 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.0.1" diff --git a/deployment/helm/datamate/charts/backend/.helmignore b/deployment/helm/datamate/charts/backend/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/deployment/helm/datamate/charts/backend/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deployment/helm/datamate/charts/backend/Chart.yaml b/deployment/helm/datamate/charts/backend/Chart.yaml new file mode 100644 index 0000000..3ee0b4e --- /dev/null +++ b/deployment/helm/datamate/charts/backend/Chart.yaml @@ -0,0 +1,29 @@ +apiVersion: v2 +name: backend +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.0.1 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.0.1" + +dependencies: + - name: database + repository: file://../database + version: 0.0.1 diff --git a/deployment/helm/datamate/charts/backend/templates/_helpers.tpl b/deployment/helm/datamate/charts/backend/templates/_helpers.tpl new file mode 100644 index 0000000..29d49ad --- /dev/null +++ b/deployment/helm/datamate/charts/backend/templates/_helpers.tpl @@ -0,0 +1,75 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "backend.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "backend.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "backend.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "backend.labels" -}} +helm.sh/chart: {{ include "backend.chart" . }} +{{ include "backend.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "backend.selectorLabels" -}} +app.kubernetes.io/name: {{ include "backend.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "backend.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "backend.fullname" .) .Values.serviceAccount.name -}} +{{- else }} +{{- default "default" .Values.serviceAccount.name -}} +{{- end }} +{{- end }} + +{{/* +Name of image +*/}} +{{- define "backend.image" -}} +{{- $name := default .Values.image.repository .Values.global.image.backend.name }} +{{- $tag := default .Values.image.tag .Values.global.image.backend.tag }} +{{- if .Values.global.image.repository }} +{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}:{{ $tag }} +{{- else }} +{{- $name }}:{{ $tag }} +{{- end }} +{{- end }} diff --git a/deployment/helm/datamate/charts/backend/templates/deployment.yaml b/deployment/helm/datamate/charts/backend/templates/deployment.yaml new file mode 100644 index 0000000..3d12f05 --- /dev/null +++ b/deployment/helm/datamate/charts/backend/templates/deployment.yaml @@ -0,0 +1,82 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "backend.fullname" . }} + labels: + {{- include "backend.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "backend.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "backend.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "backend.serviceAccountName" . }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: "{{ include "backend.image" . }}" + imagePullPolicy: {{ default .Values.global.image.pullPolicy .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deployment/helm/datamate/charts/backend/templates/service.yaml b/deployment/helm/datamate/charts/backend/templates/service.yaml new file mode 100644 index 0000000..4477c22 --- /dev/null +++ b/deployment/helm/datamate/charts/backend/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "backend.fullname" . }} + labels: + {{- include "backend.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.service.port }} + protocol: TCP + name: {{ .Chart.Name }} + selector: + {{- include "backend.selectorLabels" . | nindent 4 }} diff --git a/deployment/helm/datamate/charts/backend/templates/serviceaccount.yaml b/deployment/helm/datamate/charts/backend/templates/serviceaccount.yaml new file mode 100644 index 0000000..0977e16 --- /dev/null +++ b/deployment/helm/datamate/charts/backend/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "backend.serviceAccountName" . }} + labels: + {{- include "backend.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/deployment/helm/datamate/charts/backend/values.yaml b/deployment/helm/datamate/charts/backend/values.yaml new file mode 100644 index 0000000..cf42954 --- /dev/null +++ b/deployment/helm/datamate/charts/backend/values.yaml @@ -0,0 +1,114 @@ +# Default values for datamate. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ +replicaCount: 1 + +# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ +image: + repository: "datamate-backend" + # This sets the pull policy for images. + pullPolicy: "IfNotPresent" + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +imagePullSecrets: [] +# This is to override the chart name. +nameOverride: "datamate-backend" +fullnameOverride: "datamate-backend" + +env: + - name: namespace + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SPRING_CONFIG_LOCATION + value: file:/opt/backend/application.yml + +# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/ +serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +# This is for setting Kubernetes Annotations to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} +# This is for setting Kubernetes Labels to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +podLabels: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/ +service: + # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: ClusterIP + # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports + port: 8080 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ +# livenessProbe: +# httpGet: +# path: / +# port: http +# readinessProbe: +# httpGet: +# path: / +# port: http + +# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/ +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +# Additional volumes on the output Deployment definition. +volumes: [] +# - name: foo +# secret: +# secretName: mysecret +# optional: false + +# Additional volumeMounts on the output Deployment definition. +volumeMounts: [] +# - name: foo +# mountPath: "/etc/foo" +# readOnly: true + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/deployment/helm/datamate/charts/database/.helmignore b/deployment/helm/datamate/charts/database/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/deployment/helm/datamate/charts/database/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deployment/helm/datamate/charts/database/Chart.yaml b/deployment/helm/datamate/charts/database/Chart.yaml new file mode 100644 index 0000000..85fb58f --- /dev/null +++ b/deployment/helm/datamate/charts/database/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: database +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.0.1 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.0.1" diff --git a/deployment/helm/datamate/charts/database/templates/_helpers.tpl b/deployment/helm/datamate/charts/database/templates/_helpers.tpl new file mode 100644 index 0000000..24ff282 --- /dev/null +++ b/deployment/helm/datamate/charts/database/templates/_helpers.tpl @@ -0,0 +1,75 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "database.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "database.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "database.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "database.labels" -}} +helm.sh/chart: {{ include "database.chart" . }} +{{ include "database.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "database.selectorLabels" -}} +app.kubernetes.io/name: {{ include "database.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "database.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "database.fullname" .) .Values.serviceAccount.name -}} +{{- else }} +{{- default "default" .Values.serviceAccount.name -}} +{{- end }} +{{- end }} + +{{/* +Name of image +*/}} +{{- define "database.image" -}} +{{- $name := default .Values.image.repository .Values.global.image.database.name }} +{{- $tag := default .Values.image.tag .Values.global.image.database.tag }} +{{- if .Values.global.image.repository }} +{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}:{{ $tag }} +{{- else }} +{{- $name }}:{{ $tag }} +{{- end }} +{{- end }} diff --git a/deployment/helm/datamate/charts/database/templates/configmap.yaml b/deployment/helm/datamate/charts/database/templates/configmap.yaml new file mode 100644 index 0000000..c3ee49b --- /dev/null +++ b/deployment/helm/datamate/charts/database/templates/configmap.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: datamate-mysql-utf8-config +data: + utf8.cnf: | + [mysqld] + # 设置服务器默认字符集为 utf8mb4 (推荐，支持完整的 UTF-8，包括 emoji) + character-set-server = utf8mb4 + # 设置默认排序规则 + collation-server = utf8mb4_unicode_ci + # 或者使用 utf8_general_ci (性能稍好，但排序规则稍宽松) + default-time-zone = 'Asia/Shanghai' + log_error=/var/log/datamate/database/error.log + + [client] + # 设置客户端连接默认字符集 + default-character-set = utf8mb4 + + [mysql] + # 设置 mysql 命令行客户端默认字符集 + default-character-set = utf8mb4 diff --git a/deployment/helm/datamate/charts/database/templates/deployment.yaml b/deployment/helm/datamate/charts/database/templates/deployment.yaml new file mode 100644 index 0000000..d7ee6c8 --- /dev/null +++ b/deployment/helm/datamate/charts/database/templates/deployment.yaml @@ -0,0 +1,82 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "database.fullname" . }} + labels: + {{- include "database.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "database.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "database.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "database.serviceAccountName" . }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: "{{ include "database.image" . }}" + imagePullPolicy: {{ default .Values.global.image.pullPolicy .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deployment/helm/datamate/charts/database/templates/service.yaml b/deployment/helm/datamate/charts/database/templates/service.yaml new file mode 100644 index 0000000..817aa26 --- /dev/null +++ b/deployment/helm/datamate/charts/database/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "database.fullname" . }} + labels: + {{- include "database.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.service.port }} + protocol: TCP + name: {{ .Chart.Name }} + selector: + {{- include "database.selectorLabels" . | nindent 4 }} diff --git a/deployment/helm/datamate/charts/database/templates/serviceaccount.yaml b/deployment/helm/datamate/charts/database/templates/serviceaccount.yaml new file mode 100644 index 0000000..ed718a9 --- /dev/null +++ b/deployment/helm/datamate/charts/database/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "database.serviceAccountName" . }} + labels: + {{- include "database.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/deployment/helm/datamate/charts/database/values.yaml b/deployment/helm/datamate/charts/database/values.yaml new file mode 100644 index 0000000..c11ff4c --- /dev/null +++ b/deployment/helm/datamate/charts/database/values.yaml @@ -0,0 +1,110 @@ +# Default values for datamate. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ +replicaCount: 1 + +# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ +image: + repository: "mysql" + # This sets the pull policy for images. + pullPolicy: "IfNotPresent" + # Overrides the image tag whose default is the chart appVersion. + tag: "8" + +# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +imagePullSecrets: [] +# This is to override the chart name. +nameOverride: "datamate-database" +fullnameOverride: "datamate-database" + +env: + - name: MYSQL_ROOT_PASSWORD + value: "Huawei@123" + +# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/ +serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +# This is for setting Kubernetes Annotations to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} +# This is for setting Kubernetes Labels to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +podLabels: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/ +service: + # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: ClusterIP + # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports + port: 3306 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ +# livenessProbe: +# httpGet: +# path: / +# port: http +# readinessProbe: +# httpGet: +# path: / +# port: http + +# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/ +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +# Additional volumes on the output Deployment definition. +volumes: [] +# - name: foo +# secret: +# secretName: mysecret +# optional: false + +# Additional volumeMounts on the output Deployment definition. +volumeMounts: [] +# - name: foo +# mountPath: "/etc/foo" +# readOnly: true + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/deployment/helm/datamate/charts/frontend/.helmignore b/deployment/helm/datamate/charts/frontend/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/deployment/helm/datamate/charts/frontend/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deployment/helm/datamate/charts/frontend/Chart.yaml b/deployment/helm/datamate/charts/frontend/Chart.yaml new file mode 100644 index 0000000..98025e3 --- /dev/null +++ b/deployment/helm/datamate/charts/frontend/Chart.yaml @@ -0,0 +1,29 @@ +apiVersion: v2 +name: frontend +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.0.1 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.0.1" + +dependencies: + - name: backend + repository: file://../backend + version: 0.0.1 diff --git a/deployment/helm/datamate/charts/frontend/templates/_helpers.tpl b/deployment/helm/datamate/charts/frontend/templates/_helpers.tpl new file mode 100644 index 0000000..8bc9b2e --- /dev/null +++ b/deployment/helm/datamate/charts/frontend/templates/_helpers.tpl @@ -0,0 +1,75 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "frontend.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "frontend.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "frontend.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "frontend.labels" -}} +helm.sh/chart: {{ include "frontend.chart" . }} +{{ include "frontend.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "frontend.selectorLabels" -}} +app.kubernetes.io/name: {{ include "frontend.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "frontend.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "frontend.fullname" .) .Values.serviceAccount.name -}} +{{- else }} +{{- default "default" .Values.serviceAccount.name -}} +{{- end }} +{{- end }} + +{{/* +Name of image +*/}} +{{- define "frontend.image" -}} +{{- $name := default .Values.image.repository .Values.global.image.frontend.name }} +{{- $tag := default .Values.image.tag .Values.global.image.frontend.tag }} +{{- if .Values.global.image.repository }} +{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}:{{ $tag }} +{{- else }} +{{- $name }}:{{ $tag }} +{{- end }} +{{- end }} diff --git a/deployment/helm/datamate/charts/frontend/templates/deployment.yaml b/deployment/helm/datamate/charts/frontend/templates/deployment.yaml new file mode 100644 index 0000000..1aa5c4f --- /dev/null +++ b/deployment/helm/datamate/charts/frontend/templates/deployment.yaml @@ -0,0 +1,82 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "frontend.fullname" . }} + labels: + {{- include "frontend.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "frontend.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "frontend.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "frontend.serviceAccountName" . }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: "{{ include "frontend.image" . }}" + imagePullPolicy: {{ default .Values.global.image.pullPolicy .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deployment/helm/datamate/charts/frontend/templates/service.yaml b/deployment/helm/datamate/charts/frontend/templates/service.yaml new file mode 100644 index 0000000..ae5a5dd --- /dev/null +++ b/deployment/helm/datamate/charts/frontend/templates/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "frontend.fullname" . }} + labels: + {{- include "frontend.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.service.port }} + protocol: TCP + name: {{ .Chart.Name }} + {{- if eq .Values.service.type "NodePort" }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + selector: + {{- include "frontend.selectorLabels" . | nindent 4 }} diff --git a/deployment/helm/datamate/charts/frontend/templates/serviceaccount.yaml b/deployment/helm/datamate/charts/frontend/templates/serviceaccount.yaml new file mode 100644 index 0000000..22facbd --- /dev/null +++ b/deployment/helm/datamate/charts/frontend/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "frontend.serviceAccountName" . }} + labels: + {{- include "frontend.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/deployment/helm/datamate/charts/frontend/values.yaml b/deployment/helm/datamate/charts/frontend/values.yaml new file mode 100644 index 0000000..6691490 --- /dev/null +++ b/deployment/helm/datamate/charts/frontend/values.yaml @@ -0,0 +1,109 @@ +# Default values for datamate. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ +replicaCount: 1 + +# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ +image: + repository: "datamate-frontend" + # This sets the pull policy for images. + pullPolicy: "IfNotPresent" + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +imagePullSecrets: [] +# This is to override the chart name. +nameOverride: "datamate-frontend" +fullnameOverride: "datamate-frontend" + +env: [] + +# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/ +serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +# This is for setting Kubernetes Annotations to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} +# This is for setting Kubernetes Labels to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +podLabels: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/ +service: + # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: NodePort + # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports + port: 80 + nodePort: 30000 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ +# livenessProbe: +# httpGet: +# path: / +# port: http +# readinessProbe: +# httpGet: +# path: / +# port: http + +# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/ +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +# Additional volumes on the output Deployment definition. +volumes: [] +# - name: foo +# secret: +# secretName: mysecret +# optional: false + +# Additional volumeMounts on the output Deployment definition. +volumeMounts: [] +# - name: foo +# mountPath: "/etc/foo" +# readOnly: true + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/deployment/helm/ray/kuberay-operator/Chart.yaml b/deployment/helm/datamate/charts/kuberay-operator/Chart.yaml similarity index 100% rename from deployment/helm/ray/kuberay-operator/Chart.yaml rename to deployment/helm/datamate/charts/kuberay-operator/Chart.yaml diff --git a/deployment/helm/ray/kuberay-operator/crds/ray.io_rayclusters.yaml b/deployment/helm/datamate/charts/kuberay-operator/crds/ray.io_rayclusters.yaml similarity index 100% rename from deployment/helm/ray/kuberay-operator/crds/ray.io_rayclusters.yaml rename to deployment/helm/datamate/charts/kuberay-operator/crds/ray.io_rayclusters.yaml diff --git a/deployment/helm/ray/kuberay-operator/crds/ray.io_rayjobs.yaml b/deployment/helm/datamate/charts/kuberay-operator/crds/ray.io_rayjobs.yaml similarity index 100% rename from deployment/helm/ray/kuberay-operator/crds/ray.io_rayjobs.yaml rename to deployment/helm/datamate/charts/kuberay-operator/crds/ray.io_rayjobs.yaml diff --git a/deployment/helm/ray/kuberay-operator/crds/ray.io_rayservices.yaml b/deployment/helm/datamate/charts/kuberay-operator/crds/ray.io_rayservices.yaml similarity index 100% rename from deployment/helm/ray/kuberay-operator/crds/ray.io_rayservices.yaml rename to deployment/helm/datamate/charts/kuberay-operator/crds/ray.io_rayservices.yaml diff --git a/deployment/helm/ray/kuberay-operator/templates/_helpers.tpl b/deployment/helm/datamate/charts/kuberay-operator/templates/_helpers.tpl similarity index 100% rename from deployment/helm/ray/kuberay-operator/templates/_helpers.tpl rename to deployment/helm/datamate/charts/kuberay-operator/templates/_helpers.tpl diff --git a/deployment/helm/ray/kuberay-operator/templates/deployment.yaml b/deployment/helm/datamate/charts/kuberay-operator/templates/deployment.yaml similarity index 100% rename from deployment/helm/ray/kuberay-operator/templates/deployment.yaml rename to deployment/helm/datamate/charts/kuberay-operator/templates/deployment.yaml diff --git a/deployment/helm/ray/kuberay-operator/templates/leader_election_role.yaml b/deployment/helm/datamate/charts/kuberay-operator/templates/leader_election_role.yaml similarity index 100% rename from deployment/helm/ray/kuberay-operator/templates/leader_election_role.yaml rename to deployment/helm/datamate/charts/kuberay-operator/templates/leader_election_role.yaml diff --git a/deployment/helm/ray/kuberay-operator/templates/leader_election_role_binding.yaml b/deployment/helm/datamate/charts/kuberay-operator/templates/leader_election_role_binding.yaml similarity index 100% rename from deployment/helm/ray/kuberay-operator/templates/leader_election_role_binding.yaml rename to deployment/helm/datamate/charts/kuberay-operator/templates/leader_election_role_binding.yaml diff --git a/deployment/helm/ray/kuberay-operator/templates/multiple_namespaces_role.yaml b/deployment/helm/datamate/charts/kuberay-operator/templates/multiple_namespaces_role.yaml similarity index 100% rename from deployment/helm/ray/kuberay-operator/templates/multiple_namespaces_role.yaml rename to deployment/helm/datamate/charts/kuberay-operator/templates/multiple_namespaces_role.yaml diff --git a/deployment/helm/ray/kuberay-operator/templates/multiple_namespaces_rolebinding.yaml b/deployment/helm/datamate/charts/kuberay-operator/templates/multiple_namespaces_rolebinding.yaml similarity index 100% rename from deployment/helm/ray/kuberay-operator/templates/multiple_namespaces_rolebinding.yaml rename to deployment/helm/datamate/charts/kuberay-operator/templates/multiple_namespaces_rolebinding.yaml diff --git a/deployment/helm/ray/kuberay-operator/templates/ray_rayjob_editor_role.yaml b/deployment/helm/datamate/charts/kuberay-operator/templates/ray_rayjob_editor_role.yaml similarity index 100% rename from deployment/helm/ray/kuberay-operator/templates/ray_rayjob_editor_role.yaml rename to deployment/helm/datamate/charts/kuberay-operator/templates/ray_rayjob_editor_role.yaml diff --git a/deployment/helm/ray/kuberay-operator/templates/ray_rayjob_viewer_role.yaml b/deployment/helm/datamate/charts/kuberay-operator/templates/ray_rayjob_viewer_role.yaml similarity index 100% rename from deployment/helm/ray/kuberay-operator/templates/ray_rayjob_viewer_role.yaml rename to deployment/helm/datamate/charts/kuberay-operator/templates/ray_rayjob_viewer_role.yaml diff --git a/deployment/helm/ray/kuberay-operator/templates/ray_rayservice_editor_role.yaml b/deployment/helm/datamate/charts/kuberay-operator/templates/ray_rayservice_editor_role.yaml similarity index 100% rename from deployment/helm/ray/kuberay-operator/templates/ray_rayservice_editor_role.yaml rename to deployment/helm/datamate/charts/kuberay-operator/templates/ray_rayservice_editor_role.yaml diff --git a/deployment/helm/ray/kuberay-operator/templates/ray_rayservice_viewer_role.yaml b/deployment/helm/datamate/charts/kuberay-operator/templates/ray_rayservice_viewer_role.yaml similarity index 100% rename from deployment/helm/ray/kuberay-operator/templates/ray_rayservice_viewer_role.yaml rename to deployment/helm/datamate/charts/kuberay-operator/templates/ray_rayservice_viewer_role.yaml diff --git a/deployment/helm/ray/kuberay-operator/templates/role.yaml b/deployment/helm/datamate/charts/kuberay-operator/templates/role.yaml similarity index 100% rename from deployment/helm/ray/kuberay-operator/templates/role.yaml rename to deployment/helm/datamate/charts/kuberay-operator/templates/role.yaml diff --git a/deployment/helm/ray/kuberay-operator/templates/rolebinding.yaml b/deployment/helm/datamate/charts/kuberay-operator/templates/rolebinding.yaml similarity index 100% rename from deployment/helm/ray/kuberay-operator/templates/rolebinding.yaml rename to deployment/helm/datamate/charts/kuberay-operator/templates/rolebinding.yaml diff --git a/deployment/helm/ray/kuberay-operator/templates/service.yaml b/deployment/helm/datamate/charts/kuberay-operator/templates/service.yaml similarity index 100% rename from deployment/helm/ray/kuberay-operator/templates/service.yaml rename to deployment/helm/datamate/charts/kuberay-operator/templates/service.yaml diff --git a/deployment/helm/ray/kuberay-operator/templates/serviceaccount.yaml b/deployment/helm/datamate/charts/kuberay-operator/templates/serviceaccount.yaml similarity index 100% rename from deployment/helm/ray/kuberay-operator/templates/serviceaccount.yaml rename to deployment/helm/datamate/charts/kuberay-operator/templates/serviceaccount.yaml diff --git a/deployment/helm/ray/kuberay-operator/templates/servicemonitor.yaml b/deployment/helm/datamate/charts/kuberay-operator/templates/servicemonitor.yaml similarity index 100% rename from deployment/helm/ray/kuberay-operator/templates/servicemonitor.yaml rename to deployment/helm/datamate/charts/kuberay-operator/templates/servicemonitor.yaml diff --git a/deployment/helm/ray/kuberay-operator/values.yaml b/deployment/helm/datamate/charts/kuberay-operator/values.yaml similarity index 100% rename from deployment/helm/ray/kuberay-operator/values.yaml rename to deployment/helm/datamate/charts/kuberay-operator/values.yaml diff --git a/deployment/helm/ray/ray-cluster/Chart.yaml b/deployment/helm/datamate/charts/ray-cluster/Chart.yaml similarity index 100% rename from deployment/helm/ray/ray-cluster/Chart.yaml rename to deployment/helm/datamate/charts/ray-cluster/Chart.yaml diff --git a/deployment/helm/ray/ray-cluster/templates/_helpers.tpl b/deployment/helm/datamate/charts/ray-cluster/templates/_helpers.tpl similarity index 100% rename from deployment/helm/ray/ray-cluster/templates/_helpers.tpl rename to deployment/helm/datamate/charts/ray-cluster/templates/_helpers.tpl diff --git a/deployment/helm/ray/ray-cluster/templates/raycluster-cluster.yaml b/deployment/helm/datamate/charts/ray-cluster/templates/raycluster-cluster.yaml similarity index 100% rename from deployment/helm/ray/ray-cluster/templates/raycluster-cluster.yaml rename to deployment/helm/datamate/charts/ray-cluster/templates/raycluster-cluster.yaml diff --git a/deployment/helm/ray/service.yaml b/deployment/helm/datamate/charts/ray-cluster/templates/service.yaml similarity index 100% rename from deployment/helm/ray/service.yaml rename to deployment/helm/datamate/charts/ray-cluster/templates/service.yaml diff --git a/deployment/helm/ray/ray-cluster/values.yaml b/deployment/helm/datamate/charts/ray-cluster/values.yaml similarity index 100% rename from deployment/helm/ray/ray-cluster/values.yaml rename to deployment/helm/datamate/charts/ray-cluster/values.yaml diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml new file mode 100644 index 0000000..2bf849e --- /dev/null +++ b/deployment/helm/datamate/values.yaml @@ -0,0 +1,83 @@ +# Default values for datamate. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +global: + image: + repository: "" + pullPolicy: "IfNotPresent" + backend: + name: "datamate-backend" + tag: "latest" + frontend: + name: "datamate-frontend" + tag: "latest" + database: + name: "mysql" + tag: "8" + +datasetVolume: &datasetVolume + name: dataset-volume + hostPath: + path: /opt/datamate/data/dataset + type: DirectoryOrCreate + +flowVolume: &flowVolume + name: flow-volume + hostPath: + path: /opt/datamate/data/flow + type: DirectoryOrCreate + +logVolume: &logVolume + name: log-volume + hostPath: + path: /opt/datamate/data/log + type: DirectoryOrCreate + +dataVolume: &dataVolume + name: data-volume + hostPath: + path: /opt/datamate/data/mysql + type: DirectoryOrCreate + +backend: + volumes: + - *datasetVolume + - *flowVolume + - *logVolume + volumeMounts: + - name: dataset-volume + mountPath: /dataset + - name: flow-volume + mountPath: /flow + - name: log-volume + mountPath: /var/log/datamate + +frontend: + volumes: + - *logVolume + volumeMounts: + - name: log-volume + mountPath: /var/log/datamate/frontend + subPath: frontend + +database: + volumes: + - *dataVolume + - *logVolume + - name: init-sql + configMap: + name: datamate-init-sql + - name: mysql-utf8-config + configMap: + name: datamate-mysql-utf8-config + volumeMounts: + - name: data-volume + mountPath: /var/lib/mysql + - name: log-volume + mountPath: /var/log/datamate/database + subPath: database + - name: init-sql + mountPath: /docker-entrypoint-initdb.d + - name: mysql-utf8-config + mountPath: /etc/mysql/conf.d diff --git a/deployment/kubernetes/backend/deploy.yaml b/deployment/kubernetes/backend/deploy.yaml index cc20358..c14aa37 100644 --- a/deployment/kubernetes/backend/deploy.yaml +++ b/deployment/kubernetes/backend/deploy.yaml @@ -1,33 +1,4 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app: datamate - tier: backend - name: datamate-backend -rules: - - verbs: - - create - - list - - get - - delete - apiGroups: - - batch - resources: - - jobs - - verbs: - - list - apiGroups: - - "" - resources: - - pods - - verbs: - - get - - list - apiGroups: - - "" - resources: - - pods/log + --- apiVersion: v1 @@ -39,20 +10,7 @@ metadata: name: datamate-backend --- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - labels: - app: datamate - tier: backend - name: datamate-backend -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: datamate-backend -subjects: - - kind: ServiceAccount - name: datamate-backend + --- apiVersion: apps/v1 diff --git a/scripts/db/data-collection-init.sql b/scripts/db/data-collection-init.sql index dbd923f..62a611a 100644 --- a/scripts/db/data-collection-init.sql +++ b/scripts/db/data-collection-init.sql @@ -31,7 +31,10 @@ CREATE TABLE t_dc_task_executions ( error_message TEXT COMMENT '错误信息', datax_job_id TEXT COMMENT 'datax任务ID', result TEXT COMMENT '执行结果', - created_at TIMESTAMP NULL COMMENT '创建时间', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + created_by VARCHAR(255) COMMENT '创建者', + updated_by VARCHAR(255) COMMENT '更新者', INDEX idx_task_id (task_id), INDEX idx_status (status), INDEX idx_started_at (started_at)