diff --git a/.github/workflows/docker-image-gateway.yml b/.github/workflows/docker-image-gateway.yml new file mode 100644 index 0000000..a3ba63b --- /dev/null +++ b/.github/workflows/docker-image-gateway.yml @@ -0,0 +1,30 @@ +name: Gateway Docker Image CI + +on: + push: + branches: [ "main" ] + paths: + - 'backend/**' + - 'scripts/images/gateway/**' + - '.github/workflows/docker-image-gateway.yml' + - '.github/workflows/docker-images-reusable.yml' + pull_request: + branches: [ "main" ] + paths: + - 'backend/**' + - 'scripts/images/gateway/**' + - '.github/workflows/docker-image-gateway.yml' + - '.github/workflows/docker-images-reusable.yml' + workflow_dispatch: + workflow_call: + +jobs: + call-docker-build: + name: Build and Push Gateway Docker Image + uses: ./.github/workflows/docker-images-reusable.yml + permissions: + contents: read + packages: write + with: + service_name: gateway + build_dir: . diff --git a/.github/workflows/docker-images-release.yml b/.github/workflows/docker-images-release.yml index a30c893..266cac9 100644 --- a/.github/workflows/docker-images-release.yml +++ b/.github/workflows/docker-images-release.yml @@ -14,6 +14,13 @@ jobs: service_name: frontend build_dir: . + gateway: + name: Gateway Image + uses: ./.github/workflows/docker-images-reusable.yml + with: + service_name: gateway + build_dir: . + backend: name: Backend Image uses: ./.github/workflows/docker-images-reusable.yml diff --git a/.github/workflows/docker-push-to-huaweicloud.yml b/.github/workflows/docker-push-to-huaweicloud.yml index e1c5657..dc18f9c 100644 --- a/.github/workflows/docker-push-to-huaweicloud.yml +++ b/.github/workflows/docker-push-to-huaweicloud.yml @@ -18,6 +18,15 @@ jobs: secrets: HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}} + gateway: + name: Push Gateway Image + uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml + with: + service_name: datamate-gateway + version: ${{ inputs.version}} + secrets: + HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}} + backend: name: Push Backend Image uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml diff --git a/.github/workflows/enterprise-package.yml b/.github/workflows/enterprise-package.yml index 1292999..5efb39f 100644 --- a/.github/workflows/enterprise-package.yml +++ b/.github/workflows/enterprise-package.yml @@ -61,7 +61,7 @@ jobs: run: | mkdir -p images/datamate LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') - services=("runtime" "backend" "frontend" "database" "backend-python") + services=("runtime" "backend" "frontend" "database" "backend-python" "gateway") for service in "${services[@]}"; do docker pull ghcr.io/$LOWERCASE_REPO/datamate-$service:latest --platform arm64 docker tag ghcr.io/$LOWERCASE_REPO/datamate-$service:latest datamate-$service:latest diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 1aaf834..9719933 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -34,6 +34,12 @@ jobs: with: service_name: backend-python + gateway-docker-build: + name: Build and Push Frontend Docker Image + uses: ./.github/workflows/docker-image-save.yml + with: + service_name: gateway + package-all: needs: - backend-docker-build @@ -41,6 +47,7 @@ jobs: - database-docker-build - backend-python-docker-build - runtime-docker-build + - gateway-docker-build runs-on: ubuntu-latest steps: - name: Checkout diff --git a/backend/api-gateway/pom.xml b/backend/api-gateway/pom.xml index 87c3cf4..5825964 100644 --- a/backend/api-gateway/pom.xml +++ b/backend/api-gateway/pom.xml @@ -13,34 +13,38 @@ api-gateway + jar API Gateway API网关服务 + + 3.5.6 + 2025.0.0 + + org.springframework.cloud spring-cloud-starter-gateway - org.springframework.boot - spring-boot-starter-data-redis-reactive + com.terrabase + enterprise-impl-commercial + 1.0.0 + + + spring-web + org.springframework + + + spring-boot-starter-web + org.springframework.boot + + - org.springframework.boot - spring-boot-starter-security - - - org.springframework.boot - spring-boot-starter-actuator - - - org.springframework.cloud - spring-cloud-starter-loadbalancer - - - org.springframework.boot - spring-boot-starter-test - test + com.alibaba.fastjson2 + fastjson2 @@ -49,6 +53,18 @@ org.springframework.boot spring-boot-maven-plugin + ${spring-boot.version} + + gateway + com.datamate.gateway.ApiGatewayApplication + + + + + repackage + + + diff --git a/backend/api-gateway/src/main/java/com/datamate/gateway/ApiGatewayApplication.java b/backend/api-gateway/src/main/java/com/datamate/gateway/ApiGatewayApplication.java index a9073ce..0d954f0 100644 --- a/backend/api-gateway/src/main/java/com/datamate/gateway/ApiGatewayApplication.java +++ b/backend/api-gateway/src/main/java/com/datamate/gateway/ApiGatewayApplication.java @@ -5,7 +5,6 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.cloud.gateway.route.RouteLocator; import org.springframework.cloud.gateway.route.builder.RouteLocatorBuilder; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; /** * API Gateway & Auth Service Application @@ -13,10 +12,6 @@ import org.springframework.context.annotation.ComponentScan; * 提供路由、鉴权、限流等功能 */ @SpringBootApplication -@ComponentScan(basePackages = { - "com.datamate.gateway", - "com.datamate.shared" -}) public class ApiGatewayApplication { public static void main(String[] args) { @@ -26,51 +21,21 @@ public class ApiGatewayApplication { @Bean public RouteLocator customRouteLocator(RouteLocatorBuilder builder) { return builder.routes() - // 数据归集服务路由 - .route("data-collection", r -> r.path("/api/data-collection/**") - .uri("lb://data-collection-service")) - - // 数据管理服务路由 - .route("data-management", r -> r.path("/api/data-management/**") - .uri("lb://data-management-service")) - - // 算子市场服务路由 - .route("operator-market", r -> r.path("/api/operators/**") - .uri("lb://operator-market-service")) - - // 数据清洗服务路由 - .route("data-cleaning", r -> r.path("/api/cleaning/**") - .uri("lb://data-cleaning-service")) - // 数据合成服务路由 .route("data-synthesis", r -> r.path("/api/synthesis/**") - .uri("lb://data-synthesis-service")) + .uri("http://datamate-backend-python:18000")) // 数据标注服务路由 .route("data-annotation", r -> r.path("/api/annotation/**") - .uri("lb://data-annotation-service")) + .uri("http://datamate-backend-python:18000")) // 数据评估服务路由 .route("data-evaluation", r -> r.path("/api/evaluation/**") - .uri("lb://data-evaluation-service")) + .uri("http://datamate-backend-python:18000")) - // 流程编排服务路由 - .route("pipeline-orchestration", r -> r.path("/api/pipelines/**") - .uri("lb://pipeline-orchestration-service")) - - // 执行引擎服务路由 - .route("execution-engine", r -> r.path("/api/execution/**") - .uri("lb://execution-engine-service")) - - // 认证服务路由 - .route("auth-service", r -> r.path("/api/auth/**") - .uri("lb://auth-service")) - - // RAG服务路由 - .route("rag-indexer", r -> r.path("/api/rag/indexer/**") - .uri("lb://rag-indexer-service")) - .route("rag-query", r -> r.path("/api/rag/query/**") - .uri("lb://rag-query-service")) + // 其他后端服务 + .route("default", r -> r.path("/api/**") + .uri("http://datamate-backend:8080")) .build(); } diff --git a/backend/api-gateway/src/main/java/com/datamate/gateway/filter/UserContextFilter.java b/backend/api-gateway/src/main/java/com/datamate/gateway/filter/UserContextFilter.java new file mode 100644 index 0000000..59b7a8e --- /dev/null +++ b/backend/api-gateway/src/main/java/com/datamate/gateway/filter/UserContextFilter.java @@ -0,0 +1,51 @@ +package com.datamate.gateway.filter; + +import com.terrabase.enterprise.api.dto.LoginUserDto; +import com.terrabase.enterprise.api.sdk.TerrabaseSDK; +import com.terrabase.enterprise.api.sdk.TerrabaseSDKConfig; +import jakarta.annotation.PostConstruct; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.cloud.gateway.filter.GatewayFilterChain; +import org.springframework.cloud.gateway.filter.GlobalFilter; +import org.springframework.stereotype.Component; +import org.springframework.web.server.ServerWebExchange; +import reactor.core.publisher.Mono; + +/** + * 用户信息过滤器 + * + * @since 2025/12/22 + */ +@Slf4j +@Component +public class UserContextFilter implements GlobalFilter { + @Value("${terrabase.jar.path:/opt/terrabase}") + private String jarPath; + + @Value("${commercial.switch:false}") + private boolean isCommercial; + + private TerrabaseSDK terrabaseSDK; + + @PostConstruct + public void init() { + TerrabaseSDKConfig sdkConfig = TerrabaseSDKConfig.createDefault(); + sdkConfig.setJarPath(jarPath); + terrabaseSDK = TerrabaseSDK.init(sdkConfig); + } + + @Override + public Mono filter(ServerWebExchange exchange, GatewayFilterChain chain) { + if (!isCommercial) { + return chain.filter(exchange); + } + try { + LoginUserDto loginUserDto = terrabaseSDK.userManagement().getCurrentUserInfo().getData().getFirst(); + } catch (Exception e) { + log.error("get current user info error", e); + return chain.filter(exchange); + } + return chain.filter(exchange); + } +} diff --git a/backend/pom.xml b/backend/pom.xml index 6ff019a..f57ea15 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -38,35 +38,24 @@ 5.4.0 2.21.1 1.26.1 + 2.0.52 - - shared/domain-common - shared/security-common - - services/data-management-service - services/data-collection-service - services/operator-market-service - services/data-cleaning-service - services/data-synthesis-service - services/data-annotation-service - services/data-evaluation-service - services/pipeline-orchestration-service - services/execution-engine-service - - - services/rag-indexer-service - services/rag-query-service - - - services/main-application + services api-gateway + + + org.projectlombok + lombok + + + @@ -158,70 +147,15 @@ ${commons-compress.version} + + com.alibaba.fastjson2 + fastjson2 + ${fastjson-version} + + - - - org.springframework.boot - spring-boot-starter-web - ${spring-boot.version} - - - org.springframework.boot - spring-boot-starter-logging - - - - - - org.springframework.boot - spring-boot-starter - ${spring-boot.version} - - - org.springframework.boot - spring-boot-starter-logging - - - - - - com.baomidou - mybatis-plus-spring-boot3-starter - ${mybatis-plus.version} - - - com.baomidou - mybatis-plus-jsqlparser - - - - - org.springframework.boot - spring-boot-starter-log4j2 - ${spring-boot.version} - - - - org.projectlombok - lombok - ${lombok.version} - - - - org.mapstruct - mapstruct - ${mapstruct.version} - - - - org.apache.poi - poi - ${poi.version} - - - diff --git a/backend/services/data-annotation-service/pom.xml b/backend/services/data-annotation-service/pom.xml index 935426f..0de8bb5 100644 --- a/backend/services/data-annotation-service/pom.xml +++ b/backend/services/data-annotation-service/pom.xml @@ -7,9 +7,9 @@ com.datamate - datamate + services 1.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml data-annotation-service diff --git a/backend/services/data-cleaning-service/pom.xml b/backend/services/data-cleaning-service/pom.xml index aed8ad6..bd50163 100644 --- a/backend/services/data-cleaning-service/pom.xml +++ b/backend/services/data-cleaning-service/pom.xml @@ -7,9 +7,9 @@ com.datamate - datamate + services 1.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml data-cleaning-service @@ -53,10 +53,6 @@ org.openapitools jackson-databind-nullable - - com.baomidou - mybatis-plus-spring-boot3-starter - com.mysql mysql-connector-j diff --git a/backend/services/data-collection-service/pom.xml b/backend/services/data-collection-service/pom.xml index fd48dc4..e0979b7 100644 --- a/backend/services/data-collection-service/pom.xml +++ b/backend/services/data-collection-service/pom.xml @@ -6,9 +6,9 @@ com.datamate - datamate + services 1.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml data-collection-service @@ -31,12 +31,6 @@ spring-boot-starter-actuator - - - com.baomidou - mybatis-plus-spring-boot3-starter - - com.mysql @@ -109,13 +103,6 @@ jakarta.validation-api - - - org.projectlombok - lombok - true - - org.springframework.boot diff --git a/backend/services/data-evaluation-service/pom.xml b/backend/services/data-evaluation-service/pom.xml index 738e711..61001be 100644 --- a/backend/services/data-evaluation-service/pom.xml +++ b/backend/services/data-evaluation-service/pom.xml @@ -7,9 +7,9 @@ com.datamate - datamate + services 1.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml data-evaluation-service diff --git a/backend/services/data-management-service/pom.xml b/backend/services/data-management-service/pom.xml index b46830e..581635c 100644 --- a/backend/services/data-management-service/pom.xml +++ b/backend/services/data-management-service/pom.xml @@ -7,9 +7,9 @@ com.datamate - datamate + services 1.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml data-management-service @@ -26,10 +26,6 @@ org.springframework.boot spring-boot-starter-web - - com.baomidou - mybatis-plus-spring-boot3-starter - org.springframework.boot spring-boot-starter-data-redis diff --git a/backend/services/data-synthesis-service/pom.xml b/backend/services/data-synthesis-service/pom.xml index 6518bba..2647d20 100644 --- a/backend/services/data-synthesis-service/pom.xml +++ b/backend/services/data-synthesis-service/pom.xml @@ -7,9 +7,9 @@ com.datamate - datamate + services 1.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml data-synthesis-service diff --git a/backend/services/execution-engine-service/pom.xml b/backend/services/execution-engine-service/pom.xml index 558c2a1..7d29d4f 100644 --- a/backend/services/execution-engine-service/pom.xml +++ b/backend/services/execution-engine-service/pom.xml @@ -7,9 +7,9 @@ com.datamate - datamate + services 1.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml execution-engine-service diff --git a/backend/services/main-application/pom.xml b/backend/services/main-application/pom.xml index 20af9c4..24ffb22 100644 --- a/backend/services/main-application/pom.xml +++ b/backend/services/main-application/pom.xml @@ -7,9 +7,9 @@ com.datamate - datamate + services 1.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml main-application @@ -109,12 +109,6 @@ ${project.version} - - - com.baomidou - mybatis-plus-spring-boot3-starter - - com.mysql diff --git a/backend/services/operator-market-service/pom.xml b/backend/services/operator-market-service/pom.xml index 22cc523..5e2b246 100644 --- a/backend/services/operator-market-service/pom.xml +++ b/backend/services/operator-market-service/pom.xml @@ -7,9 +7,9 @@ com.datamate - datamate + services 1.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml operator-market-service @@ -55,10 +55,6 @@ jakarta.validation jakarta.validation-api - - com.baomidou - mybatis-plus-spring-boot3-starter - org.projectlombok lombok @@ -70,11 +66,6 @@ 1.26.1 - - org.mapstruct - mapstruct - - org.mapstruct mapstruct-processor diff --git a/backend/services/pipeline-orchestration-service/pom.xml b/backend/services/pipeline-orchestration-service/pom.xml index dac69cf..b25c7f4 100644 --- a/backend/services/pipeline-orchestration-service/pom.xml +++ b/backend/services/pipeline-orchestration-service/pom.xml @@ -7,9 +7,9 @@ com.datamate - datamate + services 1.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml pipeline-orchestration-service diff --git a/backend/services/pom.xml b/backend/services/pom.xml new file mode 100644 index 0000000..fd40146 --- /dev/null +++ b/backend/services/pom.xml @@ -0,0 +1,109 @@ + + + 4.0.0 + + + com.datamate + datamate + 1.0.0-SNAPSHOT + ../pom.xml + + + services + pom + Services + Services + + + + ../shared/domain-common + ../shared/security-common + + + data-management-service + data-collection-service + operator-market-service + data-cleaning-service + data-synthesis-service + data-annotation-service + data-evaluation-service + pipeline-orchestration-service + execution-engine-service + + + rag-indexer-service + rag-query-service + + + main-application + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-logging + + + + + + org.springframework.boot + spring-boot-starter + + + org.springframework.boot + spring-boot-starter-logging + + + + + + com.baomidou + mybatis-plus-spring-boot3-starter + + + com.baomidou + mybatis-plus-jsqlparser + + + + org.mapstruct + mapstruct + + + + org.apache.poi + poi + + + + + + + org.springframework.boot + spring-boot-maven-plugin + ${spring-boot.version} + + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 + + ${maven.compiler.source} + ${maven.compiler.target} + true + + -parameters + + + + + + diff --git a/backend/services/rag-indexer-service/pom.xml b/backend/services/rag-indexer-service/pom.xml index cef1760..5f7a4fb 100644 --- a/backend/services/rag-indexer-service/pom.xml +++ b/backend/services/rag-indexer-service/pom.xml @@ -7,9 +7,9 @@ com.datamate - datamate + services 1.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml rag-indexer-service diff --git a/backend/services/rag-query-service/pom.xml b/backend/services/rag-query-service/pom.xml index 79c527e..c5aa8ef 100644 --- a/backend/services/rag-query-service/pom.xml +++ b/backend/services/rag-query-service/pom.xml @@ -7,9 +7,9 @@ com.datamate - datamate + services 1.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml rag-query-service diff --git a/backend/shared/domain-common/pom.xml b/backend/shared/domain-common/pom.xml index da9ba1a..f6b4ee6 100644 --- a/backend/shared/domain-common/pom.xml +++ b/backend/shared/domain-common/pom.xml @@ -7,9 +7,9 @@ com.datamate - datamate + services 1.0.0-SNAPSHOT - ../../pom.xml + ../../services/pom.xml domain-common diff --git a/backend/shared/security-common/pom.xml b/backend/shared/security-common/pom.xml index 5f3a8ca..5ad7a3a 100644 --- a/backend/shared/security-common/pom.xml +++ b/backend/shared/security-common/pom.xml @@ -7,9 +7,9 @@ com.datamate - datamate + services 1.0.0-SNAPSHOT - ../../pom.xml + ../../services/pom.xml security-common diff --git a/deployment/docker/datamate/backend-with-deer-flow.conf b/deployment/docker/datamate/backend-with-deer-flow.conf index 240ce34..ad3ca5b 100644 --- a/deployment/docker/datamate/backend-with-deer-flow.conf +++ b/deployment/docker/datamate/backend-with-deer-flow.conf @@ -9,29 +9,8 @@ server { add_header Set-Cookie "NEXT_LOCALE=zh"; - location /api/synthesis/ { - proxy_pass http://datamate-backend-python:18000/api/synthesis/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api/annotation/ { - proxy_pass http://datamate-backend-python:18000/api/annotation/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api/evaluation/ { - proxy_pass http://datamate-backend-python:18000/api/evaluation/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - location /api/ { - proxy_pass http://datamate-backend:8080/api/; + proxy_pass http://datamate-gateway:8080/api/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/deployment/docker/datamate/backend.conf b/deployment/docker/datamate/backend.conf index 795c852..d73e8ca 100644 --- a/deployment/docker/datamate/backend.conf +++ b/deployment/docker/datamate/backend.conf @@ -7,29 +7,8 @@ server { client_max_body_size 1024M; - location /api/synthesis/ { - proxy_pass http://datamate-backend-python:18000/api/synthesis/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api/evaluation/ { - proxy_pass http://datamate-backend-python:18000/api/evaluation/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api/annotation/ { - proxy_pass http://datamate-backend-python:18000/api/annotation/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - location /api/ { - proxy_pass http://datamate-backend:8080/api/; + proxy_pass http://datamate-gateway:8080/api/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/deployment/docker/datamate/docker-compose.yml b/deployment/docker/datamate/docker-compose.yml index 084182c..d6972c2 100644 --- a/deployment/docker/datamate/docker-compose.yml +++ b/deployment/docker/datamate/docker-compose.yml @@ -31,6 +31,13 @@ services: depends_on: - datamate-database + datamate-gateway: + container_name: datamate-gateway + image: ${REGISTRY:-}datamate-gateway + restart: on-failure + privileged: true + networks: [ datamate ] + # 2) frontend（NodePort 30000） datamate-frontend: container_name: datamate-frontend diff --git a/deployment/helm/datamate/charts/frontend/templates/configmap.yaml b/deployment/helm/datamate/charts/frontend/templates/configmap.yaml index bb4dec9..457e388 100644 --- a/deployment/helm/datamate/charts/frontend/templates/configmap.yaml +++ b/deployment/helm/datamate/charts/frontend/templates/configmap.yaml @@ -16,27 +16,6 @@ data: add_header Set-Cookie "NEXT_LOCALE=zh"; - location /api/synthesis/ { - proxy_pass http://datamate-backend-python:18000/api/synthesis/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api/evaluation/ { - proxy_pass http://datamate-backend-python:18000/api/evaluation/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api/annotation/ { - proxy_pass http://datamate-backend-python:18000/api/annotation/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - location /api/ { proxy_pass http://datamate-backend:8080/api/; proxy_set_header Host $host; @@ -91,27 +70,6 @@ data: client_max_body_size 1024M; - location /api/synthesis/ { - proxy_pass http://datamate-backend-python:18000/api/synthesis/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api/evaluation/ { - proxy_pass http://datamate-backend-python:18000/api/evaluation/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api/annotation/ { - proxy_pass http://datamate-backend-python:18000/api/annotation/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - location /api/ { proxy_pass http://datamate-backend:8080/api/; proxy_set_header Host $host; diff --git a/deployment/helm/datamate/charts/gateway/.helmignore b/deployment/helm/datamate/charts/gateway/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/deployment/helm/datamate/charts/gateway/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deployment/helm/datamate/charts/gateway/Chart.yaml b/deployment/helm/datamate/charts/gateway/Chart.yaml new file mode 100644 index 0000000..2ff0a22 --- /dev/null +++ b/deployment/helm/datamate/charts/gateway/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: gateway +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.0.1 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.0.1" diff --git a/deployment/helm/datamate/charts/gateway/templates/_helpers.tpl b/deployment/helm/datamate/charts/gateway/templates/_helpers.tpl new file mode 100644 index 0000000..fa6eafe --- /dev/null +++ b/deployment/helm/datamate/charts/gateway/templates/_helpers.tpl @@ -0,0 +1,76 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "gateway.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "gateway.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "gateway.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "gateway.labels" -}} +helm.sh/chart: {{ include "gateway.chart" . }} +app: {{ .Release.Name }} +{{ include "gateway.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "gateway.selectorLabels" -}} +app.kubernetes.io/name: {{ include "gateway.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "gateway.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "gateway.fullname" .) .Values.serviceAccount.name -}} +{{- else }} +{{- default "default" .Values.serviceAccount.name -}} +{{- end }} +{{- end }} + +{{/* +Name of image +*/}} +{{- define "gateway.image" -}} +{{- $name := default .Values.image.repository .Values.global.image.gateway.name }} +{{- $tag := default .Values.image.tag .Values.global.image.gateway.tag }} +{{- if .Values.global.image.repository }} +{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}:{{ $tag }} +{{- else }} +{{- $name }}:{{ $tag }} +{{- end }} +{{- end }} diff --git a/deployment/helm/datamate/charts/gateway/templates/deployment.yaml b/deployment/helm/datamate/charts/gateway/templates/deployment.yaml new file mode 100644 index 0000000..772fb21 --- /dev/null +++ b/deployment/helm/datamate/charts/gateway/templates/deployment.yaml @@ -0,0 +1,82 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "gateway.fullname" . }} + labels: + {{- include "gateway.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "gateway.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "gateway.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "gateway.serviceAccountName" . }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: "{{ include "gateway.image" . }}" + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.global.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deployment/helm/datamate/charts/gateway/templates/service.yaml b/deployment/helm/datamate/charts/gateway/templates/service.yaml new file mode 100644 index 0000000..f2752fe --- /dev/null +++ b/deployment/helm/datamate/charts/gateway/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "gateway.fullname" . }} + labels: + {{- include "gateway.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.service.port }} + protocol: TCP + name: {{ .Chart.Name }} + selector: + {{- include "gateway.selectorLabels" . | nindent 4 }} diff --git a/deployment/helm/datamate/charts/gateway/templates/serviceaccount.yaml b/deployment/helm/datamate/charts/gateway/templates/serviceaccount.yaml new file mode 100644 index 0000000..c61f93d --- /dev/null +++ b/deployment/helm/datamate/charts/gateway/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "gateway.serviceAccountName" . }} + labels: + {{- include "gateway.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/deployment/helm/datamate/charts/gateway/values.yaml b/deployment/helm/datamate/charts/gateway/values.yaml new file mode 100644 index 0000000..788afab --- /dev/null +++ b/deployment/helm/datamate/charts/gateway/values.yaml @@ -0,0 +1,108 @@ +# Default values for datamate. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ +replicaCount: 1 + +# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ +image: + repository: "datamate-gateway" + # This sets the pull policy for images. + pullPolicy: "IfNotPresent" + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +imagePullSecrets: [] +# This is to override the chart name. +nameOverride: "datamate-gateway" +fullnameOverride: "datamate-gateway" + +env: [] + +# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/ +serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +# This is for setting Kubernetes Annotations to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} +# This is for setting Kubernetes Labels to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +podLabels: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/ +service: + # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: ClusterIP + # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports + port: 8080 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ +# livenessProbe: +# httpGet: +# path: / +# port: http +# readinessProbe: +# httpGet: +# path: / +# port: http + +# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/ +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +# Additional volumes on the output Deployment definition. +volumes: [] +# - name: foo +# secret: +# secretName: mysecret +# optional: false + +# Additional volumeMounts on the output Deployment definition. +volumeMounts: [] +# - name: foo +# mountPath: "/etc/foo" +# readOnly: true + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index 7ed5da6..2162e84 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -15,6 +15,9 @@ global: backendPython: name: "datamate-backend-python" tag: "latest" + gateway: + name: "datamate-gateway" + tag: "latest" frontend: name: "datamate-frontend" tag: "latest" @@ -117,6 +120,17 @@ backend-python: - name: log-volume mountPath: /var/log/datamate +gateway: + service: + type: NodePort + nodePort: 30000 + volumes: + - *logVolume + volumeMounts: + - name: log-volume + mountPath: /var/log/datamate/gateway + subPath: gateway + frontend: service: type: NodePort diff --git a/runtime/datamate-python/app/db/models/data_evaluation.py b/runtime/datamate-python/app/db/models/data_evaluation.py index d765187..7657d5e 100644 --- a/runtime/datamate-python/app/db/models/data_evaluation.py +++ b/runtime/datamate-python/app/db/models/data_evaluation.py @@ -51,6 +51,7 @@ class EvaluationFile(Base): task_id = Column(String(36), ForeignKey('t_de_eval_task.id'), nullable=False, comment="评估任务ID") file_id = Column(String(36), ForeignKey('t_dm_dataset_files.id'), nullable=True, comment="文件ID") file_name = Column(String(255), nullable=False, comment="文件名") + error_message = Column(Text, nullable=True, comment="错误信息") total_count = Column(Integer, nullable=False, default=0, comment="总数") evaluated_count = Column(Integer, nullable=False, default=0, comment="已评估数") created_at = Column(TIMESTAMP, server_default=func.current_timestamp(), comment="创建时间") diff --git a/scripts/db/data-evaluation-init.sql b/scripts/db/data-evaluation-init.sql index 2118de9..b5a3984 100644 --- a/scripts/db/data-evaluation-init.sql +++ b/scripts/db/data-evaluation-init.sql @@ -31,6 +31,7 @@ CREATE TABLE IF NOT EXISTS t_de_eval_file ( file_name VARCHAR(255) COMMENT '文件名', total_count INT DEFAULT 0 COMMENT '总数', evaluated_count INT DEFAULT 0 COMMENT '已评估数', + error_message TEXT COMMENT '错误信息', created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', created_by VARCHAR(255) COMMENT '创建者', diff --git a/scripts/images/backend/Dockerfile b/scripts/images/backend/Dockerfile index c86eec1..3f839fe 100644 --- a/scripts/images/backend/Dockerfile +++ b/scripts/images/backend/Dockerfile @@ -15,7 +15,7 @@ FROM maven:3-eclipse-temurin-21 AS builder COPY backend/ /opt/backend -RUN cd /opt/backend && \ +RUN cd /opt/backend/services && \ mvn -U clean package -Dmaven.test.skip=true diff --git a/scripts/images/gateway/Dockerfile b/scripts/images/gateway/Dockerfile new file mode 100644 index 0000000..13a148b --- /dev/null +++ b/scripts/images/gateway/Dockerfile @@ -0,0 +1,37 @@ +FROM maven:3-eclipse-temurin-21 AS builder + +RUN apt-get update && \ + apt-get install -y git && \ + git clone https://github.com/ModelEngine-Group/Terrabase.git && \ + cd Terrabase && \ + git -c core.quotepath=false -c log.showSignature=false checkout -b pyh/feat_terrabase_develop origin/pyh/feat_terrabase_develop -- && \ + mvn -U clean package install -Dmaven.test.skip=true + +COPY backend/ /opt/gateway + +RUN cd /opt/gateway/api-gateway && \ + mvn -U clean package -Dmaven.test.skip=true && \ + ls /opt/gateway/api-gateway/target + + +FROM eclipse-temurin:21-jdk + +RUN apt-get update && \ + apt-get install -y vim wget curl dos2unix && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +COPY --from=builder /opt/gateway/api-gateway/target/gateway.jar /opt/gateway/gateway.jar +COPY --from=builder /Terrabase/enterprise-impl-commercial/target/*.jar /opt/terrabase/ + +COPY scripts/images/gateway/start.sh /opt/gateway/start.sh + +RUN dos2unix /opt/gateway/start.sh \ + && chmod +x /opt/gateway/start.sh \ + && ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime + +EXPOSE 8080 + +ENTRYPOINT ["/opt/gateway/start.sh"] + +CMD ["java", "-Duser.timezone=Asia/Shanghai", "-jar", "/opt/gateway/gateway.jar"] diff --git a/scripts/images/gateway/start.sh b/scripts/images/gateway/start.sh new file mode 100644 index 0000000..5f8f4a7 --- /dev/null +++ b/scripts/images/gateway/start.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -e + +echo "Starting main application..." +exec "$@" \ No newline at end of file