q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

add gateway (#187)

Browse Source 
* feature: add gateway
This commit is contained in:
hefanli
2025-12-22 15:41:17 +08:00
committed by GitHub
parent 46f4a8c219
commit e5b28c26b1
41 changed files with 706 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
.github/workflows/docker-image-gateway.yml vendored Normal file
View File

@@ -0,0 +1,30 @@
name: Gateway Docker Image CI
on:
push:
branches: [ "main" ]
paths:
- 'backend/**'
- 'scripts/images/gateway/**'
- '.github/workflows/docker-image-gateway.yml'
- '.github/workflows/docker-images-reusable.yml'
pull_request:
branches: [ "main" ]
paths:
- 'backend/**'
- 'scripts/images/gateway/**'
- '.github/workflows/docker-image-gateway.yml'
- '.github/workflows/docker-images-reusable.yml'
workflow_dispatch:
workflow_call:
jobs:
call-docker-build:
name: Build and Push Gateway Docker Image
uses: ./.github/workflows/docker-images-reusable.yml
permissions:
contents: read
packages: write
with:
service_name: gateway
build_dir: .

7
.github/workflows/docker-images-release.yml vendored
View File

@@ -14,6 +14,13 @@ jobs:
service_name: frontend
build_dir: .
gateway:
name: Gateway Image
uses: ./.github/workflows/docker-images-reusable.yml
with:
service_name: gateway
build_dir: .
backend:
name: Backend Image
uses: ./.github/workflows/docker-images-reusable.yml

9
.github/workflows/docker-push-to-huaweicloud.yml vendored
View File

@@ -18,6 +18,15 @@ jobs:
secrets:
HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
gateway:
name: Push Gateway Image
uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
with:
service_name: datamate-gateway
version: ${{ inputs.version}}
secrets:
HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
backend:
name: Push Backend Image
uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml

2
.github/workflows/enterprise-package.yml vendored
View File

@@ -61,7 +61,7 @@ jobs:
run: |
mkdir -p images/datamate
LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
services=("runtime" "backend" "frontend" "database" "backend-python")
services=("runtime" "backend" "frontend" "database" "backend-python" "gateway")
for service in "${services[@]}"; do
docker pull ghcr.io/$LOWERCASE_REPO/datamate-$service:latest --platform arm64
docker tag ghcr.io/$LOWERCASE_REPO/datamate-$service:latest datamate-$service:latest

7
.github/workflows/package.yml vendored
View File

@@ -34,6 +34,12 @@ jobs:
with:
service_name: backend-python
gateway-docker-build:
name: Build and Push Frontend Docker Image
uses: ./.github/workflows/docker-image-save.yml
with:
service_name: gateway
package-all:
needs:
- backend-docker-build
@@ -41,6 +47,7 @@ jobs:
- database-docker-build
- backend-python-docker-build
- runtime-docker-build
- gateway-docker-build
runs-on: ubuntu-latest
steps:
- name: Checkout

50
backend/api-gateway/pom.xml
View File

@@ -13,34 +13,38 @@
</parent>
<artifactId>api-gateway</artifactId>
<packaging>jar</packaging>
<name>API Gateway</name>
<description>API网关服务</description>
<properties>
<spring-boot.version>3.5.6</spring-boot.version>
<spring-cloud.version>2025.0.0</spring-cloud.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-gateway</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis-reactive</artifactId>
<groupId>com.terrabase</groupId>
<artifactId>enterprise-impl-commercial</artifactId>
<version>1.0.0</version>
<exclusions>
<exclusion>
<artifactId>spring-web</artifactId>
<groupId>org.springframework</groupId>
</exclusion>
<exclusion>
<artifactId>spring-boot-starter-web</artifactId>
<groupId>org.springframework.boot</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-loadbalancer</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
<groupId>com.alibaba.fastjson2</groupId>
<artifactId>fastjson2</artifactId>
</dependency>
</dependencies>
@@ -49,6 +53,18 @@
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${spring-boot.version}</version>
<configuration>
<finalName>gateway</finalName>
<mainClass>com.datamate.gateway.ApiGatewayApplication</mainClass>
</configuration>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>

47
backend/api-gateway/src/main/java/com/datamate/gateway/ApiGatewayApplication.java
View File

@@ -5,7 +5,6 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.gateway.route.RouteLocator;
import org.springframework.cloud.gateway.route.builder.RouteLocatorBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
/**
* API Gateway & Auth Service Application
@@ -13,10 +12,6 @@ import org.springframework.context.annotation.ComponentScan;
* 提供路由、鉴权、限流等功能
*/
@SpringBootApplication
@ComponentScan(basePackages = {
"com.datamate.gateway",
"com.datamate.shared"
})
public class ApiGatewayApplication {
public static void main(String[] args) {
@@ -26,51 +21,21 @@ public class ApiGatewayApplication {
@Bean
public RouteLocator customRouteLocator(RouteLocatorBuilder builder) {
return builder.routes()
// 数据归集服务路由
.route("data-collection", r -> r.path("/api/data-collection/**")
.uri("lb://data-collection-service"))
// 数据管理服务路由
.route("data-management", r -> r.path("/api/data-management/**")
.uri("lb://data-management-service"))
// 算子市场服务路由
.route("operator-market", r -> r.path("/api/operators/**")
.uri("lb://operator-market-service"))
// 数据清洗服务路由
.route("data-cleaning", r -> r.path("/api/cleaning/**")
.uri("lb://data-cleaning-service"))
// 数据合成服务路由
.route("data-synthesis", r -> r.path("/api/synthesis/**")
.uri("lb://data-synthesis-service"))
.uri("http://datamate-backend-python:18000"))
// 数据标注服务路由
.route("data-annotation", r -> r.path("/api/annotation/**")
.uri("lb://data-annotation-service"))
.uri("http://datamate-backend-python:18000"))
// 数据评估服务路由
.route("data-evaluation", r -> r.path("/api/evaluation/**")
.uri("lb://data-evaluation-service"))
.uri("http://datamate-backend-python:18000"))
// 流程编排服务路由
.route("pipeline-orchestration", r -> r.path("/api/pipelines/**")
.uri("lb://pipeline-orchestration-service"))
// 执行引擎服务路由
.route("execution-engine", r -> r.path("/api/execution/**")
.uri("lb://execution-engine-service"))
// 认证服务路由
.route("auth-service", r -> r.path("/api/auth/**")
.uri("lb://auth-service"))
// RAG服务路由
.route("rag-indexer", r -> r.path("/api/rag/indexer/**")
.uri("lb://rag-indexer-service"))
.route("rag-query", r -> r.path("/api/rag/query/**")
.uri("lb://rag-query-service"))
// 其他后端服务
.route("default", r -> r.path("/api/**")
.uri("http://datamate-backend:8080"))
.build();
}

51
backend/api-gateway/src/main/java/com/datamate/gateway/filter/UserContextFilter.java Normal file
View File

@@ -0,0 +1,51 @@
package com.datamate.gateway.filter;
import com.terrabase.enterprise.api.dto.LoginUserDto;
import com.terrabase.enterprise.api.sdk.TerrabaseSDK;
import com.terrabase.enterprise.api.sdk.TerrabaseSDKConfig;
import jakarta.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cloud.gateway.filter.GatewayFilterChain;
import org.springframework.cloud.gateway.filter.GlobalFilter;
import org.springframework.stereotype.Component;
import org.springframework.web.server.ServerWebExchange;
import reactor.core.publisher.Mono;
/**
* 用户信息过滤器
*
* @since 2025/12/22
*/
@Slf4j
@Component
public class UserContextFilter implements GlobalFilter {
@Value("${terrabase.jar.path:/opt/terrabase}")
private String jarPath;
@Value("${commercial.switch:false}")
private boolean isCommercial;
private TerrabaseSDK terrabaseSDK;
@PostConstruct
public void init() {
TerrabaseSDKConfig sdkConfig = TerrabaseSDKConfig.createDefault();
sdkConfig.setJarPath(jarPath);
terrabaseSDK = TerrabaseSDK.init(sdkConfig);
}
@Override
public Mono<Void> filter(ServerWebExchange exchange, GatewayFilterChain chain) {
if (!isCommercial) {
return chain.filter(exchange);
}
try {
LoginUserDto loginUserDto = terrabaseSDK.userManagement().getCurrentUserInfo().getData().getFirst();
} catch (Exception e) {
log.error("get current user info error", e);
return chain.filter(exchange);
}
return chain.filter(exchange);
}
}

96
backend/pom.xml
View File

@@ -38,35 +38,24 @@
<poi.version>5.4.0</poi.version>
<log4j2.version>2.21.1</log4j2.version>
<commons-compress.version>1.26.1</commons-compress.version>
<fastjson-version>2.0.52</fastjson-version>
</properties>
<modules>
<!-- 共享库 -->
<module>shared/domain-common</module>
<module>shared/security-common</module>
<!-- 核心服务 -->
<module>services/data-management-service</module>
<module>services/data-collection-service</module>
<module>services/operator-market-service</module>
<module>services/data-cleaning-service</module>
<module>services/data-synthesis-service</module>
<module>services/data-annotation-service</module>
<module>services/data-evaluation-service</module>
<module>services/pipeline-orchestration-service</module>
<module>services/execution-engine-service</module>
<!-- RAG服务 -->
<module>services/rag-indexer-service</module>
<module>services/rag-query-service</module>
<!-- 主启动模块 -->
<module>services/main-application</module>
<module>services</module>
<!-- API Gateway微服务 -->
<module>api-gateway</module>
</modules>
<dependencies>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
@@ -158,70 +147,15 @@
<version>${commons-compress.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba.fastjson2</groupId>
<artifactId>fastjson2</artifactId>
<version>${fastjson-version}</version>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<version>${spring-boot.version}</version>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
<version>${spring-boot.version}</version>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-spring-boot3-starter</artifactId>
<version>${mybatis-plus.version}</version>
</dependency>
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-jsqlparser</artifactId>
</dependency>
<!-- Log4j2 API -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-log4j2</artifactId>
<version>${spring-boot.version}</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok.version}</version>
</dependency>
<dependency>
<groupId>org.mapstruct</groupId>
<artifactId>mapstruct</artifactId>
<version>${mapstruct.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>${poi.version}</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>

4
backend/services/data-annotation-service/pom.xml
View File

@@ -7,9 +7,9 @@
<parent>
<groupId>com.datamate</groupId>
<artifactId>datamate</artifactId>
<artifactId>services</artifactId>
<version>1.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>data-annotation-service</artifactId>

8
backend/services/data-cleaning-service/pom.xml
View File

@@ -7,9 +7,9 @@
<parent>
<groupId>com.datamate</groupId>
<artifactId>datamate</artifactId>
<artifactId>services</artifactId>
<version>1.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>data-cleaning-service</artifactId>
@@ -53,10 +53,6 @@
<groupId>org.openapitools</groupId>
<artifactId>jackson-databind-nullable</artifactId>
</dependency>
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-spring-boot3-starter</artifactId>
</dependency>
<dependency>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>

17
backend/services/data-collection-service/pom.xml
View File

@@ -6,9 +6,9 @@
<parent>
<groupId>com.datamate</groupId>
<artifactId>datamate</artifactId>
<artifactId>services</artifactId>
<version>1.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>data-collection-service</artifactId>
@@ -31,12 +31,6 @@
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<!-- MyBatis Dependencies -->
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-spring-boot3-starter</artifactId>
</dependency>
<!-- Database -->
<dependency>
<groupId>com.mysql</groupId>
@@ -109,13 +103,6 @@
<artifactId>jakarta.validation-api</artifactId>
</dependency>
<!-- Lombok -->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
<!-- Test Dependencies -->
<dependency>
<groupId>org.springframework.boot</groupId>

4
backend/services/data-evaluation-service/pom.xml
View File

@@ -7,9 +7,9 @@
<parent>
<groupId>com.datamate</groupId>
<artifactId>datamate</artifactId>
<artifactId>services</artifactId>
<version>1.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>data-evaluation-service</artifactId>

8
backend/services/data-management-service/pom.xml
View File

@@ -7,9 +7,9 @@
<parent>
<groupId>com.datamate</groupId>
<artifactId>datamate</artifactId>
<artifactId>services</artifactId>
<version>1.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>data-management-service</artifactId>
@@ -26,10 +26,6 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-spring-boot3-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>

4
backend/services/data-synthesis-service/pom.xml
View File

@@ -7,9 +7,9 @@
<parent>
<groupId>com.datamate</groupId>
<artifactId>datamate</artifactId>
<artifactId>services</artifactId>
<version>1.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>data-synthesis-service</artifactId>

4
backend/services/execution-engine-service/pom.xml
View File

@@ -7,9 +7,9 @@
<parent>
<groupId>com.datamate</groupId>
<artifactId>datamate</artifactId>
<artifactId>services</artifactId>
<version>1.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>execution-engine-service</artifactId>

10
backend/services/main-application/pom.xml
View File

@@ -7,9 +7,9 @@
<parent>
<groupId>com.datamate</groupId>
<artifactId>datamate</artifactId>
<artifactId>services</artifactId>
<version>1.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>main-application</artifactId>
@@ -109,12 +109,6 @@
<version>${project.version}</version>
</dependency>
<!-- MyBatis Dependencies -->
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-spring-boot3-starter</artifactId>
</dependency>
<!-- Database -->
<dependency>
<groupId>com.mysql</groupId>

13
backend/services/operator-market-service/pom.xml
View File

@@ -7,9 +7,9 @@
<parent>
<groupId>com.datamate</groupId>
<artifactId>datamate</artifactId>
<artifactId>services</artifactId>
<version>1.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>operator-market-service</artifactId>
@@ -55,10 +55,6 @@
<groupId>jakarta.validation</groupId>
<artifactId>jakarta.validation-api</artifactId>
</dependency>
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-spring-boot3-starter</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
@@ -70,11 +66,6 @@
<version>1.26.1</version>
</dependency>
<dependency>
<groupId>org.mapstruct</groupId>
<artifactId>mapstruct</artifactId>
</dependency>
<dependency>
<groupId>org.mapstruct</groupId>
<artifactId>mapstruct-processor</artifactId>

4
backend/services/pipeline-orchestration-service/pom.xml
View File

@@ -7,9 +7,9 @@
<parent>
<groupId>com.datamate</groupId>
<artifactId>datamate</artifactId>
<artifactId>services</artifactId>
<version>1.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>pipeline-orchestration-service</artifactId>

109
backend/services/pom.xml Normal file
View File

@@ -0,0 +1,109 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.datamate</groupId>
<artifactId>datamate</artifactId>
<version>1.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>services</artifactId>
<packaging>pom</packaging>
<name>Services</name>
<description>Services</description>
<modules>
<!-- 共享库 -->
<module>../shared/domain-common</module>
<module>../shared/security-common</module>
<!-- 核心服务 -->
<module>data-management-service</module>
<module>data-collection-service</module>
<module>operator-market-service</module>
<module>data-cleaning-service</module>
<module>data-synthesis-service</module>
<module>data-annotation-service</module>
<module>data-evaluation-service</module>
<module>pipeline-orchestration-service</module>
<module>execution-engine-service</module>
<!-- RAG服务 -->
<module>rag-indexer-service</module>
<module>rag-query-service</module>
<!-- 主启动模块 -->
<module>main-application</module>
</modules>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-spring-boot3-starter</artifactId>
</dependency>
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-jsqlparser</artifactId>
</dependency>
<dependency>
<groupId>org.mapstruct</groupId>
<artifactId>mapstruct</artifactId>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${spring-boot.version}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.11.0</version>
<configuration>
<source>${maven.compiler.source}</source>
<target>${maven.compiler.target}</target>
<parameters>true</parameters>
<compilerArgs>
<arg>-parameters</arg>
</compilerArgs>
</configuration>
</plugin>
</plugins>
</build>
</project>

4
backend/services/rag-indexer-service/pom.xml
View File

@@ -7,9 +7,9 @@
<parent>
<groupId>com.datamate</groupId>
<artifactId>datamate</artifactId>
<artifactId>services</artifactId>
<version>1.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>rag-indexer-service</artifactId>

4
backend/services/rag-query-service/pom.xml
View File

@@ -7,9 +7,9 @@
<parent>
<groupId>com.datamate</groupId>
<artifactId>datamate</artifactId>
<artifactId>services</artifactId>
<version>1.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>rag-query-service</artifactId>

4
backend/shared/domain-common/pom.xml
View File

@@ -7,9 +7,9 @@
<parent>
<groupId>com.datamate</groupId>
<artifactId>datamate</artifactId>
<artifactId>services</artifactId>
<version>1.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
<relativePath>../../services/pom.xml</relativePath>
</parent>
<artifactId>domain-common</artifactId>

4
backend/shared/security-common/pom.xml
View File

@@ -7,9 +7,9 @@
<parent>
<groupId>com.datamate</groupId>
<artifactId>datamate</artifactId>
<artifactId>services</artifactId>
<version>1.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
<relativePath>../../services/pom.xml</relativePath>
</parent>
<artifactId>security-common</artifactId>

23
deployment/docker/datamate/backend-with-deer-flow.conf
View File

@@ -9,29 +9,8 @@ server {
add_header Set-Cookie "NEXT_LOCALE=zh";
location /api/synthesis/ {
proxy_pass http://datamate-backend-python:18000/api/synthesis/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /api/annotation/ {
proxy_pass http://datamate-backend-python:18000/api/annotation/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /api/evaluation/ {
proxy_pass http://datamate-backend-python:18000/api/evaluation/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /api/ {
proxy_pass http://datamate-backend:8080/api/;
proxy_pass http://datamate-gateway:8080/api/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

23
deployment/docker/datamate/backend.conf
View File

@@ -7,29 +7,8 @@ server {
client_max_body_size 1024M;
location /api/synthesis/ {
proxy_pass http://datamate-backend-python:18000/api/synthesis/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /api/evaluation/ {
proxy_pass http://datamate-backend-python:18000/api/evaluation/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /api/annotation/ {
proxy_pass http://datamate-backend-python:18000/api/annotation/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /api/ {
proxy_pass http://datamate-backend:8080/api/;
proxy_pass http://datamate-gateway:8080/api/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

7
deployment/docker/datamate/docker-compose.yml
View File

@@ -31,6 +31,13 @@ services:
depends_on:
- datamate-database
datamate-gateway:
container_name: datamate-gateway
image: ${REGISTRY:-}datamate-gateway
restart: on-failure
privileged: true
networks: [ datamate ]
# 2) frontend（NodePort 30000）
datamate-frontend:
container_name: datamate-frontend

42
deployment/helm/datamate/charts/frontend/templates/configmap.yaml
View File

@@ -16,27 +16,6 @@ data:
add_header Set-Cookie "NEXT_LOCALE=zh";
location /api/synthesis/ {
proxy_pass http://datamate-backend-python:18000/api/synthesis/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /api/evaluation/ {
proxy_pass http://datamate-backend-python:18000/api/evaluation/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /api/annotation/ {
proxy_pass http://datamate-backend-python:18000/api/annotation/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /api/ {
proxy_pass http://datamate-backend:8080/api/;
proxy_set_header Host $host;
@@ -91,27 +70,6 @@ data:
client_max_body_size 1024M;
location /api/synthesis/ {
proxy_pass http://datamate-backend-python:18000/api/synthesis/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /api/evaluation/ {
proxy_pass http://datamate-backend-python:18000/api/evaluation/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /api/annotation/ {
proxy_pass http://datamate-backend-python:18000/api/annotation/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /api/ {
proxy_pass http://datamate-backend:8080/api/;
proxy_set_header Host $host;

23
deployment/helm/datamate/charts/gateway/.helmignore Normal file
View File

@@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/

24
deployment/helm/datamate/charts/gateway/Chart.yaml Normal file
View File

@@ -0,0 +1,24 @@
apiVersion: v2
name: gateway
description: A Helm chart for Kubernetes
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.0.1
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "0.0.1"

76
deployment/helm/datamate/charts/gateway/templates/_helpers.tpl Normal file
View File

@@ -0,0 +1,76 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "gateway.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "gateway.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "gateway.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end }}
{{/*
Common labels
*/}}
{{- define "gateway.labels" -}}
helm.sh/chart: {{ include "gateway.chart" . }}
app: {{ .Release.Name }}
{{ include "gateway.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "gateway.selectorLabels" -}}
app.kubernetes.io/name: {{ include "gateway.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "gateway.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "gateway.fullname" .) .Values.serviceAccount.name -}}
{{- else }}
{{- default "default" .Values.serviceAccount.name -}}
{{- end }}
{{- end }}
{{/*
Name of image
*/}}
{{- define "gateway.image" -}}
{{- $name := default .Values.image.repository .Values.global.image.gateway.name }}
{{- $tag := default .Values.image.tag .Values.global.image.gateway.tag }}
{{- if .Values.global.image.repository }}
{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}:{{ $tag }}
{{- else }}
{{- $name }}:{{ $tag }}
{{- end }}
{{- end }}

82
deployment/helm/datamate/charts/gateway/templates/deployment.yaml Normal file
View File

@@ -0,0 +1,82 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "gateway.fullname" . }}
labels:
{{- include "gateway.labels" . | nindent 4 }}
spec:
{{- if not .Values.autoscaling.enabled }}
replicas: {{ .Values.replicaCount }}
{{- end }}
selector:
matchLabels:
{{- include "gateway.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "gateway.labels" . | nindent 8 }}
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "gateway.serviceAccountName" . }}
{{- with .Values.podSecurityContext }}
securityContext:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: {{ .Chart.Name }}
{{- with .Values.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
image: "{{ include "gateway.image" . }}"
imagePullPolicy: {{ default .Values.image.pullPolicy .Values.global.image.pullPolicy }}
ports:
- name: http
containerPort: {{ .Values.service.port }}
protocol: TCP
{{- with .Values.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.readinessProbe }}
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.env }}
env:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.volumeMounts }}
volumeMounts:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.volumes }}
volumes:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}

15
deployment/helm/datamate/charts/gateway/templates/service.yaml Normal file
View File

@@ -0,0 +1,15 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "gateway.fullname" . }}
labels:
{{- include "gateway.labels" . | nindent 4 }}
spec:
type: {{ .Values.service.type }}
ports:
- port: {{ .Values.service.port }}
targetPort: {{ .Values.service.port }}
protocol: TCP
name: {{ .Chart.Name }}
selector:
{{- include "gateway.selectorLabels" . | nindent 4 }}

13
deployment/helm/datamate/charts/gateway/templates/serviceaccount.yaml Normal file
View File

@@ -0,0 +1,13 @@
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "gateway.serviceAccountName" . }}
labels:
{{- include "gateway.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
{{- end }}

108
deployment/helm/datamate/charts/gateway/values.yaml Normal file
View File

@@ -0,0 +1,108 @@
# Default values for datamate.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
replicaCount: 1
# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
image:
repository: "datamate-gateway"
# This sets the pull policy for images.
pullPolicy: "IfNotPresent"
# Overrides the image tag whose default is the chart appVersion.
tag: "latest"
# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
imagePullSecrets: []
# This is to override the chart name.
nameOverride: "datamate-gateway"
fullnameOverride: "datamate-gateway"
env: []
# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
serviceAccount:
# Specifies whether a service account should be created
create: true
# Automatically mount a ServiceAccount's API credentials?
automount: true
# Annotations to add to the service account
annotations: {}
# The name of the service account to use.
# If not set and create is true, a name is generated using the fullname template
name: ""
# This is for setting Kubernetes Annotations to a Pod.
# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
podAnnotations: {}
# This is for setting Kubernetes Labels to a Pod.
# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
podLabels: {}
podSecurityContext: {}
# fsGroup: 2000
securityContext: {}
# capabilities:
# drop:
# - ALL
# readOnlyRootFilesystem: true
# runAsNonRoot: true
# runAsUser: 1000
# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/
service:
# This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types
type: ClusterIP
# This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports
port: 8080
resources: {}
# We usually recommend not to specify default resources and to leave this as a conscious
# choice for the user. This also increases chances charts run on environments with little
# resources, such as Minikube. If you do want to specify resources, uncomment the following
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
# livenessProbe:
# httpGet:
# path: /
# port: http
# readinessProbe:
# httpGet:
# path: /
# port: http
# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/
autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 100
targetCPUUtilizationPercentage: 80
# targetMemoryUtilizationPercentage: 80
# Additional volumes on the output Deployment definition.
volumes: []
# - name: foo
# secret:
# secretName: mysecret
# optional: false
# Additional volumeMounts on the output Deployment definition.
volumeMounts: []
# - name: foo
# mountPath: "/etc/foo"
# readOnly: true
nodeSelector: {}
tolerations: []
affinity: {}

14
deployment/helm/datamate/values.yaml
View File

@@ -15,6 +15,9 @@ global:
backendPython:
name: "datamate-backend-python"
tag: "latest"
gateway:
name: "datamate-gateway"
tag: "latest"
frontend:
name: "datamate-frontend"
tag: "latest"
@@ -117,6 +120,17 @@ backend-python:
- name: log-volume
mountPath: /var/log/datamate
gateway:
service:
type: NodePort
nodePort: 30000
volumes:
- *logVolume
volumeMounts:
- name: log-volume
mountPath: /var/log/datamate/gateway
subPath: gateway
frontend:
service:
type: NodePort

1
runtime/datamate-python/app/db/models/data_evaluation.py
View File

@@ -51,6 +51,7 @@ class EvaluationFile(Base):
task_id = Column(String(36), ForeignKey('t_de_eval_task.id'), nullable=False, comment="评估任务ID")
file_id = Column(String(36), ForeignKey('t_dm_dataset_files.id'), nullable=True, comment="文件ID")
file_name = Column(String(255), nullable=False, comment="文件名")
error_message = Column(Text, nullable=True, comment="错误信息")
total_count = Column(Integer, nullable=False, default=0, comment="总数")
evaluated_count = Column(Integer, nullable=False, default=0, comment="已评估数")
created_at = Column(TIMESTAMP, server_default=func.current_timestamp(), comment="创建时间")

1
scripts/db/data-evaluation-init.sql
View File

@@ -31,6 +31,7 @@ CREATE TABLE IF NOT EXISTS t_de_eval_file (
file_name VARCHAR(255) COMMENT '文件名',
total_count INT DEFAULT 0 COMMENT '总数',
evaluated_count INT DEFAULT 0 COMMENT '已评估数',
error_message TEXT COMMENT '错误信息',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
created_by VARCHAR(255) COMMENT '创建者',

2
scripts/images/backend/Dockerfile
View File

@@ -15,7 +15,7 @@ FROM maven:3-eclipse-temurin-21 AS builder
COPY backend/ /opt/backend
RUN cd /opt/backend && \
RUN cd /opt/backend/services && \
mvn -U clean package -Dmaven.test.skip=true

37
scripts/images/gateway/Dockerfile Normal file
View File

@@ -0,0 +1,37 @@
FROM maven:3-eclipse-temurin-21 AS builder
RUN apt-get update && \
apt-get install -y git && \
git clone https://github.com/ModelEngine-Group/Terrabase.git && \
cd Terrabase && \
git -c core.quotepath=false -c log.showSignature=false checkout -b pyh/feat_terrabase_develop origin/pyh/feat_terrabase_develop -- && \
mvn -U clean package install -Dmaven.test.skip=true
COPY backend/ /opt/gateway
RUN cd /opt/gateway/api-gateway && \
mvn -U clean package -Dmaven.test.skip=true && \
ls /opt/gateway/api-gateway/target
FROM eclipse-temurin:21-jdk
RUN apt-get update && \
apt-get install -y vim wget curl dos2unix && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
COPY --from=builder /opt/gateway/api-gateway/target/gateway.jar /opt/gateway/gateway.jar
COPY --from=builder /Terrabase/enterprise-impl-commercial/target/*.jar /opt/terrabase/
COPY scripts/images/gateway/start.sh /opt/gateway/start.sh
RUN dos2unix /opt/gateway/start.sh \
&& chmod +x /opt/gateway/start.sh \
&& ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
EXPOSE 8080
ENTRYPOINT ["/opt/gateway/start.sh"]
CMD ["java", "-Duser.timezone=Asia/Shanghai", "-jar", "/opt/gateway/gateway.jar"]

6
scripts/images/gateway/start.sh Normal file
View File

@@ -0,0 +1,6 @@
#!/bin/bash
set -e
echo "Starting main application..."
exec "$@"