合并拉取请求 #74

* feat: Implement system parameter management with Redis integration
This commit is contained in:
Dallas98
2025-11-11 22:13:14 +08:00
committed by GitHub
parent c5ccc56cca
commit aa01f52535
30 changed files with 343 additions and 73 deletions

View File

@@ -7,7 +7,7 @@
![GitHub Stars](https://img.shields.io/github/stars/ModelEngine-Group/DataMate)
![GitHub Forks](https://img.shields.io/github/forks/ModelEngine-Group/DataMate)
![GitHub Issues](https://img.shields.io/github/issues/ModelEngine-Group/DataMate)
![GitHub License](https://img.shields.io/github/license/ModelEngine-Group/DataMate)
![GitHub License](https://img.shields.io/github/license/ModelEngine-Group/datamate-docs)
**DataMate is an enterprise-level data processing platform for model fine-tuning and RAG retrieval, supporting core
functions such as data collection, data management, operator marketplace, data cleaning, data synthesis, data

View File

@@ -134,6 +134,10 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
</dependencies>
<build>

View File

@@ -3,6 +3,7 @@ package com.datamate.main;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.annotation.EnableScheduling;
@@ -21,6 +22,7 @@ import org.springframework.transaction.annotation.EnableTransactionManagement;
@EnableTransactionManagement
@EnableAsync
@EnableScheduling
@EnableCaching
public class DataMatePlatformApplication {
public static void main(String[] args) {
SpringApplication.run(DataMatePlatformApplication.class, args);

View File

@@ -22,14 +22,6 @@ spring:
idle-timeout: 600000
max-lifetime: 1800000
# Elasticsearch配置
elasticsearch:
uris: ${ES_URIS:http://localhost:9200}
username: ${ES_USERNAME:}
password: ${ES_PASSWORD:}
connection-timeout: 10s
socket-timeout: 30s
# Jackson配置
jackson:
time-zone: Asia/Shanghai
@@ -61,6 +53,21 @@ spring:
- classpath:config/application-datacollection.yml
- classpath:config/application-datamanagement.yml
# Redis配置
data:
redis:
host: datamate-redis
port: 6379
timeout: 2000
password: ${REDIS_PASSWORD:password}
lettuce:
pool:
max-active: 20
max-idle: 10
min-idle: 5
max-wait: 1000ms
# MyBatis配置(需在顶层,不在 spring 下)
mybatis-plus:
configuration:

View File

@@ -99,7 +99,7 @@ public class KnowledgeBaseService {
return ragFile;
}).toList();
ragFileRepository.saveBatch(ragFiles, 100);
eventPublisher.publishEvent(new DataInsertedEvent(knowledgeBase, request.getProcessType()));
eventPublisher.publishEvent(new DataInsertedEvent(knowledgeBase, request));
}
public PagedResponse<RagFile> listFiles(String knowledgeBaseId, RagFileReq request) {

View File

@@ -1,7 +1,7 @@
package com.datamate.rag.indexer.infrastructure.event;
import com.datamate.rag.indexer.domain.model.KnowledgeBase;
import com.datamate.rag.indexer.interfaces.dto.ProcessType;
import com.datamate.rag.indexer.interfaces.dto.AddFilesReq;
/**
* 数据插入事件
@@ -9,5 +9,5 @@ import com.datamate.rag.indexer.interfaces.dto.ProcessType;
* @author dallas
* @since 2025-10-29
*/
public record DataInsertedEvent(KnowledgeBase knowledgeBase, ProcessType processType) {
public record DataInsertedEvent(KnowledgeBase knowledgeBase, AddFilesReq addFilesReq) {
}

View File

@@ -1,8 +1,8 @@
package com.datamate.rag.indexer.infrastructure.event;
import com.datamate.common.models.domain.entity.ModelConfig;
import com.datamate.common.models.domain.repository.ModelConfigRepository;
import com.datamate.common.models.infrastructure.client.ModelClient;
import com.datamate.common.setting.domain.entity.ModelConfig;
import com.datamate.common.setting.domain.repository.ModelConfigRepository;
import com.datamate.common.setting.infrastructure.client.ModelClient;
import com.datamate.rag.indexer.domain.model.FileStatus;
import com.datamate.rag.indexer.domain.model.RagFile;
import com.datamate.rag.indexer.domain.repository.RagFileRepository;
@@ -109,7 +109,7 @@ public class RagEtlService {
document = new HtmlToTextDocumentTransformer().transform(document);
}
// 使用文档分块器对文档进行分块
DocumentSplitter splitter = documentSplitter(event.processType());
DocumentSplitter splitter = documentSplitter(event.addFilesReq().getProcessType());
List<TextSegment> split = splitter.split(document);
// 更新分块数量

View File

@@ -16,6 +16,9 @@ import java.util.List;
public class AddFilesReq {
private String knowledgeBaseId;
private ProcessType processType;
private Integer chunkSize;
private Integer overlapSize;
private String customSeparator;
private List<FileInfo> files;
public record FileInfo(String id, String name) {

View File

@@ -26,10 +26,6 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
<dependency>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>

View File

@@ -38,5 +38,9 @@
<artifactId>langchain4j-open-ai</artifactId>
<version>1.8.0</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
</dependencies>
</project>

View File

@@ -1,15 +1,14 @@
package com.datamate.common.models.application;
package com.datamate.common.setting.application;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.datamate.common.infrastructure.exception.BusinessAssert;
import com.datamate.common.interfaces.PagedResponse;
import com.datamate.common.models.domain.entity.ModelConfig;
import com.datamate.common.models.domain.repository.ModelConfigRepository;
import com.datamate.common.models.infrastructure.client.ModelClient;
import com.datamate.common.models.infrastructure.exception.ModelsErrorCode;
import com.datamate.common.models.interfaces.rest.dto.CreateModelRequest;
import com.datamate.common.models.interfaces.rest.dto.QueryModelRequest;
import dev.langchain4j.model.chat.ChatModel;
import com.datamate.common.setting.domain.entity.ModelConfig;
import com.datamate.common.setting.domain.repository.ModelConfigRepository;
import com.datamate.common.setting.infrastructure.client.ModelClient;
import com.datamate.common.setting.infrastructure.exception.ModelsErrorCode;
import com.datamate.common.setting.interfaces.rest.dto.CreateModelRequest;
import com.datamate.common.setting.interfaces.rest.dto.QueryModelRequest;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
@@ -33,7 +32,7 @@ public class ModelConfigApplicationService {
providers.add(ModelConfig.builder().provider("ModelEngine").baseUrl("http://localhost:9981").build());
providers.add(ModelConfig.builder().provider("Ollama").baseUrl("http://localhost:11434").build());
providers.add(ModelConfig.builder().provider("OpenAI").baseUrl("https://api.openai.com/v1").build());
providers.add(ModelConfig.builder().provider("DeepSeek").baseUrl("https://api.deepseek.cn/v1").build());
providers.add(ModelConfig.builder().provider("DeepSeek").baseUrl("https://api.deepseek.com/v1").build());
providers.add(ModelConfig.builder().provider("火山方舟").baseUrl("https://ark.cn-beijing.volces.com/api/v3").build());
providers.add(ModelConfig.builder().provider("阿里云百炼").baseUrl("https://dashscope.aliyuncs.com/compatible-mode/v1").build());
providers.add(ModelConfig.builder().provider("硅基流动").baseUrl("https://api.siliconflow.cn/v1").build());

View File

@@ -0,0 +1,70 @@
package com.datamate.common.setting.application;
import com.datamate.common.infrastructure.exception.BusinessAssert;
import com.datamate.common.infrastructure.exception.SystemErrorCode;
import com.datamate.common.setting.domain.entity.SysParam;
import com.datamate.common.setting.domain.repository.SysParamRepository;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.util.List;
/**
* 系统参数应用服务
*
* @author dallas
* @since 2025-11-04
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class SysParamApplicationService {
private final SysParamRepository sysParamRepository;
private final StringRedisTemplate redisTemplate;
/**
* 列表查询系统参数
*
* @return 系统参数列表
*/
public List<SysParam> list() {
return sysParamRepository.list();
}
/**
* 根据参数id修改系统参数值
*
* @param paramId 参数id
* @param paramValue 参数值
*/
public void updateParamValueById(String paramId, String paramValue) {
SysParam sysParam = sysParamRepository.getById(paramId);
BusinessAssert.notNull(sysParam, SystemErrorCode.RESOURCE_NOT_FOUND);
sysParam.setParamValue(paramValue);
sysParamRepository.updateById(sysParam);
redisTemplate.opsForValue().set(sysParam.getParamKey(), paramValue);
}
public void deleteParamById(String paramId) {
SysParam sysParam = sysParamRepository.getById(paramId);
BusinessAssert.notNull(sysParam, SystemErrorCode.RESOURCE_NOT_FOUND);
sysParamRepository.removeById(paramId);
redisTemplate.delete(sysParam.getParamKey());
}
/**
* 初始化系统参数到Redis
*/
@PostConstruct
public void init() {
try {
List<SysParam> sysParams = sysParamRepository.list();
sysParams.forEach(sysParam -> redisTemplate.opsForValue().set(sysParam.getParamKey(), sysParam.getParamValue()));
} catch (Exception e) {
log.error("Init sys params to redis error", e);
}
}
}

View File

@@ -1,4 +1,4 @@
package com.datamate.common.models.domain.entity;
package com.datamate.common.setting.domain.entity;
import com.baomidou.mybatisplus.annotation.TableName;
import com.datamate.common.domain.model.base.BaseEntity;

View File

@@ -1,4 +1,4 @@
package com.datamate.common.models.domain.entity;
package com.datamate.common.setting.domain.entity;
/**
* 模型类型枚举类

View File

@@ -0,0 +1,57 @@
package com.datamate.common.setting.domain.entity;
import com.baomidou.mybatisplus.annotation.TableName;
import com.datamate.common.domain.model.base.BaseEntity;
import lombok.Getter;
import lombok.Setter;
/**
* 系统参数设置实体类
*
* @author dallas
* @since 2025-10-27
*/
@Setter
@Getter
@TableName("t_sys_param")
public class SysParam extends BaseEntity<String> {
/**
* 设置项键(唯一)
*/
private String paramKey;
/**
* 设置项值
*/
private String paramValue;
/**
* 设置项类型(如 string、integer、boolean)
*/
private String paramType;
/**
* 选项列表(JSON格式,仅对enum类型有效)
*/
private String optionList;
/**
* 设置项描述
*/
private String description;
/**
* 是否内置:1-是,0-否
*/
private Boolean isBuiltIn;
/**
* 是否可修改:1-可修改,0-不可修改
*/
private Boolean canModify;
/**
* 是否启用:1-启用,0-禁用
*/
private Boolean isEnabled;
}

View File

@@ -1,9 +1,9 @@
package com.datamate.common.models.domain.repository;
package com.datamate.common.setting.domain.repository;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.repository.IRepository;
import com.datamate.common.models.domain.entity.ModelConfig;
import com.datamate.common.models.interfaces.rest.dto.QueryModelRequest;
import com.datamate.common.setting.domain.entity.ModelConfig;
import com.datamate.common.setting.interfaces.rest.dto.QueryModelRequest;
/**
* 模型配置仓库接口

View File

@@ -0,0 +1,13 @@
package com.datamate.common.setting.domain.repository;
import com.baomidou.mybatisplus.extension.repository.IRepository;
import com.datamate.common.setting.domain.entity.SysParam;
/**
* 系统参数仓库接口
*
* @author dallas
* @since 2025-11-04
*/
public interface SysParamRepository extends IRepository<SysParam> {
}

View File

@@ -1,8 +1,8 @@
package com.datamate.common.models.infrastructure.client;
package com.datamate.common.setting.infrastructure.client;
import com.datamate.common.infrastructure.exception.BusinessException;
import com.datamate.common.models.domain.entity.ModelConfig;
import com.datamate.common.models.infrastructure.exception.ModelsErrorCode;
import com.datamate.common.setting.domain.entity.ModelConfig;
import com.datamate.common.setting.infrastructure.exception.ModelsErrorCode;
import dev.langchain4j.model.chat.ChatModel;
import dev.langchain4j.model.embedding.EmbeddingModel;
import dev.langchain4j.model.openai.OpenAiChatModel;

View File

@@ -1,4 +1,4 @@
package com.datamate.common.models.infrastructure.exception;
package com.datamate.common.setting.infrastructure.exception;
import com.datamate.common.infrastructure.exception.ErrorCode;
import lombok.AllArgsConstructor;

View File

@@ -1,13 +1,13 @@
package com.datamate.common.models.infrastructure.persistence.impl;
package com.datamate.common.setting.infrastructure.persistence.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.repository.CrudRepository;
import com.datamate.common.models.domain.entity.ModelConfig;
import com.datamate.common.models.domain.repository.ModelConfigRepository;
import com.datamate.common.models.infrastructure.persistence.mapper.ModelConfigMapper;
import com.datamate.common.models.interfaces.rest.dto.QueryModelRequest;
import com.datamate.common.setting.domain.entity.ModelConfig;
import com.datamate.common.setting.domain.repository.ModelConfigRepository;
import com.datamate.common.setting.infrastructure.persistence.mapper.ModelConfigMapper;
import com.datamate.common.setting.interfaces.rest.dto.QueryModelRequest;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Repository;
import org.springframework.util.StringUtils;

View File

@@ -0,0 +1,17 @@
package com.datamate.common.setting.infrastructure.persistence.impl;
import com.baomidou.mybatisplus.extension.repository.CrudRepository;
import com.datamate.common.setting.domain.entity.SysParam;
import com.datamate.common.setting.domain.repository.SysParamRepository;
import com.datamate.common.setting.infrastructure.persistence.mapper.SysParamMapper;
import org.springframework.stereotype.Repository;
/**
* 系统参数仓储实现类
*
* @author dallas
* @since 2025-11-04
*/
@Repository
public class SysParamRepositoryImpl extends CrudRepository<SysParamMapper, SysParam> implements SysParamRepository {
}

View File

@@ -1,7 +1,7 @@
package com.datamate.common.models.infrastructure.persistence.mapper;
package com.datamate.common.setting.infrastructure.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.datamate.common.models.domain.entity.ModelConfig;
import com.datamate.common.setting.domain.entity.ModelConfig;
import org.apache.ibatis.annotations.Mapper;
/**

View File

@@ -0,0 +1,15 @@
package com.datamate.common.setting.infrastructure.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.datamate.common.setting.domain.entity.SysParam;
import org.apache.ibatis.annotations.Mapper;
/**
* 系统参数映射器
*
* @author dallas
* @since 2025-11-04
*/
@Mapper
public interface SysParamMapper extends BaseMapper<SysParam> {
}

View File

@@ -1,11 +1,11 @@
package com.datamate.common.models.interfaces.rest;
package com.datamate.common.setting.interfaces.rest;
import com.datamate.common.interfaces.PagedResponse;
import com.datamate.common.models.application.ModelConfigApplicationService;
import com.datamate.common.models.domain.entity.ModelConfig;
import com.datamate.common.models.interfaces.rest.dto.CreateModelRequest;
import com.datamate.common.models.interfaces.rest.dto.QueryModelRequest;
import com.datamate.common.setting.application.ModelConfigApplicationService;
import com.datamate.common.setting.domain.entity.ModelConfig;
import com.datamate.common.setting.interfaces.rest.dto.CreateModelRequest;
import com.datamate.common.setting.interfaces.rest.dto.QueryModelRequest;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.*;

View File

@@ -0,0 +1,52 @@
package com.datamate.common.setting.interfaces.rest;
import com.datamate.common.setting.application.SysParamApplicationService;
import com.datamate.common.setting.domain.entity.SysParam;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.*;
import java.util.List;
/**
* 系统参数控制器
*
* @author dallas
* @since 2025-11-04
*/
@RestController
@RequestMapping("/sys-param")
@RequiredArgsConstructor
public class SysParamController {
private final SysParamApplicationService sysParamApplicationService;
/**
* 获取系统参数列表
*
* @return 系统参数列表
*/
@GetMapping("/list")
public List<SysParam> list() {
return sysParamApplicationService.list();
}
/**
* 根据参数id修改系统参数值
*
* @param paramId 参数id
* @param paramValue 参数值
*/
@PutMapping("/{paramId}")
public void updateParamValueById(@PathVariable("paramId") String paramId, @RequestBody String paramValue) {
sysParamApplicationService.updateParamValueById(paramId, paramValue);
}
/**
* 根据参数id删除系统参数
*
* @param paramId 参数id
*/
@DeleteMapping("/{paramId}")
public void deleteParamById(@PathVariable("paramId") String paramId) {
sysParamApplicationService.deleteParamById(paramId);
}
}

View File

@@ -1,6 +1,6 @@
package com.datamate.common.models.interfaces.rest.dto;
package com.datamate.common.setting.interfaces.rest.dto;
import com.datamate.common.models.domain.entity.ModelType;
import com.datamate.common.setting.domain.entity.ModelType;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import lombok.Getter;

View File

@@ -1,7 +1,7 @@
package com.datamate.common.models.interfaces.rest.dto;
package com.datamate.common.setting.interfaces.rest.dto;
import com.datamate.common.interfaces.PagingQuery;
import com.datamate.common.models.domain.entity.ModelType;
import com.datamate.common.setting.domain.entity.ModelType;
import lombok.Getter;
import lombok.Setter;

View File

@@ -113,6 +113,15 @@ services:
networks: [ datamate ]
profiles: [ mineru ]
# 5) redis
datamate-redis:
container_name: datamate-redis
image: redis:8.2.3
restart: on-failure
ports:
- "6379:6379"
networks: [ datamate ]
volumes:
dataset_volume:
name: datamate-dataset-volume

View File

@@ -1,19 +0,0 @@
USE datamate;
CREATE TABLE t_model_config
(
id VARCHAR(36) PRIMARY KEY COMMENT '主键ID',
model_name VARCHAR(100) NOT NULL COMMENT '模型名称(如 qwen2)',
provider VARCHAR(50) NOT NULL COMMENT '模型提供商(如 Ollama、OpenAI、DeepSeek)',
base_url VARCHAR(255) NOT NULL COMMENT 'API 基础地址',
api_key VARCHAR(512) DEFAULT '' COMMENT 'API 密钥(无密钥则为空)',
type VARCHAR(50) NOT NULL COMMENT '模型类型(如 chat、embedding)',
is_enabled TINYINT DEFAULT 1 COMMENT '是否启用:1-启用,0-禁用',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
created_by VARCHAR(255) COMMENT '创建者',
updated_by VARCHAR(255) COMMENT '更新者',
UNIQUE KEY uk_model_provider (model_name, provider) COMMENT '避免同一提供商下模型名称重复'
) ENGINE = InnoDB
DEFAULT CHARSET = utf8mb4 COMMENT ='模型配置表';

View File

@@ -0,0 +1,41 @@
USE datamate;
CREATE TABLE IF NOT EXISTS t_model_config
(
id VARCHAR(36) PRIMARY KEY COMMENT '主键ID',
model_name VARCHAR(100) NOT NULL COMMENT '模型名称(如 qwen2)',
provider VARCHAR(50) NOT NULL COMMENT '模型提供商(如 Ollama、OpenAI、DeepSeek)',
base_url VARCHAR(255) NOT NULL COMMENT 'API 基础地址',
api_key VARCHAR(512) DEFAULT '' COMMENT 'API 密钥(无密钥则为空)',
type VARCHAR(50) NOT NULL COMMENT '模型类型(如 chat、embedding)',
is_enabled TINYINT DEFAULT 1 COMMENT '是否启用:1-启用,0-禁用',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
created_by VARCHAR(255) COMMENT '创建者',
updated_by VARCHAR(255) COMMENT '更新者',
UNIQUE KEY uk_model_provider (model_name, provider) COMMENT '避免同一提供商下模型名称重复'
) ENGINE = InnoDB
DEFAULT CHARSET = utf8mb4 COMMENT ='模型配置表';
CREATE TABLE IF NOT EXISTS t_sys_param
(
id VARCHAR(36) PRIMARY KEY COMMENT '主键ID',
param_key VARCHAR(100) NOT NULL COMMENT '设置项键',
param_value TEXT NOT NULL COMMENT '设置项值',
param_type VARCHAR(50) DEFAULT 'string' COMMENT '设置项类型(如 string、integer、boolean)',
option_list TEXT COMMENT '选项列表(如 JSON 格式,仅对 enum 类型有效)',
description VARCHAR(255) DEFAULT '' COMMENT '设置项描述',
is_built_in TINYINT DEFAULT 0 COMMENT '是否内置:1-是,0-否',
can_modify TINYINT DEFAULT 1 COMMENT '是否可修改:1-可修改,0-不可修改',
is_enabled TINYINT DEFAULT 1 COMMENT '是否启用:1-启用,0-禁用',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
created_by VARCHAR(255) COMMENT '创建者',
updated_by VARCHAR(255) COMMENT '更新者',
UNIQUE KEY uk_sys_param (param_key) COMMENT '避免设置项键重复'
) ENGINE = InnoDB
DEFAULT CHARSET = utf8mb4 COMMENT ='设置管理表';
insert into t_sys_param (id, param_key, param_value, param_type, option_list, description, is_built_in, can_modify,
is_enabled, created_by, updated_by)
values ('1', 'sys.knowledge.base.count', '200', 'integer', '', '知识库最大数量', 1, 1, 1, 'system', 'system');