q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feature: 增加对redis未部署时异常捕获 (#131)

Browse Source 
* feature: 增加download-deer-flow

* feature: 增加对redis未部署时异常捕获

* feature: clean code
This commit is contained in:
hhhhsc701
2025-12-04 16:09:29 +08:00
committed by GitHub
parent 977d16339b
commit 7a9530c1e3
10 changed files with 107 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
Makefile
View File

@@ -464,6 +464,14 @@ download:
exit 1; \
fi
DEER_FLOW_IMAGES := \
deer-flow-backend \
deer-flow-frontend
.PHONY: download-deer-flow
download-deer-flow:
$(MAKE) download DOWNLOAD_IMAGES="$(DEER_FLOW_IMAGES)"
# Load all downloaded images from dist/ directory
.PHONY: load-images
load-images:

6
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java
View File

@@ -3,6 +3,7 @@ package com.datamate.datamanagement.application;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.datamate.common.domain.utils.ChunksSaver;
import com.datamate.common.setting.application.SysParamApplicationService;
import com.datamate.datamanagement.interfaces.dto.*;
import com.datamate.common.infrastructure.exception.BusinessAssert;
import com.datamate.common.interfaces.PagedResponse;
@@ -21,7 +22,6 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@@ -49,7 +49,7 @@ public class DatasetApplicationService {
private final DatasetFileRepository datasetFileRepository;
private final CollectionTaskClient collectionTaskClient;
private final DatasetFileApplicationService datasetFileApplicationService;
private final StringRedisTemplate redisTemplate;
private final SysParamApplicationService sysParamService;
@Value("${datamate.data-management.base-path:/dataset}")
private String datasetBasePath;
@@ -80,7 +80,7 @@ public class DatasetApplicationService {
}
public String getDatasetPvcName() {
return redisTemplate.opsForValue().get(DATASET_PVC_NAME);
return sysParamService.getParamByKey(DATASET_PVC_NAME);
}
public Dataset updateDataset(String datasetId, UpdateDatasetRequest updateDatasetRequest) {

2
backend/services/main-application/pom.xml
View File

@@ -161,7 +161,7 @@
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${spring-boot.version}</version>
<configuration>
<finalName>data-mate</finalName>
<finalName>datamate</finalName>
<mainClass>com.datamate.main.DataMateApplication</mainClass>
</configuration>
<executions>

27
backend/shared/domain-common/src/main/java/com/datamate/common/setting/application/SysParamApplicationService.java
View File

@@ -4,10 +4,10 @@ import com.datamate.common.infrastructure.exception.BusinessAssert;
import com.datamate.common.infrastructure.exception.SystemErrorCode;
import com.datamate.common.setting.domain.entity.SysParam;
import com.datamate.common.setting.domain.repository.SysParamRepository;
import com.datamate.common.setting.infrastructure.client.RedisClient;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.util.Comparator;
@@ -24,7 +24,7 @@ import java.util.List;
@RequiredArgsConstructor
public class SysParamApplicationService {
private final SysParamRepository sysParamRepository;
private final StringRedisTemplate redisTemplate;
private final RedisClient redisClient;
/**
* 列表查询系统参数
@@ -48,14 +48,25 @@ public class SysParamApplicationService {
BusinessAssert.notNull(sysParam, SystemErrorCode.RESOURCE_NOT_FOUND);
sysParam.setParamValue(paramValue);
sysParamRepository.updateById(sysParam);
redisTemplate.opsForValue().set(sysParam.getParamKey(), paramValue);
redisClient.setParam(sysParam.getId(), paramValue);
}
public void deleteParamById(String paramId) {
SysParam sysParam = sysParamRepository.getById(paramId);
public void deleteParamById(String paramKey) {
SysParam sysParam = sysParamRepository.getById(paramKey);
BusinessAssert.notNull(sysParam, SystemErrorCode.RESOURCE_NOT_FOUND);
sysParamRepository.removeById(paramId);
redisTemplate.delete(sysParam.getParamKey());
sysParamRepository.removeById(paramKey);
redisClient.delParam(sysParam.getId());
}
public String getParamByKey(String paramId) {
String value = redisClient.getParam(paramId);
if (value == null) {
SysParam sysParam = sysParamRepository.getById(paramId);
if (sysParam != null) {
value = sysParam.getParamValue();
}
}
return value;
}
/**
@@ -65,7 +76,7 @@ public class SysParamApplicationService {
public void init() {
try {
List<SysParam> sysParams = sysParamRepository.list();
sysParams.forEach(sysParam -> redisTemplate.opsForValue().set(sysParam.getParamKey(), sysParam.getParamValue()));
sysParams.forEach(sysParam -> redisClient.setParam(sysParam.getId(), sysParam.getParamValue()));
} catch (Exception e) {
log.error("Init sys params to redis error", e);
}

5
backend/shared/domain-common/src/main/java/com/datamate/common/setting/domain/entity/SysParam.java
View File

@@ -15,11 +15,6 @@ import lombok.Setter;
@Getter
@TableName("t_sys_param")
public class SysParam extends BaseEntity<String> {
/**
* 设置项键（唯一）
*/
private String paramKey;
/**
* 设置项值
*/

26
backend/shared/domain-common/src/main/java/com/datamate/common/setting/infrastructure/client/RedisClient.java Normal file
View File

@@ -0,0 +1,26 @@
package com.datamate.common.setting.infrastructure.client;
import com.datamate.common.setting.infrastructure.utils.FunctionUtil;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
@Slf4j
@Component
@AllArgsConstructor
public class RedisClient {
private final StringRedisTemplate redisTemplate;
public void setParam(String key, String value) {
FunctionUtil.doWithoutThrow((k, v) -> redisTemplate.opsForValue().set(k, v), key, value);
}
public String getParam(String key) {
return FunctionUtil.getWithoutThrow((k) -> redisTemplate.opsForValue().get(k), key);
}
public void delParam(String key) {
FunctionUtil.doWithoutThrow(redisTemplate::delete, key);
}
}

35
backend/shared/domain-common/src/main/java/com/datamate/common/setting/infrastructure/utils/FunctionUtil.java Normal file
View File

@@ -0,0 +1,35 @@
package com.datamate.common.setting.infrastructure.utils;
import lombok.extern.slf4j.Slf4j;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Function;
@Slf4j
public class FunctionUtil {
public static <T, R> R getWithoutThrow(Function<T, R> action, T key) {
try {
return action.apply(key);
} catch (Exception e) {
log.warn(e.getMessage());
return null;
}
}
public static <T> void doWithoutThrow(Consumer<T> action, T key) {
try {
action.accept(key);
} catch (Exception e) {
log.warn(e.getMessage());
}
}
public static <T, R> void doWithoutThrow(BiConsumer<T, R> action, T t, R r) {
try {
action.accept(t, r);
} catch (Exception e) {
log.warn(e.getMessage());
}
}
}

8
frontend/src/pages/SettingsPage/SystemConfig.tsx
View File

@@ -1,10 +1,9 @@
import { Divider, Input, Select, Switch, Button, Table, Spin } from "antd";
import { Input, Select, Switch, Button, Table, Spin } from "antd";
import { useEffect, useState } from "react";
import { getSysParamList, updateSysParamValue } from './settings.apis';
interface SystemParam {
id: string;
paramKey: string;
paramValue: string;
description: string;
isEnabled: boolean;
@@ -120,10 +119,9 @@ export default function SystemConfig() {
const columns = [
{
title: "参数名",
dataIndex: "paramKey",
key: "paramKey",
dataIndex: "id",
key: "id",
width: 180,
},
{
title: "参数值",

24
scripts/db/setting-management-init.sql
View File

@@ -20,8 +20,7 @@ CREATE TABLE IF NOT EXISTS t_model_config
CREATE TABLE IF NOT EXISTS t_sys_param
(
id VARCHAR(36) PRIMARY KEY COMMENT '主键ID',
param_key VARCHAR(100) NOT NULL COMMENT '设置项键',
id VARCHAR(100) PRIMARY KEY COMMENT '主键ID,设置项键',
param_value TEXT NOT NULL COMMENT '设置项值',
param_type VARCHAR(50) DEFAULT 'string' COMMENT '设置项类型（仅 string、number、boolean 三种类型）',
option_list TEXT COMMENT '选项列表（逗号分隔，仅对 enum 类型有效）',
@@ -32,20 +31,17 @@ CREATE TABLE IF NOT EXISTS t_sys_param
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
created_by VARCHAR(255) COMMENT '创建者',
updated_by VARCHAR(255) COMMENT '更新者',
UNIQUE KEY uk_sys_param (param_key) COMMENT '避免设置项键重复'
updated_by VARCHAR(255) COMMENT '更新者'
) ENGINE = InnoDB
DEFAULT CHARSET = utf8mb4 COMMENT ='设置管理表';
insert ignore into t_sys_param (id, param_key, param_value, param_type, option_list, description, is_built_in,
insert ignore into t_sys_param (id, param_value, param_type, option_list, description, is_built_in,
can_modify,
is_enabled, created_by, updated_by)
values ('1', 'sys.knowledge.base.count', '200', 'number', '10,200,500', '知识库最大数量', 1, 1, 1, 'system', 'system'),
('2', 'SEARCH_API', 'tavily', 'string', '', 'deer-flow使用的搜索引擎', 1, 1, 1, 'system', 'system'),
('3', 'TAVILY_API_KEY', 'tvly-dev-xxx', 'string', '', 'deer-flow使用的搜索引擎所需的apiKey', 1, 1, 1, 'system',
'system'),
('4', 'BRAVE_SEARCH_API_KEY', 'api-xxx', 'string', '', 'deer-flow使用的搜索引擎所需的apiKey', 1, 1, 1, 'system',
'system'),
('5', 'JINA_API_KEY', '', 'string', '', 'deer-flow使用的JINA搜索引擎所需的apiKey', 1, 1, 1, 'system', 'system'),
('6', 'sys.management.dataset.pvc.name', 'dataset-pvc', 'string', '', '数据集所在pvc名称', 1, 0, 1, 'system', 'system'),
('7', 'test_bool', 'true', 'boolean', '', '测试布尔值', 1, 1, 1, 'system', 'system');
values ('sys.knowledge.base.count', '200', 'number', '10,200,500', '知识库最大数量', 1, 1, 1, 'system', 'system'),
('SEARCH_API', 'tavily', 'string', 'tavily,infoquest,duckduckgo,brave_search,arxiv', 'deer-flow使用的搜索引擎', 1, 1, 1, 'system', 'system'),
('TAVILY_API_KEY', 'tvly-dev-xxx', 'string', '', 'deer-flow使用的搜索引擎所需的apiKey', 1, 1, 1, 'system', 'system'),
('BRAVE_SEARCH_API_KEY', 'api-xxx', 'string', '', 'deer-flow使用的搜索引擎所需的apiKey', 1, 1, 1, 'system', 'system'),
('JINA_API_KEY', '', 'string', '', 'deer-flow使用的JINA搜索引擎所需的apiKey', 1, 1, 1, 'system', 'system'),
('sys.management.dataset.pvc.name', 'dataset-pvc', 'string', '', '数据集所在pvc名称', 1, 0, 1, 'system', 'system'),
('test_bool', 'true', 'boolean', '', '测试布尔值', 1, 1, 1, 'system', 'system');

4
scripts/images/backend/Dockerfile
View File

@@ -26,7 +26,7 @@ RUN apt-get update && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
COPY --from=builder /opt/backend/services/main-application/target/data-mate.jar /opt/backend/data-mate.jar
COPY --from=builder /opt/backend/services/main-application/target/datamate.jar /opt/backend/datamate.jar
COPY --from=datax-builder /DataX/target/datax/datax /opt/datax
COPY scripts/images/backend/start.sh /opt/backend/start.sh
@@ -39,4 +39,4 @@ EXPOSE 8080
ENTRYPOINT ["/opt/backend/start.sh"]
CMD ["java", "-Duser.timezone=Asia/Shanghai", "-jar", "/opt/backend/data-mate.jar"]
CMD ["java", "-Duser.timezone=Asia/Shanghai", "-jar", "/opt/backend/datamate.jar"]