From 7a9530c1e351c82a17504de78ef514a0acb1b219 Mon Sep 17 00:00:00 2001 From: hhhhsc701 <56435672+hhhhsc701@users.noreply.github.com> Date: Thu, 4 Dec 2025 16:09:29 +0800 Subject: [PATCH] =?UTF-8?q?feature:=20=E5=A2=9E=E5=8A=A0=E5=AF=B9redis?= =?UTF-8?q?=E6=9C=AA=E9=83=A8=E7=BD=B2=E6=97=B6=E5=BC=82=E5=B8=B8=E6=8D=95?= =?UTF-8?q?=E8=8E=B7=20(#131)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feature: 增加download-deer-flow * feature: 增加对redis未部署时异常捕获 * feature: clean code --- Makefile | 8 +++++ .../DatasetApplicationService.java | 6 ++-- backend/services/main-application/pom.xml | 2 +- .../SysParamApplicationService.java | 27 +++++++++----- .../setting/domain/entity/SysParam.java | 5 --- .../infrastructure/client/RedisClient.java | 26 ++++++++++++++ .../infrastructure/utils/FunctionUtil.java | 35 +++++++++++++++++++ .../src/pages/SettingsPage/SystemConfig.tsx | 8 ++--- scripts/db/setting-management-init.sql | 24 ++++++------- scripts/images/backend/Dockerfile | 4 +-- 10 files changed, 107 insertions(+), 38 deletions(-) create mode 100644 backend/shared/domain-common/src/main/java/com/datamate/common/setting/infrastructure/client/RedisClient.java create mode 100644 backend/shared/domain-common/src/main/java/com/datamate/common/setting/infrastructure/utils/FunctionUtil.java diff --git a/Makefile b/Makefile index e8d13c7..46b12ec 100644 --- a/Makefile +++ b/Makefile @@ -464,6 +464,14 @@ download: exit 1; \ fi +DEER_FLOW_IMAGES := \ + deer-flow-backend \ + deer-flow-frontend + +.PHONY: download-deer-flow +download-deer-flow: + $(MAKE) download DOWNLOAD_IMAGES="$(DEER_FLOW_IMAGES)" + # Load all downloaded images from dist/ directory .PHONY: load-images load-images: diff --git a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java index 43ca54b..e170a51 100644 --- a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java +++ b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java @@ -3,6 +3,7 @@ package com.datamate.datamanagement.application; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.datamate.common.domain.utils.ChunksSaver; +import com.datamate.common.setting.application.SysParamApplicationService; import com.datamate.datamanagement.interfaces.dto.*; import com.datamate.common.infrastructure.exception.BusinessAssert; import com.datamate.common.interfaces.PagedResponse; @@ -21,7 +22,6 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Value; -import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -49,7 +49,7 @@ public class DatasetApplicationService { private final DatasetFileRepository datasetFileRepository; private final CollectionTaskClient collectionTaskClient; private final DatasetFileApplicationService datasetFileApplicationService; - private final StringRedisTemplate redisTemplate; + private final SysParamApplicationService sysParamService; @Value("${datamate.data-management.base-path:/dataset}") private String datasetBasePath; @@ -80,7 +80,7 @@ public class DatasetApplicationService { } public String getDatasetPvcName() { - return redisTemplate.opsForValue().get(DATASET_PVC_NAME); + return sysParamService.getParamByKey(DATASET_PVC_NAME); } public Dataset updateDataset(String datasetId, UpdateDatasetRequest updateDatasetRequest) { diff --git a/backend/services/main-application/pom.xml b/backend/services/main-application/pom.xml index b5b31ee..1d31862 100644 --- a/backend/services/main-application/pom.xml +++ b/backend/services/main-application/pom.xml @@ -161,7 +161,7 @@ spring-boot-maven-plugin ${spring-boot.version} - data-mate + datamate com.datamate.main.DataMateApplication diff --git a/backend/shared/domain-common/src/main/java/com/datamate/common/setting/application/SysParamApplicationService.java b/backend/shared/domain-common/src/main/java/com/datamate/common/setting/application/SysParamApplicationService.java index d83a9bb..7de8a1d 100644 --- a/backend/shared/domain-common/src/main/java/com/datamate/common/setting/application/SysParamApplicationService.java +++ b/backend/shared/domain-common/src/main/java/com/datamate/common/setting/application/SysParamApplicationService.java @@ -4,10 +4,10 @@ import com.datamate.common.infrastructure.exception.BusinessAssert; import com.datamate.common.infrastructure.exception.SystemErrorCode; import com.datamate.common.setting.domain.entity.SysParam; import com.datamate.common.setting.domain.repository.SysParamRepository; +import com.datamate.common.setting.infrastructure.client.RedisClient; import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Service; import java.util.Comparator; @@ -24,7 +24,7 @@ import java.util.List; @RequiredArgsConstructor public class SysParamApplicationService { private final SysParamRepository sysParamRepository; - private final StringRedisTemplate redisTemplate; + private final RedisClient redisClient; /** * 列表查询系统参数 @@ -48,14 +48,25 @@ public class SysParamApplicationService { BusinessAssert.notNull(sysParam, SystemErrorCode.RESOURCE_NOT_FOUND); sysParam.setParamValue(paramValue); sysParamRepository.updateById(sysParam); - redisTemplate.opsForValue().set(sysParam.getParamKey(), paramValue); + redisClient.setParam(sysParam.getId(), paramValue); } - public void deleteParamById(String paramId) { - SysParam sysParam = sysParamRepository.getById(paramId); + public void deleteParamById(String paramKey) { + SysParam sysParam = sysParamRepository.getById(paramKey); BusinessAssert.notNull(sysParam, SystemErrorCode.RESOURCE_NOT_FOUND); - sysParamRepository.removeById(paramId); - redisTemplate.delete(sysParam.getParamKey()); + sysParamRepository.removeById(paramKey); + redisClient.delParam(sysParam.getId()); + } + + public String getParamByKey(String paramId) { + String value = redisClient.getParam(paramId); + if (value == null) { + SysParam sysParam = sysParamRepository.getById(paramId); + if (sysParam != null) { + value = sysParam.getParamValue(); + } + } + return value; } /** @@ -65,7 +76,7 @@ public class SysParamApplicationService { public void init() { try { List sysParams = sysParamRepository.list(); - sysParams.forEach(sysParam -> redisTemplate.opsForValue().set(sysParam.getParamKey(), sysParam.getParamValue())); + sysParams.forEach(sysParam -> redisClient.setParam(sysParam.getId(), sysParam.getParamValue())); } catch (Exception e) { log.error("Init sys params to redis error", e); } diff --git a/backend/shared/domain-common/src/main/java/com/datamate/common/setting/domain/entity/SysParam.java b/backend/shared/domain-common/src/main/java/com/datamate/common/setting/domain/entity/SysParam.java index 8d526ca..78f8d7c 100644 --- a/backend/shared/domain-common/src/main/java/com/datamate/common/setting/domain/entity/SysParam.java +++ b/backend/shared/domain-common/src/main/java/com/datamate/common/setting/domain/entity/SysParam.java @@ -15,11 +15,6 @@ import lombok.Setter; @Getter @TableName("t_sys_param") public class SysParam extends BaseEntity { - /** - * 设置项键（唯一） - */ - private String paramKey; - /** * 设置项值 */ diff --git a/backend/shared/domain-common/src/main/java/com/datamate/common/setting/infrastructure/client/RedisClient.java b/backend/shared/domain-common/src/main/java/com/datamate/common/setting/infrastructure/client/RedisClient.java new file mode 100644 index 0000000..9201e02 --- /dev/null +++ b/backend/shared/domain-common/src/main/java/com/datamate/common/setting/infrastructure/client/RedisClient.java @@ -0,0 +1,26 @@ +package com.datamate.common.setting.infrastructure.client; + +import com.datamate.common.setting.infrastructure.utils.FunctionUtil; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +@AllArgsConstructor +public class RedisClient { + private final StringRedisTemplate redisTemplate; + + public void setParam(String key, String value) { + FunctionUtil.doWithoutThrow((k, v) -> redisTemplate.opsForValue().set(k, v), key, value); + } + + public String getParam(String key) { + return FunctionUtil.getWithoutThrow((k) -> redisTemplate.opsForValue().get(k), key); + } + + public void delParam(String key) { + FunctionUtil.doWithoutThrow(redisTemplate::delete, key); + } +} diff --git a/backend/shared/domain-common/src/main/java/com/datamate/common/setting/infrastructure/utils/FunctionUtil.java b/backend/shared/domain-common/src/main/java/com/datamate/common/setting/infrastructure/utils/FunctionUtil.java new file mode 100644 index 0000000..aaab638 --- /dev/null +++ b/backend/shared/domain-common/src/main/java/com/datamate/common/setting/infrastructure/utils/FunctionUtil.java @@ -0,0 +1,35 @@ +package com.datamate.common.setting.infrastructure.utils; + +import lombok.extern.slf4j.Slf4j; + +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Function; + +@Slf4j +public class FunctionUtil { + public static R getWithoutThrow(Function action, T key) { + try { + return action.apply(key); + } catch (Exception e) { + log.warn(e.getMessage()); + return null; + } + } + + public static void doWithoutThrow(Consumer action, T key) { + try { + action.accept(key); + } catch (Exception e) { + log.warn(e.getMessage()); + } + } + + public static void doWithoutThrow(BiConsumer action, T t, R r) { + try { + action.accept(t, r); + } catch (Exception e) { + log.warn(e.getMessage()); + } + } +} diff --git a/frontend/src/pages/SettingsPage/SystemConfig.tsx b/frontend/src/pages/SettingsPage/SystemConfig.tsx index 7e3f1fa..2d12e00 100644 --- a/frontend/src/pages/SettingsPage/SystemConfig.tsx +++ b/frontend/src/pages/SettingsPage/SystemConfig.tsx @@ -1,10 +1,9 @@ -import { Divider, Input, Select, Switch, Button, Table, Spin } from "antd"; +import { Input, Select, Switch, Button, Table, Spin } from "antd"; import { useEffect, useState } from "react"; import { getSysParamList, updateSysParamValue } from './settings.apis'; interface SystemParam { id: string; - paramKey: string; paramValue: string; description: string; isEnabled: boolean; @@ -120,10 +119,9 @@ export default function SystemConfig() { const columns = [ { title: "参数名", - dataIndex: "paramKey", - key: "paramKey", + dataIndex: "id", + key: "id", width: 180, - }, { title: "参数值", diff --git a/scripts/db/setting-management-init.sql b/scripts/db/setting-management-init.sql index 785df98..a53066b 100644 --- a/scripts/db/setting-management-init.sql +++ b/scripts/db/setting-management-init.sql @@ -20,8 +20,7 @@ CREATE TABLE IF NOT EXISTS t_model_config CREATE TABLE IF NOT EXISTS t_sys_param ( - id VARCHAR(36) PRIMARY KEY COMMENT '主键ID', - param_key VARCHAR(100) NOT NULL COMMENT '设置项键', + id VARCHAR(100) PRIMARY KEY COMMENT '主键ID,设置项键', param_value TEXT NOT NULL COMMENT '设置项值', param_type VARCHAR(50) DEFAULT 'string' COMMENT '设置项类型（仅 string、number、boolean 三种类型）', option_list TEXT COMMENT '选项列表（逗号分隔，仅对 enum 类型有效）', @@ -32,20 +31,17 @@ CREATE TABLE IF NOT EXISTS t_sys_param created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', created_by VARCHAR(255) COMMENT '创建者', - updated_by VARCHAR(255) COMMENT '更新者', - UNIQUE KEY uk_sys_param (param_key) COMMENT '避免设置项键重复' + updated_by VARCHAR(255) COMMENT '更新者' ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COMMENT ='设置管理表'; -insert ignore into t_sys_param (id, param_key, param_value, param_type, option_list, description, is_built_in, +insert ignore into t_sys_param (id, param_value, param_type, option_list, description, is_built_in, can_modify, is_enabled, created_by, updated_by) -values ('1', 'sys.knowledge.base.count', '200', 'number', '10,200,500', '知识库最大数量', 1, 1, 1, 'system', 'system'), - ('2', 'SEARCH_API', 'tavily', 'string', '', 'deer-flow使用的搜索引擎', 1, 1, 1, 'system', 'system'), - ('3', 'TAVILY_API_KEY', 'tvly-dev-xxx', 'string', '', 'deer-flow使用的搜索引擎所需的apiKey', 1, 1, 1, 'system', - 'system'), - ('4', 'BRAVE_SEARCH_API_KEY', 'api-xxx', 'string', '', 'deer-flow使用的搜索引擎所需的apiKey', 1, 1, 1, 'system', - 'system'), - ('5', 'JINA_API_KEY', '', 'string', '', 'deer-flow使用的JINA搜索引擎所需的apiKey', 1, 1, 1, 'system', 'system'), - ('6', 'sys.management.dataset.pvc.name', 'dataset-pvc', 'string', '', '数据集所在pvc名称', 1, 0, 1, 'system', 'system'), - ('7', 'test_bool', 'true', 'boolean', '', '测试布尔值', 1, 1, 1, 'system', 'system'); +values ('sys.knowledge.base.count', '200', 'number', '10,200,500', '知识库最大数量', 1, 1, 1, 'system', 'system'), + ('SEARCH_API', 'tavily', 'string', 'tavily,infoquest,duckduckgo,brave_search,arxiv', 'deer-flow使用的搜索引擎', 1, 1, 1, 'system', 'system'), + ('TAVILY_API_KEY', 'tvly-dev-xxx', 'string', '', 'deer-flow使用的搜索引擎所需的apiKey', 1, 1, 1, 'system', 'system'), + ('BRAVE_SEARCH_API_KEY', 'api-xxx', 'string', '', 'deer-flow使用的搜索引擎所需的apiKey', 1, 1, 1, 'system', 'system'), + ('JINA_API_KEY', '', 'string', '', 'deer-flow使用的JINA搜索引擎所需的apiKey', 1, 1, 1, 'system', 'system'), + ('sys.management.dataset.pvc.name', 'dataset-pvc', 'string', '', '数据集所在pvc名称', 1, 0, 1, 'system', 'system'), + ('test_bool', 'true', 'boolean', '', '测试布尔值', 1, 1, 1, 'system', 'system'); diff --git a/scripts/images/backend/Dockerfile b/scripts/images/backend/Dockerfile index 7c4bf89..968efab 100644 --- a/scripts/images/backend/Dockerfile +++ b/scripts/images/backend/Dockerfile @@ -26,7 +26,7 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -COPY --from=builder /opt/backend/services/main-application/target/data-mate.jar /opt/backend/data-mate.jar +COPY --from=builder /opt/backend/services/main-application/target/datamate.jar /opt/backend/datamate.jar COPY --from=datax-builder /DataX/target/datax/datax /opt/datax COPY scripts/images/backend/start.sh /opt/backend/start.sh @@ -39,4 +39,4 @@ EXPOSE 8080 ENTRYPOINT ["/opt/backend/start.sh"] -CMD ["java", "-Duser.timezone=Asia/Shanghai", "-jar", "/opt/backend/data-mate.jar"] +CMD ["java", "-Duser.timezone=Asia/Shanghai", "-jar", "/opt/backend/datamate.jar"]