q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(knowledge-management): 添加知识管理搜索功能和统计接口

Browse Source 
- 新增知识条目搜索查询和响应DTO
- 实现知识管理统计功能，包括总数、文件数和总大小
- 添加数据库查询方法支持文件搜索和统计计算
- 创建知识条目搜索控制器提供REST API
- 在前端添加知识管理搜索页面和相关组件
- 更新前端路由配置添加搜索页面入口
- 移除RAG索引服务中的重复统计功能
- 优化前端页面统计数据显示和刷新逻辑
This commit is contained in:
Jerry Yan
2026-01-31 09:30:37 +08:00
parent 97170a90fe
commit 790385bd80
24 changed files with 598 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/KnowledgeItemApplicationService.java
View File

@@ -25,6 +25,9 @@ import com.datamate.datamanagement.interfaces.dto.CreateKnowledgeItemRequest;
import com.datamate.datamanagement.interfaces.dto.ImportKnowledgeItemsRequest;
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemPagingQuery;
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemResponse;
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemSearchQuery;
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemSearchResponse;
import com.datamate.datamanagement.interfaces.dto.KnowledgeManagementStatisticsResponse;
import com.datamate.datamanagement.interfaces.dto.ReplaceKnowledgeItemFileRequest;
import com.datamate.datamanagement.interfaces.dto.UpdateKnowledgeItemRequest;
import com.datamate.datamanagement.interfaces.dto.UploadKnowledgeItemsRequest;
@@ -196,6 +199,39 @@ public class KnowledgeItemApplicationService {
return PagedResponse.of(responses, page.getCurrent(), page.getTotal(), page.getPages());
}
@Transactional(readOnly = true)
public KnowledgeManagementStatisticsResponse getKnowledgeManagementStatistics() {
KnowledgeManagementStatisticsResponse response = new KnowledgeManagementStatisticsResponse();
response.setTotalKnowledgeSets(knowledgeSetRepository.count());
long totalFiles = knowledgeItemRepository.countBySourceTypes(List.of(
KnowledgeSourceType.DATASET_FILE,
KnowledgeSourceType.FILE_UPLOAD
));
response.setTotalFiles(totalFiles);
long datasetFileSize = safeLong(knowledgeItemRepository.sumDatasetFileSize());
long uploadFileSize = calculateUploadFileTotalSize();
response.setTotalSize(datasetFileSize + uploadFileSize);
return response;
}
@Transactional(readOnly = true)
public PagedResponse<KnowledgeItemSearchResponse> searchKnowledgeItems(KnowledgeItemSearchQuery query) {
BusinessAssert.notNull(query, CommonErrorCode.PARAM_ERROR);
String keyword = StringUtils.trimToEmpty(query.getKeyword());
BusinessAssert.isTrue(StringUtils.isNotBlank(keyword), CommonErrorCode.PARAM_ERROR);
IPage<KnowledgeItemSearchResponse> page = new Page<>(query.getPage(), query.getSize());
IPage<KnowledgeItemSearchResponse> result = knowledgeItemRepository.searchFileItems(page, keyword);
List<KnowledgeItemSearchResponse> responses = result.getRecords()
.stream()
.map(this::normalizeSearchResponse)
.toList();
return PagedResponse.of(responses, result.getCurrent(), result.getTotal(), result.getPages());
}
public List<KnowledgeItem> importKnowledgeItems(String setId, ImportKnowledgeItemsRequest request) {
KnowledgeSet knowledgeSet = requireKnowledgeSet(setId);
BusinessAssert.isTrue(!isReadOnlyStatus(knowledgeSet.getStatus()),
@@ -447,6 +483,58 @@ public class KnowledgeItemApplicationService {
return target;
}
private KnowledgeItemSearchResponse normalizeSearchResponse(KnowledgeItemSearchResponse item) {
BusinessAssert.notNull(item, CommonErrorCode.PARAM_ERROR);
if (item.getSourceType() == KnowledgeSourceType.FILE_UPLOAD) {
item.setFileSize(resolveUploadFileSize(item.getContent()));
if (StringUtils.isBlank(item.getFileName())) {
item.setFileName(item.getSourceFileId());
}
}
if (item.getSourceType() == KnowledgeSourceType.DATASET_FILE) {
if (item.getFileSize() == null) {
item.setFileSize(0L);
}
if (StringUtils.isBlank(item.getFileName())) {
item.setFileName(item.getSourceFileId());
}
}
item.setContent(null);
return item;
}
private long calculateUploadFileTotalSize() {
List<KnowledgeItem> items = knowledgeItemRepository.findFileUploadItems();
if (CollectionUtils.isEmpty(items)) {
return 0L;
}
long total = 0L;
for (KnowledgeItem item : items) {
total += resolveUploadFileSize(item.getContent());
}
return total;
}
private long resolveUploadFileSize(String relativePath) {
if (StringUtils.isBlank(relativePath)) {
return 0L;
}
try {
Path filePath = resolveKnowledgeItemStoragePath(relativePath);
if (!Files.exists(filePath) || !Files.isRegularFile(filePath)) {
return 0L;
}
return Files.size(filePath);
} catch (Exception e) {
log.warn("resolve knowledge item file size error, path: {}", relativePath, e);
return 0L;
}
}
private long safeLong(Long value) {
return value == null ? 0L : value;
}
private String buildRelativeFilePath(String setId, String storedName) {
String relativePath = Paths.get(KNOWLEDGE_ITEM_UPLOAD_DIR, setId, storedName).toString();
return relativePath.replace(File.separatorChar, '/');

40
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/mapper/KnowledgeItemMapper.java
View File

@@ -1,9 +1,49 @@
package com.datamate.datamanagement.infrastructure.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItem;
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemSearchResponse;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
@Mapper
public interface KnowledgeItemMapper extends BaseMapper<KnowledgeItem> {
@Select("""
SELECT
ki.id AS id,
ki.set_id AS setId,
ks.name AS setName,
ki.content_type AS contentType,
ki.source_type AS sourceType,
ki.source_dataset_id AS sourceDatasetId,
ki.source_file_id AS sourceFileId,
CASE
WHEN ki.source_type = 'DATASET_FILE' THEN df.file_name
ELSE ki.source_file_id
END AS fileName,
df.file_size AS fileSize,
CASE
WHEN ki.source_type = 'FILE_UPLOAD' THEN ki.content
ELSE NULL
END AS content,
ki.created_at AS createdAt,
ki.updated_at AS updatedAt
FROM t_dm_knowledge_items ki
LEFT JOIN t_dm_knowledge_sets ks ON ki.set_id = ks.id
LEFT JOIN t_dm_dataset_files df ON ki.source_file_id = df.id AND ki.source_type = 'DATASET_FILE'
WHERE (ki.source_type = 'FILE_UPLOAD' AND ki.source_file_id LIKE CONCAT('%', #{keyword}, '%'))
OR (ki.source_type = 'DATASET_FILE' AND df.file_name LIKE CONCAT('%', #{keyword}, '%'))
ORDER BY ki.created_at DESC
""")
IPage<KnowledgeItemSearchResponse> searchFileItems(IPage<?> page, @Param("keyword") String keyword);
@Select("""
SELECT COALESCE(SUM(df.file_size), 0)
FROM t_dm_knowledge_items ki
LEFT JOIN t_dm_dataset_files df ON ki.source_file_id = df.id
WHERE ki.source_type = 'DATASET_FILE'
""")
Long sumDatasetFileSize();
}

11
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/repository/KnowledgeItemRepository.java
View File

@@ -2,8 +2,11 @@ package com.datamate.datamanagement.infrastructure.persistence.repository;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.repository.IRepository;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.datamate.datamanagement.common.enums.KnowledgeSourceType;
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItem;
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemPagingQuery;
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemSearchResponse;
import java.util.List;
/**
@@ -15,4 +18,12 @@ public interface KnowledgeItemRepository extends IRepository<KnowledgeItem> {
long countBySetId(String setId);
List<KnowledgeItem> findAllBySetId(String setId);
long countBySourceTypes(List<KnowledgeSourceType> sourceTypes);
List<KnowledgeItem> findFileUploadItems();
IPage<KnowledgeItemSearchResponse> searchFileItems(IPage<?> page, String keyword);
Long sumDatasetFileSize();
}

25
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/repository/impl/KnowledgeItemRepositoryImpl.java
View File

@@ -3,10 +3,12 @@ package com.datamate.datamanagement.infrastructure.persistence.repository.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.repository.CrudRepository;
import com.datamate.datamanagement.common.enums.KnowledgeSourceType;
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItem;
import com.datamate.datamanagement.infrastructure.persistence.mapper.KnowledgeItemMapper;
import com.datamate.datamanagement.infrastructure.persistence.repository.KnowledgeItemRepository;
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemPagingQuery;
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemSearchResponse;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Repository;
@@ -52,4 +54,27 @@ public class KnowledgeItemRepositoryImpl extends CrudRepository<KnowledgeItemMap
.eq(KnowledgeItem::getSetId, setId)
.orderByDesc(KnowledgeItem::getCreatedAt));
}
@Override
public long countBySourceTypes(List<KnowledgeSourceType> sourceTypes) {
return knowledgeItemMapper.selectCount(new LambdaQueryWrapper<KnowledgeItem>()
.in(KnowledgeItem::getSourceType, sourceTypes));
}
@Override
public List<KnowledgeItem> findFileUploadItems() {
return knowledgeItemMapper.selectList(new LambdaQueryWrapper<KnowledgeItem>()
.eq(KnowledgeItem::getSourceType, KnowledgeSourceType.FILE_UPLOAD)
.select(KnowledgeItem::getId, KnowledgeItem::getContent, KnowledgeItem::getSourceFileId));
}
@Override
public IPage<KnowledgeItemSearchResponse> searchFileItems(IPage<?> page, String keyword) {
return knowledgeItemMapper.searchFileItems(page, keyword);
}
@Override
public Long sumDatasetFileSize() {
return knowledgeItemMapper.sumDatasetFileSize();
}
}

17
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/dto/KnowledgeItemSearchQuery.java Normal file
View File

@@ -0,0 +1,17 @@
package com.datamate.datamanagement.interfaces.dto;
import com.datamate.common.interfaces.PagingQuery;
import lombok.Getter;
import lombok.Setter;
/**
* 知识条目文件搜索请求
*/
@Getter
@Setter
public class KnowledgeItemSearchQuery extends PagingQuery {
/**
* 文件名关键词
*/
private String keyword;
}

31
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/dto/KnowledgeItemSearchResponse.java Normal file
View File

@@ -0,0 +1,31 @@
package com.datamate.datamanagement.interfaces.dto;
import com.datamate.datamanagement.common.enums.KnowledgeContentType;
import com.datamate.datamanagement.common.enums.KnowledgeSourceType;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Getter;
import lombok.Setter;
import java.time.LocalDateTime;
/**
* 知识条目文件搜索响应
*/
@Getter
@Setter
public class KnowledgeItemSearchResponse {
private String id;
private String setId;
private String setName;
private KnowledgeContentType contentType;
private KnowledgeSourceType sourceType;
private String sourceDatasetId;
private String sourceFileId;
private String fileName;
private Long fileSize;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
@JsonIgnore
private String content;
}

15
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/dto/KnowledgeManagementStatisticsResponse.java Normal file
View File

@@ -0,0 +1,15 @@
package com.datamate.datamanagement.interfaces.dto;
import lombok.Getter;
import lombok.Setter;
/**
* 知识管理统计响应
*/
@Getter
@Setter
public class KnowledgeManagementStatisticsResponse {
private Long totalKnowledgeSets = 0L;
private Long totalFiles = 0L;
private Long totalSize = 0L;
}

27
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/rest/KnowledgeItemSearchController.java Normal file
View File

@@ -0,0 +1,27 @@
package com.datamate.datamanagement.interfaces.rest;
import com.datamate.common.interfaces.PagedResponse;
import com.datamate.datamanagement.application.KnowledgeItemApplicationService;
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemSearchQuery;
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemSearchResponse;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* 知识条目搜索控制器
*/
@Slf4j
@RestController
@RequiredArgsConstructor
@RequestMapping("/data-management/knowledge-items")
public class KnowledgeItemSearchController {
private final KnowledgeItemApplicationService knowledgeItemApplicationService;
@GetMapping("/search")
public PagedResponse<KnowledgeItemSearchResponse> search(KnowledgeItemSearchQuery query) {
return knowledgeItemApplicationService.searchKnowledgeItems(query);
}
}

8
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/rest/KnowledgeSetController.java
View File

@@ -1,10 +1,12 @@
package com.datamate.datamanagement.interfaces.rest;
import com.datamate.common.interfaces.PagedResponse;
import com.datamate.datamanagement.application.KnowledgeItemApplicationService;
import com.datamate.datamanagement.application.KnowledgeSetApplicationService;
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeSet;
import com.datamate.datamanagement.interfaces.converter.KnowledgeConverter;
import com.datamate.datamanagement.interfaces.dto.CreateKnowledgeSetRequest;
import com.datamate.datamanagement.interfaces.dto.KnowledgeManagementStatisticsResponse;
import com.datamate.datamanagement.interfaces.dto.KnowledgeSetPagingQuery;
import com.datamate.datamanagement.interfaces.dto.KnowledgeSetResponse;
import com.datamate.datamanagement.interfaces.dto.UpdateKnowledgeSetRequest;
@@ -22,6 +24,7 @@ import org.springframework.web.bind.annotation.*;
@RequestMapping("/data-management/knowledge-sets")
public class KnowledgeSetController {
private final KnowledgeSetApplicationService knowledgeSetApplicationService;
private final KnowledgeItemApplicationService knowledgeItemApplicationService;
@GetMapping
public PagedResponse<KnowledgeSetResponse> getKnowledgeSets(KnowledgeSetPagingQuery query) {
@@ -51,4 +54,9 @@ public class KnowledgeSetController {
public void deleteKnowledgeSet(@PathVariable("setId") String setId) {
knowledgeSetApplicationService.deleteKnowledgeSet(setId);
}
@GetMapping("/statistics")
public KnowledgeManagementStatisticsResponse getKnowledgeManagementStatistics() {
return knowledgeItemApplicationService.getKnowledgeManagementStatistics();
}
}

11
backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/application/KnowledgeBaseService.java
View File

@@ -140,17 +140,6 @@ public class KnowledgeBaseService {
return PagedResponse.of(respList, page.getCurrent(), page.getTotal(), page.getPages());
}
public KnowledgeBaseStatisticsResp getStatistics() {
KnowledgeBaseStatisticsResp resp = new KnowledgeBaseStatisticsResp();
resp.setTotalKnowledgeBases(knowledgeBaseRepository.count());
RagFileStatistics fileStatistics = ragFileRepository.getStatistics();
if (fileStatistics != null) {
resp.setTotalFiles(fileStatistics.getTotalFiles() != null ? fileStatistics.getTotalFiles() : 0L);
resp.setTotalSize(fileStatistics.getTotalSize() != null ? fileStatistics.getTotalSize() : 0L);
}
return resp;
}
@Transactional(rollbackFor = Exception.class)
public void addFiles(AddFilesReq request) {

3
backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/domain/repository/RagFileRepository.java
View File

@@ -5,7 +5,6 @@ import com.baomidou.mybatisplus.extension.repository.IRepository;
import com.datamate.rag.indexer.domain.model.RagFile;
import com.datamate.rag.indexer.interfaces.dto.KnowledgeBaseFileSearchReq;
import com.datamate.rag.indexer.interfaces.dto.RagFileReq;
import com.datamate.rag.indexer.interfaces.dto.RagFileStatistics;
import java.util.List;
@@ -25,6 +24,4 @@ public interface RagFileRepository extends IRepository<RagFile> {
IPage<RagFile> page(IPage<RagFile> page, RagFileReq request);
IPage<RagFile> searchPage(IPage<RagFile> page, KnowledgeBaseFileSearchReq request);
RagFileStatistics getStatistics();
}

6
backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/infrastructure/persistence/impl/RagFileRepositoryImpl.java
View File

@@ -8,7 +8,6 @@ import com.datamate.rag.indexer.domain.repository.RagFileRepository;
import com.datamate.rag.indexer.infrastructure.persistence.mapper.RagFileMapper;
import com.datamate.rag.indexer.interfaces.dto.KnowledgeBaseFileSearchReq;
import com.datamate.rag.indexer.interfaces.dto.RagFileReq;
import com.datamate.rag.indexer.interfaces.dto.RagFileStatistics;
import org.springframework.stereotype.Repository;
import org.springframework.util.StringUtils;
@@ -61,11 +60,6 @@ public class RagFileRepositoryImpl extends CrudRepository<RagFileMapper, RagFile
.page(page);
}
@Override
public RagFileStatistics getStatistics() {
return baseMapper.getStatistics();
}
private String normalizeRelativePath(String relativePath) {
if (!StringUtils.hasText(relativePath)) {
return "";

7
backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/infrastructure/persistence/mapper/RagFileMapper.java
View File

@@ -3,9 +3,7 @@ package com.datamate.rag.indexer.infrastructure.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.datamate.rag.indexer.domain.model.RagFile;
import com.datamate.rag.indexer.interfaces.dto.RagFileStatistics;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Select;
/**
* RAG文件映射器接口
@@ -15,9 +13,4 @@ import org.apache.ibatis.annotations.Select;
*/
@Mapper
public interface RagFileMapper extends BaseMapper<RagFile> {
@Select("SELECT COUNT(*) AS totalFiles, " +
"COALESCE(SUM(df.file_size), 0) AS totalSize " +
"FROM t_rag_file rf " +
"LEFT JOIN t_dm_dataset_files df ON rf.file_id = df.id")
RagFileStatistics getStatistics();
}

9
backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/interfaces/KnowledgeBaseController.java
View File

@@ -80,15 +80,6 @@ public class KnowledgeBaseController {
return knowledgeBaseService.list(request);
}
/**
* 获取知识库统计信息
*
* @return 知识库统计信息
*/
@GetMapping("/statistics")
public KnowledgeBaseStatisticsResp statistics() {
return knowledgeBaseService.getStatistics();
}
/**
* 添加文件到知识库

15
backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/interfaces/dto/KnowledgeBaseStatisticsResp.java
View File

@@ -1,15 +0,0 @@
package com.datamate.rag.indexer.interfaces.dto;
import lombok.Getter;
import lombok.Setter;
/**
* 知识库统计响应
*/
@Getter
@Setter
public class KnowledgeBaseStatisticsResp {
private Long totalKnowledgeBases = 0L;
private Long totalFiles = 0L;
private Long totalSize = 0L;
}

14
backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/interfaces/dto/RagFileStatistics.java
View File

@@ -1,14 +0,0 @@
package com.datamate.rag.indexer.interfaces.dto;
import lombok.Getter;
import lombok.Setter;
/**
* 知识库文件统计
*/
@Getter
@Setter
public class RagFileStatistics {
private Long totalFiles = 0L;
private Long totalSize = 0L;
}