q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(knowledge-base): 添加知识库文件全库检索功能

Browse Source 
- 新增相对路径字段替代原有的metadata存储方式
- 实现跨知识库文件检索接口searchFiles
- 添加前端全库检索页面和相关API调用
- 优化文件路径处理和数据库索引配置
- 统一请求参数类型定义为RequestPayload和RequestParams
- 简化RagFile模型中的元数据结构设计
This commit is contained in:
Jerry Yan
2026-01-30 22:23:52 +08:00
parent cbad129ce4
commit 76f70a6847
14 changed files with 403 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/application/KnowledgeBaseService.java
View File

@@ -35,10 +35,10 @@ import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.StringUtils;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
/**
* 知识库服务类
@@ -49,7 +49,6 @@ import java.util.Optional;
@Service
@RequiredArgsConstructor
public class KnowledgeBaseService {
private static final String RELATIVE_PATH_KEY = "relativePath";
private static final String PATH_SEPARATOR = "/";
private final KnowledgeBaseRepository knowledgeBaseRepository;
private final RagFileRepository ragFileRepository;
@@ -150,12 +149,7 @@ public class KnowledgeBaseService {
ragFile.setKnowledgeBaseId(knowledgeBase.getId());
ragFile.setFileId(fileInfo.id());
ragFile.setFileName(fileInfo.fileName());
String relativePath = normalizeRelativePath(fileInfo.relativePath());
if (StringUtils.hasText(relativePath)) {
Map<String, Object> metadata = new HashMap<>();
metadata.put(RELATIVE_PATH_KEY, relativePath);
ragFile.setMetadata(metadata);
}
ragFile.setRelativePath(normalizeRelativePath(fileInfo.relativePath()));
ragFile.setStatus(FileStatus.UNPROCESSED);
return ragFile;
}).toList();
@@ -181,6 +175,41 @@ public class KnowledgeBaseService {
return PagedResponse.of(page.getRecords(), page.getCurrent(), page.getTotal(), page.getPages());
}
public PagedResponse<KnowledgeBaseFileSearchResp> searchFiles(KnowledgeBaseFileSearchReq request) {
IPage<RagFile> page = new Page<>(request.getPage(), request.getSize());
page = ragFileRepository.searchPage(page, request);
List<RagFile> records = page.getRecords();
if (records.isEmpty()) {
return PagedResponse.of(Collections.emptyList(), page.getCurrent(), page.getTotal(), page.getPages());
}
List<String> knowledgeBaseIds = records.stream()
.map(RagFile::getKnowledgeBaseId)
.filter(StringUtils::hasText)
.distinct()
.toList();
Map<String, String> knowledgeBaseNameMap = knowledgeBaseRepository.listByIds(knowledgeBaseIds).stream()
.collect(Collectors.toMap(KnowledgeBase::getId, KnowledgeBase::getName));
List<KnowledgeBaseFileSearchResp> responses = records.stream()
.map(file -> {
KnowledgeBaseFileSearchResp resp = new KnowledgeBaseFileSearchResp();
resp.setId(file.getId());
resp.setKnowledgeBaseId(file.getKnowledgeBaseId());
resp.setKnowledgeBaseName(knowledgeBaseNameMap.getOrDefault(file.getKnowledgeBaseId(), ""));
resp.setFileName(file.getFileName());
resp.setRelativePath(file.getRelativePath());
resp.setChunkCount(file.getChunkCount());
resp.setStatus(file.getStatus());
resp.setCreatedAt(file.getCreatedAt());
resp.setUpdatedAt(file.getUpdatedAt());
return resp;
})
.toList();
return PagedResponse.of(responses, page.getCurrent(), page.getTotal(), page.getPages());
}
@Transactional(rollbackFor = Exception.class)
public void deleteFiles(String knowledgeBaseId, DeleteFilesReq request) {
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))

4
backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/domain/model/RagFile.java
View File

@@ -28,6 +28,10 @@ public class RagFile extends BaseEntity<String> {
* 文件名
*/
private String fileName;
/**
* 相对路径
*/
private String relativePath;
/**
* 文件ID
*/

3
backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/domain/repository/RagFileRepository.java
View File

@@ -3,6 +3,7 @@ package com.datamate.rag.indexer.domain.repository;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.repository.IRepository;
import com.datamate.rag.indexer.domain.model.RagFile;
import com.datamate.rag.indexer.interfaces.dto.KnowledgeBaseFileSearchReq;
import com.datamate.rag.indexer.interfaces.dto.RagFileReq;
import java.util.List;
@@ -21,4 +22,6 @@ public interface RagFileRepository extends IRepository<RagFile> {
List<RagFile> findAllByKnowledgeBaseId(String knowledgeBaseId);
IPage<RagFile> page(IPage<RagFile> page, RagFileReq request);
IPage<RagFile> searchPage(IPage<RagFile> page, KnowledgeBaseFileSearchReq request);
}

21
backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/infrastructure/persistence/impl/RagFileRepositoryImpl.java
View File

@@ -6,6 +6,7 @@ import com.datamate.rag.indexer.domain.model.FileStatus;
import com.datamate.rag.indexer.domain.model.RagFile;
import com.datamate.rag.indexer.domain.repository.RagFileRepository;
import com.datamate.rag.indexer.infrastructure.persistence.mapper.RagFileMapper;
import com.datamate.rag.indexer.interfaces.dto.KnowledgeBaseFileSearchReq;
import com.datamate.rag.indexer.interfaces.dto.RagFileReq;
import org.springframework.stereotype.Repository;
import org.springframework.util.StringUtils;
@@ -20,7 +21,6 @@ import java.util.List;
*/
@Repository
public class RagFileRepositoryImpl extends CrudRepository<RagFileMapper, RagFile> implements RagFileRepository {
private static final String RELATIVE_PATH_KEY = "\"relativePath\":\"";
private static final String PATH_SEPARATOR = "/";
@Override
public void removeByKnowledgeBaseId(String knowledgeBaseId) {
@@ -44,15 +44,23 @@ public class RagFileRepositoryImpl extends CrudRepository<RagFileMapper, RagFile
@Override
public IPage<RagFile> page(IPage<RagFile> page, RagFileReq request) {
String relativePathPattern = buildRelativePathPattern(request.getRelativePath());
return lambdaQuery()
.eq(RagFile::getKnowledgeBaseId, request.getKnowledgeBaseId())
.like(StringUtils.hasText(request.getFileName()), RagFile::getFileName, request.getFileName())
.like(StringUtils.hasText(relativePathPattern), RagFile::getMetadata, relativePathPattern)
.likeRight(StringUtils.hasText(request.getRelativePath()), RagFile::getRelativePath, normalizeRelativePath(request.getRelativePath()))
.page(page);
}
private String buildRelativePathPattern(String relativePath) {
@Override
public IPage<RagFile> searchPage(IPage<RagFile> page, KnowledgeBaseFileSearchReq request) {
return lambdaQuery()
.eq(StringUtils.hasText(request.getKnowledgeBaseId()), RagFile::getKnowledgeBaseId, request.getKnowledgeBaseId())
.like(StringUtils.hasText(request.getFileName()), RagFile::getFileName, request.getFileName())
.likeRight(StringUtils.hasText(request.getRelativePath()), RagFile::getRelativePath, normalizeRelativePath(request.getRelativePath()))
.page(page);
}
private String normalizeRelativePath(String relativePath) {
if (!StringUtils.hasText(relativePath)) {
return "";
}
@@ -60,9 +68,6 @@ public class RagFileRepositoryImpl extends CrudRepository<RagFileMapper, RagFile
while (normalized.startsWith(PATH_SEPARATOR)) {
normalized = normalized.substring(1);
}
if (!StringUtils.hasText(normalized)) {
return "";
}
return RELATIVE_PATH_KEY + normalized;
return normalized;
}
}

13
backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/interfaces/KnowledgeBaseController.java
View File

@@ -105,6 +105,17 @@ public class KnowledgeBaseController {
return knowledgeBaseService.listFiles(knowledgeBaseId, request);
}
/**
* 全库检索知识库文件（跨知识库）
*
* @param request 检索请求
* @return 文件列表
*/
@GetMapping("/files/search")
public PagedResponse<KnowledgeBaseFileSearchResp> searchFiles(KnowledgeBaseFileSearchReq request) {
return knowledgeBaseService.searchFiles(request);
}
/**
* 删除知识库文件
*
@@ -141,4 +152,4 @@ public class KnowledgeBaseController {
public List<SearchResp.SearchResult> retrieve(@RequestBody @Valid RetrieveReq request) {
return knowledgeBaseService.retrieve(request);
}
}
}

19
backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/interfaces/dto/KnowledgeBaseFileSearchReq.java Normal file
View File

@@ -0,0 +1,19 @@
package com.datamate.rag.indexer.interfaces.dto;
import com.datamate.common.interfaces.PagingQuery;
import lombok.Getter;
import lombok.Setter;
/**
* 知识库文件全库检索请求
*
* @author dallas
* @since 2026-01-30
*/
@Getter
@Setter
public class KnowledgeBaseFileSearchReq extends PagingQuery {
private String fileName;
private String relativePath;
private String knowledgeBaseId;
}

27
backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/interfaces/dto/KnowledgeBaseFileSearchResp.java Normal file
View File

@@ -0,0 +1,27 @@
package com.datamate.rag.indexer.interfaces.dto;
import com.datamate.rag.indexer.domain.model.FileStatus;
import lombok.Getter;
import lombok.Setter;
import java.time.LocalDateTime;
/**
* 知识库文件全库检索响应
*
* @author dallas
* @since 2026-01-30
*/
@Getter
@Setter
public class KnowledgeBaseFileSearchResp {
private String id;
private String knowledgeBaseId;
private String knowledgeBaseName;
private String fileName;
private String relativePath;
private Integer chunkCount;
private FileStatus status;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}