|
|
|
|
@@ -2,8 +2,11 @@ package com.datamate.rag.indexer.application;
|
|
|
|
|
|
|
|
|
|
import com.baomidou.mybatisplus.core.metadata.IPage;
|
|
|
|
|
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
|
|
|
|
import com.datamate.common.auth.application.ResourceAccessService;
|
|
|
|
|
import com.datamate.common.infrastructure.exception.BusinessAssert;
|
|
|
|
|
import com.datamate.common.infrastructure.exception.BusinessException;
|
|
|
|
|
import com.datamate.common.infrastructure.exception.KnowledgeBaseErrorCode;
|
|
|
|
|
import com.datamate.common.infrastructure.exception.SystemErrorCode;
|
|
|
|
|
import com.datamate.common.interfaces.PagedResponse;
|
|
|
|
|
import com.datamate.common.interfaces.PagingQuery;
|
|
|
|
|
import com.datamate.common.setting.domain.entity.ModelConfig;
|
|
|
|
|
@@ -55,6 +58,7 @@ public class KnowledgeBaseService {
|
|
|
|
|
private final ApplicationEventPublisher eventPublisher;
|
|
|
|
|
private final ModelConfigRepository modelConfigRepository;
|
|
|
|
|
private final MilvusService milvusService;
|
|
|
|
|
private final ResourceAccessService resourceAccessService;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 创建知识库
|
|
|
|
|
@@ -77,8 +81,7 @@ public class KnowledgeBaseService {
|
|
|
|
|
*/
|
|
|
|
|
@Transactional(rollbackFor = Exception.class)
|
|
|
|
|
public void update(String knowledgeBaseId, KnowledgeBaseUpdateReq request) {
|
|
|
|
|
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
|
|
|
|
|
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
|
|
|
|
|
KnowledgeBase knowledgeBase = getKnowledgeBaseWithAccessCheck(knowledgeBaseId);
|
|
|
|
|
if (StringUtils.hasText(request.getName()) && !knowledgeBase.getName().equals(request.getName())) {
|
|
|
|
|
milvusService.getMilvusClient().renameCollection(RenameCollectionReq.builder()
|
|
|
|
|
.collectionName(knowledgeBase.getName())
|
|
|
|
|
@@ -98,16 +101,14 @@ public class KnowledgeBaseService {
|
|
|
|
|
*/
|
|
|
|
|
@Transactional(rollbackFor = Exception.class)
|
|
|
|
|
public void delete(String knowledgeBaseId) {
|
|
|
|
|
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
|
|
|
|
|
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
|
|
|
|
|
KnowledgeBase knowledgeBase = getKnowledgeBaseWithAccessCheck(knowledgeBaseId);
|
|
|
|
|
knowledgeBaseRepository.removeById(knowledgeBaseId);
|
|
|
|
|
ragFileRepository.removeByKnowledgeBaseId(knowledgeBaseId);
|
|
|
|
|
milvusService.getMilvusClient().dropCollection(DropCollectionReq.builder().collectionName(knowledgeBase.getName()).build());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public KnowledgeBaseResp getById(String knowledgeBaseId) {
|
|
|
|
|
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
|
|
|
|
|
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
|
|
|
|
|
KnowledgeBase knowledgeBase = getKnowledgeBaseWithAccessCheck(knowledgeBaseId);
|
|
|
|
|
KnowledgeBaseResp resp = getKnowledgeBaseResp(knowledgeBase);
|
|
|
|
|
resp.setEmbedding(modelConfigRepository.getById(knowledgeBase.getEmbeddingModel()));
|
|
|
|
|
resp.setChat(modelConfigRepository.getById(knowledgeBase.getChatModel()));
|
|
|
|
|
@@ -133,7 +134,8 @@ public class KnowledgeBaseService {
|
|
|
|
|
|
|
|
|
|
public PagedResponse<KnowledgeBaseResp> list(KnowledgeBaseQueryReq request) {
|
|
|
|
|
IPage<KnowledgeBase> page = new Page<>(request.getPage(), request.getSize());
|
|
|
|
|
page = knowledgeBaseRepository.page(page, request);
|
|
|
|
|
String ownerFilterUserId = resourceAccessService.resolveOwnerFilterUserId();
|
|
|
|
|
page = knowledgeBaseRepository.page(page, request, ownerFilterUserId);
|
|
|
|
|
|
|
|
|
|
// 将 KnowledgeBase 转换为 KnowledgeBaseResp,并计算 fileCount 和 chunkCount
|
|
|
|
|
List<KnowledgeBaseResp> respList = page.getRecords().stream().map(this::getKnowledgeBaseResp).toList();
|
|
|
|
|
@@ -143,8 +145,7 @@ public class KnowledgeBaseService {
|
|
|
|
|
|
|
|
|
|
@Transactional(rollbackFor = Exception.class)
|
|
|
|
|
public void addFiles(AddFilesReq request) {
|
|
|
|
|
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(request.getKnowledgeBaseId()))
|
|
|
|
|
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
|
|
|
|
|
KnowledgeBase knowledgeBase = getKnowledgeBaseWithAccessCheck(request.getKnowledgeBaseId());
|
|
|
|
|
List<RagFile> ragFiles = request.getFiles().stream().map(fileInfo -> {
|
|
|
|
|
RagFile ragFile = new RagFile();
|
|
|
|
|
ragFile.setKnowledgeBaseId(knowledgeBase.getId());
|
|
|
|
|
@@ -170,6 +171,7 @@ public class KnowledgeBaseService {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public PagedResponse<RagFile> listFiles(String knowledgeBaseId, RagFileReq request) {
|
|
|
|
|
getKnowledgeBaseWithAccessCheck(knowledgeBaseId);
|
|
|
|
|
IPage<RagFile> page = new Page<>(request.getPage(), request.getSize());
|
|
|
|
|
request.setKnowledgeBaseId(knowledgeBaseId);
|
|
|
|
|
page = ragFileRepository.page(page, request);
|
|
|
|
|
@@ -177,8 +179,13 @@ public class KnowledgeBaseService {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public PagedResponse<KnowledgeBaseFileSearchResp> searchFiles(KnowledgeBaseFileSearchReq request) {
|
|
|
|
|
boolean admin = resourceAccessService.isAdmin();
|
|
|
|
|
List<String> scopedKnowledgeBaseIds = resolveSearchScopeKnowledgeBaseIds(request, admin);
|
|
|
|
|
if (!admin && scopedKnowledgeBaseIds.isEmpty()) {
|
|
|
|
|
return PagedResponse.of(Collections.emptyList(), request.getPage(), 0L, 0);
|
|
|
|
|
}
|
|
|
|
|
IPage<RagFile> page = new Page<>(request.getPage(), request.getSize());
|
|
|
|
|
page = ragFileRepository.searchPage(page, request);
|
|
|
|
|
page = ragFileRepository.searchPage(page, request, scopedKnowledgeBaseIds);
|
|
|
|
|
List<RagFile> records = page.getRecords();
|
|
|
|
|
if (records.isEmpty()) {
|
|
|
|
|
return PagedResponse.of(Collections.emptyList(), page.getCurrent(), page.getTotal(), page.getPages());
|
|
|
|
|
@@ -213,8 +220,7 @@ public class KnowledgeBaseService {
|
|
|
|
|
|
|
|
|
|
@Transactional(rollbackFor = Exception.class)
|
|
|
|
|
public void deleteFiles(String knowledgeBaseId, DeleteFilesReq request) {
|
|
|
|
|
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
|
|
|
|
|
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
|
|
|
|
|
KnowledgeBase knowledgeBase = getKnowledgeBaseWithAccessCheck(knowledgeBaseId);
|
|
|
|
|
ragFileRepository.removeByIds(request.getIds());
|
|
|
|
|
milvusService.getMilvusClient().delete(DeleteReq.builder()
|
|
|
|
|
.collectionName(knowledgeBase.getName())
|
|
|
|
|
@@ -223,8 +229,7 @@ public class KnowledgeBaseService {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public PagedResponse<RagChunk> getChunks(String knowledgeBaseId, String ragFileId, PagingQuery pagingQuery) {
|
|
|
|
|
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
|
|
|
|
|
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
|
|
|
|
|
KnowledgeBase knowledgeBase = getKnowledgeBaseWithAccessCheck(knowledgeBaseId);
|
|
|
|
|
QueryResp results = milvusService.getMilvusClient().query(QueryReq.builder()
|
|
|
|
|
.collectionName(knowledgeBase.getName())
|
|
|
|
|
.filter("metadata[\"rag_file_id\"] == \"" + ragFileId + "\"")
|
|
|
|
|
@@ -259,8 +264,7 @@ public class KnowledgeBaseService {
|
|
|
|
|
* @return 检索结果
|
|
|
|
|
*/
|
|
|
|
|
public List<SearchResp.SearchResult> retrieve(RetrieveReq request) {
|
|
|
|
|
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(request.getKnowledgeBaseIds().getFirst()))
|
|
|
|
|
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
|
|
|
|
|
KnowledgeBase knowledgeBase = getKnowledgeBaseWithAccessCheck(request.getKnowledgeBaseIds().getFirst());
|
|
|
|
|
ModelConfig modelConfig = modelConfigRepository.getById(knowledgeBase.getEmbeddingModel());
|
|
|
|
|
EmbeddingModel embeddingModel = ModelClient.invokeEmbeddingModel(modelConfig);
|
|
|
|
|
Embedding embedding = embeddingModel.embed(request.getQuery()).content();
|
|
|
|
|
@@ -273,4 +277,27 @@ public class KnowledgeBaseService {
|
|
|
|
|
});
|
|
|
|
|
return searchResults;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private KnowledgeBase getKnowledgeBaseWithAccessCheck(String knowledgeBaseId) {
|
|
|
|
|
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
|
|
|
|
|
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
|
|
|
|
|
resourceAccessService.assertOwnerAccess(knowledgeBase.getCreatedBy());
|
|
|
|
|
return knowledgeBase;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private List<String> resolveSearchScopeKnowledgeBaseIds(KnowledgeBaseFileSearchReq request, boolean admin) {
|
|
|
|
|
if (admin) {
|
|
|
|
|
return Collections.emptyList();
|
|
|
|
|
}
|
|
|
|
|
String currentUserId = resourceAccessService.requireCurrentUserId();
|
|
|
|
|
List<String> ownedKnowledgeBaseIds = knowledgeBaseRepository.listIdsByCreatedBy(currentUserId);
|
|
|
|
|
if (!StringUtils.hasText(request.getKnowledgeBaseId())) {
|
|
|
|
|
return ownedKnowledgeBaseIds;
|
|
|
|
|
}
|
|
|
|
|
BusinessAssert.isTrue(
|
|
|
|
|
ownedKnowledgeBaseIds.contains(request.getKnowledgeBaseId()),
|
|
|
|
|
SystemErrorCode.INSUFFICIENT_PERMISSIONS
|
|
|
|
|
);
|
|
|
|
|
return Collections.singletonList(request.getKnowledgeBaseId());
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|