q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

合并拉取请求 #74

Browse Source 
* feat: Implement system parameter management with Redis integration
This commit is contained in:
Dallas98
2025-11-11 22:13:14 +08:00
committed by GitHub
parent c5ccc56cca
commit aa01f52535
30 changed files with 343 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/application/KnowledgeBaseService.java
View File

@@ -99,7 +99,7 @@ public class KnowledgeBaseService {
return ragFile;
}).toList();
ragFileRepository.saveBatch(ragFiles, 100);
eventPublisher.publishEvent(new DataInsertedEvent(knowledgeBase, request.getProcessType()));
eventPublisher.publishEvent(new DataInsertedEvent(knowledgeBase, request));
}
public PagedResponse<RagFile> listFiles(String knowledgeBaseId, RagFileReq request) {

4
backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/infrastructure/event/DataInsertedEvent.java
View File

@@ -1,7 +1,7 @@
package com.datamate.rag.indexer.infrastructure.event;
import com.datamate.rag.indexer.domain.model.KnowledgeBase;
import com.datamate.rag.indexer.interfaces.dto.ProcessType;
import com.datamate.rag.indexer.interfaces.dto.AddFilesReq;
/**
* 数据插入事件
@@ -9,5 +9,5 @@ import com.datamate.rag.indexer.interfaces.dto.ProcessType;
* @author dallas
* @since 2025-10-29
*/
public record DataInsertedEvent(KnowledgeBase knowledgeBase, ProcessType processType) {
public record DataInsertedEvent(KnowledgeBase knowledgeBase, AddFilesReq addFilesReq) {
}

8
backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/infrastructure/event/RagEtlService.java
View File

@@ -1,8 +1,8 @@
package com.datamate.rag.indexer.infrastructure.event;
import com.datamate.common.models.domain.entity.ModelConfig;
import com.datamate.common.models.domain.repository.ModelConfigRepository;
import com.datamate.common.models.infrastructure.client.ModelClient;
import com.datamate.common.setting.domain.entity.ModelConfig;
import com.datamate.common.setting.domain.repository.ModelConfigRepository;
import com.datamate.common.setting.infrastructure.client.ModelClient;
import com.datamate.rag.indexer.domain.model.FileStatus;
import com.datamate.rag.indexer.domain.model.RagFile;
import com.datamate.rag.indexer.domain.repository.RagFileRepository;
@@ -109,7 +109,7 @@ public class RagEtlService {
document = new HtmlToTextDocumentTransformer().transform(document);
}
// 使用文档分块器对文档进行分块
DocumentSplitter splitter = documentSplitter(event.processType());
DocumentSplitter splitter = documentSplitter(event.addFilesReq().getProcessType());
List<TextSegment> split = splitter.split(document);
// 更新分块数量

3
backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/interfaces/dto/AddFilesReq.java
View File

@@ -16,6 +16,9 @@ import java.util.List;
public class AddFilesReq {
private String knowledgeBaseId;
private ProcessType processType;
private Integer chunkSize;
private Integer overlapSize;
private String customSeparator;
private List<FileInfo> files;
public record FileInfo(String id, String name) {