q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Enhance AddDataDialog with dataset file selection and improved upload process (#91)

Browse Source
This commit is contained in:
Dallas98
2025-11-18 20:48:28 +08:00
committed by GitHub
parent 5da992d312
commit 4506fa8a91
5 changed files with 770 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/application/KnowledgeBaseService.java
View File

@@ -14,8 +14,8 @@ import com.datamate.rag.indexer.domain.model.RagFile;
import com.datamate.rag.indexer.domain.repository.KnowledgeBaseRepository;
import com.datamate.rag.indexer.domain.repository.RagFileRepository;
import com.datamate.rag.indexer.infrastructure.event.DataInsertedEvent;
import com.datamate.rag.indexer.infrastructure.milvus.MilvusService;
import com.datamate.rag.indexer.interfaces.dto.*;
import io.milvus.client.MilvusClient;
import io.milvus.param.collection.DropCollectionParam;
import io.milvus.param.dml.DeleteParam;
import lombok.RequiredArgsConstructor;
@@ -42,7 +42,7 @@ public class KnowledgeBaseService {
private final RagFileRepository ragFileRepository;
private final ApplicationEventPublisher eventPublisher;
private final ModelConfigRepository modelConfigRepository;
private final MilvusClient milvusClient;
private final MilvusService milvusService;
/**
* 创建知识库
@@ -81,7 +81,7 @@ public class KnowledgeBaseService {
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
knowledgeBaseRepository.removeById(knowledgeBaseId);
ragFileRepository.removeByKnowledgeBaseId(knowledgeBaseId);
milvusClient.dropCollection(DropCollectionParam.newBuilder().withCollectionName(knowledgeBase.getName()).build());
milvusService.getMilvusClient().dropCollection(DropCollectionParam.newBuilder().withCollectionName(knowledgeBase.getName()).build());
}
public KnowledgeBaseResp getById(String knowledgeBaseId) {
@@ -147,7 +147,7 @@ public class KnowledgeBaseService {
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
ragFileRepository.removeByIds(request.getIds());
milvusClient.delete(DeleteParam.newBuilder()
milvusService.getMilvusClient().delete(DeleteParam.newBuilder()
.withCollectionName(knowledgeBase.getName())
.withExpr("metadata[\"rag_file_id\"] in [" + org.apache.commons.lang3.StringUtils.join(request.getIds().stream().map(id -> "\"" + id + "\"").toArray(), ",") + "]")
.build());

31
backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/infrastructure/milvus/MilvusService.java
View File

@@ -7,8 +7,8 @@ import dev.langchain4j.store.embedding.milvus.MilvusEmbeddingStore;
import io.milvus.client.MilvusClient;
import io.milvus.client.MilvusServiceClient;
import io.milvus.param.ConnectParam;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
/**
@@ -17,6 +17,7 @@ import org.springframework.stereotype.Component;
* @author dallas
* @since 2025-11-17
*/
@Slf4j
@Component
public class MilvusService {
@Value("${datamate.rag.milvus-host:milvus-standalone}")
@@ -24,6 +25,8 @@ public class MilvusService {
@Value("${datamate.rag.milvus-port:19530}")
private int milvusPort;
private volatile MilvusClient milvusClient;
public EmbeddingStore<TextSegment> embeddingStore(EmbeddingModel embeddingModel, String knowledgeBaseName) {
return MilvusEmbeddingStore.builder()
.host(milvusHost)
@@ -33,12 +36,24 @@ public class MilvusService {
.build();
}
@Bean
public MilvusClient milvusClient() {
ConnectParam connectParam = ConnectParam.newBuilder()
.withHost(milvusHost)
.withPort(milvusPort)
.build();
return new MilvusServiceClient(connectParam);
public MilvusClient getMilvusClient() {
if (milvusClient == null) {
synchronized (this) {
if (milvusClient == null) {
try {
ConnectParam connectParam = ConnectParam.newBuilder()
.withHost(milvusHost)
.withPort(milvusPort)
.build();
milvusClient = new MilvusServiceClient(connectParam);
log.info("Milvus client connected successfully");
} catch (Exception e) {
log.error("Milvus client connection failed: {}", e.getMessage());
throw new RuntimeException("Milvus client connection failed", e);
}
}
}
}
return milvusClient;
}
}