feat: Enhance AddDataDialog with dataset file selection and improved upload process (#91)

This commit is contained in:
Dallas98
2025-11-18 20:48:28 +08:00
committed by GitHub
parent 5da992d312
commit 4506fa8a91
5 changed files with 770 additions and 268 deletions

View File

@@ -52,9 +52,13 @@ public class DatasetRepositoryImpl extends CrudRepository<DatasetMapper, Dataset
public IPage<Dataset> findByCriteria(IPage<Dataset> page, DatasetPagingQuery query) { public IPage<Dataset> findByCriteria(IPage<Dataset> page, DatasetPagingQuery query) {
LambdaQueryWrapper<Dataset> wrapper = new LambdaQueryWrapper<Dataset>() LambdaQueryWrapper<Dataset> wrapper = new LambdaQueryWrapper<Dataset>()
.eq(query.getType() != null, Dataset::getDatasetType, query.getType()) .eq(query.getType() != null, Dataset::getDatasetType, query.getType())
.eq(query.getStatus() != null, Dataset::getStatus, query.getStatus()) .eq(query.getStatus() != null, Dataset::getStatus, query.getStatus());
.like(StringUtils.isNotBlank(query.getKeyword()), Dataset::getName, query.getKeyword())
.like(StringUtils.isNotBlank(query.getKeyword()), Dataset::getDescription, query.getKeyword()); if (StringUtils.isNotBlank(query.getKeyword())) {
wrapper.and(w ->
w.like(Dataset::getName, query.getKeyword()).or()
.like(Dataset::getDescription, query.getKeyword()));
}
/* /*
标签过滤 {@link Tag} 标签过滤 {@link Tag}

View File

@@ -14,8 +14,8 @@ import com.datamate.rag.indexer.domain.model.RagFile;
import com.datamate.rag.indexer.domain.repository.KnowledgeBaseRepository; import com.datamate.rag.indexer.domain.repository.KnowledgeBaseRepository;
import com.datamate.rag.indexer.domain.repository.RagFileRepository; import com.datamate.rag.indexer.domain.repository.RagFileRepository;
import com.datamate.rag.indexer.infrastructure.event.DataInsertedEvent; import com.datamate.rag.indexer.infrastructure.event.DataInsertedEvent;
import com.datamate.rag.indexer.infrastructure.milvus.MilvusService;
import com.datamate.rag.indexer.interfaces.dto.*; import com.datamate.rag.indexer.interfaces.dto.*;
import io.milvus.client.MilvusClient;
import io.milvus.param.collection.DropCollectionParam; import io.milvus.param.collection.DropCollectionParam;
import io.milvus.param.dml.DeleteParam; import io.milvus.param.dml.DeleteParam;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@@ -42,7 +42,7 @@ public class KnowledgeBaseService {
private final RagFileRepository ragFileRepository; private final RagFileRepository ragFileRepository;
private final ApplicationEventPublisher eventPublisher; private final ApplicationEventPublisher eventPublisher;
private final ModelConfigRepository modelConfigRepository; private final ModelConfigRepository modelConfigRepository;
private final MilvusClient milvusClient; private final MilvusService milvusService;
/** /**
* 创建知识库 * 创建知识库
@@ -81,7 +81,7 @@ public class KnowledgeBaseService {
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND)); .orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
knowledgeBaseRepository.removeById(knowledgeBaseId); knowledgeBaseRepository.removeById(knowledgeBaseId);
ragFileRepository.removeByKnowledgeBaseId(knowledgeBaseId); ragFileRepository.removeByKnowledgeBaseId(knowledgeBaseId);
milvusClient.dropCollection(DropCollectionParam.newBuilder().withCollectionName(knowledgeBase.getName()).build()); milvusService.getMilvusClient().dropCollection(DropCollectionParam.newBuilder().withCollectionName(knowledgeBase.getName()).build());
} }
public KnowledgeBaseResp getById(String knowledgeBaseId) { public KnowledgeBaseResp getById(String knowledgeBaseId) {
@@ -147,7 +147,7 @@ public class KnowledgeBaseService {
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId)) KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND)); .orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
ragFileRepository.removeByIds(request.getIds()); ragFileRepository.removeByIds(request.getIds());
milvusClient.delete(DeleteParam.newBuilder() milvusService.getMilvusClient().delete(DeleteParam.newBuilder()
.withCollectionName(knowledgeBase.getName()) .withCollectionName(knowledgeBase.getName())
.withExpr("metadata[\"rag_file_id\"] in [" + org.apache.commons.lang3.StringUtils.join(request.getIds().stream().map(id -> "\"" + id + "\"").toArray(), ",") + "]") .withExpr("metadata[\"rag_file_id\"] in [" + org.apache.commons.lang3.StringUtils.join(request.getIds().stream().map(id -> "\"" + id + "\"").toArray(), ",") + "]")
.build()); .build());

View File

@@ -7,8 +7,8 @@ import dev.langchain4j.store.embedding.milvus.MilvusEmbeddingStore;
import io.milvus.client.MilvusClient; import io.milvus.client.MilvusClient;
import io.milvus.client.MilvusServiceClient; import io.milvus.client.MilvusServiceClient;
import io.milvus.param.ConnectParam; import io.milvus.param.ConnectParam;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
/** /**
@@ -17,6 +17,7 @@ import org.springframework.stereotype.Component;
* @author dallas * @author dallas
* @since 2025-11-17 * @since 2025-11-17
*/ */
@Slf4j
@Component @Component
public class MilvusService { public class MilvusService {
@Value("${datamate.rag.milvus-host:milvus-standalone}") @Value("${datamate.rag.milvus-host:milvus-standalone}")
@@ -24,6 +25,8 @@ public class MilvusService {
@Value("${datamate.rag.milvus-port:19530}") @Value("${datamate.rag.milvus-port:19530}")
private int milvusPort; private int milvusPort;
private volatile MilvusClient milvusClient;
public EmbeddingStore<TextSegment> embeddingStore(EmbeddingModel embeddingModel, String knowledgeBaseName) { public EmbeddingStore<TextSegment> embeddingStore(EmbeddingModel embeddingModel, String knowledgeBaseName) {
return MilvusEmbeddingStore.builder() return MilvusEmbeddingStore.builder()
.host(milvusHost) .host(milvusHost)
@@ -33,12 +36,24 @@ public class MilvusService {
.build(); .build();
} }
@Bean public MilvusClient getMilvusClient() {
public MilvusClient milvusClient() { if (milvusClient == null) {
synchronized (this) {
if (milvusClient == null) {
try {
ConnectParam connectParam = ConnectParam.newBuilder() ConnectParam connectParam = ConnectParam.newBuilder()
.withHost(milvusHost) .withHost(milvusHost)
.withPort(milvusPort) .withPort(milvusPort)
.build(); .build();
return new MilvusServiceClient(connectParam); milvusClient = new MilvusServiceClient(connectParam);
log.info("Milvus client connected successfully");
} catch (Exception e) {
log.error("Milvus client connection failed: {}", e.getMessage());
throw new RuntimeException("Milvus client connection failed", e);
}
}
}
}
return milvusClient;
} }
} }

View File

@@ -221,7 +221,7 @@ const KnowledgeBaseDetailPage: React.FC = () => {
showReload={false} showReload={false}
/> />
</div> </div>
<AddDataDialog knowledgeBase={knowledgeBase} /> <AddDataDialog knowledgeBase={knowledgeBase} onDataAdded={handleRefreshPage} />
</div> </div>
<Table <Table

File diff suppressed because it is too large Load Diff