You've already forked DataMate
feat: Enhance AddDataDialog with dataset file selection and improved upload process (#91)
This commit is contained in:
@@ -51,10 +51,14 @@ public class DatasetRepositoryImpl extends CrudRepository<DatasetMapper, Dataset
|
|||||||
@Override
|
@Override
|
||||||
public IPage<Dataset> findByCriteria(IPage<Dataset> page, DatasetPagingQuery query) {
|
public IPage<Dataset> findByCriteria(IPage<Dataset> page, DatasetPagingQuery query) {
|
||||||
LambdaQueryWrapper<Dataset> wrapper = new LambdaQueryWrapper<Dataset>()
|
LambdaQueryWrapper<Dataset> wrapper = new LambdaQueryWrapper<Dataset>()
|
||||||
.eq(query.getType() != null, Dataset::getDatasetType, query.getType())
|
.eq(query.getType() != null, Dataset::getDatasetType, query.getType())
|
||||||
.eq(query.getStatus() != null, Dataset::getStatus, query.getStatus())
|
.eq(query.getStatus() != null, Dataset::getStatus, query.getStatus());
|
||||||
.like(StringUtils.isNotBlank(query.getKeyword()), Dataset::getName, query.getKeyword())
|
|
||||||
.like(StringUtils.isNotBlank(query.getKeyword()), Dataset::getDescription, query.getKeyword());
|
if (StringUtils.isNotBlank(query.getKeyword())) {
|
||||||
|
wrapper.and(w ->
|
||||||
|
w.like(Dataset::getName, query.getKeyword()).or()
|
||||||
|
.like(Dataset::getDescription, query.getKeyword()));
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
标签过滤 {@link Tag}
|
标签过滤 {@link Tag}
|
||||||
|
|||||||
@@ -14,8 +14,8 @@ import com.datamate.rag.indexer.domain.model.RagFile;
|
|||||||
import com.datamate.rag.indexer.domain.repository.KnowledgeBaseRepository;
|
import com.datamate.rag.indexer.domain.repository.KnowledgeBaseRepository;
|
||||||
import com.datamate.rag.indexer.domain.repository.RagFileRepository;
|
import com.datamate.rag.indexer.domain.repository.RagFileRepository;
|
||||||
import com.datamate.rag.indexer.infrastructure.event.DataInsertedEvent;
|
import com.datamate.rag.indexer.infrastructure.event.DataInsertedEvent;
|
||||||
|
import com.datamate.rag.indexer.infrastructure.milvus.MilvusService;
|
||||||
import com.datamate.rag.indexer.interfaces.dto.*;
|
import com.datamate.rag.indexer.interfaces.dto.*;
|
||||||
import io.milvus.client.MilvusClient;
|
|
||||||
import io.milvus.param.collection.DropCollectionParam;
|
import io.milvus.param.collection.DropCollectionParam;
|
||||||
import io.milvus.param.dml.DeleteParam;
|
import io.milvus.param.dml.DeleteParam;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
@@ -42,7 +42,7 @@ public class KnowledgeBaseService {
|
|||||||
private final RagFileRepository ragFileRepository;
|
private final RagFileRepository ragFileRepository;
|
||||||
private final ApplicationEventPublisher eventPublisher;
|
private final ApplicationEventPublisher eventPublisher;
|
||||||
private final ModelConfigRepository modelConfigRepository;
|
private final ModelConfigRepository modelConfigRepository;
|
||||||
private final MilvusClient milvusClient;
|
private final MilvusService milvusService;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 创建知识库
|
* 创建知识库
|
||||||
@@ -81,7 +81,7 @@ public class KnowledgeBaseService {
|
|||||||
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
|
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
|
||||||
knowledgeBaseRepository.removeById(knowledgeBaseId);
|
knowledgeBaseRepository.removeById(knowledgeBaseId);
|
||||||
ragFileRepository.removeByKnowledgeBaseId(knowledgeBaseId);
|
ragFileRepository.removeByKnowledgeBaseId(knowledgeBaseId);
|
||||||
milvusClient.dropCollection(DropCollectionParam.newBuilder().withCollectionName(knowledgeBase.getName()).build());
|
milvusService.getMilvusClient().dropCollection(DropCollectionParam.newBuilder().withCollectionName(knowledgeBase.getName()).build());
|
||||||
}
|
}
|
||||||
|
|
||||||
public KnowledgeBaseResp getById(String knowledgeBaseId) {
|
public KnowledgeBaseResp getById(String knowledgeBaseId) {
|
||||||
@@ -147,7 +147,7 @@ public class KnowledgeBaseService {
|
|||||||
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
|
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
|
||||||
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
|
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
|
||||||
ragFileRepository.removeByIds(request.getIds());
|
ragFileRepository.removeByIds(request.getIds());
|
||||||
milvusClient.delete(DeleteParam.newBuilder()
|
milvusService.getMilvusClient().delete(DeleteParam.newBuilder()
|
||||||
.withCollectionName(knowledgeBase.getName())
|
.withCollectionName(knowledgeBase.getName())
|
||||||
.withExpr("metadata[\"rag_file_id\"] in [" + org.apache.commons.lang3.StringUtils.join(request.getIds().stream().map(id -> "\"" + id + "\"").toArray(), ",") + "]")
|
.withExpr("metadata[\"rag_file_id\"] in [" + org.apache.commons.lang3.StringUtils.join(request.getIds().stream().map(id -> "\"" + id + "\"").toArray(), ",") + "]")
|
||||||
.build());
|
.build());
|
||||||
|
|||||||
@@ -7,8 +7,8 @@ import dev.langchain4j.store.embedding.milvus.MilvusEmbeddingStore;
|
|||||||
import io.milvus.client.MilvusClient;
|
import io.milvus.client.MilvusClient;
|
||||||
import io.milvus.client.MilvusServiceClient;
|
import io.milvus.client.MilvusServiceClient;
|
||||||
import io.milvus.param.ConnectParam;
|
import io.milvus.param.ConnectParam;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
import org.springframework.context.annotation.Bean;
|
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -17,6 +17,7 @@ import org.springframework.stereotype.Component;
|
|||||||
* @author dallas
|
* @author dallas
|
||||||
* @since 2025-11-17
|
* @since 2025-11-17
|
||||||
*/
|
*/
|
||||||
|
@Slf4j
|
||||||
@Component
|
@Component
|
||||||
public class MilvusService {
|
public class MilvusService {
|
||||||
@Value("${datamate.rag.milvus-host:milvus-standalone}")
|
@Value("${datamate.rag.milvus-host:milvus-standalone}")
|
||||||
@@ -24,6 +25,8 @@ public class MilvusService {
|
|||||||
@Value("${datamate.rag.milvus-port:19530}")
|
@Value("${datamate.rag.milvus-port:19530}")
|
||||||
private int milvusPort;
|
private int milvusPort;
|
||||||
|
|
||||||
|
private volatile MilvusClient milvusClient;
|
||||||
|
|
||||||
public EmbeddingStore<TextSegment> embeddingStore(EmbeddingModel embeddingModel, String knowledgeBaseName) {
|
public EmbeddingStore<TextSegment> embeddingStore(EmbeddingModel embeddingModel, String knowledgeBaseName) {
|
||||||
return MilvusEmbeddingStore.builder()
|
return MilvusEmbeddingStore.builder()
|
||||||
.host(milvusHost)
|
.host(milvusHost)
|
||||||
@@ -33,12 +36,24 @@ public class MilvusService {
|
|||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Bean
|
public MilvusClient getMilvusClient() {
|
||||||
public MilvusClient milvusClient() {
|
if (milvusClient == null) {
|
||||||
ConnectParam connectParam = ConnectParam.newBuilder()
|
synchronized (this) {
|
||||||
.withHost(milvusHost)
|
if (milvusClient == null) {
|
||||||
.withPort(milvusPort)
|
try {
|
||||||
.build();
|
ConnectParam connectParam = ConnectParam.newBuilder()
|
||||||
return new MilvusServiceClient(connectParam);
|
.withHost(milvusHost)
|
||||||
|
.withPort(milvusPort)
|
||||||
|
.build();
|
||||||
|
milvusClient = new MilvusServiceClient(connectParam);
|
||||||
|
log.info("Milvus client connected successfully");
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("Milvus client connection failed: {}", e.getMessage());
|
||||||
|
throw new RuntimeException("Milvus client connection failed", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return milvusClient;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -221,7 +221,7 @@ const KnowledgeBaseDetailPage: React.FC = () => {
|
|||||||
showReload={false}
|
showReload={false}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
<AddDataDialog knowledgeBase={knowledgeBase} />
|
<AddDataDialog knowledgeBase={knowledgeBase} onDataAdded={handleRefreshPage} />
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<Table
|
<Table
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user