You've already forked DataMate
fix:修复配比任务操作问题 (#66)
* fix:配比任务需要能够跳转到目标数据集 * feature:增加配比任务详情接口 * fix:删除不存在的配比详情页面 * fix:使用正式的逻辑来展示标签 * fix:参数默认值去掉多余的- * fix:修复配比任务相关操作
This commit is contained in:
@@ -2,8 +2,10 @@ package com.datamate.datamanagement.domain.model.dataset;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import lombok.*;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
@@ -17,6 +19,7 @@ import java.util.List;
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@Slf4j
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@TableName("t_dm_dataset_files")
|
||||
@@ -42,11 +45,12 @@ public class DatasetFile {
|
||||
*
|
||||
* @return 标签列表
|
||||
*/
|
||||
public List<String> analyzeTag() {
|
||||
public List<FileTag> analyzeTag() {
|
||||
try {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
return mapper.readValue(tags, List.class);
|
||||
return mapper.readValue(tags, new TypeReference<List<FileTag>>() {});
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package com.datamate.datamanagement.domain.model.dataset;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
public class FileTag {
|
||||
private Map<String, Object> value;
|
||||
|
||||
private String type;
|
||||
|
||||
private String id;
|
||||
|
||||
private String fromName;
|
||||
|
||||
public List<String> getTags() {
|
||||
List<String> tags = new ArrayList<>();
|
||||
Object tagValues = value.get(type);
|
||||
if (tagValues instanceof List) {
|
||||
for (Object tag : (List<?>) tagValues) {
|
||||
if (tag instanceof String) {
|
||||
tags.add((String) tag);
|
||||
}
|
||||
}
|
||||
} else if (tagValues instanceof String) {
|
||||
tags.add((String) tagValues);
|
||||
}
|
||||
if(StringUtils.isNotEmpty(fromName)) {
|
||||
return tags.stream().map(tag -> fromName + " " + tag).toList();
|
||||
}
|
||||
return tags;
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package com.datamate.datamanagement.interfaces.converter;
|
||||
|
||||
import com.datamate.common.infrastructure.exception.BusinessException;
|
||||
import com.datamate.common.infrastructure.exception.SystemErrorCode;
|
||||
import com.datamate.datamanagement.domain.model.dataset.FileTag;
|
||||
import com.datamate.datamanagement.interfaces.dto.CreateDatasetRequest;
|
||||
import com.datamate.datamanagement.interfaces.dto.DatasetFileResponse;
|
||||
import com.datamate.datamanagement.interfaces.dto.DatasetResponse;
|
||||
@@ -71,12 +72,12 @@ public interface DatasetConverter {
|
||||
return distribution;
|
||||
}
|
||||
for (DatasetFile datasetFile : datasetFiles) {
|
||||
List<String> tags = datasetFile.analyzeTag();
|
||||
List<FileTag> tags = datasetFile.analyzeTag();
|
||||
if (CollectionUtils.isEmpty(tags)) {
|
||||
continue;
|
||||
return distribution;
|
||||
}
|
||||
for (String tag : tags) {
|
||||
distribution.put(tag, distribution.getOrDefault(tag, 0L) + 1);
|
||||
for (FileTag tag : tags) {
|
||||
tag.getTags().forEach(tagName -> distribution.put(tagName, distribution.getOrDefault(tagName, 0L) + 1));
|
||||
}
|
||||
}
|
||||
return distribution;
|
||||
|
||||
@@ -51,9 +51,9 @@ import java.util.concurrent.Semaphore;
|
||||
public class RagEtlService {
|
||||
private static final Semaphore SEMAPHORE = new Semaphore(10);
|
||||
|
||||
@Value("${datamate.rag.milvus-host:-milvus-standalone}")
|
||||
@Value("${datamate.rag.milvus-host:milvus-standalone}")
|
||||
private String milvusHost;
|
||||
@Value("${datamate.rag.milvus-port:-19530}")
|
||||
@Value("${datamate.rag.milvus-port:19530}")
|
||||
private int milvusPort;
|
||||
|
||||
private final RagFileRepository ragFileRepository;
|
||||
|
||||
Reference in New Issue
Block a user