You've already forked DataMate
feature: add data-evaluation
* feature: add evaluation task management function * feature: add evaluation task detail page * fix: delete duplicate definition for table t_model_config * refactor: rename package synthesis to ratio * refactor: add eval file table and refactor related code * fix: calling large models in parallel during evaluation
This commit is contained in:
@@ -21,6 +21,7 @@ import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.scheduling.annotation.Async;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
@@ -42,11 +43,13 @@ import java.util.stream.Stream;
|
||||
@Transactional
|
||||
@RequiredArgsConstructor
|
||||
public class DatasetApplicationService {
|
||||
private static final String DATASET_PVC_NAME = "sys.management.dataset.pvc.name";
|
||||
private final DatasetRepository datasetRepository;
|
||||
private final TagMapper tagMapper;
|
||||
private final DatasetFileRepository datasetFileRepository;
|
||||
private final CollectionTaskClient collectionTaskClient;
|
||||
private final DatasetFileApplicationService datasetFileApplicationService;
|
||||
private final StringRedisTemplate redisTemplate;
|
||||
|
||||
@Value("${datamate.data-management.base-path:/dataset}")
|
||||
private String datasetBasePath;
|
||||
@@ -76,6 +79,10 @@ public class DatasetApplicationService {
|
||||
return dataset;
|
||||
}
|
||||
|
||||
public String getDatasetPvcName() {
|
||||
return redisTemplate.opsForValue().get(DATASET_PVC_NAME);
|
||||
}
|
||||
|
||||
public Dataset updateDataset(String datasetId, UpdateDatasetRequest updateDatasetRequest) {
|
||||
Dataset dataset = datasetRepository.getById(datasetId);
|
||||
BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND);
|
||||
@@ -130,7 +137,10 @@ public class DatasetApplicationService {
|
||||
public PagedResponse<DatasetResponse> getDatasets(DatasetPagingQuery query) {
|
||||
IPage<Dataset> page = new Page<>(query.getPage(), query.getSize());
|
||||
page = datasetRepository.findByCriteria(page, query);
|
||||
return PagedResponse.of(DatasetConverter.INSTANCE.convertToResponse(page.getRecords()), page.getCurrent(), page.getTotal(), page.getPages());
|
||||
String datasetPvcName = getDatasetPvcName();
|
||||
List<DatasetResponse> datasetResponses = DatasetConverter.INSTANCE.convertToResponse(page.getRecords());
|
||||
datasetResponses.forEach(dataset -> dataset.setPvcName(datasetPvcName));
|
||||
return PagedResponse.of(datasetResponses, page.getCurrent(), page.getTotal(), page.getPages());
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -28,7 +28,7 @@ public class FileTag {
|
||||
|
||||
private String fromName;
|
||||
|
||||
public List<String> getTags() {
|
||||
public List<String> getTagValues() {
|
||||
List<String> tags = new ArrayList<>();
|
||||
Object tagValues = values.get(type);
|
||||
if (tagValues instanceof List) {
|
||||
@@ -40,9 +40,6 @@ public class FileTag {
|
||||
} else if (tagValues instanceof String) {
|
||||
tags.add((String) tagValues);
|
||||
}
|
||||
if(StringUtils.isNotEmpty(fromName)) {
|
||||
return tags.stream().map(tag -> fromName + " " + tag).toList();
|
||||
}
|
||||
return tags;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -70,8 +70,8 @@ public interface DatasetConverter {
|
||||
* @return 标签分布
|
||||
*/
|
||||
@Named("getDistribution")
|
||||
default Map<String, Long> getDistribution(List<DatasetFile> datasetFiles) {
|
||||
Map<String, Long> distribution = new HashMap<>();
|
||||
default Map<String, Map<String, Long>> getDistribution(List<DatasetFile> datasetFiles) {
|
||||
Map<String, Map<String, Long>> distribution = new HashMap<>();
|
||||
if (CollectionUtils.isEmpty(datasetFiles)) {
|
||||
return distribution;
|
||||
}
|
||||
@@ -81,7 +81,9 @@ public interface DatasetConverter {
|
||||
return distribution;
|
||||
}
|
||||
for (FileTag tag : tags) {
|
||||
tag.getTags().forEach(tagName -> distribution.put(tagName, distribution.getOrDefault(tagName, 0L) + 1));
|
||||
Map<String, Long> tagValueMap = distribution.getOrDefault(tag.getFromName(), new HashMap<>());
|
||||
tag.getTagValues().forEach(tagValue -> tagValueMap.put(tagValue, tagValueMap.getOrDefault(tagValue, 0L) + 1));
|
||||
distribution.put(tag.getFromName(), tagValueMap);
|
||||
}
|
||||
}
|
||||
return distribution;
|
||||
|
||||
@@ -46,5 +46,7 @@ public class DatasetResponse {
|
||||
/** 更新者 */
|
||||
private String updatedBy;
|
||||
/** 分布 */
|
||||
private Map<String, Long> distribution ;
|
||||
private Map<String, Map<String, Long>> distribution;
|
||||
/** 数据集pvc名称 */
|
||||
private String pvcName;
|
||||
}
|
||||
|
||||
@@ -58,8 +58,9 @@ public class DatasetController {
|
||||
*/
|
||||
@GetMapping("/{datasetId}")
|
||||
public DatasetResponse getDatasetById(@PathVariable("datasetId") String datasetId) {
|
||||
Dataset dataset = datasetApplicationService.getDataset(datasetId);
|
||||
return DatasetConverter.INSTANCE.convertToResponse(dataset);
|
||||
DatasetResponse dataset = DatasetConverter.INSTANCE.convertToResponse(datasetApplicationService.getDataset(datasetId));
|
||||
dataset.setPvcName(datasetApplicationService.getDatasetPvcName());
|
||||
return dataset;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user