q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化部分问题 (#126)

Browse Source 
* feature: 支持相对路径引用

* feature: 优化本地部署命令

* feature: 优化算子编排展示

* feature: 优化清洗任务失败后重试
This commit is contained in:
hhhhsc701
2025-12-03 16:41:48 +08:00
committed by GitHub
parent 04cff3f7e6
commit c22683d635
5 changed files with 19 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/CleaningTaskService.java
View File

@@ -171,10 +171,10 @@ public class CleaningTaskService {
}
public void executeTask(String taskId) {
List<CleaningResultDto> failed = cleaningResultRepo.findByInstanceId(taskId, "FAILED");
Set<String> failedSet = failed.stream().map(CleaningResultDto::getSrcFileId).collect(Collectors.toSet());
List<CleaningResultDto> succeed = cleaningResultRepo.findByInstanceId(taskId, "COMPLETED");
Set<String> succeedSet = succeed.stream().map(CleaningResultDto::getSrcFileId).collect(Collectors.toSet());
CleaningTaskDto task = cleaningTaskRepo.findTaskById(taskId);
scanDataset(taskId, task.getSrcDatasetId(), failedSet);
scanDataset(taskId, task.getSrcDatasetId(), succeedSet);
cleaningResultRepo.deleteByInstanceId(taskId, "FAILED");
taskScheduler.executeTask(taskId);
}
@@ -232,7 +232,7 @@ public class CleaningTaskService {
} while (pageNumber < datasetFiles.getTotalPages());
}
private void scanDataset(String taskId, String srcDatasetId, Set<String> failedFiles) {
private void scanDataset(String taskId, String srcDatasetId, Set<String> succeedFiles) {
int pageNumber = 0;
int pageSize = 500;
PagingQuery pageRequest = new PagingQuery(pageNumber, pageSize);
@@ -243,7 +243,7 @@ public class CleaningTaskService {
break;
}
List<Map<String, Object>> files = datasetFiles.getContent().stream()
.filter(content -> failedFiles.contains(content.getId()))
.filter(content -> !succeedFiles.contains(content.getId()))
.map(content -> Map.of("fileName", (Object) content.getFileName(),
"fileSize", content.getFileSize(),
"filePath", content.getFilePath(),