feature: 清洗任务详情页 (#73)

* feature: 清洗任务详情

* fix: 取消构建镜像,改为直接拉取

* fix: 增加清洗任务详情页

* fix: 增加清洗任务详情页

* fix: 算子列表可点击

* fix: 模板详情和更新
This commit is contained in:
hhhhsc701
2025-11-12 18:00:19 +08:00
committed by GitHub
parent 442e561817
commit 6bbde0ec56
46 changed files with 1065 additions and 795 deletions

View File

@@ -11,10 +11,7 @@ import com.datamate.cleaning.domain.repository.CleaningTaskRepository;
import com.datamate.cleaning.domain.repository.OperatorInstanceRepository;
import com.datamate.cleaning.infrastructure.validator.CleanTaskValidator;
import com.datamate.cleaning.interfaces.dto.CleaningProcess;
import com.datamate.cleaning.interfaces.dto.CleaningTaskDto;
import com.datamate.cleaning.interfaces.dto.CreateCleaningTaskRequest;
import com.datamate.cleaning.interfaces.dto.OperatorInstanceDto;
import com.datamate.cleaning.interfaces.dto.*;
import com.datamate.common.infrastructure.exception.BusinessException;
import com.datamate.common.infrastructure.exception.SystemErrorCode;
import com.datamate.datamanagement.application.DatasetApplicationService;
@@ -40,15 +37,19 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
@Slf4j
@Service
@RequiredArgsConstructor
public class CleaningTaskService {
private final CleaningTaskRepository CleaningTaskRepo;
private final CleaningTaskRepository cleaningTaskRepo;
private final OperatorInstanceRepository operatorInstanceRepo;
@@ -66,19 +67,24 @@ public class CleaningTaskService {
private final String FLOW_PATH = "/flow";
private final Pattern LEVEL_PATTERN = Pattern.compile(
"\\b(TRACE|DEBUG|INFO|WARN|WARNING|ERROR|FATAL)\\b",
Pattern.CASE_INSENSITIVE
);
public List<CleaningTaskDto> getTasks(String status, String keywords, Integer page, Integer size) {
List<CleaningTaskDto> tasks = CleaningTaskRepo.findTasks(status, keywords, page, size);
List<CleaningTaskDto> tasks = cleaningTaskRepo.findTasks(status, keywords, page, size);
tasks.forEach(this::setProcess);
return tasks;
}
private void setProcess(CleaningTaskDto task) {
int count = cleaningResultRepo.countByInstanceId(task.getId());
task.setProgress(CleaningProcess.of(task.getFileCount(), count));
int[] count = cleaningResultRepo.countByInstanceId(task.getId());
task.setProgress(CleaningProcess.of(task.getFileCount(), count[0], count[1]));
}
public int countTasks(String status, String keywords) {
return CleaningTaskRepo.findTasks(status, keywords, null, null).size();
return cleaningTaskRepo.findTasks(status, keywords, null, null).size();
}
@Transactional
@@ -105,7 +111,7 @@ public class CleaningTaskService {
task.setDestDatasetName(destDataset.getName());
task.setBeforeSize(srcDataset.getSizeBytes());
task.setFileCount(srcDataset.getFileCount().intValue());
CleaningTaskRepo.insertTask(task);
cleaningTaskRepo.insertTask(task);
operatorInstanceRepo.insertInstance(taskId, request.getInstance());
@@ -116,14 +122,50 @@ public class CleaningTaskService {
}
public CleaningTaskDto getTask(String taskId) {
CleaningTaskDto task = CleaningTaskRepo.findTaskById(taskId);
CleaningTaskDto task = cleaningTaskRepo.findTaskById(taskId);
setProcess(task);
task.setInstance(operatorInstanceRepo.findOperatorByInstanceId(taskId));
return task;
}
public List<CleaningResultDto> getTaskResults(String taskId) {
return cleaningResultRepo.findByInstanceId(taskId);
}
public List<CleaningTaskLog> getTaskLog(String taskId) {
String logPath = FLOW_PATH + "/" + taskId + "/output.log";
try (Stream<String> lines = Files.lines(Paths.get(logPath))) {
List<CleaningTaskLog> logs = new ArrayList<>();
AtomicReference<String> lastLevel = new AtomicReference<>("INFO");
lines.forEach(line -> {
lastLevel.set(getLogLevel(line, lastLevel.get()));
CleaningTaskLog log = new CleaningTaskLog();
log.setLevel(lastLevel.get());
log.setMessage(line);
logs.add(log);
});
return logs;
} catch (IOException e) {
log.error("Fail to read log file {}", logPath, e);
return Collections.emptyList();
}
}
private String getLogLevel(String logLine, String defaultLevel) {
if (logLine == null || logLine.trim().isEmpty()) {
return defaultLevel;
}
Matcher matcher = LEVEL_PATTERN.matcher(logLine);
if (matcher.find()) {
return matcher.group(1).toUpperCase();
}
return defaultLevel;
}
@Transactional
public void deleteTask(String taskId) {
CleaningTaskRepo.deleteTaskById(taskId);
cleaningTaskRepo.deleteTaskById(taskId);
operatorInstanceRepo.deleteByInstanceId(taskId);
cleaningResultRepo.deleteByInstanceId(taskId);
}
@@ -190,7 +232,7 @@ public class CleaningTaskService {
try (BufferedWriter writer = new BufferedWriter(new FileWriter(fileName))) {
if (!mapList.isEmpty()) { // 检查列表是否为空,避免异常
String jsonString = objectMapper.writeValueAsString(mapList.get(0));
String jsonString = objectMapper.writeValueAsString(mapList.getFirst());
writer.write(jsonString);
for (int i = 1; i < mapList.size(); i++) {

View File

@@ -5,7 +5,7 @@ import com.datamate.cleaning.domain.repository.CleaningTemplateRepository;
import com.datamate.cleaning.domain.repository.OperatorInstanceRepository;
import com.datamate.cleaning.interfaces.dto.*;
import com.datamate.cleaning.domain.model.entity.TemplateWithInstance;
import com.datamate.operator.domain.repository.OperatorRepository;
import com.datamate.operator.domain.repository.OperatorViewRepository;
import com.datamate.operator.interfaces.dto.OperatorDto;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
@@ -26,10 +26,11 @@ public class CleaningTemplateService {
private final OperatorInstanceRepository operatorInstanceRepo;
private final OperatorRepository operatorRepo;
private final OperatorViewRepository operatorViewRepo;
public List<CleaningTemplateDto> getTemplates(String keywords) {
List<OperatorDto> allOperators = operatorRepo.findAllOperators();
List<OperatorDto> allOperators =
operatorViewRepo.findOperatorsByCriteria(null, null, null, null, null);
Map<String, OperatorDto> operatorsMap = allOperators.stream()
.collect(Collectors.toMap(OperatorDto::getId, Function.identity()));
List<TemplateWithInstance> allTemplates = cleaningTemplateRepo.findAllTemplates(keywords);
@@ -39,8 +40,8 @@ public class CleaningTemplateService {
List<TemplateWithInstance> value = twi.getValue();
CleaningTemplateDto template = new CleaningTemplateDto();
template.setId(twi.getKey());
template.setName(value.get(0).getName());
template.setDescription(value.get(0).getDescription());
template.setName(value.getFirst().getName());
template.setDescription(value.getFirst().getDescription());
template.setInstance(value.stream().filter(v -> StringUtils.isNotBlank(v.getOperatorId()))
.sorted(Comparator.comparingInt(TemplateWithInstance::getOpIndex))
.map(v -> {
@@ -50,8 +51,8 @@ public class CleaningTemplateService {
}
return operator;
}).toList());
template.setCreatedAt(value.get(0).getCreatedAt());
template.setUpdatedAt(value.get(0).getUpdatedAt());
template.setCreatedAt(value.getFirst().getCreatedAt());
template.setUpdatedAt(value.getFirst().getUpdatedAt());
return template;
}).toList();
}
@@ -70,17 +71,22 @@ public class CleaningTemplateService {
}
public CleaningTemplateDto getTemplate(String templateId) {
return cleaningTemplateRepo.findTemplateById(templateId);
CleaningTemplateDto template = cleaningTemplateRepo.findTemplateById(templateId);
template.setInstance(operatorInstanceRepo.findOperatorByInstanceId(templateId));
return template;
}
@Transactional
public CleaningTemplateDto updateTemplate(String templateId, UpdateCleaningTemplateRequest request) {
CleaningTemplateDto template = cleaningTemplateRepo.findTemplateById(templateId);
if (template != null) {
template.setName(request.getName());
template.setDescription(request.getDescription());
cleaningTemplateRepo.updateTemplate(template);
if (template == null) {
return null;
}
template.setName(request.getName());
template.setDescription(request.getDescription());
cleaningTemplateRepo.updateTemplate(template);
operatorInstanceRepo.deleteByInstanceId(templateId);
operatorInstanceRepo.insertInstance(templateId, request.getInstance());
return template;
}

View File

@@ -16,6 +16,8 @@ import java.util.concurrent.Executors;
public class CleaningTaskScheduler {
private final CleaningTaskRepository cleaningTaskRepo;
private final RuntimeClient runtimeClient;
private final ExecutorService taskExecutor = Executors.newFixedThreadPool(5);
public void executeTask(String taskId) {
@@ -28,11 +30,11 @@ public class CleaningTaskScheduler {
task.setStatus(CleaningTaskStatusEnum.RUNNING);
task.setStartedAt(LocalDateTime.now());
cleaningTaskRepo.updateTask(task);
RuntimeClient.submitTask(taskId);
runtimeClient.submitTask(taskId);
}
public void stopTask(String taskId) {
RuntimeClient.stopTask(taskId);
runtimeClient.stopTask(taskId);
CleaningTaskDto task = new CleaningTaskDto();
task.setId(taskId);
task.setStatus(CleaningTaskStatusEnum.STOPPED);

View File

@@ -1,36 +0,0 @@
package com.datamate.cleaning.domain.model.entity;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Getter;
import lombok.Setter;
import java.time.LocalDateTime;
@Getter
@Setter
@TableName(value = "t_operator", autoResultMap = true)
public class Operator {
@TableId
private String id;
private String name;
private String description;
private String version;
private String inputs;
private String outputs;
private String runtime;
private String settings;
private Boolean isStar;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}

View File

@@ -3,9 +3,14 @@ package com.datamate.cleaning.domain.repository;
import com.baomidou.mybatisplus.extension.repository.IRepository;
import com.datamate.cleaning.domain.model.entity.CleaningResult;
import com.datamate.cleaning.interfaces.dto.CleaningResultDto;
import java.util.List;
public interface CleaningResultRepository extends IRepository<CleaningResult> {
void deleteByInstanceId(String instanceId);
int countByInstanceId(String instanceId);
int[] countByInstanceId(String instanceId);
List<CleaningResultDto> findByInstanceId(String instanceId);
}

View File

@@ -3,6 +3,7 @@ package com.datamate.cleaning.domain.repository;
import com.baomidou.mybatisplus.extension.repository.IRepository;
import com.datamate.cleaning.interfaces.dto.OperatorInstanceDto;
import com.datamate.cleaning.domain.model.entity.OperatorInstance;
import com.datamate.operator.interfaces.dto.OperatorDto;
import java.util.List;
@@ -10,4 +11,6 @@ public interface OperatorInstanceRepository extends IRepository<OperatorInstance
void insertInstance(String instanceId, List<OperatorInstanceDto> instances);
void deleteByInstanceId(String instanceId);
List<OperatorDto> findOperatorByInstanceId(String instanceId);
}

View File

@@ -0,0 +1,15 @@
package com.datamate.cleaning.infrastructure.converter;
import com.datamate.cleaning.domain.model.entity.CleaningResult;
import com.datamate.cleaning.interfaces.dto.CleaningResultDto;
import org.mapstruct.Mapper;
import org.mapstruct.factory.Mappers;
import java.util.List;
@Mapper
public interface CleaningResultConverter {
CleaningResultConverter INSTANCE = Mappers.getMapper(CleaningResultConverter.class);
List<CleaningResultDto> convertEntityToDto(List<CleaningResult> cleaningResult);
}

View File

@@ -2,10 +2,10 @@ package com.datamate.cleaning.infrastructure.converter;
import com.datamate.cleaning.domain.model.entity.OperatorInstance;
import com.datamate.cleaning.domain.model.entity.Operator;
import com.datamate.cleaning.interfaces.dto.OperatorInstanceDto;
import com.datamate.common.infrastructure.exception.BusinessException;
import com.datamate.common.infrastructure.exception.SystemErrorCode;
import com.datamate.operator.domain.model.OperatorView;
import com.datamate.operator.interfaces.dto.OperatorDto;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -14,6 +14,8 @@ import org.mapstruct.Mapping;
import org.mapstruct.Named;
import org.mapstruct.factory.Mappers;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -35,5 +37,16 @@ public interface OperatorInstanceConverter {
}
}
List<OperatorDto> fromEntityToDto(List<Operator> operator);
@Mapping(target = "categories", source = "categories", qualifiedByName = "stringToList")
OperatorDto fromEntityToDto(OperatorView operator);
List<OperatorDto> fromEntityToDto(List<OperatorView> operator);
@Named("stringToList")
default List<String> stringToList(String input) {
if (input == null || input.isEmpty()) {
return Collections.emptyList();
}
return Arrays.stream(input.split(",")).toList();
}
}

View File

@@ -3,6 +3,8 @@ package com.datamate.cleaning.infrastructure.httpclient;
import com.datamate.common.infrastructure.exception.BusinessException;
import com.datamate.common.infrastructure.exception.SystemErrorCode;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.net.URI;
@@ -13,24 +15,36 @@ import java.text.MessageFormat;
import java.time.Duration;
@Slf4j
@Component
public class RuntimeClient {
private static final String BASE_URL = "http://datamate-runtime:8081/api";
private final String CREATE_TASK_URL = "/api/task/{0}/submit";
private static final String CREATE_TASK_URL = BASE_URL + "/task/{0}/submit";
private final String STOP_TASK_URL = "/api/task/{0}/stop";
private static final String STOP_TASK_URL = BASE_URL + "/task/{0}/stop";
@Value("${runtime.protocol:http}")
private String protocol;
private static final HttpClient CLIENT = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build();
@Value("${runtime.host:datamate-runtime}")
private String host;
public static void submitTask(String taskId) {
send(MessageFormat.format(CREATE_TASK_URL, taskId));
@Value("${runtime.port:8081}")
private int port;
private final HttpClient CLIENT = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build();
public void submitTask(String taskId) {
send(MessageFormat.format(getRequestUrl(CREATE_TASK_URL), taskId));
}
public static void stopTask(String taskId) {
send(MessageFormat.format(STOP_TASK_URL, taskId));
public void stopTask(String taskId) {
send(MessageFormat.format(getRequestUrl(STOP_TASK_URL), taskId));
}
private static void send(String url) {
private String getRequestUrl(String url) {
return protocol + "://" + host + ":" + port + url;
}
private void send(String url) {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(30))

View File

@@ -2,12 +2,18 @@ package com.datamate.cleaning.infrastructure.persistence.Impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.repository.CrudRepository;
import com.datamate.cleaning.common.enums.CleaningTaskStatusEnum;
import com.datamate.cleaning.domain.model.entity.CleaningResult;
import com.datamate.cleaning.domain.repository.CleaningResultRepository;
import com.datamate.cleaning.infrastructure.converter.CleaningResultConverter;
import com.datamate.cleaning.infrastructure.persistence.mapper.CleaningResultMapper;
import com.datamate.cleaning.interfaces.dto.CleaningResultDto;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
@RequiredArgsConstructor
public class CleaningResultRepositoryImpl extends CrudRepository<CleaningResultMapper, CleaningResult>
@@ -22,9 +28,20 @@ public class CleaningResultRepositoryImpl extends CrudRepository<CleaningResultM
}
@Override
public int countByInstanceId(String instanceId) {
public int[] countByInstanceId(String instanceId) {
LambdaQueryWrapper<CleaningResult> lambdaWrapper = new LambdaQueryWrapper<>();
lambdaWrapper.eq(CleaningResult::getInstanceId, instanceId);
return mapper.selectCount(lambdaWrapper).intValue();
List<CleaningResult> cleaningResults = mapper.selectList(lambdaWrapper);
int succeed = Math.toIntExact(cleaningResults.stream()
.filter(result ->
StringUtils.equals(result.getStatus(), CleaningTaskStatusEnum.COMPLETED.getValue()))
.count());
return new int[] {succeed, cleaningResults.size() - succeed};
}
public List<CleaningResultDto> findByInstanceId(String instanceId) {
LambdaQueryWrapper<CleaningResult> queryWrapper = new LambdaQueryWrapper<>();
queryWrapper.eq(CleaningResult::getInstanceId, instanceId);
return CleaningResultConverter.INSTANCE.convertEntityToDto(mapper.selectList(queryWrapper));
}
}

View File

@@ -7,6 +7,7 @@ import com.datamate.cleaning.interfaces.dto.OperatorInstanceDto;
import com.datamate.cleaning.domain.model.entity.OperatorInstance;
import com.datamate.cleaning.domain.repository.OperatorInstanceRepository;
import com.datamate.cleaning.infrastructure.persistence.mapper.OperatorInstanceMapper;
import com.datamate.operator.interfaces.dto.OperatorDto;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Repository;
@@ -37,4 +38,8 @@ public class OperatorInstanceRepositoryImpl extends CrudRepository<OperatorInsta
lambdaWrapper.eq(OperatorInstance::getInstanceId, instanceId);
mapper.delete(lambdaWrapper);
}
public List<OperatorDto> findOperatorByInstanceId(String instanceId) {
return OperatorInstanceConverter.INSTANCE.fromEntityToDto(mapper.findOperatorByInstanceId(instanceId));
}
}

View File

@@ -2,9 +2,22 @@ package com.datamate.cleaning.infrastructure.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.datamate.cleaning.domain.model.entity.OperatorInstance;
import com.datamate.operator.domain.model.OperatorView;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Select;
import java.util.List;
@Mapper
public interface OperatorInstanceMapper extends BaseMapper<OperatorInstance> {
@Select("SELECT o.operator_id as id, o.operator_name as name, description, version, inputs, outputs, runtime, " +
" settings, created_at, updated_at, " +
"GROUP_CONCAT(category_id ORDER BY created_at DESC SEPARATOR ',') AS categories " +
"FROM t_operator_instance toi LEFT JOIN v_operator o ON toi.operator_id = o.operator_id " +
"WHERE toi.instance_id = #{instanceId} " +
"GROUP BY o.operator_id, o.operator_name, description, version, inputs, outputs, runtime," +
" settings, created_at, updated_at, op_index " +
"ORDER BY toi.op_index")
List<OperatorView> findOperatorByInstanceId(String instanceId);
}

View File

@@ -16,23 +16,37 @@ import java.math.RoundingMode;
public class CleaningProcess {
private Float process;
private Float successRate;
private Integer totalFileNum;
private Integer succeedFileNum;
private Integer failedFileNum;
private Integer finishedFileNum;
public CleaningProcess(int totalFileNum, int finishedFileNum) {
public CleaningProcess(int totalFileNum, int succeedFileNum, int failedFileNum) {
this.totalFileNum = totalFileNum;
this.finishedFileNum = finishedFileNum;
this.succeedFileNum = succeedFileNum;
this.failedFileNum = failedFileNum;
this.finishedFileNum = succeedFileNum + failedFileNum;
if (totalFileNum == 0) {
this.process = 0.0f;
} else {
this.process = BigDecimal.valueOf(finishedFileNum * 100L)
.divide(BigDecimal.valueOf(totalFileNum), 2, RoundingMode.HALF_UP).floatValue();
}
if (finishedFileNum == 0) {
this.successRate = 0f;
} else {
this.successRate = BigDecimal.valueOf(succeedFileNum * 100L)
.divide(BigDecimal.valueOf(finishedFileNum), 2, RoundingMode.HALF_UP).floatValue();
}
}
public static CleaningProcess of(int totalFileNum, int finishedFileNum) {
return new CleaningProcess(totalFileNum, finishedFileNum);
public static CleaningProcess of(int totalFileNum, int succeedFileNum, int failedFileNum) {
return new CleaningProcess(totalFileNum, succeedFileNum, failedFileNum);
}
}

View File

@@ -0,0 +1,30 @@
package com.datamate.cleaning.interfaces.dto;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class CleaningResultDto {
private String instanceId;
private String srcFileId;
private String destFileId;
private String srcName;
private String destName;
private String srcType;
private String destType;
private long srcSize;
private long destSize;
private String status;
private String result;
}

View File

@@ -0,0 +1,12 @@
package com.datamate.cleaning.interfaces.dto;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class CleaningTaskLog {
private String level;
private String message;
}

View File

@@ -1,7 +1,9 @@
package com.datamate.cleaning.interfaces.rest;
import com.datamate.cleaning.application.CleaningTaskService;
import com.datamate.cleaning.interfaces.dto.CleaningResultDto;
import com.datamate.cleaning.interfaces.dto.CleaningTaskDto;
import com.datamate.cleaning.interfaces.dto.CleaningTaskLog;
import com.datamate.cleaning.interfaces.dto.CreateCleaningTaskRequest;
import com.datamate.common.interfaces.PagedResponse;
import lombok.RequiredArgsConstructor;
@@ -54,4 +56,14 @@ public class CleaningTaskController {
cleaningTaskService.deleteTask(taskId);
return taskId;
}
@GetMapping("/{taskId}/result")
public List<CleaningResultDto> cleaningTasksTaskIdGetResult(@PathVariable("taskId") String taskId) {
return cleaningTaskService.getTaskResults(taskId);
}
@GetMapping("/{taskId}/log")
public List<CleaningTaskLog> cleaningTasksTaskIdGetLog(@PathVariable("taskId") String taskId) {
return cleaningTaskService.getTaskLog(taskId);
}
}