You've already forked DataMate
feature: 清洗任务详情页 (#73)
* feature: 清洗任务详情 * fix: 取消构建镜像,改为直接拉取 * fix: 增加清洗任务详情页 * fix: 增加清洗任务详情页 * fix: 算子列表可点击 * fix: 模板详情和更新
This commit is contained in:
27
.github/workflows/docker-image-save.yml
vendored
Normal file
27
.github/workflows/docker-image-save.yml
vendored
Normal file
@@ -0,0 +1,27 @@
|
||||
name: docker-image-save.yml
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
service_name:
|
||||
required: true
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
pull-and-save:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Pull Docker Image
|
||||
run: |
|
||||
LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
|
||||
docker pull ghcr.io/$LOWERCASE_REPO/datamate-${{ inputs.service_name }}:latest
|
||||
docker tag ghcr.io/$LOWERCASE_REPO/datamate-${{ inputs.service_name }}:latest datamate-${{ inputs.service_name }}:latest
|
||||
|
||||
- name: Save Docker Image
|
||||
run: |
|
||||
docker save -o datamate-${{ inputs.service_name }}.tar datamate-${{ inputs.service_name }}:latest
|
||||
|
||||
- name: Upload Docker Image
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: datamate-${{ inputs.service_name }}
|
||||
path: datamate-${{ inputs.service_name }}.tar
|
||||
16
.github/workflows/docker-images-reusable.yml
vendored
16
.github/workflows/docker-images-reusable.yml
vendored
@@ -47,19 +47,7 @@ jobs:
|
||||
make build-${{ inputs.service_name }} VERSION=latest
|
||||
|
||||
- name: Tag & Push Docker Image
|
||||
if: github.event_name != 'pull_request' && !startsWith(github.workflow, 'Package')
|
||||
if: github.event_name != 'pull_request'
|
||||
run: |
|
||||
docker tag datamate-${{ inputs.service_name }}:latest ${{ steps.set-tag.outputs.TAGS }}
|
||||
docker push ${{ steps.set-tag.outputs.TAGS }}
|
||||
|
||||
- name: Save Docker Image
|
||||
if: startsWith(github.workflow, 'Package')
|
||||
run: |
|
||||
docker save -o datamate-${{ inputs.service_name }}.tar datamate-${{ inputs.service_name }}:latest
|
||||
|
||||
- name: Upload Docker Image
|
||||
if: startsWith(github.workflow, 'Package')
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: datamate-${{ inputs.service_name }}
|
||||
path: datamate-${{ inputs.service_name }}.tar
|
||||
docker push ${{ steps.set-tag.outputs.TAGS }}
|
||||
22
.github/workflows/package.yml
vendored
22
.github/workflows/package.yml
vendored
@@ -6,23 +6,33 @@ on:
|
||||
jobs:
|
||||
backend-docker-build:
|
||||
name: Build and Push Backend Docker Image
|
||||
uses: ./.github/workflows/docker-image-backend.yml
|
||||
uses: ./.github/workflows/docker-image-save.yml
|
||||
with:
|
||||
service_name: backend
|
||||
|
||||
frontend-docker-build:
|
||||
name: Build and Push Frontend Docker Image
|
||||
uses: ./.github/workflows/docker-image-frontend.yml
|
||||
uses: ./.github/workflows/docker-image-save.yml
|
||||
with:
|
||||
service_name: frontend
|
||||
|
||||
database-docker-build:
|
||||
name: Build and Push Database Docker Image
|
||||
uses: ./.github/workflows/docker-image-database.yml
|
||||
uses: ./.github/workflows/docker-image-save.yml
|
||||
with:
|
||||
service_name: database
|
||||
|
||||
runtime-docker-build:
|
||||
name: Build and Push Runtime Docker Image
|
||||
uses: ./.github/workflows/docker-image-runtime.yml
|
||||
uses: ./.github/workflows/docker-image-save.yml
|
||||
with:
|
||||
service_name: runtime
|
||||
|
||||
backend-python-docker-build:
|
||||
name: Build and Push Backend Python Docker Image
|
||||
uses: ./.github/workflows/docker-image-backend-python.yml
|
||||
uses: ./.github/workflows/docker-image-save.yml
|
||||
with:
|
||||
service_name: backend-python
|
||||
|
||||
package-all:
|
||||
needs:
|
||||
@@ -54,7 +64,7 @@ jobs:
|
||||
- name: Upload Package
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: datamate
|
||||
name: DataMate
|
||||
include-hidden-files: true
|
||||
path: |
|
||||
deployment/
|
||||
|
||||
@@ -11,10 +11,7 @@ import com.datamate.cleaning.domain.repository.CleaningTaskRepository;
|
||||
import com.datamate.cleaning.domain.repository.OperatorInstanceRepository;
|
||||
|
||||
import com.datamate.cleaning.infrastructure.validator.CleanTaskValidator;
|
||||
import com.datamate.cleaning.interfaces.dto.CleaningProcess;
|
||||
import com.datamate.cleaning.interfaces.dto.CleaningTaskDto;
|
||||
import com.datamate.cleaning.interfaces.dto.CreateCleaningTaskRequest;
|
||||
import com.datamate.cleaning.interfaces.dto.OperatorInstanceDto;
|
||||
import com.datamate.cleaning.interfaces.dto.*;
|
||||
import com.datamate.common.infrastructure.exception.BusinessException;
|
||||
import com.datamate.common.infrastructure.exception.SystemErrorCode;
|
||||
import com.datamate.datamanagement.application.DatasetApplicationService;
|
||||
@@ -40,15 +37,19 @@ import java.io.BufferedWriter;
|
||||
import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class CleaningTaskService {
|
||||
private final CleaningTaskRepository CleaningTaskRepo;
|
||||
private final CleaningTaskRepository cleaningTaskRepo;
|
||||
|
||||
private final OperatorInstanceRepository operatorInstanceRepo;
|
||||
|
||||
@@ -66,19 +67,24 @@ public class CleaningTaskService {
|
||||
|
||||
private final String FLOW_PATH = "/flow";
|
||||
|
||||
private final Pattern LEVEL_PATTERN = Pattern.compile(
|
||||
"\\b(TRACE|DEBUG|INFO|WARN|WARNING|ERROR|FATAL)\\b",
|
||||
Pattern.CASE_INSENSITIVE
|
||||
);
|
||||
|
||||
public List<CleaningTaskDto> getTasks(String status, String keywords, Integer page, Integer size) {
|
||||
List<CleaningTaskDto> tasks = CleaningTaskRepo.findTasks(status, keywords, page, size);
|
||||
List<CleaningTaskDto> tasks = cleaningTaskRepo.findTasks(status, keywords, page, size);
|
||||
tasks.forEach(this::setProcess);
|
||||
return tasks;
|
||||
}
|
||||
|
||||
private void setProcess(CleaningTaskDto task) {
|
||||
int count = cleaningResultRepo.countByInstanceId(task.getId());
|
||||
task.setProgress(CleaningProcess.of(task.getFileCount(), count));
|
||||
int[] count = cleaningResultRepo.countByInstanceId(task.getId());
|
||||
task.setProgress(CleaningProcess.of(task.getFileCount(), count[0], count[1]));
|
||||
}
|
||||
|
||||
public int countTasks(String status, String keywords) {
|
||||
return CleaningTaskRepo.findTasks(status, keywords, null, null).size();
|
||||
return cleaningTaskRepo.findTasks(status, keywords, null, null).size();
|
||||
}
|
||||
|
||||
@Transactional
|
||||
@@ -105,7 +111,7 @@ public class CleaningTaskService {
|
||||
task.setDestDatasetName(destDataset.getName());
|
||||
task.setBeforeSize(srcDataset.getSizeBytes());
|
||||
task.setFileCount(srcDataset.getFileCount().intValue());
|
||||
CleaningTaskRepo.insertTask(task);
|
||||
cleaningTaskRepo.insertTask(task);
|
||||
|
||||
operatorInstanceRepo.insertInstance(taskId, request.getInstance());
|
||||
|
||||
@@ -116,14 +122,50 @@ public class CleaningTaskService {
|
||||
}
|
||||
|
||||
public CleaningTaskDto getTask(String taskId) {
|
||||
CleaningTaskDto task = CleaningTaskRepo.findTaskById(taskId);
|
||||
CleaningTaskDto task = cleaningTaskRepo.findTaskById(taskId);
|
||||
setProcess(task);
|
||||
task.setInstance(operatorInstanceRepo.findOperatorByInstanceId(taskId));
|
||||
return task;
|
||||
}
|
||||
|
||||
public List<CleaningResultDto> getTaskResults(String taskId) {
|
||||
return cleaningResultRepo.findByInstanceId(taskId);
|
||||
}
|
||||
|
||||
public List<CleaningTaskLog> getTaskLog(String taskId) {
|
||||
String logPath = FLOW_PATH + "/" + taskId + "/output.log";
|
||||
try (Stream<String> lines = Files.lines(Paths.get(logPath))) {
|
||||
List<CleaningTaskLog> logs = new ArrayList<>();
|
||||
AtomicReference<String> lastLevel = new AtomicReference<>("INFO");
|
||||
lines.forEach(line -> {
|
||||
lastLevel.set(getLogLevel(line, lastLevel.get()));
|
||||
CleaningTaskLog log = new CleaningTaskLog();
|
||||
log.setLevel(lastLevel.get());
|
||||
log.setMessage(line);
|
||||
logs.add(log);
|
||||
});
|
||||
return logs;
|
||||
} catch (IOException e) {
|
||||
log.error("Fail to read log file {}", logPath, e);
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
|
||||
private String getLogLevel(String logLine, String defaultLevel) {
|
||||
if (logLine == null || logLine.trim().isEmpty()) {
|
||||
return defaultLevel;
|
||||
}
|
||||
|
||||
Matcher matcher = LEVEL_PATTERN.matcher(logLine);
|
||||
if (matcher.find()) {
|
||||
return matcher.group(1).toUpperCase();
|
||||
}
|
||||
return defaultLevel;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void deleteTask(String taskId) {
|
||||
CleaningTaskRepo.deleteTaskById(taskId);
|
||||
cleaningTaskRepo.deleteTaskById(taskId);
|
||||
operatorInstanceRepo.deleteByInstanceId(taskId);
|
||||
cleaningResultRepo.deleteByInstanceId(taskId);
|
||||
}
|
||||
@@ -190,7 +232,7 @@ public class CleaningTaskService {
|
||||
|
||||
try (BufferedWriter writer = new BufferedWriter(new FileWriter(fileName))) {
|
||||
if (!mapList.isEmpty()) { // 检查列表是否为空,避免异常
|
||||
String jsonString = objectMapper.writeValueAsString(mapList.get(0));
|
||||
String jsonString = objectMapper.writeValueAsString(mapList.getFirst());
|
||||
writer.write(jsonString);
|
||||
|
||||
for (int i = 1; i < mapList.size(); i++) {
|
||||
|
||||
@@ -5,7 +5,7 @@ import com.datamate.cleaning.domain.repository.CleaningTemplateRepository;
|
||||
import com.datamate.cleaning.domain.repository.OperatorInstanceRepository;
|
||||
import com.datamate.cleaning.interfaces.dto.*;
|
||||
import com.datamate.cleaning.domain.model.entity.TemplateWithInstance;
|
||||
import com.datamate.operator.domain.repository.OperatorRepository;
|
||||
import com.datamate.operator.domain.repository.OperatorViewRepository;
|
||||
import com.datamate.operator.interfaces.dto.OperatorDto;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
@@ -26,10 +26,11 @@ public class CleaningTemplateService {
|
||||
|
||||
private final OperatorInstanceRepository operatorInstanceRepo;
|
||||
|
||||
private final OperatorRepository operatorRepo;
|
||||
private final OperatorViewRepository operatorViewRepo;
|
||||
|
||||
public List<CleaningTemplateDto> getTemplates(String keywords) {
|
||||
List<OperatorDto> allOperators = operatorRepo.findAllOperators();
|
||||
List<OperatorDto> allOperators =
|
||||
operatorViewRepo.findOperatorsByCriteria(null, null, null, null, null);
|
||||
Map<String, OperatorDto> operatorsMap = allOperators.stream()
|
||||
.collect(Collectors.toMap(OperatorDto::getId, Function.identity()));
|
||||
List<TemplateWithInstance> allTemplates = cleaningTemplateRepo.findAllTemplates(keywords);
|
||||
@@ -39,8 +40,8 @@ public class CleaningTemplateService {
|
||||
List<TemplateWithInstance> value = twi.getValue();
|
||||
CleaningTemplateDto template = new CleaningTemplateDto();
|
||||
template.setId(twi.getKey());
|
||||
template.setName(value.get(0).getName());
|
||||
template.setDescription(value.get(0).getDescription());
|
||||
template.setName(value.getFirst().getName());
|
||||
template.setDescription(value.getFirst().getDescription());
|
||||
template.setInstance(value.stream().filter(v -> StringUtils.isNotBlank(v.getOperatorId()))
|
||||
.sorted(Comparator.comparingInt(TemplateWithInstance::getOpIndex))
|
||||
.map(v -> {
|
||||
@@ -50,8 +51,8 @@ public class CleaningTemplateService {
|
||||
}
|
||||
return operator;
|
||||
}).toList());
|
||||
template.setCreatedAt(value.get(0).getCreatedAt());
|
||||
template.setUpdatedAt(value.get(0).getUpdatedAt());
|
||||
template.setCreatedAt(value.getFirst().getCreatedAt());
|
||||
template.setUpdatedAt(value.getFirst().getUpdatedAt());
|
||||
return template;
|
||||
}).toList();
|
||||
}
|
||||
@@ -70,17 +71,22 @@ public class CleaningTemplateService {
|
||||
}
|
||||
|
||||
public CleaningTemplateDto getTemplate(String templateId) {
|
||||
return cleaningTemplateRepo.findTemplateById(templateId);
|
||||
CleaningTemplateDto template = cleaningTemplateRepo.findTemplateById(templateId);
|
||||
template.setInstance(operatorInstanceRepo.findOperatorByInstanceId(templateId));
|
||||
return template;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public CleaningTemplateDto updateTemplate(String templateId, UpdateCleaningTemplateRequest request) {
|
||||
CleaningTemplateDto template = cleaningTemplateRepo.findTemplateById(templateId);
|
||||
if (template != null) {
|
||||
template.setName(request.getName());
|
||||
template.setDescription(request.getDescription());
|
||||
cleaningTemplateRepo.updateTemplate(template);
|
||||
if (template == null) {
|
||||
return null;
|
||||
}
|
||||
template.setName(request.getName());
|
||||
template.setDescription(request.getDescription());
|
||||
cleaningTemplateRepo.updateTemplate(template);
|
||||
operatorInstanceRepo.deleteByInstanceId(templateId);
|
||||
operatorInstanceRepo.insertInstance(templateId, request.getInstance());
|
||||
return template;
|
||||
}
|
||||
|
||||
|
||||
@@ -16,6 +16,8 @@ import java.util.concurrent.Executors;
|
||||
public class CleaningTaskScheduler {
|
||||
private final CleaningTaskRepository cleaningTaskRepo;
|
||||
|
||||
private final RuntimeClient runtimeClient;
|
||||
|
||||
private final ExecutorService taskExecutor = Executors.newFixedThreadPool(5);
|
||||
|
||||
public void executeTask(String taskId) {
|
||||
@@ -28,11 +30,11 @@ public class CleaningTaskScheduler {
|
||||
task.setStatus(CleaningTaskStatusEnum.RUNNING);
|
||||
task.setStartedAt(LocalDateTime.now());
|
||||
cleaningTaskRepo.updateTask(task);
|
||||
RuntimeClient.submitTask(taskId);
|
||||
runtimeClient.submitTask(taskId);
|
||||
}
|
||||
|
||||
public void stopTask(String taskId) {
|
||||
RuntimeClient.stopTask(taskId);
|
||||
runtimeClient.stopTask(taskId);
|
||||
CleaningTaskDto task = new CleaningTaskDto();
|
||||
task.setId(taskId);
|
||||
task.setStatus(CleaningTaskStatusEnum.STOPPED);
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
package com.datamate.cleaning.domain.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@TableName(value = "t_operator", autoResultMap = true)
|
||||
public class Operator {
|
||||
@TableId
|
||||
private String id;
|
||||
|
||||
private String name;
|
||||
|
||||
private String description;
|
||||
|
||||
private String version;
|
||||
|
||||
private String inputs;
|
||||
|
||||
private String outputs;
|
||||
|
||||
private String runtime;
|
||||
|
||||
private String settings;
|
||||
|
||||
private Boolean isStar;
|
||||
|
||||
private LocalDateTime createdAt;
|
||||
|
||||
private LocalDateTime updatedAt;
|
||||
}
|
||||
@@ -3,9 +3,14 @@ package com.datamate.cleaning.domain.repository;
|
||||
|
||||
import com.baomidou.mybatisplus.extension.repository.IRepository;
|
||||
import com.datamate.cleaning.domain.model.entity.CleaningResult;
|
||||
import com.datamate.cleaning.interfaces.dto.CleaningResultDto;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface CleaningResultRepository extends IRepository<CleaningResult> {
|
||||
void deleteByInstanceId(String instanceId);
|
||||
|
||||
int countByInstanceId(String instanceId);
|
||||
int[] countByInstanceId(String instanceId);
|
||||
|
||||
List<CleaningResultDto> findByInstanceId(String instanceId);
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package com.datamate.cleaning.domain.repository;
|
||||
import com.baomidou.mybatisplus.extension.repository.IRepository;
|
||||
import com.datamate.cleaning.interfaces.dto.OperatorInstanceDto;
|
||||
import com.datamate.cleaning.domain.model.entity.OperatorInstance;
|
||||
import com.datamate.operator.interfaces.dto.OperatorDto;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@@ -10,4 +11,6 @@ public interface OperatorInstanceRepository extends IRepository<OperatorInstance
|
||||
void insertInstance(String instanceId, List<OperatorInstanceDto> instances);
|
||||
|
||||
void deleteByInstanceId(String instanceId);
|
||||
|
||||
List<OperatorDto> findOperatorByInstanceId(String instanceId);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
package com.datamate.cleaning.infrastructure.converter;
|
||||
|
||||
import com.datamate.cleaning.domain.model.entity.CleaningResult;
|
||||
import com.datamate.cleaning.interfaces.dto.CleaningResultDto;
|
||||
import org.mapstruct.Mapper;
|
||||
import org.mapstruct.factory.Mappers;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
public interface CleaningResultConverter {
|
||||
CleaningResultConverter INSTANCE = Mappers.getMapper(CleaningResultConverter.class);
|
||||
|
||||
List<CleaningResultDto> convertEntityToDto(List<CleaningResult> cleaningResult);
|
||||
}
|
||||
@@ -2,10 +2,10 @@ package com.datamate.cleaning.infrastructure.converter;
|
||||
|
||||
|
||||
import com.datamate.cleaning.domain.model.entity.OperatorInstance;
|
||||
import com.datamate.cleaning.domain.model.entity.Operator;
|
||||
import com.datamate.cleaning.interfaces.dto.OperatorInstanceDto;
|
||||
import com.datamate.common.infrastructure.exception.BusinessException;
|
||||
import com.datamate.common.infrastructure.exception.SystemErrorCode;
|
||||
import com.datamate.operator.domain.model.OperatorView;
|
||||
import com.datamate.operator.interfaces.dto.OperatorDto;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
@@ -14,6 +14,8 @@ import org.mapstruct.Mapping;
|
||||
import org.mapstruct.Named;
|
||||
import org.mapstruct.factory.Mappers;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@@ -35,5 +37,16 @@ public interface OperatorInstanceConverter {
|
||||
}
|
||||
}
|
||||
|
||||
List<OperatorDto> fromEntityToDto(List<Operator> operator);
|
||||
@Mapping(target = "categories", source = "categories", qualifiedByName = "stringToList")
|
||||
OperatorDto fromEntityToDto(OperatorView operator);
|
||||
|
||||
List<OperatorDto> fromEntityToDto(List<OperatorView> operator);
|
||||
|
||||
@Named("stringToList")
|
||||
default List<String> stringToList(String input) {
|
||||
if (input == null || input.isEmpty()) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
return Arrays.stream(input.split(",")).toList();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,8 @@ package com.datamate.cleaning.infrastructure.httpclient;
|
||||
import com.datamate.common.infrastructure.exception.BusinessException;
|
||||
import com.datamate.common.infrastructure.exception.SystemErrorCode;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
@@ -13,24 +15,36 @@ import java.text.MessageFormat;
|
||||
import java.time.Duration;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class RuntimeClient {
|
||||
private static final String BASE_URL = "http://datamate-runtime:8081/api";
|
||||
private final String CREATE_TASK_URL = "/api/task/{0}/submit";
|
||||
|
||||
private static final String CREATE_TASK_URL = BASE_URL + "/task/{0}/submit";
|
||||
private final String STOP_TASK_URL = "/api/task/{0}/stop";
|
||||
|
||||
private static final String STOP_TASK_URL = BASE_URL + "/task/{0}/stop";
|
||||
@Value("${runtime.protocol:http}")
|
||||
private String protocol;
|
||||
|
||||
private static final HttpClient CLIENT = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build();
|
||||
@Value("${runtime.host:datamate-runtime}")
|
||||
private String host;
|
||||
|
||||
public static void submitTask(String taskId) {
|
||||
send(MessageFormat.format(CREATE_TASK_URL, taskId));
|
||||
@Value("${runtime.port:8081}")
|
||||
private int port;
|
||||
|
||||
private final HttpClient CLIENT = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build();
|
||||
|
||||
public void submitTask(String taskId) {
|
||||
send(MessageFormat.format(getRequestUrl(CREATE_TASK_URL), taskId));
|
||||
}
|
||||
|
||||
public static void stopTask(String taskId) {
|
||||
send(MessageFormat.format(STOP_TASK_URL, taskId));
|
||||
public void stopTask(String taskId) {
|
||||
send(MessageFormat.format(getRequestUrl(STOP_TASK_URL), taskId));
|
||||
}
|
||||
|
||||
private static void send(String url) {
|
||||
private String getRequestUrl(String url) {
|
||||
return protocol + "://" + host + ":" + port + url;
|
||||
}
|
||||
|
||||
private void send(String url) {
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(URI.create(url))
|
||||
.timeout(Duration.ofSeconds(30))
|
||||
|
||||
@@ -2,12 +2,18 @@ package com.datamate.cleaning.infrastructure.persistence.Impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.extension.repository.CrudRepository;
|
||||
import com.datamate.cleaning.common.enums.CleaningTaskStatusEnum;
|
||||
import com.datamate.cleaning.domain.model.entity.CleaningResult;
|
||||
import com.datamate.cleaning.domain.repository.CleaningResultRepository;
|
||||
import com.datamate.cleaning.infrastructure.converter.CleaningResultConverter;
|
||||
import com.datamate.cleaning.infrastructure.persistence.mapper.CleaningResultMapper;
|
||||
import com.datamate.cleaning.interfaces.dto.CleaningResultDto;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Repository
|
||||
@RequiredArgsConstructor
|
||||
public class CleaningResultRepositoryImpl extends CrudRepository<CleaningResultMapper, CleaningResult>
|
||||
@@ -22,9 +28,20 @@ public class CleaningResultRepositoryImpl extends CrudRepository<CleaningResultM
|
||||
}
|
||||
|
||||
@Override
|
||||
public int countByInstanceId(String instanceId) {
|
||||
public int[] countByInstanceId(String instanceId) {
|
||||
LambdaQueryWrapper<CleaningResult> lambdaWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaWrapper.eq(CleaningResult::getInstanceId, instanceId);
|
||||
return mapper.selectCount(lambdaWrapper).intValue();
|
||||
List<CleaningResult> cleaningResults = mapper.selectList(lambdaWrapper);
|
||||
int succeed = Math.toIntExact(cleaningResults.stream()
|
||||
.filter(result ->
|
||||
StringUtils.equals(result.getStatus(), CleaningTaskStatusEnum.COMPLETED.getValue()))
|
||||
.count());
|
||||
return new int[] {succeed, cleaningResults.size() - succeed};
|
||||
}
|
||||
|
||||
public List<CleaningResultDto> findByInstanceId(String instanceId) {
|
||||
LambdaQueryWrapper<CleaningResult> queryWrapper = new LambdaQueryWrapper<>();
|
||||
queryWrapper.eq(CleaningResult::getInstanceId, instanceId);
|
||||
return CleaningResultConverter.INSTANCE.convertEntityToDto(mapper.selectList(queryWrapper));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import com.datamate.cleaning.interfaces.dto.OperatorInstanceDto;
|
||||
import com.datamate.cleaning.domain.model.entity.OperatorInstance;
|
||||
import com.datamate.cleaning.domain.repository.OperatorInstanceRepository;
|
||||
import com.datamate.cleaning.infrastructure.persistence.mapper.OperatorInstanceMapper;
|
||||
import com.datamate.operator.interfaces.dto.OperatorDto;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
@@ -37,4 +38,8 @@ public class OperatorInstanceRepositoryImpl extends CrudRepository<OperatorInsta
|
||||
lambdaWrapper.eq(OperatorInstance::getInstanceId, instanceId);
|
||||
mapper.delete(lambdaWrapper);
|
||||
}
|
||||
|
||||
public List<OperatorDto> findOperatorByInstanceId(String instanceId) {
|
||||
return OperatorInstanceConverter.INSTANCE.fromEntityToDto(mapper.findOperatorByInstanceId(instanceId));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,9 +2,22 @@ package com.datamate.cleaning.infrastructure.persistence.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.datamate.cleaning.domain.model.entity.OperatorInstance;
|
||||
import com.datamate.operator.domain.model.OperatorView;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
||||
@Mapper
|
||||
public interface OperatorInstanceMapper extends BaseMapper<OperatorInstance> {
|
||||
@Select("SELECT o.operator_id as id, o.operator_name as name, description, version, inputs, outputs, runtime, " +
|
||||
" settings, created_at, updated_at, " +
|
||||
"GROUP_CONCAT(category_id ORDER BY created_at DESC SEPARATOR ',') AS categories " +
|
||||
"FROM t_operator_instance toi LEFT JOIN v_operator o ON toi.operator_id = o.operator_id " +
|
||||
"WHERE toi.instance_id = #{instanceId} " +
|
||||
"GROUP BY o.operator_id, o.operator_name, description, version, inputs, outputs, runtime," +
|
||||
" settings, created_at, updated_at, op_index " +
|
||||
"ORDER BY toi.op_index")
|
||||
List<OperatorView> findOperatorByInstanceId(String instanceId);
|
||||
}
|
||||
|
||||
@@ -16,23 +16,37 @@ import java.math.RoundingMode;
|
||||
public class CleaningProcess {
|
||||
private Float process;
|
||||
|
||||
private Float successRate;
|
||||
|
||||
private Integer totalFileNum;
|
||||
|
||||
private Integer succeedFileNum;
|
||||
|
||||
private Integer failedFileNum;
|
||||
|
||||
private Integer finishedFileNum;
|
||||
|
||||
public CleaningProcess(int totalFileNum, int finishedFileNum) {
|
||||
public CleaningProcess(int totalFileNum, int succeedFileNum, int failedFileNum) {
|
||||
this.totalFileNum = totalFileNum;
|
||||
this.finishedFileNum = finishedFileNum;
|
||||
this.succeedFileNum = succeedFileNum;
|
||||
this.failedFileNum = failedFileNum;
|
||||
this.finishedFileNum = succeedFileNum + failedFileNum;
|
||||
if (totalFileNum == 0) {
|
||||
this.process = 0.0f;
|
||||
} else {
|
||||
this.process = BigDecimal.valueOf(finishedFileNum * 100L)
|
||||
.divide(BigDecimal.valueOf(totalFileNum), 2, RoundingMode.HALF_UP).floatValue();
|
||||
}
|
||||
if (finishedFileNum == 0) {
|
||||
this.successRate = 0f;
|
||||
} else {
|
||||
this.successRate = BigDecimal.valueOf(succeedFileNum * 100L)
|
||||
.divide(BigDecimal.valueOf(finishedFileNum), 2, RoundingMode.HALF_UP).floatValue();
|
||||
}
|
||||
}
|
||||
|
||||
public static CleaningProcess of(int totalFileNum, int finishedFileNum) {
|
||||
return new CleaningProcess(totalFileNum, finishedFileNum);
|
||||
public static CleaningProcess of(int totalFileNum, int succeedFileNum, int failedFileNum) {
|
||||
return new CleaningProcess(totalFileNum, succeedFileNum, failedFileNum);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package com.datamate.cleaning.interfaces.dto;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public class CleaningResultDto {
|
||||
private String instanceId;
|
||||
|
||||
private String srcFileId;
|
||||
|
||||
private String destFileId;
|
||||
|
||||
private String srcName;
|
||||
|
||||
private String destName;
|
||||
|
||||
private String srcType;
|
||||
|
||||
private String destType;
|
||||
|
||||
private long srcSize;
|
||||
|
||||
private long destSize;
|
||||
|
||||
private String status;
|
||||
|
||||
private String result;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package com.datamate.cleaning.interfaces.dto;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public class CleaningTaskLog {
|
||||
private String level;
|
||||
|
||||
private String message;
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
package com.datamate.cleaning.interfaces.rest;
|
||||
|
||||
import com.datamate.cleaning.application.CleaningTaskService;
|
||||
import com.datamate.cleaning.interfaces.dto.CleaningResultDto;
|
||||
import com.datamate.cleaning.interfaces.dto.CleaningTaskDto;
|
||||
import com.datamate.cleaning.interfaces.dto.CleaningTaskLog;
|
||||
import com.datamate.cleaning.interfaces.dto.CreateCleaningTaskRequest;
|
||||
import com.datamate.common.interfaces.PagedResponse;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@@ -54,4 +56,14 @@ public class CleaningTaskController {
|
||||
cleaningTaskService.deleteTask(taskId);
|
||||
return taskId;
|
||||
}
|
||||
|
||||
@GetMapping("/{taskId}/result")
|
||||
public List<CleaningResultDto> cleaningTasksTaskIdGetResult(@PathVariable("taskId") String taskId) {
|
||||
return cleaningTaskService.getTaskResults(taskId);
|
||||
}
|
||||
|
||||
@GetMapping("/{taskId}/log")
|
||||
public List<CleaningTaskLog> cleaningTasksTaskIdGetLog(@PathVariable("taskId") String taskId) {
|
||||
return cleaningTaskService.getTaskLog(taskId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,9 +50,7 @@ public class OperatorService {
|
||||
|
||||
public List<OperatorDto> getOperators(Integer page, Integer size, List<String> categories,
|
||||
String operatorName, Boolean isStar) {
|
||||
List<OperatorView> filteredOperators = operatorViewRepo.findOperatorsByCriteria(page, size, operatorName,
|
||||
categories, isStar);
|
||||
return filteredOperators.stream().map(OperatorConverter.INSTANCE::fromEntityToDto).toList();
|
||||
return operatorViewRepo.findOperatorsByCriteria(page, size, operatorName, categories, isStar);
|
||||
}
|
||||
|
||||
public int getOperatorsCount(List<String> categories, String operatorName, Boolean isStar) {
|
||||
|
||||
@@ -2,12 +2,13 @@ package com.datamate.operator.domain.repository;
|
||||
|
||||
import com.baomidou.mybatisplus.extension.repository.IRepository;
|
||||
import com.datamate.operator.domain.model.OperatorView;
|
||||
import com.datamate.operator.interfaces.dto.OperatorDto;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface OperatorViewRepository extends IRepository<OperatorView> {
|
||||
List<OperatorView> findOperatorsByCriteria(Integer page, Integer size, String operatorName,
|
||||
List<String> categories, Boolean isStar);
|
||||
List<OperatorDto> findOperatorsByCriteria(Integer page, Integer size, String operatorName,
|
||||
List<String> categories, Boolean isStar);
|
||||
|
||||
Integer countOperatorsByCriteria(String operatorName, List<String> categories, Boolean isStar);
|
||||
|
||||
|
||||
@@ -21,6 +21,8 @@ public interface OperatorConverter {
|
||||
@Mapping(target = "categories", source = "categories", qualifiedByName = "stringToList")
|
||||
OperatorDto fromEntityToDto(OperatorView operator);
|
||||
|
||||
List<OperatorDto> fromEntityViewToDto(List<OperatorView> operator);
|
||||
|
||||
List<OperatorDto> fromEntityToDto(List<Operator> operator);
|
||||
|
||||
@Named("stringToList")
|
||||
|
||||
@@ -7,7 +7,9 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import com.baomidou.mybatisplus.extension.repository.CrudRepository;
|
||||
import com.datamate.operator.domain.model.OperatorView;
|
||||
import com.datamate.operator.domain.repository.OperatorViewRepository;
|
||||
import com.datamate.operator.infrastructure.converter.OperatorConverter;
|
||||
import com.datamate.operator.infrastructure.persistence.mapper.OperatorViewMapper;
|
||||
import com.datamate.operator.interfaces.dto.OperatorDto;
|
||||
import io.micrometer.common.util.StringUtils;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
@@ -21,20 +23,23 @@ public class OperatorViewRepositoryImpl extends CrudRepository<OperatorViewMappe
|
||||
private final OperatorViewMapper mapper;
|
||||
|
||||
@Override
|
||||
public List<OperatorView> findOperatorsByCriteria(Integer page, Integer size, String operatorName,
|
||||
List<String> categories, Boolean isStar) {
|
||||
public List<OperatorDto> findOperatorsByCriteria(Integer page, Integer size, String operatorName,
|
||||
List<String> categories, Boolean isStar) {
|
||||
QueryWrapper<OperatorView> queryWrapper = Wrappers.query();
|
||||
queryWrapper.in(CollectionUtils.isNotEmpty(categories), "category_id", categories)
|
||||
.like(StringUtils.isNotBlank(operatorName), "operator_name", operatorName)
|
||||
.eq(isStar != null, "is_star", isStar)
|
||||
.groupBy("operator_id")
|
||||
.orderByDesc("created_at");
|
||||
Page<OperatorView> queryPage = null;
|
||||
Page<OperatorView> queryPage;
|
||||
if (size != null && page != null) {
|
||||
queryPage = new Page<>(page + 1, size);
|
||||
} else {
|
||||
queryPage = new Page<>(1, -1);
|
||||
}
|
||||
IPage<OperatorView> operators = mapper.findOperatorsByCriteria(queryPage, queryWrapper);
|
||||
return operators.getRecords();
|
||||
|
||||
return OperatorConverter.INSTANCE.fromEntityViewToDto(operators.getRecords());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -1,13 +1,18 @@
|
||||
import { useState } from "react";
|
||||
import { Card, Button, Steps, Form, Divider } from "antd";
|
||||
import { Link, useNavigate } from "react-router";
|
||||
import {useEffect, useState} from "react";
|
||||
import {Button, Steps, Form, message} from "antd";
|
||||
import {Link, useNavigate, useParams} from "react-router";
|
||||
|
||||
import { ArrowLeft } from "lucide-react";
|
||||
import { createCleaningTemplateUsingPost } from "../cleansing.api";
|
||||
import {
|
||||
createCleaningTemplateUsingPost,
|
||||
queryCleaningTemplateByIdUsingGet,
|
||||
updateCleaningTemplateByIdUsingPut
|
||||
} from "../cleansing.api";
|
||||
import CleansingTemplateStepOne from "./components/CreateTemplateStepOne";
|
||||
import { useCreateStepTwo } from "./hooks/useCreateStepTwo";
|
||||
|
||||
export default function CleansingTemplateCreate() {
|
||||
const { id = "" } = useParams()
|
||||
const navigate = useNavigate();
|
||||
const [form] = Form.useForm();
|
||||
const [templateConfig, setTemplateConfig] = useState({
|
||||
@@ -15,6 +20,21 @@ export default function CleansingTemplateCreate() {
|
||||
description: "",
|
||||
});
|
||||
|
||||
const fetchTemplateDetail = async () => {
|
||||
if (!id) return;
|
||||
try {
|
||||
const { data } = await queryCleaningTemplateByIdUsingGet(id);
|
||||
setTemplateConfig(data);
|
||||
} catch (error) {
|
||||
message.error("获取任务详情失败");
|
||||
navigate("/data/cleansing");
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
fetchTemplateDetail()
|
||||
}, [id]);
|
||||
|
||||
const handleSave = async () => {
|
||||
const template = {
|
||||
...templateConfig,
|
||||
@@ -27,7 +47,8 @@ export default function CleansingTemplateCreate() {
|
||||
})),
|
||||
};
|
||||
|
||||
await createCleaningTemplateUsingPost(template);
|
||||
!id && await createCleaningTemplateUsingPost(template) && message.success("模板创建成功");
|
||||
id && await updateCleaningTemplateByIdUsingPut(id, template) && message.success("模板更新成功");
|
||||
navigate("/data/cleansing?view=template");
|
||||
};
|
||||
|
||||
@@ -79,7 +100,7 @@ export default function CleansingTemplateCreate() {
|
||||
<ArrowLeft className="w-4 h-4 mr-1" />
|
||||
</Button>
|
||||
</Link>
|
||||
<h1 className="text-xl font-bold">创建清洗模板</h1>
|
||||
<h1 className="text-xl font-bold">{id ? '更新清洗模板' : '创建清洗模板'}</h1>
|
||||
</div>
|
||||
<div className="w-1/2">
|
||||
<Steps
|
||||
@@ -101,7 +122,7 @@ export default function CleansingTemplateCreate() {
|
||||
onClick={handleSave}
|
||||
disabled={!canProceed()}
|
||||
>
|
||||
创建模板
|
||||
{id ? '更新模板' : '创建模板'}
|
||||
</Button>
|
||||
) : (
|
||||
<Button
|
||||
@@ -1,4 +1,5 @@
|
||||
import { Input, Form } from "antd";
|
||||
import {useEffect} from "react";
|
||||
|
||||
const { TextArea } = Input;
|
||||
|
||||
@@ -16,6 +17,11 @@ export default function CreateTemplateStepOne({
|
||||
const handleValuesChange = (_, allValues) => {
|
||||
setTemplateConfig({ ...templateConfig, ...allValues });
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
form.setFieldsValue(templateConfig);
|
||||
}, [templateConfig]);
|
||||
|
||||
return (
|
||||
<Form
|
||||
form={form}
|
||||
|
||||
@@ -1,17 +1,9 @@
|
||||
import React, { useMemo, useState } from "react";
|
||||
import {
|
||||
Card,
|
||||
Input,
|
||||
Select,
|
||||
Tooltip,
|
||||
Collapse,
|
||||
Tag,
|
||||
Checkbox,
|
||||
Button,
|
||||
} from "antd";
|
||||
import { StarFilled, StarOutlined, SearchOutlined } from "@ant-design/icons";
|
||||
import { CategoryI, OperatorI } from "@/pages/OperatorMarket/operator.model";
|
||||
import { Layers } from "lucide-react";
|
||||
import React, {useEffect, useMemo, useState} from "react";
|
||||
import {Button, Card, Checkbox, Collapse, Input, Select, Tag, Tooltip,} from "antd";
|
||||
import {SearchOutlined, StarFilled, StarOutlined} from "@ant-design/icons";
|
||||
import {CategoryI, OperatorI} from "@/pages/OperatorMarket/operator.model";
|
||||
import {Layers} from "lucide-react";
|
||||
import {updateOperatorByIdUsingPut} from "@/pages/OperatorMarket/operator.api.ts";
|
||||
|
||||
interface OperatorListProps {
|
||||
operators: OperatorI[];
|
||||
@@ -27,12 +19,20 @@ interface OperatorListProps {
|
||||
) => void;
|
||||
}
|
||||
|
||||
const handleStar = async (operator: OperatorI, toggleFavorite: (id: string) => void) => {
|
||||
const data = {
|
||||
id: operator.id,
|
||||
isStar: !operator.isStar
|
||||
};
|
||||
await updateOperatorByIdUsingPut(operator.id, data);
|
||||
toggleFavorite(operator.id)
|
||||
}
|
||||
|
||||
const OperatorList: React.FC<OperatorListProps> = ({
|
||||
operators,
|
||||
favorites,
|
||||
toggleFavorite,
|
||||
toggleOperator,
|
||||
showPoppular,
|
||||
selectedOperators,
|
||||
onDragOperator,
|
||||
}) => (
|
||||
@@ -56,17 +56,9 @@ const OperatorList: React.FC<OperatorListProps> = ({
|
||||
{operator.name}
|
||||
</span>
|
||||
</div>
|
||||
{showPoppular && operator.isStar && (
|
||||
<Tag color="gold" className="text-xs">
|
||||
热门
|
||||
</Tag>
|
||||
)}
|
||||
<span
|
||||
className="cursor-pointer"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
toggleFavorite(operator.id);
|
||||
}}
|
||||
onClick={() => handleStar(operator, toggleFavorite)}
|
||||
>
|
||||
{favorites.has(operator.id) ? (
|
||||
<StarFilled style={{ color: "#FFD700" }} />
|
||||
@@ -156,10 +148,9 @@ const OperatorLibrary: React.FC<OperatorLibraryProps> = ({
|
||||
|
||||
// 过滤算子
|
||||
const filteredOperators = useMemo(() => {
|
||||
const filtered = Object.values(groupedOperators).flatMap(
|
||||
return Object.values(groupedOperators).flatMap(
|
||||
(category) => category.operators
|
||||
);
|
||||
return filtered;
|
||||
}, [groupedOperators]);
|
||||
|
||||
// 收藏切换
|
||||
@@ -173,6 +164,18 @@ const OperatorLibrary: React.FC<OperatorLibraryProps> = ({
|
||||
setFavorites(newFavorites);
|
||||
};
|
||||
|
||||
const fetchFavorite = async () => {
|
||||
const newFavorites = new Set(favorites);
|
||||
operatorList.forEach(item => {
|
||||
item.isStar && newFavorites.add(item.id);
|
||||
});
|
||||
setFavorites(newFavorites);
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
fetchFavorite()
|
||||
}, [operatorList]);
|
||||
|
||||
// 全选分类算子
|
||||
const handleSelectAll = (operators: OperatorI[]) => {
|
||||
const newSelected = [...selectedOperators];
|
||||
@@ -257,7 +260,6 @@ const OperatorLibrary: React.FC<OperatorLibraryProps> = ({
|
||||
}
|
||||
>
|
||||
<OperatorList
|
||||
showPoppular
|
||||
selectedOperators={selectedOperators}
|
||||
operators={category.operators}
|
||||
favorites={favorites}
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
import React, { useState } from "react";
|
||||
import React, {useMemo, useState} from "react";
|
||||
import { Card, Input, Tag, Select, Button } from "antd";
|
||||
import { DeleteOutlined } from "@ant-design/icons";
|
||||
import { CleansingTemplate } from "../../cleansing.model";
|
||||
import { Workflow } from "lucide-react";
|
||||
import { OperatorI } from "@/pages/OperatorMarket/operator.model";
|
||||
import {CategoryI, OperatorI} from "@/pages/OperatorMarket/operator.model";
|
||||
|
||||
interface OperatorFlowProps {
|
||||
selectedOperators: OperatorI[];
|
||||
configOperator: OperatorI | null;
|
||||
templates: CleansingTemplate[];
|
||||
currentTemplate: CleansingTemplate | null;
|
||||
categoryOptions: [];
|
||||
setCurrentTemplate: (template: CleansingTemplate | null) => void;
|
||||
removeOperator: (id: string) => void;
|
||||
setSelectedOperators: (operators: OperatorI[]) => void;
|
||||
@@ -33,6 +34,7 @@ const OperatorFlow: React.FC<OperatorFlowProps> = ({
|
||||
configOperator,
|
||||
templates,
|
||||
currentTemplate,
|
||||
categoryOptions,
|
||||
setSelectedOperators,
|
||||
setConfigOperator,
|
||||
removeOperator,
|
||||
@@ -47,6 +49,16 @@ const OperatorFlow: React.FC<OperatorFlowProps> = ({
|
||||
}) => {
|
||||
const [editingIndex, setEditingIndex] = useState<string | null>(null);
|
||||
|
||||
const categoryMap = useMemo(() => {
|
||||
const map: { [key: string]: CategoryI } = {};
|
||||
categoryOptions.forEach((cat: any) => {
|
||||
map[cat.id] = {
|
||||
...cat,
|
||||
};
|
||||
});
|
||||
return map;
|
||||
}, [categoryOptions]);
|
||||
|
||||
// 添加编号修改处理函数
|
||||
const handleIndexChange = (operatorId: string, newIndex: string) => {
|
||||
const index = Number.parseInt(newIndex);
|
||||
@@ -167,8 +179,9 @@ const OperatorFlow: React.FC<OperatorFlowProps> = ({
|
||||
{operator.name}
|
||||
</span>
|
||||
</div>
|
||||
{/* 分类标签 */}
|
||||
<Tag color="default">分类</Tag>
|
||||
{operator?.categories?.map((categoryId) => {
|
||||
return <Tag color="default">{categoryMap[categoryId].name}</Tag>
|
||||
})}
|
||||
{/* 参数状态指示 */}
|
||||
{Object.values(operator.configs).some(
|
||||
(param: any) =>
|
||||
@@ -192,7 +205,7 @@ const OperatorFlow: React.FC<OperatorFlowProps> = ({
|
||||
))}
|
||||
{selectedOperators.length === 0 && (
|
||||
<div className="text-center py-16 text-gray-400 border-2 border-dashed border-gray-100 rounded-lg">
|
||||
<Workflow className="w-full w-10 h-10 mb-4 opacity-50" />
|
||||
<Workflow className="w-full h-10 mb-4 opacity-50" />
|
||||
<div className="text-lg font-medium mb-2">开始构建您的算子流程</div>
|
||||
<div className="text-sm">
|
||||
从左侧算子库拖拽算子到此处,或点击算子添加
|
||||
|
||||
@@ -55,6 +55,7 @@ export function useCreateStepTwo() {
|
||||
configOperator={configOperator}
|
||||
templates={templates}
|
||||
currentTemplate={currentTemplate}
|
||||
categoryOptions={categoryOptions}
|
||||
setSelectedOperators={setSelectedOperators}
|
||||
setConfigOperator={setConfigOperator}
|
||||
setCurrentTemplate={setCurrentTemplate}
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { OperatorI } from "@/pages/OperatorMarket/operator.model";
|
||||
import { CleansingTemplate } from "../../cleansing.model";
|
||||
import { queryCleaningTemplatesUsingGet } from "../../cleansing.api";
|
||||
import {queryCleaningTemplateByIdUsingGet, queryCleaningTemplatesUsingGet} from "../../cleansing.api";
|
||||
import {
|
||||
queryCategoryTreeUsingGet,
|
||||
queryOperatorsUsingPost,
|
||||
} from "@/pages/OperatorMarket/operator.api";
|
||||
import {useParams} from "react-router";
|
||||
|
||||
export function useOperatorOperations() {
|
||||
const { id = "" } = useParams();
|
||||
const [currentStep, setCurrentStep] = useState(1);
|
||||
|
||||
const [operators, setOperators] = useState<OperatorI[]>([]);
|
||||
@@ -21,7 +23,7 @@ export function useOperatorOperations() {
|
||||
// 将后端返回的算子数据映射为前端需要的格式
|
||||
const mapOperator = (op: OperatorI) => {
|
||||
const configs =
|
||||
op.settings && typeof op.settings === "string"
|
||||
op.settings
|
||||
? JSON.parse(op.settings)
|
||||
: {};
|
||||
const defaultParams: Record<string, string> = {};
|
||||
@@ -64,14 +66,26 @@ export function useOperatorOperations() {
|
||||
};
|
||||
|
||||
const initTemplates = async () => {
|
||||
const { data } = await queryCleaningTemplatesUsingGet();
|
||||
const newTemplates =
|
||||
data.content?.map?.((item) => ({
|
||||
...item,
|
||||
label: item.name,
|
||||
value: item.id,
|
||||
})) || [];
|
||||
setTemplates(newTemplates);
|
||||
if (id) {
|
||||
const { data } = await queryCleaningTemplateByIdUsingGet(id);
|
||||
const template = {
|
||||
...data,
|
||||
label: data.name,
|
||||
value: data.id,
|
||||
}
|
||||
setTemplates([template])
|
||||
setCurrentTemplate(template)
|
||||
} else {
|
||||
const { data } = await queryCleaningTemplatesUsingGet();
|
||||
const newTemplates =
|
||||
data.content?.map?.((item) => ({
|
||||
...item,
|
||||
label: item.name,
|
||||
value: item.id,
|
||||
})) || [];
|
||||
setTemplates(newTemplates);
|
||||
setCurrentTemplate(newTemplates?.[0])
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
|
||||
@@ -1,29 +1,30 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { Card, Breadcrumb, App } from "antd";
|
||||
import {Breadcrumb, App, Tabs} from "antd";
|
||||
import {
|
||||
Play,
|
||||
Pause,
|
||||
Clock,
|
||||
CheckCircle,
|
||||
AlertCircle,
|
||||
Database,
|
||||
Trash2,
|
||||
Activity,
|
||||
Activity, LayoutList,
|
||||
} from "lucide-react";
|
||||
import DetailHeader from "@/components/DetailHeader";
|
||||
import { Link, useNavigate, useParams } from "react-router";
|
||||
import {
|
||||
deleteCleaningTaskByIdUsingDelete,
|
||||
executeCleaningTaskUsingPost,
|
||||
queryCleaningTaskByIdUsingGet,
|
||||
queryCleaningTaskByIdUsingGet, queryCleaningTaskLogByIdUsingGet, queryCleaningTaskResultByIdUsingGet,
|
||||
stopCleaningTaskUsingPost,
|
||||
} from "../cleansing.api";
|
||||
import { TaskStatusMap } from "../cleansing.const";
|
||||
import { TaskStatus } from "@/pages/DataCleansing/cleansing.model";
|
||||
import {mapTask, TaskStatusMap} from "../cleansing.const";
|
||||
import {CleansingResult, TaskStatus} from "@/pages/DataCleansing/cleansing.model";
|
||||
import BasicInfo from "./components/BasicInfo";
|
||||
import OperatorTable from "./components/OperatorTable";
|
||||
import FileTable from "./components/FileTable";
|
||||
import LogsTable from "./components/LogsTable";
|
||||
import {formatExecutionDuration} from "@/utils/unit.ts";
|
||||
import {ReloadOutlined} from "@ant-design/icons";
|
||||
|
||||
// 任务详情页面组件
|
||||
export default function CleansingTaskDetail() {
|
||||
@@ -35,7 +36,7 @@ export default function CleansingTaskDetail() {
|
||||
if (!id) return;
|
||||
try {
|
||||
const { data } = await queryCleaningTaskByIdUsingGet(id);
|
||||
setTask(data);
|
||||
setTask(mapTask(data));
|
||||
} catch (error) {
|
||||
message.error("获取任务详情失败");
|
||||
navigate("/data/cleansing");
|
||||
@@ -60,6 +61,38 @@ export default function CleansingTaskDetail() {
|
||||
navigate("/data/cleansing");
|
||||
};
|
||||
|
||||
const [result, setResult] = useState<CleansingResult[]>();
|
||||
|
||||
const fetchTaskResult = async () => {
|
||||
if (!id) return;
|
||||
try {
|
||||
const { data } = await queryCleaningTaskResultByIdUsingGet(id);
|
||||
setResult(data);
|
||||
} catch (error) {
|
||||
message.error("获取清洗结果失败");
|
||||
navigate("/data/cleansing/task-detail/" + id);
|
||||
}
|
||||
};
|
||||
|
||||
const [taskLog, setTaskLog] = useState();
|
||||
|
||||
const fetchTaskLog = async () => {
|
||||
if (!id) return;
|
||||
try {
|
||||
const { data } = await queryCleaningTaskLogByIdUsingGet(id);
|
||||
setTaskLog(data);
|
||||
} catch (error) {
|
||||
message.error("获取清洗日志失败");
|
||||
navigate("/data/cleansing/task-detail/" + id);
|
||||
}
|
||||
};
|
||||
|
||||
const handleRefresh = async () => {
|
||||
fetchTaskDetail();
|
||||
{activeTab === "files" && await fetchTaskResult()}
|
||||
{activeTab === "logs" && await fetchTaskLog()}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
fetchTaskDetail();
|
||||
}, [id]);
|
||||
@@ -69,9 +102,9 @@ export default function CleansingTaskDetail() {
|
||||
|
||||
const headerData = {
|
||||
...task,
|
||||
icon: <Database className="w-8 h-8" />,
|
||||
icon: <LayoutList className="w-8 h-8" />,
|
||||
status: TaskStatusMap[task?.status],
|
||||
createdAt: task?.startTime,
|
||||
createdAt: task?.createdAt,
|
||||
lastUpdated: task?.updatedAt,
|
||||
};
|
||||
|
||||
@@ -79,22 +112,24 @@ export default function CleansingTaskDetail() {
|
||||
{
|
||||
icon: <Clock className="w-4 h-4 text-blue-500" />,
|
||||
label: "总耗时",
|
||||
value: task?.duration || "--",
|
||||
value: formatExecutionDuration(task?.startedAt, task?.finishedAt) || "--",
|
||||
},
|
||||
{
|
||||
icon: <CheckCircle className="w-4 h-4 text-green-500" />,
|
||||
label: "成功文件",
|
||||
value: task?.successFiles || "--",
|
||||
value: task?.progress?.succeedFileNum || "0",
|
||||
},
|
||||
{
|
||||
icon: <AlertCircle className="w-4 h-4 text-red-500" />,
|
||||
label: "失败文件",
|
||||
value: task?.failedFiles || "--",
|
||||
value: (task?.status.value === TaskStatus.RUNNING || task?.status.value === TaskStatus.PENDING) ?
|
||||
task?.progress.failedFileNum :
|
||||
task?.progress?.totalFileNum - task?.progress.succeedFileNum,
|
||||
},
|
||||
{
|
||||
icon: <Activity className="w-4 h-4 text-purple-500" />,
|
||||
label: "成功率",
|
||||
value: `${task?.progress}%`,
|
||||
value: task?.progress?.successRate ? task?.progress?.successRate + "%" : "--",
|
||||
},
|
||||
];
|
||||
|
||||
@@ -109,7 +144,7 @@ export default function CleansingTaskDetail() {
|
||||
},
|
||||
]
|
||||
: []),
|
||||
...(task?.status === TaskStatus.PENDING
|
||||
...([TaskStatus.PENDING, TaskStatus.STOPPED, TaskStatus.FAILED].includes(task?.status?.value)
|
||||
? [
|
||||
{
|
||||
key: "start",
|
||||
@@ -119,6 +154,12 @@ export default function CleansingTaskDetail() {
|
||||
},
|
||||
]
|
||||
: []),
|
||||
{
|
||||
key: "refresh",
|
||||
label: "更新任务",
|
||||
icon: <ReloadOutlined className="w-4 h-4" />,
|
||||
onClick: handleRefresh,
|
||||
},
|
||||
{
|
||||
key: "delete",
|
||||
label: "删除任务",
|
||||
@@ -131,20 +172,20 @@ export default function CleansingTaskDetail() {
|
||||
const tabList = [
|
||||
{
|
||||
key: "basic",
|
||||
tab: "基本信息",
|
||||
children: <BasicInfo task={task} />,
|
||||
label: "基本信息",
|
||||
},
|
||||
{
|
||||
key: "operators",
|
||||
tab: "处理算子",
|
||||
children: <OperatorTable task={task} />,
|
||||
label: "处理算子",
|
||||
},
|
||||
{
|
||||
key: "files",
|
||||
tab: "处理文件",
|
||||
children: <FileTable task={task} />,
|
||||
label: "处理文件",
|
||||
},
|
||||
{
|
||||
key: "logs",
|
||||
label: "运行日志",
|
||||
},
|
||||
{ key: "logs", tab: "运行日志", children: <LogsTable task={task} /> },
|
||||
];
|
||||
|
||||
const breadItems = [
|
||||
@@ -157,7 +198,7 @@ export default function CleansingTaskDetail() {
|
||||
];
|
||||
|
||||
return (
|
||||
<div className="min-h-screen">
|
||||
<>
|
||||
<Breadcrumb items={breadItems} />
|
||||
<div className="mb-4 mt-4">
|
||||
<DetailHeader
|
||||
@@ -166,11 +207,17 @@ export default function CleansingTaskDetail() {
|
||||
operations={operations}
|
||||
/>
|
||||
</div>
|
||||
<Card
|
||||
tabList={tabList}
|
||||
activeTabKey={activeTab}
|
||||
onTabChange={setActiveTab}
|
||||
></Card>
|
||||
</div>
|
||||
<div className="flex-overflow-auto p-6 pt-2 bg-white rounded-md shadow">
|
||||
<Tabs activeKey={activeTab} items={tabList} onChange={setActiveTab} />
|
||||
<div className="h-full flex-1 overflow-auto">
|
||||
{activeTab === "basic" && (
|
||||
<BasicInfo task={task} />
|
||||
)}
|
||||
{activeTab === "operators" && <OperatorTable task={task} />}
|
||||
{activeTab === "files" && <FileTable result={result} fetchTaskResult={fetchTaskResult} />}
|
||||
{activeTab === "logs" && <LogsTable taskLog={taskLog} fetchTaskLog={fetchTaskLog} />}
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
122
frontend/src/pages/DataCleansing/Detail/TemplateDetail.tsx
Normal file
122
frontend/src/pages/DataCleansing/Detail/TemplateDetail.tsx
Normal file
@@ -0,0 +1,122 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import {Breadcrumb, App, Tabs} from "antd";
|
||||
import {
|
||||
Trash2,
|
||||
LayoutList,
|
||||
} from "lucide-react";
|
||||
import DetailHeader from "@/components/DetailHeader";
|
||||
import { Link, useNavigate, useParams } from "react-router";
|
||||
import {
|
||||
deleteCleaningTemplateByIdUsingDelete,
|
||||
queryCleaningTemplateByIdUsingGet,
|
||||
} from "../cleansing.api";
|
||||
import {mapTemplate} from "../cleansing.const";
|
||||
import OperatorTable from "./components/OperatorTable";
|
||||
import {EditOutlined, ReloadOutlined, NumberOutlined} from "@ant-design/icons";
|
||||
|
||||
// 任务详情页面组件
|
||||
export default function CleansingTemplateDetail() {
|
||||
const { id = "" } = useParams(); // 获取动态路由参数
|
||||
const { message } = App.useApp();
|
||||
const navigate = useNavigate();
|
||||
const [template, setTemplate] = useState();
|
||||
|
||||
const fetchTemplateDetail = async () => {
|
||||
if (!id) return;
|
||||
try {
|
||||
const { data } = await queryCleaningTemplateByIdUsingGet(id);
|
||||
setTemplate(mapTemplate(data));
|
||||
} catch (error) {
|
||||
message.error("获取任务详情失败");
|
||||
navigate("/data/cleansing");
|
||||
}
|
||||
};
|
||||
|
||||
const deleteTemplate = async () => {
|
||||
await deleteCleaningTemplateByIdUsingDelete(id);
|
||||
message.success("模板已删除");
|
||||
navigate("/data/cleansing");
|
||||
};
|
||||
|
||||
const handleRefresh = async () => {
|
||||
fetchTemplateDetail();
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
fetchTemplateDetail();
|
||||
}, [id]);
|
||||
|
||||
const [activeTab, setActiveTab] = useState("operators");
|
||||
|
||||
const headerData = {
|
||||
...template,
|
||||
icon: <LayoutList className="w-8 h-8" />,
|
||||
createdAt: template?.createdAt,
|
||||
lastUpdated: template?.updatedAt,
|
||||
};
|
||||
|
||||
const statistics = [
|
||||
{
|
||||
icon: <NumberOutlined className="w-4 h-4 text-green-500" />,
|
||||
label: "算子数量",
|
||||
value: template?.instance?.length || 0,
|
||||
},
|
||||
];
|
||||
|
||||
const operations = [
|
||||
{
|
||||
key: "update",
|
||||
label: "更新任务",
|
||||
icon: <EditOutlined className="w-4 h-4" />,
|
||||
onClick: () => navigate(`/data/cleansing/update-template/${id}`),
|
||||
},
|
||||
{
|
||||
key: "refresh",
|
||||
label: "更新任务",
|
||||
icon: <ReloadOutlined className="w-4 h-4" />,
|
||||
onClick: handleRefresh,
|
||||
},
|
||||
{
|
||||
key: "delete",
|
||||
label: "删除任务",
|
||||
icon: <Trash2 className="w-4 h-4" />,
|
||||
danger: true,
|
||||
onClick: deleteTemplate,
|
||||
},
|
||||
];
|
||||
|
||||
const tabList = [
|
||||
{
|
||||
key: "operators",
|
||||
label: "处理算子",
|
||||
},
|
||||
];
|
||||
|
||||
const breadItems = [
|
||||
{
|
||||
title: <Link to="/data/cleansing">数据清洗</Link>,
|
||||
},
|
||||
{
|
||||
title: "模板详情",
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
<>
|
||||
<Breadcrumb items={breadItems} />
|
||||
<div className="mb-4 mt-4">
|
||||
<DetailHeader
|
||||
data={headerData}
|
||||
statistics={statistics}
|
||||
operations={operations}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex-overflow-auto p-6 pt-2 bg-white rounded-md shadow">
|
||||
<Tabs activeKey={activeTab} items={tabList} onChange={setActiveTab} />
|
||||
<div className="h-full flex-1 overflow-auto">
|
||||
<OperatorTable task={template} />
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
import type { CleansingTask } from "@/pages/DataCleansing/cleansing.model";
|
||||
import { OperatorI } from "@/pages/OperatorMarket/operator.model";
|
||||
import { Button, Card, Descriptions, Progress, Tag } from "antd";
|
||||
import {CleansingTask, TaskStatus} from "@/pages/DataCleansing/cleansing.model";
|
||||
import { Button, Card, Descriptions, Progress } from "antd";
|
||||
import { Activity, AlertCircle, CheckCircle, Clock } from "lucide-react";
|
||||
import { useNavigate } from "react-router";
|
||||
import {formatExecutionDuration} from "@/utils/unit.ts";
|
||||
|
||||
export default function BasicInfo({ task }: { task: CleansingTask }) {
|
||||
const navigate = useNavigate();
|
||||
@@ -11,7 +11,7 @@ export default function BasicInfo({ task }: { task: CleansingTask }) {
|
||||
{
|
||||
key: "id",
|
||||
label: "任务ID",
|
||||
children: <span className="font-mono">#{task?.id}</span>,
|
||||
children: <span className="font-mono">{task?.id}</span>,
|
||||
},
|
||||
{ key: "name", label: "任务名称", children: task?.name },
|
||||
{
|
||||
@@ -19,6 +19,7 @@ export default function BasicInfo({ task }: { task: CleansingTask }) {
|
||||
label: "源数据集",
|
||||
children: (
|
||||
<Button
|
||||
style={{ paddingLeft: 0, marginLeft: 0 }}
|
||||
type="link"
|
||||
size="small"
|
||||
onClick={() =>
|
||||
@@ -34,6 +35,7 @@ export default function BasicInfo({ task }: { task: CleansingTask }) {
|
||||
label: "目标数据集",
|
||||
children: (
|
||||
<Button
|
||||
style={{ paddingLeft: 0, marginLeft: 0 }}
|
||||
type="link"
|
||||
size="small"
|
||||
onClick={() =>
|
||||
@@ -44,26 +46,12 @@ export default function BasicInfo({ task }: { task: CleansingTask }) {
|
||||
</Button>
|
||||
),
|
||||
},
|
||||
{ key: "template", label: "使用模板", children: task?.template },
|
||||
{ key: "startTime", label: "开始时间", children: task?.startedAt },
|
||||
{ key: "estimatedTime", label: "预计用时", children: task?.estimatedTime },
|
||||
{
|
||||
key: "description",
|
||||
label: "任务描述",
|
||||
children: (
|
||||
<span className="text-gray-600">{task?.description || "暂无描述"}</span>
|
||||
),
|
||||
span: 2,
|
||||
},
|
||||
{
|
||||
key: "rules",
|
||||
label: "处理算子",
|
||||
children: (
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{task?.instance?.map?.((op: OperatorI) => (
|
||||
<Tag key={op.id}>{op.name}</Tag>
|
||||
))}
|
||||
</div>
|
||||
<span className="text-gray-600">{task?.description || "--"}</span>
|
||||
),
|
||||
span: 2,
|
||||
},
|
||||
@@ -77,28 +65,30 @@ export default function BasicInfo({ task }: { task: CleansingTask }) {
|
||||
<div className="text-center p-4 bg-gradient-to-br from-blue-50 to-blue-100 rounded-lg">
|
||||
<Clock className="w-8 h-8 text-blue-500 mb-2 mx-auto" />
|
||||
<div className="text-xl font-bold text-blue-500">
|
||||
{task?.duration || "--"}
|
||||
{formatExecutionDuration(task?.startedAt, task?.finishedAt) || "--"}
|
||||
</div>
|
||||
<div className="text-sm text-gray-600">总耗时</div>
|
||||
</div>
|
||||
<div className="text-center p-4 bg-gradient-to-br from-green-50 to-green-100 rounded-lg">
|
||||
<CheckCircle className="w-8 h-8 text-green-500 mb-2 mx-auto" />
|
||||
<div className="text-xl font-bold text-green-500">
|
||||
{task?.successFiles || "--"}
|
||||
{task?.progress?.succeedFileNum || "0"}
|
||||
</div>
|
||||
<div className="text-sm text-gray-600">成功文件</div>
|
||||
</div>
|
||||
<div className="text-center p-4 bg-gradient-to-br from-red-50 to-red-100 rounded-lg">
|
||||
<AlertCircle className="w-8 h-8 text-red-500 mb-2 mx-auto" />
|
||||
<div className="text-xl font-bold text-red-500">
|
||||
{task?.failedFiles || "--"}
|
||||
{(task?.status.value === TaskStatus.RUNNING || task?.status.value === TaskStatus.PENDING) ?
|
||||
task?.progress.failedFileNum :
|
||||
task?.progress?.totalFileNum - task?.progress.succeedFileNum}
|
||||
</div>
|
||||
<div className="text-sm text-gray-600">失败文件</div>
|
||||
</div>
|
||||
<div className="text-center p-4 bg-gradient-to-br from-purple-50 to-purple-100 rounded-lg">
|
||||
<Activity className="w-8 h-8 text-purple-500 mb-2 mx-auto" />
|
||||
<div className="text-xl font-bold text-purple-500">
|
||||
{task?.progress || "--"}
|
||||
{task?.progress?.successRate ? task?.progress?.successRate + "%" : "--"}
|
||||
</div>
|
||||
<div className="text-sm text-gray-600">成功率</div>
|
||||
</div>
|
||||
@@ -120,25 +110,22 @@ export default function BasicInfo({ task }: { task: CleansingTask }) {
|
||||
{/* 处理进度 */}
|
||||
<div>
|
||||
<h3 className="text-lg font-semibold text-gray-900 mb-4">处理进度</h3>
|
||||
<Progress percent={task?.progress} showInfo />
|
||||
<Progress percent={task?.progress?.process} showInfo />
|
||||
<div className="grid grid-cols-2 gap-4 text-sm mt-4">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="w-3 h-3 bg-green-500 rounded-full inline-block" />
|
||||
<span>已完成: {task?.processedFiles || "--"}</span>
|
||||
<span>已完成: {task?.progress?.succeedFileNum || "0"}</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="w-3 h-3 bg-blue-500 rounded-full inline-block" />
|
||||
<span>处理中: {task?.processingFiles || "--"}</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="w-3 h-3 bg-gray-300 rounded-full inline-block" />
|
||||
<span>
|
||||
待处理: {task?.totalFiles - task?.processedFiles || "--"}
|
||||
</span>
|
||||
<span>处理中: {(task?.status.value === TaskStatus.RUNNING || task?.status.value === TaskStatus.PENDING) ?
|
||||
task?.progress?.totalFileNum - task?.progress.succeedFileNum : 0}</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="w-3 h-3 bg-red-500 rounded-full inline-block" />
|
||||
<span>失败: {task?.failedFiles || "--"}</span>
|
||||
<span>失败: {(task?.status.value === TaskStatus.RUNNING || task?.status.value === TaskStatus.PENDING) ?
|
||||
task?.progress.failedFileNum :
|
||||
task?.progress?.totalFileNum - task?.progress.succeedFileNum}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -1,70 +1,30 @@
|
||||
import { Button, Modal, Table, Badge, Input } from "antd";
|
||||
import { Download, FileText } from "lucide-react";
|
||||
import { useState } from "react";
|
||||
import {Button, Modal, Table, Badge, Input} from "antd";
|
||||
import { Download } from "lucide-react";
|
||||
import {useEffect, useState} from "react";
|
||||
import {useParams} from "react-router";
|
||||
import {TaskStatus} from "@/pages/DataCleansing/cleansing.model.ts";
|
||||
import {TaskStatusMap} from "@/pages/DataCleansing/cleansing.const.tsx";
|
||||
|
||||
// 模拟文件列表数据
|
||||
const fileList = [
|
||||
{
|
||||
id: 1,
|
||||
fileName: "lung_cancer_001.svs",
|
||||
originalSize: "15.2MB",
|
||||
processedSize: "8.5MB",
|
||||
status: "已完成",
|
||||
duration: "2分15秒",
|
||||
processedAt: "2024-01-20 09:32:40",
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
fileName: "lung_cancer_002.svs",
|
||||
originalSize: "18.7MB",
|
||||
processedSize: "10.2MB",
|
||||
status: "已完成",
|
||||
duration: "2分38秒",
|
||||
processedAt: "2024-01-20 09:35:18",
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
fileName: "lung_cancer_003.svs",
|
||||
originalSize: "12.3MB",
|
||||
processedSize: "6.8MB",
|
||||
status: "已完成",
|
||||
duration: "1分52秒",
|
||||
processedAt: "2024-01-20 09:37:10",
|
||||
},
|
||||
{
|
||||
id: 4,
|
||||
fileName: "lung_cancer_004.svs",
|
||||
originalSize: "20.1MB",
|
||||
processedSize: "-",
|
||||
status: "失败",
|
||||
duration: "0分45秒",
|
||||
processedAt: "2024-01-20 09:38:55",
|
||||
},
|
||||
{
|
||||
id: 5,
|
||||
fileName: "lung_cancer_005.svs",
|
||||
originalSize: "16.8MB",
|
||||
processedSize: "9.3MB",
|
||||
status: "已完成",
|
||||
duration: "2分22秒",
|
||||
processedAt: "2024-01-20 09:41:17",
|
||||
},
|
||||
];
|
||||
|
||||
export default function FileTable() {
|
||||
export default function FileTable({result, fetchTaskResult}) {
|
||||
const { id = "" } = useParams();
|
||||
const [showFileCompareDialog, setShowFileCompareDialog] = useState(false);
|
||||
const [showFileLogDialog, setShowFileLogDialog] = useState(false);
|
||||
const [selectedFile, setSelectedFile] = useState<any>(null);
|
||||
const [selectedFileIds, setSelectedFileIds] = useState<number[]>([]);
|
||||
const [selectedFileIds, setSelectedFileIds] = useState<string[]>([]);
|
||||
|
||||
useEffect(() => {
|
||||
fetchTaskResult();
|
||||
}, [id]);
|
||||
|
||||
const handleSelectAllFiles = (checked: boolean) => {
|
||||
if (checked) {
|
||||
setSelectedFileIds(fileList.map((file) => file.id));
|
||||
setSelectedFileIds(result.map((file) => file.instanceId));
|
||||
} else {
|
||||
setSelectedFileIds([]);
|
||||
}
|
||||
};
|
||||
|
||||
const handleSelectFile = (fileId: number, checked: boolean) => {
|
||||
const handleSelectFile = (fileId: string, checked: boolean) => {
|
||||
if (checked) {
|
||||
setSelectedFileIds([...selectedFileIds, fileId]);
|
||||
} else {
|
||||
@@ -79,116 +39,16 @@ export default function FileTable() {
|
||||
// 实际下载逻辑
|
||||
};
|
||||
|
||||
const handleBatchDeleteFiles = () => {
|
||||
// 实际删除逻辑
|
||||
setSelectedFileIds([]);
|
||||
};
|
||||
const handleViewFileLog = (file: any) => {
|
||||
setSelectedFile(file);
|
||||
setShowFileLogDialog(true);
|
||||
};
|
||||
function formatFileSize(bytes: number, decimals: number = 2): string {
|
||||
if (bytes === 0) return '0 Bytes';
|
||||
|
||||
// 模拟单个文件的处理日志
|
||||
const getFileProcessLog = (fileName: string) => [
|
||||
{
|
||||
time: "09:30:18",
|
||||
step: "开始处理",
|
||||
operator: "格式转换",
|
||||
status: "INFO",
|
||||
message: `开始处理文件: ${fileName}`,
|
||||
},
|
||||
{
|
||||
time: "09:30:19",
|
||||
step: "文件验证",
|
||||
operator: "格式转换",
|
||||
status: "INFO",
|
||||
message: "验证文件格式和完整性",
|
||||
},
|
||||
{
|
||||
time: "09:30:20",
|
||||
step: "格式解析",
|
||||
operator: "格式转换",
|
||||
status: "INFO",
|
||||
message: "解析SVS格式文件",
|
||||
},
|
||||
{
|
||||
time: "09:30:25",
|
||||
step: "格式转换",
|
||||
operator: "格式转换",
|
||||
status: "SUCCESS",
|
||||
message: "成功转换为JPEG格式",
|
||||
},
|
||||
{
|
||||
time: "09:30:26",
|
||||
step: "噪声检测",
|
||||
operator: "噪声去除",
|
||||
status: "INFO",
|
||||
message: "检测图像噪声水平",
|
||||
},
|
||||
{
|
||||
time: "09:30:28",
|
||||
step: "噪声去除",
|
||||
operator: "噪声去除",
|
||||
status: "INFO",
|
||||
message: "应用高斯滤波去除噪声",
|
||||
},
|
||||
{
|
||||
time: "09:30:31",
|
||||
step: "噪声去除完成",
|
||||
operator: "噪声去除",
|
||||
status: "SUCCESS",
|
||||
message: "噪声去除处理完成",
|
||||
},
|
||||
{
|
||||
time: "09:30:32",
|
||||
step: "尺寸检测",
|
||||
operator: "尺寸标准化",
|
||||
status: "INFO",
|
||||
message: "检测当前图像尺寸: 2048x1536",
|
||||
},
|
||||
{
|
||||
time: "09:30:33",
|
||||
step: "尺寸调整",
|
||||
operator: "尺寸标准化",
|
||||
status: "INFO",
|
||||
message: "调整图像尺寸至512x512",
|
||||
},
|
||||
{
|
||||
time: "09:30:35",
|
||||
step: "尺寸标准化完成",
|
||||
operator: "尺寸标准化",
|
||||
status: "SUCCESS",
|
||||
message: "图像尺寸标准化完成",
|
||||
},
|
||||
{
|
||||
time: "09:30:36",
|
||||
step: "质量检查",
|
||||
operator: "质量检查",
|
||||
status: "INFO",
|
||||
message: "检查图像质量指标",
|
||||
},
|
||||
{
|
||||
time: "09:30:38",
|
||||
step: "分辨率检查",
|
||||
operator: "质量检查",
|
||||
status: "SUCCESS",
|
||||
message: "分辨率符合要求",
|
||||
},
|
||||
{
|
||||
time: "09:30:39",
|
||||
step: "清晰度检查",
|
||||
operator: "质量检查",
|
||||
status: "SUCCESS",
|
||||
message: "图像清晰度良好",
|
||||
},
|
||||
{
|
||||
time: "09:30:40",
|
||||
step: "处理完成",
|
||||
operator: "质量检查",
|
||||
status: "SUCCESS",
|
||||
message: `文件 ${fileName} 处理完成`,
|
||||
},
|
||||
];
|
||||
const k = 1024;
|
||||
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
|
||||
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(decimals)) + ' ' + sizes[i];
|
||||
}
|
||||
|
||||
const fileColumns = [
|
||||
{
|
||||
@@ -196,7 +56,7 @@ export default function FileTable() {
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={
|
||||
selectedFileIds.length === fileList.length && fileList.length > 0
|
||||
selectedFileIds.length === result?.length && result?.length > 0
|
||||
}
|
||||
onChange={(e) => handleSelectAllFiles(e.target.checked)}
|
||||
className="w-4 h-4"
|
||||
@@ -205,7 +65,7 @@ export default function FileTable() {
|
||||
dataIndex: "select",
|
||||
key: "select",
|
||||
width: 50,
|
||||
render: (text: string, record: any) => (
|
||||
render: (_text: string, record: any) => (
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={selectedFileIds.includes(record.id)}
|
||||
@@ -216,8 +76,8 @@ export default function FileTable() {
|
||||
},
|
||||
{
|
||||
title: "文件名",
|
||||
dataIndex: "fileName",
|
||||
key: "fileName",
|
||||
dataIndex: "srcName",
|
||||
key: "srcName",
|
||||
filterDropdown: ({
|
||||
setSelectedKeys,
|
||||
selectedKeys,
|
||||
@@ -245,15 +105,87 @@ export default function FileTable() {
|
||||
</div>
|
||||
),
|
||||
onFilter: (value: string, record: any) =>
|
||||
record.fileName.toLowerCase().includes(value.toLowerCase()),
|
||||
record.srcName.toLowerCase().includes(value.toLowerCase()),
|
||||
render: (text: string) => (
|
||||
<span className="font-mono text-sm">{text?.replace(/\.[^/.]+$/, "")}</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: "文件类型",
|
||||
dataIndex: "srcType",
|
||||
key: "srcType",
|
||||
filterDropdown: ({
|
||||
setSelectedKeys,
|
||||
selectedKeys,
|
||||
confirm,
|
||||
clearFilters,
|
||||
}: any) => (
|
||||
<div className="p-4 w-64">
|
||||
<Input
|
||||
placeholder="搜索文件类型"
|
||||
value={selectedKeys[0]}
|
||||
onChange={(e) =>
|
||||
setSelectedKeys(e.target.value ? [e.target.value] : [])
|
||||
}
|
||||
onPressEnter={() => confirm()}
|
||||
className="mb-2"
|
||||
/>
|
||||
<div className="flex gap-2">
|
||||
<Button size="small" onClick={() => confirm()}>
|
||||
搜索
|
||||
</Button>
|
||||
<Button size="small" onClick={() => clearFilters()}>
|
||||
重置
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
),
|
||||
onFilter: (value: string, record: any) =>
|
||||
record.srcType.toLowerCase().includes(value.toLowerCase()),
|
||||
render: (text: string) => (
|
||||
<span className="font-mono text-sm">{text}</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: "清洗后文件类型",
|
||||
dataIndex: "destType",
|
||||
key: "destType",
|
||||
filterDropdown: ({
|
||||
setSelectedKeys,
|
||||
selectedKeys,
|
||||
confirm,
|
||||
clearFilters,
|
||||
}: any) => (
|
||||
<div className="p-4 w-64">
|
||||
<Input
|
||||
placeholder="搜索文件类型"
|
||||
value={selectedKeys[0]}
|
||||
onChange={(e) =>
|
||||
setSelectedKeys(e.target.value ? [e.target.value] : [])
|
||||
}
|
||||
onPressEnter={() => confirm()}
|
||||
className="mb-2"
|
||||
/>
|
||||
<div className="flex gap-2">
|
||||
<Button size="small" onClick={() => confirm()}>
|
||||
搜索
|
||||
</Button>
|
||||
<Button size="small" onClick={() => clearFilters()}>
|
||||
重置
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
),
|
||||
onFilter: (value: string, record: any) =>
|
||||
record.destType.toLowerCase().includes(value.toLowerCase()),
|
||||
render: (text: string) => (
|
||||
<span className="font-mono text-sm">{text}</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: "清洗前大小",
|
||||
dataIndex: "originalSize",
|
||||
key: "originalSize",
|
||||
dataIndex: "srcSize",
|
||||
key: "srcSize",
|
||||
sorter: (a: any, b: any) => {
|
||||
const getSizeInBytes = (size: string) => {
|
||||
if (!size || size === "-") return 0;
|
||||
@@ -265,11 +197,14 @@ export default function FileTable() {
|
||||
};
|
||||
return getSizeInBytes(a.originalSize) - getSizeInBytes(b.originalSize);
|
||||
},
|
||||
render: (number: number) => (
|
||||
<span className="font-mono text-sm">{formatFileSize(number)}</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: "清洗后大小",
|
||||
dataIndex: "processedSize",
|
||||
key: "processedSize",
|
||||
dataIndex: "destSize",
|
||||
key: "destSize",
|
||||
sorter: (a: any, b: any) => {
|
||||
const getSizeInBytes = (size: string) => {
|
||||
if (!size || size === "-") return 0;
|
||||
@@ -283,6 +218,9 @@ export default function FileTable() {
|
||||
getSizeInBytes(a.processedSize) - getSizeInBytes(b.processedSize)
|
||||
);
|
||||
},
|
||||
render: (number: number) => (
|
||||
<span className="font-mono text-sm">{formatFileSize(number)}</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: "状态",
|
||||
@@ -297,43 +235,22 @@ export default function FileTable() {
|
||||
render: (status: string) => (
|
||||
<Badge
|
||||
status={
|
||||
status === "已完成"
|
||||
status === "COMPLETED"
|
||||
? "success"
|
||||
: status === "失败"
|
||||
: status === "FAILED"
|
||||
? "error"
|
||||
: "processing"
|
||||
}
|
||||
text={status}
|
||||
text={TaskStatusMap[status as TaskStatus].label}
|
||||
/>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: "执行耗时",
|
||||
dataIndex: "duration",
|
||||
key: "duration",
|
||||
sorter: (a: any, b: any) => {
|
||||
const getTimeInSeconds = (duration: string) => {
|
||||
const parts = duration.split(/[分秒]/);
|
||||
const minutes = Number.parseInt(parts[0]) || 0;
|
||||
const seconds = Number.parseInt(parts[1]) || 0;
|
||||
return minutes * 60 + seconds;
|
||||
};
|
||||
return getTimeInSeconds(a.duration) - getTimeInSeconds(b.duration);
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "操作",
|
||||
key: "action",
|
||||
render: (text: string, record: any) => (
|
||||
render: (_text: string, record: any) => (
|
||||
<div className="flex">
|
||||
<Button
|
||||
type="link"
|
||||
size="small"
|
||||
onClick={() => handleViewFileLog(record)}
|
||||
>
|
||||
日志
|
||||
</Button>
|
||||
{record.status === "已完成" && (
|
||||
{record.status === "COMPLETED" && (
|
||||
<Button
|
||||
type="link"
|
||||
size="small"
|
||||
@@ -371,53 +288,12 @@ export default function FileTable() {
|
||||
)}
|
||||
<Table
|
||||
columns={fileColumns}
|
||||
dataSource={fileList}
|
||||
dataSource={result}
|
||||
pagination={{ pageSize: 10, showSizeChanger: true }}
|
||||
size="middle"
|
||||
rowKey="id"
|
||||
/>
|
||||
|
||||
{/* 文件日志弹窗 */}
|
||||
<Modal
|
||||
open={showFileLogDialog}
|
||||
onCancel={() => setShowFileLogDialog(false)}
|
||||
footer={null}
|
||||
width={700}
|
||||
title={
|
||||
<span>
|
||||
<FileText className="w-4 h-4 mr-2 inline" />
|
||||
文件处理日志 - {selectedFile?.fileName}
|
||||
</span>
|
||||
}
|
||||
>
|
||||
<div className="py-4">
|
||||
<div className="bg-gray-900 rounded-lg p-4 max-h-96 overflow-y-auto">
|
||||
<div className="font-mono text-sm">
|
||||
{selectedFile &&
|
||||
getFileProcessLog(selectedFile.fileName).map((log, index) => (
|
||||
<div key={index} className="flex gap-3">
|
||||
<span className="text-gray-500 min-w-20">{log.time}</span>
|
||||
<span className="text-blue-400 min-w-24">
|
||||
[{log.operator}]
|
||||
</span>
|
||||
<span
|
||||
className={`min-w-20 ${
|
||||
log.status === "ERROR"
|
||||
? "text-red-400"
|
||||
: log.status === "SUCCESS"
|
||||
? "text-green-400"
|
||||
: "text-yellow-400"
|
||||
}`}
|
||||
>
|
||||
{log.step}
|
||||
</span>
|
||||
<span className="text-gray-100">{log.message}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</Modal>
|
||||
{/* 文件对比弹窗 */}
|
||||
<Modal
|
||||
open={showFileCompareDialog}
|
||||
@@ -434,22 +310,13 @@ export default function FileTable() {
|
||||
<div className="w-16 h-16 bg-gray-300 rounded-lg mx-auto mb-2" />
|
||||
<div className="text-sm">原始文件预览</div>
|
||||
<div className="text-xs text-gray-400">
|
||||
大小: {selectedFile?.originalSize}
|
||||
大小: {formatFileSize(selectedFile?.srcSize)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="text-sm text-gray-600 mt-3 space-y-1">
|
||||
<div>
|
||||
<span className="font-medium">文件格式:</span> SVS
|
||||
</div>
|
||||
<div>
|
||||
<span className="font-medium">分辨率:</span> 2048x1536
|
||||
</div>
|
||||
<div>
|
||||
<span className="font-medium">色彩空间:</span> RGB
|
||||
</div>
|
||||
<div>
|
||||
<span className="font-medium">压缩方式:</span> 无压缩
|
||||
<span className="font-medium">文件格式:</span> {selectedFile?.srcType}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -460,22 +327,13 @@ export default function FileTable() {
|
||||
<div className="w-16 h-16 bg-blue-300 rounded-lg mx-auto mb-2" />
|
||||
<div className="text-sm">处理后文件预览</div>
|
||||
<div className="text-xs text-gray-400">
|
||||
大小: {selectedFile?.processedSize}
|
||||
大小: {formatFileSize(selectedFile?.destSize)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="text-sm text-gray-600 mt-3 space-y-1">
|
||||
<div>
|
||||
<span className="font-medium">文件格式:</span> JPEG
|
||||
</div>
|
||||
<div>
|
||||
<span className="font-medium">分辨率:</span> 512x512
|
||||
</div>
|
||||
<div>
|
||||
<span className="font-medium">色彩空间:</span> RGB
|
||||
</div>
|
||||
<div>
|
||||
<span className="font-medium">压缩方式:</span> JPEG压缩
|
||||
<span className="font-medium">文件格式:</span> {selectedFile?.destType}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -485,15 +343,7 @@ export default function FileTable() {
|
||||
<div className="grid grid-cols-3 gap-4 text-sm">
|
||||
<div className="bg-green-50 p-4 rounded-lg">
|
||||
<div className="font-medium text-green-700">文件大小优化</div>
|
||||
<div className="text-green-600">减少了 44.1%</div>
|
||||
</div>
|
||||
<div className="bg-blue-50 p-4 rounded-lg">
|
||||
<div className="font-medium text-blue-700">处理时间</div>
|
||||
<div className="text-blue-600">{selectedFile?.duration}</div>
|
||||
</div>
|
||||
<div className="bg-purple-50 p-4 rounded-lg">
|
||||
<div className="font-medium text-purple-700">质量评分</div>
|
||||
<div className="text-purple-600">优秀 (9.2/10)</div>
|
||||
<div className="text-green-600">减少了 {(100 * (selectedFile?.srcSize - selectedFile?.destSize) / selectedFile?.srcSize).toFixed(2)}%</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -1,110 +1,43 @@
|
||||
export default function LogsTable({ task }: { task: any }) {
|
||||
// 模拟运行日志
|
||||
const runLogs = [
|
||||
{
|
||||
time: "09:30:15",
|
||||
level: "INFO",
|
||||
message: "开始执行数据清洗任务: 肺癌WSI图像清洗任务",
|
||||
},
|
||||
{
|
||||
time: "09:30:16",
|
||||
level: "INFO",
|
||||
message: "加载源数据集: 肺癌WSI病理图像数据集 (1250 文件)",
|
||||
},
|
||||
{ time: "09:30:17", level: "INFO", message: "初始化算子: 格式转换" },
|
||||
{
|
||||
time: "09:30:18",
|
||||
level: "INFO",
|
||||
message: "开始处理文件: lung_cancer_001.svs",
|
||||
},
|
||||
{
|
||||
time: "09:30:25",
|
||||
level: "SUCCESS",
|
||||
message: "文件处理成功: lung_cancer_001.svs -> lung_cancer_001.jpg",
|
||||
},
|
||||
{
|
||||
time: "09:30:26",
|
||||
level: "INFO",
|
||||
message: "开始处理文件: lung_cancer_002.svs",
|
||||
},
|
||||
{
|
||||
time: "09:30:33",
|
||||
level: "SUCCESS",
|
||||
message: "文件处理成功: lung_cancer_002.svs -> lung_cancer_002.jpg",
|
||||
},
|
||||
{
|
||||
time: "09:58:42",
|
||||
level: "INFO",
|
||||
message: "格式转换完成,成功处理 1250/1250 文件",
|
||||
},
|
||||
{ time: "09:58:43", level: "INFO", message: "初始化算子: 噪声去除" },
|
||||
{
|
||||
time: "09:58:44",
|
||||
level: "INFO",
|
||||
message: "开始处理文件: lung_cancer_001.jpg",
|
||||
},
|
||||
{
|
||||
time: "09:58:51",
|
||||
level: "SUCCESS",
|
||||
message: "噪声去除成功: lung_cancer_001.jpg",
|
||||
},
|
||||
{
|
||||
time: "10:15:23",
|
||||
level: "WARNING",
|
||||
message: "文件质量较低,跳过处理: lung_cancer_156.jpg",
|
||||
},
|
||||
{
|
||||
time: "10:35:18",
|
||||
level: "INFO",
|
||||
message: "噪声去除完成,成功处理 1228/1250 文件",
|
||||
},
|
||||
{ time: "10:35:19", level: "INFO", message: "初始化算子: 尺寸标准化" },
|
||||
{
|
||||
time: "11:12:05",
|
||||
level: "INFO",
|
||||
message: "尺寸标准化完成,成功处理 1222/1228 文件",
|
||||
},
|
||||
{ time: "11:12:06", level: "INFO", message: "初始化算子: 质量检查" },
|
||||
{
|
||||
time: "11:25:33",
|
||||
level: "ERROR",
|
||||
message: "质量检查失败: lung_cancer_089.jpg - 分辨率过低",
|
||||
},
|
||||
{
|
||||
time: "11:45:32",
|
||||
level: "INFO",
|
||||
message: "质量检查完成,成功处理 1198/1222 文件",
|
||||
},
|
||||
{
|
||||
time: "11:45:33",
|
||||
level: "SUCCESS",
|
||||
message: "数据清洗任务完成!总成功率: 95.8%",
|
||||
},
|
||||
];
|
||||
import {useEffect} from "react";
|
||||
import {useParams} from "react-router";
|
||||
import {FileClock} from "lucide-react";
|
||||
|
||||
return (
|
||||
<div className="text-gray-300 p-4 border border-gray-700 bg-gray-800 rounded-lg">
|
||||
<div className="font-mono text-sm">
|
||||
{runLogs?.map?.((log, index) => (
|
||||
<div key={index} className="flex gap-3">
|
||||
<span className="text-gray-500 min-w-20">{log.time}</span>
|
||||
<span
|
||||
className={`min-w-20 ${
|
||||
log.level === "ERROR"
|
||||
? "text-red-500"
|
||||
: log.level === "WARNING"
|
||||
? "text-yellow-500"
|
||||
: log.level === "SUCCESS"
|
||||
? "text-green-500"
|
||||
: "text-blue-500"
|
||||
}`}
|
||||
>
|
||||
[{log.level}]
|
||||
</span>
|
||||
<span className="text-gray-100">{log.message}</span>
|
||||
</div>
|
||||
))}
|
||||
export default function LogsTable({taskLog, fetchTaskLog} : {taskLog: any[], fetchTaskLog: () => Promise<any>}) {
|
||||
const { id = "" } = useParams();
|
||||
|
||||
useEffect(() => {
|
||||
fetchTaskLog();
|
||||
}, [id]);
|
||||
|
||||
return taskLog?.length > 0 ? (
|
||||
<>
|
||||
<div className="text-gray-300 p-4 border border-gray-700 bg-gray-800 rounded-lg">
|
||||
<div className="font-mono text-sm">
|
||||
{taskLog?.map?.((log, index) => (
|
||||
<div key={index} className="flex gap-3">
|
||||
<span
|
||||
className={`min-w-20 ${
|
||||
log.level === "ERROR" || log.level === "FATAL"
|
||||
? "text-red-500"
|
||||
: log.level === "WARNING" || log.level === "WARN"
|
||||
? "text-yellow-500"
|
||||
: "text-green-500"
|
||||
}`}
|
||||
>
|
||||
[{log.level}]
|
||||
</span>
|
||||
<span className="text-gray-100">{log.message}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
) : (
|
||||
<div className="text-center py-12">
|
||||
<FileClock className="w-16 h-16 text-gray-300 mx-auto mb-4" />
|
||||
<h3 className="text-lg font-medium text-gray-900 mb-2">
|
||||
当前任务无可用日志
|
||||
</h3>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,103 +1,25 @@
|
||||
import { Button, Input, Table } from "antd";
|
||||
import {Steps, Typography} from "antd";
|
||||
import {useNavigate} from "react-router";
|
||||
|
||||
const operators = [
|
||||
{
|
||||
name: "格式转换",
|
||||
startTime: "09:30:15",
|
||||
endTime: "09:58:42",
|
||||
duration: "28分27秒",
|
||||
status: "成功",
|
||||
processedFiles: 1250,
|
||||
successRate: 100,
|
||||
},
|
||||
{
|
||||
name: "噪声去除",
|
||||
startTime: "09:58:42",
|
||||
endTime: "10:35:18",
|
||||
duration: "36分36秒",
|
||||
status: "成功",
|
||||
processedFiles: 1250,
|
||||
successRate: 98.2,
|
||||
},
|
||||
{
|
||||
name: "尺寸标准化",
|
||||
startTime: "10:35:18",
|
||||
endTime: "11:12:05",
|
||||
duration: "36分47秒",
|
||||
status: "成功",
|
||||
processedFiles: 1228,
|
||||
successRate: 99.5,
|
||||
},
|
||||
{
|
||||
name: "质量检查",
|
||||
startTime: "11:12:05",
|
||||
endTime: "11:45:32",
|
||||
duration: "33分27秒",
|
||||
status: "成功",
|
||||
processedFiles: 1222,
|
||||
successRate: 97.8,
|
||||
},
|
||||
];
|
||||
export default function OperatorTable({ task }: { task: any }) {
|
||||
const operatorColumns = [
|
||||
{
|
||||
title: "算子名称",
|
||||
dataIndex: "name",
|
||||
key: "name",
|
||||
fixed: "left",
|
||||
width: 200,
|
||||
filterDropdown: ({
|
||||
setSelectedKeys,
|
||||
selectedKeys,
|
||||
confirm,
|
||||
clearFilters,
|
||||
}: any) => (
|
||||
<div className="p-4 w-64">
|
||||
<Input
|
||||
placeholder="搜索算子名称"
|
||||
value={selectedKeys[0]}
|
||||
onChange={(e) =>
|
||||
setSelectedKeys(e.target.value ? [e.target.value] : [])
|
||||
}
|
||||
onPressEnter={() => confirm()}
|
||||
className="mb-2"
|
||||
/>
|
||||
<div className="flex gap-2">
|
||||
<Button size="small" onClick={() => confirm()}>
|
||||
搜索
|
||||
</Button>
|
||||
<Button size="small" onClick={() => clearFilters()}>
|
||||
重置
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
),
|
||||
onFilter: (value: string, record: any) =>
|
||||
record.name.toLowerCase().includes(value.toLowerCase()),
|
||||
},
|
||||
{
|
||||
title: "版本",
|
||||
dataIndex: "version",
|
||||
key: "version",
|
||||
},
|
||||
{
|
||||
title: "创建时间",
|
||||
dataIndex: "createdAt",
|
||||
key: "createdAt",
|
||||
},
|
||||
{
|
||||
title: "更新时间",
|
||||
dataIndex: "updatedAt",
|
||||
key: "updatedAt",
|
||||
},
|
||||
];
|
||||
const navigate = useNavigate();
|
||||
|
||||
return (
|
||||
<Table
|
||||
columns={operatorColumns}
|
||||
dataSource={task?.instance || operators}
|
||||
pagination={false}
|
||||
size="middle"
|
||||
/>
|
||||
return task?.instance?.length > 0 && (
|
||||
<>
|
||||
<Steps
|
||||
progressDot
|
||||
direction="vertical"
|
||||
items={Object.values(task?.instance).map((item) => ({
|
||||
title: <Typography.Link
|
||||
onClick={() => navigate(`/data/operator-market/plugin-detail/${item?.id}`)}
|
||||
>
|
||||
{item?.name}
|
||||
</Typography.Link>,
|
||||
description: item?.description,
|
||||
status: "finish"
|
||||
}))}
|
||||
className="overflow-auto"
|
||||
/>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -43,10 +43,6 @@ export default function TaskList() {
|
||||
handleFiltersChange,
|
||||
} = useFetchData(queryCleaningTasksUsingGet, mapTask);
|
||||
|
||||
const handleViewTask = (task: any) => {
|
||||
navigate("/data/cleansing/task-detail/" + task.id);
|
||||
};
|
||||
|
||||
const pauseTask = async (item: CleansingTask) => {
|
||||
await stopCleaningTaskUsingPost(item.id);
|
||||
message.success("任务已暂停");
|
||||
@@ -86,8 +82,12 @@ export default function TaskList() {
|
||||
onClick: startTask, // implement pause/play logic
|
||||
};
|
||||
return [
|
||||
isRunning && pauseBtn,
|
||||
showStart && startBtn,
|
||||
...(isRunning
|
||||
? [ pauseBtn ]
|
||||
: []),
|
||||
...(showStart
|
||||
? [ startBtn ]
|
||||
: []),
|
||||
{
|
||||
key: "delete",
|
||||
label: "删除",
|
||||
@@ -106,6 +106,18 @@ export default function TaskList() {
|
||||
fixed: "left",
|
||||
width: 150,
|
||||
ellipsis: true,
|
||||
render: (_, task: CleansingTask) => {
|
||||
return (
|
||||
<Button
|
||||
type="link"
|
||||
onClick={() =>
|
||||
navigate("/data/cleansing/task-detail/" + task.id)
|
||||
}
|
||||
>
|
||||
{task.name}
|
||||
</Button>
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "任务ID",
|
||||
@@ -273,6 +285,9 @@ export default function TaskList() {
|
||||
data={tableData}
|
||||
operations={taskOperations}
|
||||
pagination={pagination}
|
||||
onView={(tableData) => {
|
||||
navigate("/data/cleansing/task-detail/" + tableData.id)
|
||||
}}
|
||||
/>
|
||||
) : (
|
||||
<Card>
|
||||
|
||||
@@ -1,21 +1,102 @@
|
||||
import { DeleteOutlined } from "@ant-design/icons";
|
||||
import {DeleteOutlined, EditOutlined} from "@ant-design/icons";
|
||||
import CardView from "@/components/CardView";
|
||||
import {
|
||||
deleteCleaningTemplateByIdUsingDelete,
|
||||
queryCleaningTemplatesUsingGet,
|
||||
deleteCleaningTemplateByIdUsingDelete, queryCleaningTemplatesUsingGet,
|
||||
} from "../../cleansing.api";
|
||||
import useFetchData from "@/hooks/useFetchData";
|
||||
import { mapTemplate } from "../../cleansing.const";
|
||||
import { App } from "antd";
|
||||
import { CleansingTemplate } from "../../cleansing.model";
|
||||
import {mapTemplate} from "../../cleansing.const";
|
||||
import {App, Button, Card, Table, Tooltip} from "antd";
|
||||
import {CleansingTemplate} from "../../cleansing.model";
|
||||
import {SearchControls} from "@/components/SearchControls.tsx";
|
||||
import {useNavigate} from "react-router";
|
||||
import {useState} from "react";
|
||||
|
||||
export default function TemplateList() {
|
||||
const navigate = useNavigate();
|
||||
const { message } = App.useApp();
|
||||
const [viewMode, setViewMode] = useState<"card" | "list">("list");
|
||||
|
||||
const { tableData, pagination, fetchData } = useFetchData(
|
||||
queryCleaningTemplatesUsingGet,
|
||||
mapTemplate
|
||||
);
|
||||
const {
|
||||
loading,
|
||||
tableData,
|
||||
pagination,
|
||||
searchParams,
|
||||
setSearchParams,
|
||||
fetchData,
|
||||
handleFiltersChange,
|
||||
} = useFetchData(queryCleaningTemplatesUsingGet, mapTemplate);
|
||||
|
||||
const templateOperations = () => {
|
||||
return [
|
||||
{
|
||||
key: "update",
|
||||
label: "编辑",
|
||||
icon: <EditOutlined />,
|
||||
onClick: (template: CleansingTemplate) => navigate(`/data/cleansing/update-template/${template.id}`)
|
||||
},
|
||||
{
|
||||
key: "delete",
|
||||
label: "删除",
|
||||
danger: true,
|
||||
icon: <DeleteOutlined />,
|
||||
onClick: deleteTemplate, // implement delete logic
|
||||
},
|
||||
];
|
||||
};
|
||||
|
||||
const templateColumns = [
|
||||
{
|
||||
title: "模板名称",
|
||||
dataIndex: "name",
|
||||
key: "name",
|
||||
fixed: "left",
|
||||
width: 150,
|
||||
ellipsis: true,
|
||||
render: (_, template: CleansingTemplate) => {
|
||||
return (
|
||||
<Button
|
||||
type="link"
|
||||
onClick={() =>
|
||||
navigate("/data/cleansing/template-detail/" + template.id)
|
||||
}
|
||||
>
|
||||
{template.name}
|
||||
</Button>
|
||||
);
|
||||
}},
|
||||
{
|
||||
title: "算子数量",
|
||||
dataIndex: "num",
|
||||
key: "num",
|
||||
width: 100,
|
||||
ellipsis: true,
|
||||
render: (_, template: CleansingTemplate) => {
|
||||
return template.instance?.length ?? 0;
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "操作",
|
||||
key: "action",
|
||||
fixed: "right",
|
||||
width: 20,
|
||||
render: (text: string, record: any) => (
|
||||
<div className="flex gap-2">
|
||||
{templateOperations(record).map((op) =>
|
||||
op ? (
|
||||
<Tooltip key={op.key} title={op.label}>
|
||||
<Button
|
||||
type="text"
|
||||
icon={op.icon}
|
||||
danger={op?.danger}
|
||||
onClick={() => op.onClick(record)}
|
||||
/>
|
||||
</Tooltip>
|
||||
) : null
|
||||
)}
|
||||
</div>
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
const deleteTemplate = async (template: CleansingTemplate) => {
|
||||
if (!template.id) {
|
||||
@@ -27,21 +108,43 @@ export default function TemplateList() {
|
||||
message.success("模板删除成功");
|
||||
};
|
||||
|
||||
const operations = [
|
||||
{
|
||||
key: "delete",
|
||||
label: "删除模板",
|
||||
danger: true,
|
||||
icon: <DeleteOutlined />,
|
||||
onClick: (template: CleansingTemplate) => deleteTemplate(template), // 可实现删除逻辑
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
<CardView
|
||||
data={tableData}
|
||||
operations={operations}
|
||||
pagination={pagination}
|
||||
/>
|
||||
<>
|
||||
{/* Search and Filters */}
|
||||
<SearchControls
|
||||
searchTerm={searchParams.keyword}
|
||||
onSearchChange={(keyword) =>
|
||||
setSearchParams({ ...searchParams, keyword })
|
||||
}
|
||||
searchPlaceholder="搜索模板名称、描述"
|
||||
onFiltersChange={handleFiltersChange}
|
||||
viewMode={viewMode}
|
||||
onViewModeChange={setViewMode}
|
||||
showViewToggle={true}
|
||||
onReload={fetchData}
|
||||
onClearFilters={() => setSearchParams({ ...searchParams, filter: {} })}
|
||||
/>
|
||||
{viewMode === "card" ? (
|
||||
<CardView
|
||||
data={tableData}
|
||||
operations={templateOperations}
|
||||
pagination={pagination}
|
||||
onView={(tableData) => {
|
||||
navigate("/data/cleansing/template-detail/" + tableData.id)
|
||||
}}
|
||||
/>
|
||||
) : (
|
||||
<Card>
|
||||
<Table
|
||||
columns={templateColumns}
|
||||
dataSource={tableData}
|
||||
rowKey="id"
|
||||
loading={loading}
|
||||
scroll={{ x: "max-content", y: "calc(100vh - 35rem)" }}
|
||||
pagination={pagination}
|
||||
/>
|
||||
</Card>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -13,6 +13,14 @@ export function queryCleaningTaskByIdUsingGet(taskId: string | number) {
|
||||
return get(`/api/cleaning/tasks/${taskId}`);
|
||||
}
|
||||
|
||||
export function queryCleaningTaskResultByIdUsingGet(taskId: string | number) {
|
||||
return get(`/api/cleaning/tasks/${taskId}/result`);
|
||||
}
|
||||
|
||||
export function queryCleaningTaskLogByIdUsingGet(taskId: string | number) {
|
||||
return get(`/api/cleaning/tasks/${taskId}/log`);
|
||||
}
|
||||
|
||||
export function updateCleaningTaskByIdUsingPut(taskId: string | number, data: any) {
|
||||
return put(`/api/cleaning/tasks/${taskId}`, data);
|
||||
}
|
||||
|
||||
@@ -98,6 +98,11 @@ export const mapTask = (task: CleansingTask) => {
|
||||
createdAt,
|
||||
startedAt,
|
||||
finishedAt,
|
||||
updatedAt: formatDateTime(
|
||||
new Date(Math.max(...[
|
||||
new Date(task.finishedAt).getTime(),
|
||||
new Date(task.startedAt).getTime(),
|
||||
new Date(task.createdAt).getTime()])).toISOString()),
|
||||
icon: <BrushCleaning className="w-full h-full" />,
|
||||
status,
|
||||
duration,
|
||||
|
||||
@@ -18,10 +18,13 @@ export interface CleansingTask {
|
||||
startedAt: string;
|
||||
progress: {
|
||||
finishedFileNum: number;
|
||||
process: 100,
|
||||
succeedFileNum: number;
|
||||
failedFileNum: number;
|
||||
process: 100;
|
||||
totalFileNum: number;
|
||||
successRate: 100;
|
||||
};
|
||||
operators: OperatorI[];
|
||||
instance: OperatorI[];
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
finishedAt: string;
|
||||
@@ -70,3 +73,17 @@ export enum TemplateType {
|
||||
AUDIO = "AUDIO",
|
||||
IMAGE2TEXT = "IMAGE2TEXT",
|
||||
}
|
||||
|
||||
export interface CleansingResult {
|
||||
instanceId: string;
|
||||
srcFileId: string;
|
||||
destFileId: string;
|
||||
srcName: string;
|
||||
destName: string;
|
||||
srcType: string;
|
||||
destType: string;
|
||||
srcSize: number;
|
||||
destSize: number;
|
||||
status: string;
|
||||
result: string;
|
||||
}
|
||||
@@ -110,9 +110,6 @@ export function ListView({ operators = [], pagination, operations }) {
|
||||
{operator.name}
|
||||
</span>
|
||||
<Tag color="default">v{operator.version}</Tag>
|
||||
<Badge color={getStatusBadge(operator.status).color}>
|
||||
{getStatusBadge(operator.status).label}
|
||||
</Badge>
|
||||
</div>
|
||||
}
|
||||
description={
|
||||
|
||||
@@ -33,7 +33,7 @@ export interface OperatorI {
|
||||
tags: string[];
|
||||
isStar?: boolean;
|
||||
originalId?: string; // 用于标识原始算子ID,便于去重
|
||||
categories: number[]; // 分类列表
|
||||
categories: string[]; // 分类列表
|
||||
settings: string;
|
||||
overrides?: { [key: string]: any }; // 用户配置的参数
|
||||
defaultParams?: { [key: string]: any }; // 默认参数
|
||||
@@ -50,6 +50,8 @@ export interface CategoryI {
|
||||
count: number; // 该分类下的算子数量
|
||||
type: string; // e.g., "数据源", "数据清洗", "数据分析", "数据可视化"
|
||||
parentId?: number; // 父分类ID,若无父分类则为null
|
||||
value: string;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface CategoryTreeI {
|
||||
|
||||
@@ -12,7 +12,7 @@ import DatasetDetail from "@/pages/DataManagement/Detail/DatasetDetail";
|
||||
import DataCleansing from "@/pages/DataCleansing/Home/DataCleansing";
|
||||
import CleansingTaskCreate from "@/pages/DataCleansing/Create/CreateTask";
|
||||
import CleansingTaskDetail from "@/pages/DataCleansing/Detail/TaskDetail";
|
||||
import CleansingTemplateCreate from "@/pages/DataCleansing/Create/CreateTempate";
|
||||
import CleansingTemplateCreate from "@/pages/DataCleansing/Create/CreateTemplate";
|
||||
|
||||
import DataAnnotation from "@/pages/DataAnnotation/Home/DataAnnotation";
|
||||
import AnnotationTaskCreate from "@/pages/DataAnnotation/Create/CreateTask";
|
||||
@@ -39,6 +39,7 @@ import OrchestrationPage from "@/pages/Orchestration/Orchestration";
|
||||
import WorkflowEditor from "@/pages/Orchestration/WorkflowEditor";
|
||||
import { withErrorBoundary } from "@/components/ErrorBoundary";
|
||||
import AgentPage from "@/pages/Agent/Agent.tsx";
|
||||
import CleansingTemplateDetail from "@/pages/DataCleansing/Detail/TemplateDetail.tsx";
|
||||
import RatioTaskDetail from "@/pages/RatioTask/Detail/RatioTaskDetail";
|
||||
|
||||
const router = createBrowserRouter([
|
||||
@@ -120,6 +121,14 @@ const router = createBrowserRouter([
|
||||
path: "create-template",
|
||||
Component: CleansingTemplateCreate,
|
||||
},
|
||||
{
|
||||
path: "template-detail/:id",
|
||||
Component: CleansingTemplateDetail,
|
||||
},
|
||||
{
|
||||
path: "update-template/:id",
|
||||
Component: CleansingTemplateCreate,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
@@ -160,7 +160,7 @@ class Mapper(BaseOp):
|
||||
sample["execute_status"] = execute_status
|
||||
task_info = TaskInfoPersistence()
|
||||
task_info.persistence_task_info(sample)
|
||||
return sample
|
||||
raise e
|
||||
|
||||
sample["execute_status"] = execute_status
|
||||
# 加载文件成功执行信息到数据库
|
||||
|
||||
@@ -10,14 +10,13 @@ from .scheduler import Task, TaskStatus, TaskResult, TaskScheduler
|
||||
class CommandTask(Task):
|
||||
"""命令任务包装类"""
|
||||
|
||||
def __init__(self, task_id: str, command: str, shell: bool = True,
|
||||
def __init__(self, task_id: str, command: str, log_path = None, shell: bool = True,
|
||||
timeout: Optional[int] = None, *args, **kwargs):
|
||||
super().__init__(task_id, *args, **kwargs)
|
||||
self.log_path = log_path
|
||||
self.command = command
|
||||
self.shell = shell
|
||||
self.timeout = timeout
|
||||
self.stdout = None
|
||||
self.stderr = None
|
||||
self.return_code = None
|
||||
self._process = None
|
||||
|
||||
@@ -35,56 +34,54 @@ class CommandTask(Task):
|
||||
self.status = TaskStatus.RUNNING
|
||||
self.started_at = datetime.now()
|
||||
|
||||
# 使用 asyncio.create_subprocess_shell 或 create_subprocess_exec
|
||||
if self.shell:
|
||||
process = await asyncio.create_subprocess_shell(
|
||||
self.command,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
**self.kwargs
|
||||
)
|
||||
else:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*self.command.split(),
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
**self.kwargs
|
||||
)
|
||||
|
||||
self._process = process
|
||||
|
||||
# 等待进程完成(带超时)
|
||||
try:
|
||||
if self.timeout:
|
||||
stdout, stderr = await asyncio.wait_for(
|
||||
process.communicate(),
|
||||
timeout=self.timeout
|
||||
with open(self.log_path, 'a') as f:
|
||||
# 使用 asyncio.create_subprocess_shell 或 create_subprocess_exec
|
||||
if self.shell:
|
||||
process = await asyncio.create_subprocess_shell(
|
||||
self.command,
|
||||
stdout=f,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
**self.kwargs
|
||||
)
|
||||
else:
|
||||
stdout, stderr = await process.communicate()
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*self.command.split(),
|
||||
stdout=f,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
**self.kwargs
|
||||
)
|
||||
|
||||
self.stdout = stdout.decode() if stdout else ""
|
||||
self.stderr = stderr.decode() if stderr else ""
|
||||
self.return_code = process.returncode
|
||||
self._process = process
|
||||
|
||||
if self._cancelled:
|
||||
self.status = TaskStatus.CANCELLED
|
||||
elif process.returncode == 0:
|
||||
self.status = TaskStatus.COMPLETED
|
||||
else:
|
||||
self.status = TaskStatus.FAILED
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
# 超时处理
|
||||
self._process.terminate()
|
||||
# 等待进程完成(带超时)
|
||||
try:
|
||||
await asyncio.wait_for(self._process.wait(), timeout=5.0)
|
||||
except asyncio.TimeoutError:
|
||||
self._process.kill()
|
||||
await self._process.wait()
|
||||
if self.timeout:
|
||||
await asyncio.wait_for(
|
||||
process.wait(),
|
||||
timeout=self.timeout
|
||||
)
|
||||
else:
|
||||
await process.wait()
|
||||
self.return_code = process.returncode
|
||||
|
||||
self.status = TaskStatus.FAILED
|
||||
self.stderr = f"Command timed out after {self.timeout} seconds"
|
||||
if self._cancelled:
|
||||
self.status = TaskStatus.CANCELLED
|
||||
elif process.returncode == 0:
|
||||
self.status = TaskStatus.COMPLETED
|
||||
else:
|
||||
self.status = TaskStatus.FAILED
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
# 超时处理
|
||||
self._process.terminate()
|
||||
try:
|
||||
await asyncio.wait_for(self._process.wait(), timeout=5.0)
|
||||
except asyncio.TimeoutError:
|
||||
self._process.kill()
|
||||
await self._process.wait()
|
||||
|
||||
self.status = TaskStatus.FAILED
|
||||
f.write(f"\nCommand timed out after {self.timeout} seconds\n")
|
||||
|
||||
except asyncio.CancelledError:
|
||||
# 任务被取消
|
||||
@@ -101,7 +98,7 @@ class CommandTask(Task):
|
||||
|
||||
except Exception as e:
|
||||
self.status = TaskStatus.FAILED
|
||||
self.stderr = str(e)
|
||||
logger.error(f"Task(id: {self.task_id}) run failed. Cause: {e}")
|
||||
finally:
|
||||
self.completed_at = datetime.now()
|
||||
|
||||
@@ -127,8 +124,6 @@ class CommandTask(Task):
|
||||
"""转换为结果对象"""
|
||||
self.result = {
|
||||
"command": self.command,
|
||||
"stdout": self.stdout,
|
||||
"stderr": self.stderr,
|
||||
"return_code": self.return_code,
|
||||
}
|
||||
return super().to_result()
|
||||
@@ -140,10 +135,13 @@ class CommandScheduler(TaskScheduler):
|
||||
def __init__(self, max_concurrent: int = 5):
|
||||
super().__init__(max_concurrent)
|
||||
|
||||
async def submit(self, task_id, command: str, shell: bool = True,
|
||||
async def submit(self, task_id, command: str, log_path = None, shell: bool = True,
|
||||
timeout: Optional[int] = None, **kwargs) -> str:
|
||||
if log_path is None:
|
||||
log_path = f"/flow/{task_id}/output.log"
|
||||
|
||||
"""提交命令任务"""
|
||||
task = CommandTask(task_id, command, shell, timeout, **kwargs)
|
||||
task = CommandTask(task_id, command, log_path, shell, timeout, **kwargs)
|
||||
self.tasks[task_id] = task
|
||||
|
||||
# 使用信号量限制并发
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
FROM ghcr.io/astral-sh/uv:python3.11-bookworm
|
||||
|
||||
RUN --mount=type=cache,target=/var/cache/apt \
|
||||
--mount=type=cache,target=/var/lib/apt \
|
||||
apt update \
|
||||
&& apt install -y libgl1 libglib2.0-0 vim libmagic1 libreoffice dos2unix
|
||||
|
||||
COPY runtime/python-executor /opt/runtime
|
||||
COPY runtime/ops /opt/runtime/datamate/ops
|
||||
COPY runtime/ops/user /opt/runtime/user
|
||||
@@ -7,11 +12,6 @@ COPY scripts/images/runtime/start.sh /opt/runtime/start.sh
|
||||
|
||||
ENV PYTHONPATH=/opt/runtime/datamate/
|
||||
|
||||
RUN --mount=type=cache,target=/var/cache/apt \
|
||||
--mount=type=cache,target=/var/lib/apt \
|
||||
apt update \
|
||||
&& apt install -y libgl1 libglib2.0-0 vim libmagic1 libreoffice dos2unix
|
||||
|
||||
WORKDIR /opt/runtime
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
|
||||
Reference in New Issue
Block a user