feature: 清洗任务详情页 (#73)

* feature: 清洗任务详情

* fix: 取消构建镜像,改为直接拉取

* fix: 增加清洗任务详情页

* fix: 增加清洗任务详情页

* fix: 算子列表可点击

* fix: 模板详情和更新
This commit is contained in:
hhhhsc701
2025-11-12 18:00:19 +08:00
committed by GitHub
parent 442e561817
commit 6bbde0ec56
46 changed files with 1065 additions and 795 deletions

27
.github/workflows/docker-image-save.yml vendored Normal file
View File

@@ -0,0 +1,27 @@
name: docker-image-save.yml
on:
workflow_call:
inputs:
service_name:
required: true
type: string
jobs:
pull-and-save:
runs-on: ubuntu-latest
steps:
- name: Pull Docker Image
run: |
LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
docker pull ghcr.io/$LOWERCASE_REPO/datamate-${{ inputs.service_name }}:latest
docker tag ghcr.io/$LOWERCASE_REPO/datamate-${{ inputs.service_name }}:latest datamate-${{ inputs.service_name }}:latest
- name: Save Docker Image
run: |
docker save -o datamate-${{ inputs.service_name }}.tar datamate-${{ inputs.service_name }}:latest
- name: Upload Docker Image
uses: actions/upload-artifact@v4
with:
name: datamate-${{ inputs.service_name }}
path: datamate-${{ inputs.service_name }}.tar

View File

@@ -47,19 +47,7 @@ jobs:
make build-${{ inputs.service_name }} VERSION=latest make build-${{ inputs.service_name }} VERSION=latest
- name: Tag & Push Docker Image - name: Tag & Push Docker Image
if: github.event_name != 'pull_request' && !startsWith(github.workflow, 'Package') if: github.event_name != 'pull_request'
run: | run: |
docker tag datamate-${{ inputs.service_name }}:latest ${{ steps.set-tag.outputs.TAGS }} docker tag datamate-${{ inputs.service_name }}:latest ${{ steps.set-tag.outputs.TAGS }}
docker push ${{ steps.set-tag.outputs.TAGS }} docker push ${{ steps.set-tag.outputs.TAGS }}
- name: Save Docker Image
if: startsWith(github.workflow, 'Package')
run: |
docker save -o datamate-${{ inputs.service_name }}.tar datamate-${{ inputs.service_name }}:latest
- name: Upload Docker Image
if: startsWith(github.workflow, 'Package')
uses: actions/upload-artifact@v4
with:
name: datamate-${{ inputs.service_name }}
path: datamate-${{ inputs.service_name }}.tar

View File

@@ -6,23 +6,33 @@ on:
jobs: jobs:
backend-docker-build: backend-docker-build:
name: Build and Push Backend Docker Image name: Build and Push Backend Docker Image
uses: ./.github/workflows/docker-image-backend.yml uses: ./.github/workflows/docker-image-save.yml
with:
service_name: backend
frontend-docker-build: frontend-docker-build:
name: Build and Push Frontend Docker Image name: Build and Push Frontend Docker Image
uses: ./.github/workflows/docker-image-frontend.yml uses: ./.github/workflows/docker-image-save.yml
with:
service_name: frontend
database-docker-build: database-docker-build:
name: Build and Push Database Docker Image name: Build and Push Database Docker Image
uses: ./.github/workflows/docker-image-database.yml uses: ./.github/workflows/docker-image-save.yml
with:
service_name: database
runtime-docker-build: runtime-docker-build:
name: Build and Push Runtime Docker Image name: Build and Push Runtime Docker Image
uses: ./.github/workflows/docker-image-runtime.yml uses: ./.github/workflows/docker-image-save.yml
with:
service_name: runtime
backend-python-docker-build: backend-python-docker-build:
name: Build and Push Backend Python Docker Image name: Build and Push Backend Python Docker Image
uses: ./.github/workflows/docker-image-backend-python.yml uses: ./.github/workflows/docker-image-save.yml
with:
service_name: backend-python
package-all: package-all:
needs: needs:
@@ -54,7 +64,7 @@ jobs:
- name: Upload Package - name: Upload Package
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: datamate name: DataMate
include-hidden-files: true include-hidden-files: true
path: | path: |
deployment/ deployment/

View File

@@ -11,10 +11,7 @@ import com.datamate.cleaning.domain.repository.CleaningTaskRepository;
import com.datamate.cleaning.domain.repository.OperatorInstanceRepository; import com.datamate.cleaning.domain.repository.OperatorInstanceRepository;
import com.datamate.cleaning.infrastructure.validator.CleanTaskValidator; import com.datamate.cleaning.infrastructure.validator.CleanTaskValidator;
import com.datamate.cleaning.interfaces.dto.CleaningProcess; import com.datamate.cleaning.interfaces.dto.*;
import com.datamate.cleaning.interfaces.dto.CleaningTaskDto;
import com.datamate.cleaning.interfaces.dto.CreateCleaningTaskRequest;
import com.datamate.cleaning.interfaces.dto.OperatorInstanceDto;
import com.datamate.common.infrastructure.exception.BusinessException; import com.datamate.common.infrastructure.exception.BusinessException;
import com.datamate.common.infrastructure.exception.SystemErrorCode; import com.datamate.common.infrastructure.exception.SystemErrorCode;
import com.datamate.datamanagement.application.DatasetApplicationService; import com.datamate.datamanagement.application.DatasetApplicationService;
@@ -40,15 +37,19 @@ import java.io.BufferedWriter;
import java.io.File; import java.io.File;
import java.io.FileWriter; import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.nio.file.Files;
import java.util.Map; import java.nio.file.Paths;
import java.util.UUID; import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
@Slf4j @Slf4j
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
public class CleaningTaskService { public class CleaningTaskService {
private final CleaningTaskRepository CleaningTaskRepo; private final CleaningTaskRepository cleaningTaskRepo;
private final OperatorInstanceRepository operatorInstanceRepo; private final OperatorInstanceRepository operatorInstanceRepo;
@@ -66,19 +67,24 @@ public class CleaningTaskService {
private final String FLOW_PATH = "/flow"; private final String FLOW_PATH = "/flow";
private final Pattern LEVEL_PATTERN = Pattern.compile(
"\\b(TRACE|DEBUG|INFO|WARN|WARNING|ERROR|FATAL)\\b",
Pattern.CASE_INSENSITIVE
);
public List<CleaningTaskDto> getTasks(String status, String keywords, Integer page, Integer size) { public List<CleaningTaskDto> getTasks(String status, String keywords, Integer page, Integer size) {
List<CleaningTaskDto> tasks = CleaningTaskRepo.findTasks(status, keywords, page, size); List<CleaningTaskDto> tasks = cleaningTaskRepo.findTasks(status, keywords, page, size);
tasks.forEach(this::setProcess); tasks.forEach(this::setProcess);
return tasks; return tasks;
} }
private void setProcess(CleaningTaskDto task) { private void setProcess(CleaningTaskDto task) {
int count = cleaningResultRepo.countByInstanceId(task.getId()); int[] count = cleaningResultRepo.countByInstanceId(task.getId());
task.setProgress(CleaningProcess.of(task.getFileCount(), count)); task.setProgress(CleaningProcess.of(task.getFileCount(), count[0], count[1]));
} }
public int countTasks(String status, String keywords) { public int countTasks(String status, String keywords) {
return CleaningTaskRepo.findTasks(status, keywords, null, null).size(); return cleaningTaskRepo.findTasks(status, keywords, null, null).size();
} }
@Transactional @Transactional
@@ -105,7 +111,7 @@ public class CleaningTaskService {
task.setDestDatasetName(destDataset.getName()); task.setDestDatasetName(destDataset.getName());
task.setBeforeSize(srcDataset.getSizeBytes()); task.setBeforeSize(srcDataset.getSizeBytes());
task.setFileCount(srcDataset.getFileCount().intValue()); task.setFileCount(srcDataset.getFileCount().intValue());
CleaningTaskRepo.insertTask(task); cleaningTaskRepo.insertTask(task);
operatorInstanceRepo.insertInstance(taskId, request.getInstance()); operatorInstanceRepo.insertInstance(taskId, request.getInstance());
@@ -116,14 +122,50 @@ public class CleaningTaskService {
} }
public CleaningTaskDto getTask(String taskId) { public CleaningTaskDto getTask(String taskId) {
CleaningTaskDto task = CleaningTaskRepo.findTaskById(taskId); CleaningTaskDto task = cleaningTaskRepo.findTaskById(taskId);
setProcess(task); setProcess(task);
task.setInstance(operatorInstanceRepo.findOperatorByInstanceId(taskId));
return task; return task;
} }
public List<CleaningResultDto> getTaskResults(String taskId) {
return cleaningResultRepo.findByInstanceId(taskId);
}
public List<CleaningTaskLog> getTaskLog(String taskId) {
String logPath = FLOW_PATH + "/" + taskId + "/output.log";
try (Stream<String> lines = Files.lines(Paths.get(logPath))) {
List<CleaningTaskLog> logs = new ArrayList<>();
AtomicReference<String> lastLevel = new AtomicReference<>("INFO");
lines.forEach(line -> {
lastLevel.set(getLogLevel(line, lastLevel.get()));
CleaningTaskLog log = new CleaningTaskLog();
log.setLevel(lastLevel.get());
log.setMessage(line);
logs.add(log);
});
return logs;
} catch (IOException e) {
log.error("Fail to read log file {}", logPath, e);
return Collections.emptyList();
}
}
private String getLogLevel(String logLine, String defaultLevel) {
if (logLine == null || logLine.trim().isEmpty()) {
return defaultLevel;
}
Matcher matcher = LEVEL_PATTERN.matcher(logLine);
if (matcher.find()) {
return matcher.group(1).toUpperCase();
}
return defaultLevel;
}
@Transactional @Transactional
public void deleteTask(String taskId) { public void deleteTask(String taskId) {
CleaningTaskRepo.deleteTaskById(taskId); cleaningTaskRepo.deleteTaskById(taskId);
operatorInstanceRepo.deleteByInstanceId(taskId); operatorInstanceRepo.deleteByInstanceId(taskId);
cleaningResultRepo.deleteByInstanceId(taskId); cleaningResultRepo.deleteByInstanceId(taskId);
} }
@@ -190,7 +232,7 @@ public class CleaningTaskService {
try (BufferedWriter writer = new BufferedWriter(new FileWriter(fileName))) { try (BufferedWriter writer = new BufferedWriter(new FileWriter(fileName))) {
if (!mapList.isEmpty()) { // 检查列表是否为空,避免异常 if (!mapList.isEmpty()) { // 检查列表是否为空,避免异常
String jsonString = objectMapper.writeValueAsString(mapList.get(0)); String jsonString = objectMapper.writeValueAsString(mapList.getFirst());
writer.write(jsonString); writer.write(jsonString);
for (int i = 1; i < mapList.size(); i++) { for (int i = 1; i < mapList.size(); i++) {

View File

@@ -5,7 +5,7 @@ import com.datamate.cleaning.domain.repository.CleaningTemplateRepository;
import com.datamate.cleaning.domain.repository.OperatorInstanceRepository; import com.datamate.cleaning.domain.repository.OperatorInstanceRepository;
import com.datamate.cleaning.interfaces.dto.*; import com.datamate.cleaning.interfaces.dto.*;
import com.datamate.cleaning.domain.model.entity.TemplateWithInstance; import com.datamate.cleaning.domain.model.entity.TemplateWithInstance;
import com.datamate.operator.domain.repository.OperatorRepository; import com.datamate.operator.domain.repository.OperatorViewRepository;
import com.datamate.operator.interfaces.dto.OperatorDto; import com.datamate.operator.interfaces.dto.OperatorDto;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
@@ -26,10 +26,11 @@ public class CleaningTemplateService {
private final OperatorInstanceRepository operatorInstanceRepo; private final OperatorInstanceRepository operatorInstanceRepo;
private final OperatorRepository operatorRepo; private final OperatorViewRepository operatorViewRepo;
public List<CleaningTemplateDto> getTemplates(String keywords) { public List<CleaningTemplateDto> getTemplates(String keywords) {
List<OperatorDto> allOperators = operatorRepo.findAllOperators(); List<OperatorDto> allOperators =
operatorViewRepo.findOperatorsByCriteria(null, null, null, null, null);
Map<String, OperatorDto> operatorsMap = allOperators.stream() Map<String, OperatorDto> operatorsMap = allOperators.stream()
.collect(Collectors.toMap(OperatorDto::getId, Function.identity())); .collect(Collectors.toMap(OperatorDto::getId, Function.identity()));
List<TemplateWithInstance> allTemplates = cleaningTemplateRepo.findAllTemplates(keywords); List<TemplateWithInstance> allTemplates = cleaningTemplateRepo.findAllTemplates(keywords);
@@ -39,8 +40,8 @@ public class CleaningTemplateService {
List<TemplateWithInstance> value = twi.getValue(); List<TemplateWithInstance> value = twi.getValue();
CleaningTemplateDto template = new CleaningTemplateDto(); CleaningTemplateDto template = new CleaningTemplateDto();
template.setId(twi.getKey()); template.setId(twi.getKey());
template.setName(value.get(0).getName()); template.setName(value.getFirst().getName());
template.setDescription(value.get(0).getDescription()); template.setDescription(value.getFirst().getDescription());
template.setInstance(value.stream().filter(v -> StringUtils.isNotBlank(v.getOperatorId())) template.setInstance(value.stream().filter(v -> StringUtils.isNotBlank(v.getOperatorId()))
.sorted(Comparator.comparingInt(TemplateWithInstance::getOpIndex)) .sorted(Comparator.comparingInt(TemplateWithInstance::getOpIndex))
.map(v -> { .map(v -> {
@@ -50,8 +51,8 @@ public class CleaningTemplateService {
} }
return operator; return operator;
}).toList()); }).toList());
template.setCreatedAt(value.get(0).getCreatedAt()); template.setCreatedAt(value.getFirst().getCreatedAt());
template.setUpdatedAt(value.get(0).getUpdatedAt()); template.setUpdatedAt(value.getFirst().getUpdatedAt());
return template; return template;
}).toList(); }).toList();
} }
@@ -70,17 +71,22 @@ public class CleaningTemplateService {
} }
public CleaningTemplateDto getTemplate(String templateId) { public CleaningTemplateDto getTemplate(String templateId) {
return cleaningTemplateRepo.findTemplateById(templateId); CleaningTemplateDto template = cleaningTemplateRepo.findTemplateById(templateId);
template.setInstance(operatorInstanceRepo.findOperatorByInstanceId(templateId));
return template;
} }
@Transactional @Transactional
public CleaningTemplateDto updateTemplate(String templateId, UpdateCleaningTemplateRequest request) { public CleaningTemplateDto updateTemplate(String templateId, UpdateCleaningTemplateRequest request) {
CleaningTemplateDto template = cleaningTemplateRepo.findTemplateById(templateId); CleaningTemplateDto template = cleaningTemplateRepo.findTemplateById(templateId);
if (template != null) { if (template == null) {
template.setName(request.getName()); return null;
template.setDescription(request.getDescription());
cleaningTemplateRepo.updateTemplate(template);
} }
template.setName(request.getName());
template.setDescription(request.getDescription());
cleaningTemplateRepo.updateTemplate(template);
operatorInstanceRepo.deleteByInstanceId(templateId);
operatorInstanceRepo.insertInstance(templateId, request.getInstance());
return template; return template;
} }

View File

@@ -16,6 +16,8 @@ import java.util.concurrent.Executors;
public class CleaningTaskScheduler { public class CleaningTaskScheduler {
private final CleaningTaskRepository cleaningTaskRepo; private final CleaningTaskRepository cleaningTaskRepo;
private final RuntimeClient runtimeClient;
private final ExecutorService taskExecutor = Executors.newFixedThreadPool(5); private final ExecutorService taskExecutor = Executors.newFixedThreadPool(5);
public void executeTask(String taskId) { public void executeTask(String taskId) {
@@ -28,11 +30,11 @@ public class CleaningTaskScheduler {
task.setStatus(CleaningTaskStatusEnum.RUNNING); task.setStatus(CleaningTaskStatusEnum.RUNNING);
task.setStartedAt(LocalDateTime.now()); task.setStartedAt(LocalDateTime.now());
cleaningTaskRepo.updateTask(task); cleaningTaskRepo.updateTask(task);
RuntimeClient.submitTask(taskId); runtimeClient.submitTask(taskId);
} }
public void stopTask(String taskId) { public void stopTask(String taskId) {
RuntimeClient.stopTask(taskId); runtimeClient.stopTask(taskId);
CleaningTaskDto task = new CleaningTaskDto(); CleaningTaskDto task = new CleaningTaskDto();
task.setId(taskId); task.setId(taskId);
task.setStatus(CleaningTaskStatusEnum.STOPPED); task.setStatus(CleaningTaskStatusEnum.STOPPED);

View File

@@ -1,36 +0,0 @@
package com.datamate.cleaning.domain.model.entity;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Getter;
import lombok.Setter;
import java.time.LocalDateTime;
@Getter
@Setter
@TableName(value = "t_operator", autoResultMap = true)
public class Operator {
@TableId
private String id;
private String name;
private String description;
private String version;
private String inputs;
private String outputs;
private String runtime;
private String settings;
private Boolean isStar;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}

View File

@@ -3,9 +3,14 @@ package com.datamate.cleaning.domain.repository;
import com.baomidou.mybatisplus.extension.repository.IRepository; import com.baomidou.mybatisplus.extension.repository.IRepository;
import com.datamate.cleaning.domain.model.entity.CleaningResult; import com.datamate.cleaning.domain.model.entity.CleaningResult;
import com.datamate.cleaning.interfaces.dto.CleaningResultDto;
import java.util.List;
public interface CleaningResultRepository extends IRepository<CleaningResult> { public interface CleaningResultRepository extends IRepository<CleaningResult> {
void deleteByInstanceId(String instanceId); void deleteByInstanceId(String instanceId);
int countByInstanceId(String instanceId); int[] countByInstanceId(String instanceId);
List<CleaningResultDto> findByInstanceId(String instanceId);
} }

View File

@@ -3,6 +3,7 @@ package com.datamate.cleaning.domain.repository;
import com.baomidou.mybatisplus.extension.repository.IRepository; import com.baomidou.mybatisplus.extension.repository.IRepository;
import com.datamate.cleaning.interfaces.dto.OperatorInstanceDto; import com.datamate.cleaning.interfaces.dto.OperatorInstanceDto;
import com.datamate.cleaning.domain.model.entity.OperatorInstance; import com.datamate.cleaning.domain.model.entity.OperatorInstance;
import com.datamate.operator.interfaces.dto.OperatorDto;
import java.util.List; import java.util.List;
@@ -10,4 +11,6 @@ public interface OperatorInstanceRepository extends IRepository<OperatorInstance
void insertInstance(String instanceId, List<OperatorInstanceDto> instances); void insertInstance(String instanceId, List<OperatorInstanceDto> instances);
void deleteByInstanceId(String instanceId); void deleteByInstanceId(String instanceId);
List<OperatorDto> findOperatorByInstanceId(String instanceId);
} }

View File

@@ -0,0 +1,15 @@
package com.datamate.cleaning.infrastructure.converter;
import com.datamate.cleaning.domain.model.entity.CleaningResult;
import com.datamate.cleaning.interfaces.dto.CleaningResultDto;
import org.mapstruct.Mapper;
import org.mapstruct.factory.Mappers;
import java.util.List;
@Mapper
public interface CleaningResultConverter {
CleaningResultConverter INSTANCE = Mappers.getMapper(CleaningResultConverter.class);
List<CleaningResultDto> convertEntityToDto(List<CleaningResult> cleaningResult);
}

View File

@@ -2,10 +2,10 @@ package com.datamate.cleaning.infrastructure.converter;
import com.datamate.cleaning.domain.model.entity.OperatorInstance; import com.datamate.cleaning.domain.model.entity.OperatorInstance;
import com.datamate.cleaning.domain.model.entity.Operator;
import com.datamate.cleaning.interfaces.dto.OperatorInstanceDto; import com.datamate.cleaning.interfaces.dto.OperatorInstanceDto;
import com.datamate.common.infrastructure.exception.BusinessException; import com.datamate.common.infrastructure.exception.BusinessException;
import com.datamate.common.infrastructure.exception.SystemErrorCode; import com.datamate.common.infrastructure.exception.SystemErrorCode;
import com.datamate.operator.domain.model.OperatorView;
import com.datamate.operator.interfaces.dto.OperatorDto; import com.datamate.operator.interfaces.dto.OperatorDto;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
@@ -14,6 +14,8 @@ import org.mapstruct.Mapping;
import org.mapstruct.Named; import org.mapstruct.Named;
import org.mapstruct.factory.Mappers; import org.mapstruct.factory.Mappers;
import java.util.Arrays;
import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -35,5 +37,16 @@ public interface OperatorInstanceConverter {
} }
} }
List<OperatorDto> fromEntityToDto(List<Operator> operator); @Mapping(target = "categories", source = "categories", qualifiedByName = "stringToList")
OperatorDto fromEntityToDto(OperatorView operator);
List<OperatorDto> fromEntityToDto(List<OperatorView> operator);
@Named("stringToList")
default List<String> stringToList(String input) {
if (input == null || input.isEmpty()) {
return Collections.emptyList();
}
return Arrays.stream(input.split(",")).toList();
}
} }

View File

@@ -3,6 +3,8 @@ package com.datamate.cleaning.infrastructure.httpclient;
import com.datamate.common.infrastructure.exception.BusinessException; import com.datamate.common.infrastructure.exception.BusinessException;
import com.datamate.common.infrastructure.exception.SystemErrorCode; import com.datamate.common.infrastructure.exception.SystemErrorCode;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
@@ -13,24 +15,36 @@ import java.text.MessageFormat;
import java.time.Duration; import java.time.Duration;
@Slf4j @Slf4j
@Component
public class RuntimeClient { public class RuntimeClient {
private static final String BASE_URL = "http://datamate-runtime:8081/api"; private final String CREATE_TASK_URL = "/api/task/{0}/submit";
private static final String CREATE_TASK_URL = BASE_URL + "/task/{0}/submit"; private final String STOP_TASK_URL = "/api/task/{0}/stop";
private static final String STOP_TASK_URL = BASE_URL + "/task/{0}/stop"; @Value("${runtime.protocol:http}")
private String protocol;
private static final HttpClient CLIENT = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build(); @Value("${runtime.host:datamate-runtime}")
private String host;
public static void submitTask(String taskId) { @Value("${runtime.port:8081}")
send(MessageFormat.format(CREATE_TASK_URL, taskId)); private int port;
private final HttpClient CLIENT = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build();
public void submitTask(String taskId) {
send(MessageFormat.format(getRequestUrl(CREATE_TASK_URL), taskId));
} }
public static void stopTask(String taskId) { public void stopTask(String taskId) {
send(MessageFormat.format(STOP_TASK_URL, taskId)); send(MessageFormat.format(getRequestUrl(STOP_TASK_URL), taskId));
} }
private static void send(String url) { private String getRequestUrl(String url) {
return protocol + "://" + host + ":" + port + url;
}
private void send(String url) {
HttpRequest request = HttpRequest.newBuilder() HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url)) .uri(URI.create(url))
.timeout(Duration.ofSeconds(30)) .timeout(Duration.ofSeconds(30))

View File

@@ -2,12 +2,18 @@ package com.datamate.cleaning.infrastructure.persistence.Impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.repository.CrudRepository; import com.baomidou.mybatisplus.extension.repository.CrudRepository;
import com.datamate.cleaning.common.enums.CleaningTaskStatusEnum;
import com.datamate.cleaning.domain.model.entity.CleaningResult; import com.datamate.cleaning.domain.model.entity.CleaningResult;
import com.datamate.cleaning.domain.repository.CleaningResultRepository; import com.datamate.cleaning.domain.repository.CleaningResultRepository;
import com.datamate.cleaning.infrastructure.converter.CleaningResultConverter;
import com.datamate.cleaning.infrastructure.persistence.mapper.CleaningResultMapper; import com.datamate.cleaning.infrastructure.persistence.mapper.CleaningResultMapper;
import com.datamate.cleaning.interfaces.dto.CleaningResultDto;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Repository; import org.springframework.stereotype.Repository;
import java.util.List;
@Repository @Repository
@RequiredArgsConstructor @RequiredArgsConstructor
public class CleaningResultRepositoryImpl extends CrudRepository<CleaningResultMapper, CleaningResult> public class CleaningResultRepositoryImpl extends CrudRepository<CleaningResultMapper, CleaningResult>
@@ -22,9 +28,20 @@ public class CleaningResultRepositoryImpl extends CrudRepository<CleaningResultM
} }
@Override @Override
public int countByInstanceId(String instanceId) { public int[] countByInstanceId(String instanceId) {
LambdaQueryWrapper<CleaningResult> lambdaWrapper = new LambdaQueryWrapper<>(); LambdaQueryWrapper<CleaningResult> lambdaWrapper = new LambdaQueryWrapper<>();
lambdaWrapper.eq(CleaningResult::getInstanceId, instanceId); lambdaWrapper.eq(CleaningResult::getInstanceId, instanceId);
return mapper.selectCount(lambdaWrapper).intValue(); List<CleaningResult> cleaningResults = mapper.selectList(lambdaWrapper);
int succeed = Math.toIntExact(cleaningResults.stream()
.filter(result ->
StringUtils.equals(result.getStatus(), CleaningTaskStatusEnum.COMPLETED.getValue()))
.count());
return new int[] {succeed, cleaningResults.size() - succeed};
}
public List<CleaningResultDto> findByInstanceId(String instanceId) {
LambdaQueryWrapper<CleaningResult> queryWrapper = new LambdaQueryWrapper<>();
queryWrapper.eq(CleaningResult::getInstanceId, instanceId);
return CleaningResultConverter.INSTANCE.convertEntityToDto(mapper.selectList(queryWrapper));
} }
} }

View File

@@ -7,6 +7,7 @@ import com.datamate.cleaning.interfaces.dto.OperatorInstanceDto;
import com.datamate.cleaning.domain.model.entity.OperatorInstance; import com.datamate.cleaning.domain.model.entity.OperatorInstance;
import com.datamate.cleaning.domain.repository.OperatorInstanceRepository; import com.datamate.cleaning.domain.repository.OperatorInstanceRepository;
import com.datamate.cleaning.infrastructure.persistence.mapper.OperatorInstanceMapper; import com.datamate.cleaning.infrastructure.persistence.mapper.OperatorInstanceMapper;
import com.datamate.operator.interfaces.dto.OperatorDto;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Repository; import org.springframework.stereotype.Repository;
@@ -37,4 +38,8 @@ public class OperatorInstanceRepositoryImpl extends CrudRepository<OperatorInsta
lambdaWrapper.eq(OperatorInstance::getInstanceId, instanceId); lambdaWrapper.eq(OperatorInstance::getInstanceId, instanceId);
mapper.delete(lambdaWrapper); mapper.delete(lambdaWrapper);
} }
public List<OperatorDto> findOperatorByInstanceId(String instanceId) {
return OperatorInstanceConverter.INSTANCE.fromEntityToDto(mapper.findOperatorByInstanceId(instanceId));
}
} }

View File

@@ -2,9 +2,22 @@ package com.datamate.cleaning.infrastructure.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.datamate.cleaning.domain.model.entity.OperatorInstance; import com.datamate.cleaning.domain.model.entity.OperatorInstance;
import com.datamate.operator.domain.model.OperatorView;
import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Select;
import java.util.List;
@Mapper @Mapper
public interface OperatorInstanceMapper extends BaseMapper<OperatorInstance> { public interface OperatorInstanceMapper extends BaseMapper<OperatorInstance> {
@Select("SELECT o.operator_id as id, o.operator_name as name, description, version, inputs, outputs, runtime, " +
" settings, created_at, updated_at, " +
"GROUP_CONCAT(category_id ORDER BY created_at DESC SEPARATOR ',') AS categories " +
"FROM t_operator_instance toi LEFT JOIN v_operator o ON toi.operator_id = o.operator_id " +
"WHERE toi.instance_id = #{instanceId} " +
"GROUP BY o.operator_id, o.operator_name, description, version, inputs, outputs, runtime," +
" settings, created_at, updated_at, op_index " +
"ORDER BY toi.op_index")
List<OperatorView> findOperatorByInstanceId(String instanceId);
} }

View File

@@ -16,23 +16,37 @@ import java.math.RoundingMode;
public class CleaningProcess { public class CleaningProcess {
private Float process; private Float process;
private Float successRate;
private Integer totalFileNum; private Integer totalFileNum;
private Integer succeedFileNum;
private Integer failedFileNum;
private Integer finishedFileNum; private Integer finishedFileNum;
public CleaningProcess(int totalFileNum, int finishedFileNum) { public CleaningProcess(int totalFileNum, int succeedFileNum, int failedFileNum) {
this.totalFileNum = totalFileNum; this.totalFileNum = totalFileNum;
this.finishedFileNum = finishedFileNum; this.succeedFileNum = succeedFileNum;
this.failedFileNum = failedFileNum;
this.finishedFileNum = succeedFileNum + failedFileNum;
if (totalFileNum == 0) { if (totalFileNum == 0) {
this.process = 0.0f; this.process = 0.0f;
} else { } else {
this.process = BigDecimal.valueOf(finishedFileNum * 100L) this.process = BigDecimal.valueOf(finishedFileNum * 100L)
.divide(BigDecimal.valueOf(totalFileNum), 2, RoundingMode.HALF_UP).floatValue(); .divide(BigDecimal.valueOf(totalFileNum), 2, RoundingMode.HALF_UP).floatValue();
} }
if (finishedFileNum == 0) {
this.successRate = 0f;
} else {
this.successRate = BigDecimal.valueOf(succeedFileNum * 100L)
.divide(BigDecimal.valueOf(finishedFileNum), 2, RoundingMode.HALF_UP).floatValue();
}
} }
public static CleaningProcess of(int totalFileNum, int finishedFileNum) { public static CleaningProcess of(int totalFileNum, int succeedFileNum, int failedFileNum) {
return new CleaningProcess(totalFileNum, finishedFileNum); return new CleaningProcess(totalFileNum, succeedFileNum, failedFileNum);
} }
} }

View File

@@ -0,0 +1,30 @@
package com.datamate.cleaning.interfaces.dto;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class CleaningResultDto {
private String instanceId;
private String srcFileId;
private String destFileId;
private String srcName;
private String destName;
private String srcType;
private String destType;
private long srcSize;
private long destSize;
private String status;
private String result;
}

View File

@@ -0,0 +1,12 @@
package com.datamate.cleaning.interfaces.dto;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class CleaningTaskLog {
private String level;
private String message;
}

View File

@@ -1,7 +1,9 @@
package com.datamate.cleaning.interfaces.rest; package com.datamate.cleaning.interfaces.rest;
import com.datamate.cleaning.application.CleaningTaskService; import com.datamate.cleaning.application.CleaningTaskService;
import com.datamate.cleaning.interfaces.dto.CleaningResultDto;
import com.datamate.cleaning.interfaces.dto.CleaningTaskDto; import com.datamate.cleaning.interfaces.dto.CleaningTaskDto;
import com.datamate.cleaning.interfaces.dto.CleaningTaskLog;
import com.datamate.cleaning.interfaces.dto.CreateCleaningTaskRequest; import com.datamate.cleaning.interfaces.dto.CreateCleaningTaskRequest;
import com.datamate.common.interfaces.PagedResponse; import com.datamate.common.interfaces.PagedResponse;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@@ -54,4 +56,14 @@ public class CleaningTaskController {
cleaningTaskService.deleteTask(taskId); cleaningTaskService.deleteTask(taskId);
return taskId; return taskId;
} }
@GetMapping("/{taskId}/result")
public List<CleaningResultDto> cleaningTasksTaskIdGetResult(@PathVariable("taskId") String taskId) {
return cleaningTaskService.getTaskResults(taskId);
}
@GetMapping("/{taskId}/log")
public List<CleaningTaskLog> cleaningTasksTaskIdGetLog(@PathVariable("taskId") String taskId) {
return cleaningTaskService.getTaskLog(taskId);
}
} }

View File

@@ -50,9 +50,7 @@ public class OperatorService {
public List<OperatorDto> getOperators(Integer page, Integer size, List<String> categories, public List<OperatorDto> getOperators(Integer page, Integer size, List<String> categories,
String operatorName, Boolean isStar) { String operatorName, Boolean isStar) {
List<OperatorView> filteredOperators = operatorViewRepo.findOperatorsByCriteria(page, size, operatorName, return operatorViewRepo.findOperatorsByCriteria(page, size, operatorName, categories, isStar);
categories, isStar);
return filteredOperators.stream().map(OperatorConverter.INSTANCE::fromEntityToDto).toList();
} }
public int getOperatorsCount(List<String> categories, String operatorName, Boolean isStar) { public int getOperatorsCount(List<String> categories, String operatorName, Boolean isStar) {

View File

@@ -2,12 +2,13 @@ package com.datamate.operator.domain.repository;
import com.baomidou.mybatisplus.extension.repository.IRepository; import com.baomidou.mybatisplus.extension.repository.IRepository;
import com.datamate.operator.domain.model.OperatorView; import com.datamate.operator.domain.model.OperatorView;
import com.datamate.operator.interfaces.dto.OperatorDto;
import java.util.List; import java.util.List;
public interface OperatorViewRepository extends IRepository<OperatorView> { public interface OperatorViewRepository extends IRepository<OperatorView> {
List<OperatorView> findOperatorsByCriteria(Integer page, Integer size, String operatorName, List<OperatorDto> findOperatorsByCriteria(Integer page, Integer size, String operatorName,
List<String> categories, Boolean isStar); List<String> categories, Boolean isStar);
Integer countOperatorsByCriteria(String operatorName, List<String> categories, Boolean isStar); Integer countOperatorsByCriteria(String operatorName, List<String> categories, Boolean isStar);

View File

@@ -21,6 +21,8 @@ public interface OperatorConverter {
@Mapping(target = "categories", source = "categories", qualifiedByName = "stringToList") @Mapping(target = "categories", source = "categories", qualifiedByName = "stringToList")
OperatorDto fromEntityToDto(OperatorView operator); OperatorDto fromEntityToDto(OperatorView operator);
List<OperatorDto> fromEntityViewToDto(List<OperatorView> operator);
List<OperatorDto> fromEntityToDto(List<Operator> operator); List<OperatorDto> fromEntityToDto(List<Operator> operator);
@Named("stringToList") @Named("stringToList")

View File

@@ -7,7 +7,9 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.repository.CrudRepository; import com.baomidou.mybatisplus.extension.repository.CrudRepository;
import com.datamate.operator.domain.model.OperatorView; import com.datamate.operator.domain.model.OperatorView;
import com.datamate.operator.domain.repository.OperatorViewRepository; import com.datamate.operator.domain.repository.OperatorViewRepository;
import com.datamate.operator.infrastructure.converter.OperatorConverter;
import com.datamate.operator.infrastructure.persistence.mapper.OperatorViewMapper; import com.datamate.operator.infrastructure.persistence.mapper.OperatorViewMapper;
import com.datamate.operator.interfaces.dto.OperatorDto;
import io.micrometer.common.util.StringUtils; import io.micrometer.common.util.StringUtils;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
@@ -21,20 +23,23 @@ public class OperatorViewRepositoryImpl extends CrudRepository<OperatorViewMappe
private final OperatorViewMapper mapper; private final OperatorViewMapper mapper;
@Override @Override
public List<OperatorView> findOperatorsByCriteria(Integer page, Integer size, String operatorName, public List<OperatorDto> findOperatorsByCriteria(Integer page, Integer size, String operatorName,
List<String> categories, Boolean isStar) { List<String> categories, Boolean isStar) {
QueryWrapper<OperatorView> queryWrapper = Wrappers.query(); QueryWrapper<OperatorView> queryWrapper = Wrappers.query();
queryWrapper.in(CollectionUtils.isNotEmpty(categories), "category_id", categories) queryWrapper.in(CollectionUtils.isNotEmpty(categories), "category_id", categories)
.like(StringUtils.isNotBlank(operatorName), "operator_name", operatorName) .like(StringUtils.isNotBlank(operatorName), "operator_name", operatorName)
.eq(isStar != null, "is_star", isStar) .eq(isStar != null, "is_star", isStar)
.groupBy("operator_id") .groupBy("operator_id")
.orderByDesc("created_at"); .orderByDesc("created_at");
Page<OperatorView> queryPage = null; Page<OperatorView> queryPage;
if (size != null && page != null) { if (size != null && page != null) {
queryPage = new Page<>(page + 1, size); queryPage = new Page<>(page + 1, size);
} else {
queryPage = new Page<>(1, -1);
} }
IPage<OperatorView> operators = mapper.findOperatorsByCriteria(queryPage, queryWrapper); IPage<OperatorView> operators = mapper.findOperatorsByCriteria(queryPage, queryWrapper);
return operators.getRecords();
return OperatorConverter.INSTANCE.fromEntityViewToDto(operators.getRecords());
} }
@Override @Override

View File

@@ -1,13 +1,18 @@
import { useState } from "react"; import {useEffect, useState} from "react";
import { Card, Button, Steps, Form, Divider } from "antd"; import {Button, Steps, Form, message} from "antd";
import { Link, useNavigate } from "react-router"; import {Link, useNavigate, useParams} from "react-router";
import { ArrowLeft } from "lucide-react"; import { ArrowLeft } from "lucide-react";
import { createCleaningTemplateUsingPost } from "../cleansing.api"; import {
createCleaningTemplateUsingPost,
queryCleaningTemplateByIdUsingGet,
updateCleaningTemplateByIdUsingPut
} from "../cleansing.api";
import CleansingTemplateStepOne from "./components/CreateTemplateStepOne"; import CleansingTemplateStepOne from "./components/CreateTemplateStepOne";
import { useCreateStepTwo } from "./hooks/useCreateStepTwo"; import { useCreateStepTwo } from "./hooks/useCreateStepTwo";
export default function CleansingTemplateCreate() { export default function CleansingTemplateCreate() {
const { id = "" } = useParams()
const navigate = useNavigate(); const navigate = useNavigate();
const [form] = Form.useForm(); const [form] = Form.useForm();
const [templateConfig, setTemplateConfig] = useState({ const [templateConfig, setTemplateConfig] = useState({
@@ -15,6 +20,21 @@ export default function CleansingTemplateCreate() {
description: "", description: "",
}); });
const fetchTemplateDetail = async () => {
if (!id) return;
try {
const { data } = await queryCleaningTemplateByIdUsingGet(id);
setTemplateConfig(data);
} catch (error) {
message.error("获取任务详情失败");
navigate("/data/cleansing");
}
};
useEffect(() => {
fetchTemplateDetail()
}, [id]);
const handleSave = async () => { const handleSave = async () => {
const template = { const template = {
...templateConfig, ...templateConfig,
@@ -27,7 +47,8 @@ export default function CleansingTemplateCreate() {
})), })),
}; };
await createCleaningTemplateUsingPost(template); !id && await createCleaningTemplateUsingPost(template) && message.success("模板创建成功");
id && await updateCleaningTemplateByIdUsingPut(id, template) && message.success("模板更新成功");
navigate("/data/cleansing?view=template"); navigate("/data/cleansing?view=template");
}; };
@@ -79,7 +100,7 @@ export default function CleansingTemplateCreate() {
<ArrowLeft className="w-4 h-4 mr-1" /> <ArrowLeft className="w-4 h-4 mr-1" />
</Button> </Button>
</Link> </Link>
<h1 className="text-xl font-bold"></h1> <h1 className="text-xl font-bold">{id ? '更新清洗模板' : '创建清洗模板'}</h1>
</div> </div>
<div className="w-1/2"> <div className="w-1/2">
<Steps <Steps
@@ -101,7 +122,7 @@ export default function CleansingTemplateCreate() {
onClick={handleSave} onClick={handleSave}
disabled={!canProceed()} disabled={!canProceed()}
> >
{id ? '更新模板' : '创建模板'}
</Button> </Button>
) : ( ) : (
<Button <Button

View File

@@ -1,4 +1,5 @@
import { Input, Form } from "antd"; import { Input, Form } from "antd";
import {useEffect} from "react";
const { TextArea } = Input; const { TextArea } = Input;
@@ -16,6 +17,11 @@ export default function CreateTemplateStepOne({
const handleValuesChange = (_, allValues) => { const handleValuesChange = (_, allValues) => {
setTemplateConfig({ ...templateConfig, ...allValues }); setTemplateConfig({ ...templateConfig, ...allValues });
}; };
useEffect(() => {
form.setFieldsValue(templateConfig);
}, [templateConfig]);
return ( return (
<Form <Form
form={form} form={form}

View File

@@ -1,17 +1,9 @@
import React, { useMemo, useState } from "react"; import React, {useEffect, useMemo, useState} from "react";
import { import {Button, Card, Checkbox, Collapse, Input, Select, Tag, Tooltip,} from "antd";
Card, import {SearchOutlined, StarFilled, StarOutlined} from "@ant-design/icons";
Input, import {CategoryI, OperatorI} from "@/pages/OperatorMarket/operator.model";
Select, import {Layers} from "lucide-react";
Tooltip, import {updateOperatorByIdUsingPut} from "@/pages/OperatorMarket/operator.api.ts";
Collapse,
Tag,
Checkbox,
Button,
} from "antd";
import { StarFilled, StarOutlined, SearchOutlined } from "@ant-design/icons";
import { CategoryI, OperatorI } from "@/pages/OperatorMarket/operator.model";
import { Layers } from "lucide-react";
interface OperatorListProps { interface OperatorListProps {
operators: OperatorI[]; operators: OperatorI[];
@@ -27,12 +19,20 @@ interface OperatorListProps {
) => void; ) => void;
} }
const handleStar = async (operator: OperatorI, toggleFavorite: (id: string) => void) => {
const data = {
id: operator.id,
isStar: !operator.isStar
};
await updateOperatorByIdUsingPut(operator.id, data);
toggleFavorite(operator.id)
}
const OperatorList: React.FC<OperatorListProps> = ({ const OperatorList: React.FC<OperatorListProps> = ({
operators, operators,
favorites, favorites,
toggleFavorite, toggleFavorite,
toggleOperator, toggleOperator,
showPoppular,
selectedOperators, selectedOperators,
onDragOperator, onDragOperator,
}) => ( }) => (
@@ -56,17 +56,9 @@ const OperatorList: React.FC<OperatorListProps> = ({
{operator.name} {operator.name}
</span> </span>
</div> </div>
{showPoppular && operator.isStar && (
<Tag color="gold" className="text-xs">
</Tag>
)}
<span <span
className="cursor-pointer" className="cursor-pointer"
onClick={(e) => { onClick={() => handleStar(operator, toggleFavorite)}
e.stopPropagation();
toggleFavorite(operator.id);
}}
> >
{favorites.has(operator.id) ? ( {favorites.has(operator.id) ? (
<StarFilled style={{ color: "#FFD700" }} /> <StarFilled style={{ color: "#FFD700" }} />
@@ -156,10 +148,9 @@ const OperatorLibrary: React.FC<OperatorLibraryProps> = ({
// 过滤算子 // 过滤算子
const filteredOperators = useMemo(() => { const filteredOperators = useMemo(() => {
const filtered = Object.values(groupedOperators).flatMap( return Object.values(groupedOperators).flatMap(
(category) => category.operators (category) => category.operators
); );
return filtered;
}, [groupedOperators]); }, [groupedOperators]);
// 收藏切换 // 收藏切换
@@ -173,6 +164,18 @@ const OperatorLibrary: React.FC<OperatorLibraryProps> = ({
setFavorites(newFavorites); setFavorites(newFavorites);
}; };
const fetchFavorite = async () => {
const newFavorites = new Set(favorites);
operatorList.forEach(item => {
item.isStar && newFavorites.add(item.id);
});
setFavorites(newFavorites);
}
useEffect(() => {
fetchFavorite()
}, [operatorList]);
// 全选分类算子 // 全选分类算子
const handleSelectAll = (operators: OperatorI[]) => { const handleSelectAll = (operators: OperatorI[]) => {
const newSelected = [...selectedOperators]; const newSelected = [...selectedOperators];
@@ -257,7 +260,6 @@ const OperatorLibrary: React.FC<OperatorLibraryProps> = ({
} }
> >
<OperatorList <OperatorList
showPoppular
selectedOperators={selectedOperators} selectedOperators={selectedOperators}
operators={category.operators} operators={category.operators}
favorites={favorites} favorites={favorites}

View File

@@ -1,15 +1,16 @@
import React, { useState } from "react"; import React, {useMemo, useState} from "react";
import { Card, Input, Tag, Select, Button } from "antd"; import { Card, Input, Tag, Select, Button } from "antd";
import { DeleteOutlined } from "@ant-design/icons"; import { DeleteOutlined } from "@ant-design/icons";
import { CleansingTemplate } from "../../cleansing.model"; import { CleansingTemplate } from "../../cleansing.model";
import { Workflow } from "lucide-react"; import { Workflow } from "lucide-react";
import { OperatorI } from "@/pages/OperatorMarket/operator.model"; import {CategoryI, OperatorI} from "@/pages/OperatorMarket/operator.model";
interface OperatorFlowProps { interface OperatorFlowProps {
selectedOperators: OperatorI[]; selectedOperators: OperatorI[];
configOperator: OperatorI | null; configOperator: OperatorI | null;
templates: CleansingTemplate[]; templates: CleansingTemplate[];
currentTemplate: CleansingTemplate | null; currentTemplate: CleansingTemplate | null;
categoryOptions: [];
setCurrentTemplate: (template: CleansingTemplate | null) => void; setCurrentTemplate: (template: CleansingTemplate | null) => void;
removeOperator: (id: string) => void; removeOperator: (id: string) => void;
setSelectedOperators: (operators: OperatorI[]) => void; setSelectedOperators: (operators: OperatorI[]) => void;
@@ -33,6 +34,7 @@ const OperatorFlow: React.FC<OperatorFlowProps> = ({
configOperator, configOperator,
templates, templates,
currentTemplate, currentTemplate,
categoryOptions,
setSelectedOperators, setSelectedOperators,
setConfigOperator, setConfigOperator,
removeOperator, removeOperator,
@@ -47,6 +49,16 @@ const OperatorFlow: React.FC<OperatorFlowProps> = ({
}) => { }) => {
const [editingIndex, setEditingIndex] = useState<string | null>(null); const [editingIndex, setEditingIndex] = useState<string | null>(null);
const categoryMap = useMemo(() => {
const map: { [key: string]: CategoryI } = {};
categoryOptions.forEach((cat: any) => {
map[cat.id] = {
...cat,
};
});
return map;
}, [categoryOptions]);
// 添加编号修改处理函数 // 添加编号修改处理函数
const handleIndexChange = (operatorId: string, newIndex: string) => { const handleIndexChange = (operatorId: string, newIndex: string) => {
const index = Number.parseInt(newIndex); const index = Number.parseInt(newIndex);
@@ -167,8 +179,9 @@ const OperatorFlow: React.FC<OperatorFlowProps> = ({
{operator.name} {operator.name}
</span> </span>
</div> </div>
{/* 分类标签 */} {operator?.categories?.map((categoryId) => {
<Tag color="default"></Tag> return <Tag color="default">{categoryMap[categoryId].name}</Tag>
})}
{/* 参数状态指示 */} {/* 参数状态指示 */}
{Object.values(operator.configs).some( {Object.values(operator.configs).some(
(param: any) => (param: any) =>
@@ -192,7 +205,7 @@ const OperatorFlow: React.FC<OperatorFlowProps> = ({
))} ))}
{selectedOperators.length === 0 && ( {selectedOperators.length === 0 && (
<div className="text-center py-16 text-gray-400 border-2 border-dashed border-gray-100 rounded-lg"> <div className="text-center py-16 text-gray-400 border-2 border-dashed border-gray-100 rounded-lg">
<Workflow className="w-full w-10 h-10 mb-4 opacity-50" /> <Workflow className="w-full h-10 mb-4 opacity-50" />
<div className="text-lg font-medium mb-2"></div> <div className="text-lg font-medium mb-2"></div>
<div className="text-sm"> <div className="text-sm">

View File

@@ -55,6 +55,7 @@ export function useCreateStepTwo() {
configOperator={configOperator} configOperator={configOperator}
templates={templates} templates={templates}
currentTemplate={currentTemplate} currentTemplate={currentTemplate}
categoryOptions={categoryOptions}
setSelectedOperators={setSelectedOperators} setSelectedOperators={setSelectedOperators}
setConfigOperator={setConfigOperator} setConfigOperator={setConfigOperator}
setCurrentTemplate={setCurrentTemplate} setCurrentTemplate={setCurrentTemplate}

View File

@@ -1,13 +1,15 @@
import { useEffect, useState } from "react"; import { useEffect, useState } from "react";
import { OperatorI } from "@/pages/OperatorMarket/operator.model"; import { OperatorI } from "@/pages/OperatorMarket/operator.model";
import { CleansingTemplate } from "../../cleansing.model"; import { CleansingTemplate } from "../../cleansing.model";
import { queryCleaningTemplatesUsingGet } from "../../cleansing.api"; import {queryCleaningTemplateByIdUsingGet, queryCleaningTemplatesUsingGet} from "../../cleansing.api";
import { import {
queryCategoryTreeUsingGet, queryCategoryTreeUsingGet,
queryOperatorsUsingPost, queryOperatorsUsingPost,
} from "@/pages/OperatorMarket/operator.api"; } from "@/pages/OperatorMarket/operator.api";
import {useParams} from "react-router";
export function useOperatorOperations() { export function useOperatorOperations() {
const { id = "" } = useParams();
const [currentStep, setCurrentStep] = useState(1); const [currentStep, setCurrentStep] = useState(1);
const [operators, setOperators] = useState<OperatorI[]>([]); const [operators, setOperators] = useState<OperatorI[]>([]);
@@ -21,7 +23,7 @@ export function useOperatorOperations() {
// 将后端返回的算子数据映射为前端需要的格式 // 将后端返回的算子数据映射为前端需要的格式
const mapOperator = (op: OperatorI) => { const mapOperator = (op: OperatorI) => {
const configs = const configs =
op.settings && typeof op.settings === "string" op.settings
? JSON.parse(op.settings) ? JSON.parse(op.settings)
: {}; : {};
const defaultParams: Record<string, string> = {}; const defaultParams: Record<string, string> = {};
@@ -64,14 +66,26 @@ export function useOperatorOperations() {
}; };
const initTemplates = async () => { const initTemplates = async () => {
const { data } = await queryCleaningTemplatesUsingGet(); if (id) {
const newTemplates = const { data } = await queryCleaningTemplateByIdUsingGet(id);
data.content?.map?.((item) => ({ const template = {
...item, ...data,
label: item.name, label: data.name,
value: item.id, value: data.id,
})) || []; }
setTemplates(newTemplates); setTemplates([template])
setCurrentTemplate(template)
} else {
const { data } = await queryCleaningTemplatesUsingGet();
const newTemplates =
data.content?.map?.((item) => ({
...item,
label: item.name,
value: item.id,
})) || [];
setTemplates(newTemplates);
setCurrentTemplate(newTemplates?.[0])
}
}; };
useEffect(() => { useEffect(() => {

View File

@@ -1,29 +1,30 @@
import { useEffect, useState } from "react"; import { useEffect, useState } from "react";
import { Card, Breadcrumb, App } from "antd"; import {Breadcrumb, App, Tabs} from "antd";
import { import {
Play, Play,
Pause, Pause,
Clock, Clock,
CheckCircle, CheckCircle,
AlertCircle, AlertCircle,
Database,
Trash2, Trash2,
Activity, Activity, LayoutList,
} from "lucide-react"; } from "lucide-react";
import DetailHeader from "@/components/DetailHeader"; import DetailHeader from "@/components/DetailHeader";
import { Link, useNavigate, useParams } from "react-router"; import { Link, useNavigate, useParams } from "react-router";
import { import {
deleteCleaningTaskByIdUsingDelete, deleteCleaningTaskByIdUsingDelete,
executeCleaningTaskUsingPost, executeCleaningTaskUsingPost,
queryCleaningTaskByIdUsingGet, queryCleaningTaskByIdUsingGet, queryCleaningTaskLogByIdUsingGet, queryCleaningTaskResultByIdUsingGet,
stopCleaningTaskUsingPost, stopCleaningTaskUsingPost,
} from "../cleansing.api"; } from "../cleansing.api";
import { TaskStatusMap } from "../cleansing.const"; import {mapTask, TaskStatusMap} from "../cleansing.const";
import { TaskStatus } from "@/pages/DataCleansing/cleansing.model"; import {CleansingResult, TaskStatus} from "@/pages/DataCleansing/cleansing.model";
import BasicInfo from "./components/BasicInfo"; import BasicInfo from "./components/BasicInfo";
import OperatorTable from "./components/OperatorTable"; import OperatorTable from "./components/OperatorTable";
import FileTable from "./components/FileTable"; import FileTable from "./components/FileTable";
import LogsTable from "./components/LogsTable"; import LogsTable from "./components/LogsTable";
import {formatExecutionDuration} from "@/utils/unit.ts";
import {ReloadOutlined} from "@ant-design/icons";
// 任务详情页面组件 // 任务详情页面组件
export default function CleansingTaskDetail() { export default function CleansingTaskDetail() {
@@ -35,7 +36,7 @@ export default function CleansingTaskDetail() {
if (!id) return; if (!id) return;
try { try {
const { data } = await queryCleaningTaskByIdUsingGet(id); const { data } = await queryCleaningTaskByIdUsingGet(id);
setTask(data); setTask(mapTask(data));
} catch (error) { } catch (error) {
message.error("获取任务详情失败"); message.error("获取任务详情失败");
navigate("/data/cleansing"); navigate("/data/cleansing");
@@ -60,6 +61,38 @@ export default function CleansingTaskDetail() {
navigate("/data/cleansing"); navigate("/data/cleansing");
}; };
const [result, setResult] = useState<CleansingResult[]>();
const fetchTaskResult = async () => {
if (!id) return;
try {
const { data } = await queryCleaningTaskResultByIdUsingGet(id);
setResult(data);
} catch (error) {
message.error("获取清洗结果失败");
navigate("/data/cleansing/task-detail/" + id);
}
};
const [taskLog, setTaskLog] = useState();
const fetchTaskLog = async () => {
if (!id) return;
try {
const { data } = await queryCleaningTaskLogByIdUsingGet(id);
setTaskLog(data);
} catch (error) {
message.error("获取清洗日志失败");
navigate("/data/cleansing/task-detail/" + id);
}
};
const handleRefresh = async () => {
fetchTaskDetail();
{activeTab === "files" && await fetchTaskResult()}
{activeTab === "logs" && await fetchTaskLog()}
};
useEffect(() => { useEffect(() => {
fetchTaskDetail(); fetchTaskDetail();
}, [id]); }, [id]);
@@ -69,9 +102,9 @@ export default function CleansingTaskDetail() {
const headerData = { const headerData = {
...task, ...task,
icon: <Database className="w-8 h-8" />, icon: <LayoutList className="w-8 h-8" />,
status: TaskStatusMap[task?.status], status: TaskStatusMap[task?.status],
createdAt: task?.startTime, createdAt: task?.createdAt,
lastUpdated: task?.updatedAt, lastUpdated: task?.updatedAt,
}; };
@@ -79,22 +112,24 @@ export default function CleansingTaskDetail() {
{ {
icon: <Clock className="w-4 h-4 text-blue-500" />, icon: <Clock className="w-4 h-4 text-blue-500" />,
label: "总耗时", label: "总耗时",
value: task?.duration || "--", value: formatExecutionDuration(task?.startedAt, task?.finishedAt) || "--",
}, },
{ {
icon: <CheckCircle className="w-4 h-4 text-green-500" />, icon: <CheckCircle className="w-4 h-4 text-green-500" />,
label: "成功文件", label: "成功文件",
value: task?.successFiles || "--", value: task?.progress?.succeedFileNum || "0",
}, },
{ {
icon: <AlertCircle className="w-4 h-4 text-red-500" />, icon: <AlertCircle className="w-4 h-4 text-red-500" />,
label: "失败文件", label: "失败文件",
value: task?.failedFiles || "--", value: (task?.status.value === TaskStatus.RUNNING || task?.status.value === TaskStatus.PENDING) ?
task?.progress.failedFileNum :
task?.progress?.totalFileNum - task?.progress.succeedFileNum,
}, },
{ {
icon: <Activity className="w-4 h-4 text-purple-500" />, icon: <Activity className="w-4 h-4 text-purple-500" />,
label: "成功率", label: "成功率",
value: `${task?.progress}%`, value: task?.progress?.successRate ? task?.progress?.successRate + "%" : "--",
}, },
]; ];
@@ -109,7 +144,7 @@ export default function CleansingTaskDetail() {
}, },
] ]
: []), : []),
...(task?.status === TaskStatus.PENDING ...([TaskStatus.PENDING, TaskStatus.STOPPED, TaskStatus.FAILED].includes(task?.status?.value)
? [ ? [
{ {
key: "start", key: "start",
@@ -119,6 +154,12 @@ export default function CleansingTaskDetail() {
}, },
] ]
: []), : []),
{
key: "refresh",
label: "更新任务",
icon: <ReloadOutlined className="w-4 h-4" />,
onClick: handleRefresh,
},
{ {
key: "delete", key: "delete",
label: "删除任务", label: "删除任务",
@@ -131,20 +172,20 @@ export default function CleansingTaskDetail() {
const tabList = [ const tabList = [
{ {
key: "basic", key: "basic",
tab: "基本信息", label: "基本信息",
children: <BasicInfo task={task} />,
}, },
{ {
key: "operators", key: "operators",
tab: "处理算子", label: "处理算子",
children: <OperatorTable task={task} />,
}, },
{ {
key: "files", key: "files",
tab: "处理文件", label: "处理文件",
children: <FileTable task={task} />, },
{
key: "logs",
label: "运行日志",
}, },
{ key: "logs", tab: "运行日志", children: <LogsTable task={task} /> },
]; ];
const breadItems = [ const breadItems = [
@@ -157,7 +198,7 @@ export default function CleansingTaskDetail() {
]; ];
return ( return (
<div className="min-h-screen"> <>
<Breadcrumb items={breadItems} /> <Breadcrumb items={breadItems} />
<div className="mb-4 mt-4"> <div className="mb-4 mt-4">
<DetailHeader <DetailHeader
@@ -166,11 +207,17 @@ export default function CleansingTaskDetail() {
operations={operations} operations={operations}
/> />
</div> </div>
<Card <div className="flex-overflow-auto p-6 pt-2 bg-white rounded-md shadow">
tabList={tabList} <Tabs activeKey={activeTab} items={tabList} onChange={setActiveTab} />
activeTabKey={activeTab} <div className="h-full flex-1 overflow-auto">
onTabChange={setActiveTab} {activeTab === "basic" && (
></Card> <BasicInfo task={task} />
</div> )}
{activeTab === "operators" && <OperatorTable task={task} />}
{activeTab === "files" && <FileTable result={result} fetchTaskResult={fetchTaskResult} />}
{activeTab === "logs" && <LogsTable taskLog={taskLog} fetchTaskLog={fetchTaskLog} />}
</div>
</div>
</>
); );
} }

View File

@@ -0,0 +1,122 @@
import { useEffect, useState } from "react";
import {Breadcrumb, App, Tabs} from "antd";
import {
Trash2,
LayoutList,
} from "lucide-react";
import DetailHeader from "@/components/DetailHeader";
import { Link, useNavigate, useParams } from "react-router";
import {
deleteCleaningTemplateByIdUsingDelete,
queryCleaningTemplateByIdUsingGet,
} from "../cleansing.api";
import {mapTemplate} from "../cleansing.const";
import OperatorTable from "./components/OperatorTable";
import {EditOutlined, ReloadOutlined, NumberOutlined} from "@ant-design/icons";
// 任务详情页面组件
export default function CleansingTemplateDetail() {
const { id = "" } = useParams(); // 获取动态路由参数
const { message } = App.useApp();
const navigate = useNavigate();
const [template, setTemplate] = useState();
const fetchTemplateDetail = async () => {
if (!id) return;
try {
const { data } = await queryCleaningTemplateByIdUsingGet(id);
setTemplate(mapTemplate(data));
} catch (error) {
message.error("获取任务详情失败");
navigate("/data/cleansing");
}
};
const deleteTemplate = async () => {
await deleteCleaningTemplateByIdUsingDelete(id);
message.success("模板已删除");
navigate("/data/cleansing");
};
const handleRefresh = async () => {
fetchTemplateDetail();
};
useEffect(() => {
fetchTemplateDetail();
}, [id]);
const [activeTab, setActiveTab] = useState("operators");
const headerData = {
...template,
icon: <LayoutList className="w-8 h-8" />,
createdAt: template?.createdAt,
lastUpdated: template?.updatedAt,
};
const statistics = [
{
icon: <NumberOutlined className="w-4 h-4 text-green-500" />,
label: "算子数量",
value: template?.instance?.length || 0,
},
];
const operations = [
{
key: "update",
label: "更新任务",
icon: <EditOutlined className="w-4 h-4" />,
onClick: () => navigate(`/data/cleansing/update-template/${id}`),
},
{
key: "refresh",
label: "更新任务",
icon: <ReloadOutlined className="w-4 h-4" />,
onClick: handleRefresh,
},
{
key: "delete",
label: "删除任务",
icon: <Trash2 className="w-4 h-4" />,
danger: true,
onClick: deleteTemplate,
},
];
const tabList = [
{
key: "operators",
label: "处理算子",
},
];
const breadItems = [
{
title: <Link to="/data/cleansing"></Link>,
},
{
title: "模板详情",
},
];
return (
<>
<Breadcrumb items={breadItems} />
<div className="mb-4 mt-4">
<DetailHeader
data={headerData}
statistics={statistics}
operations={operations}
/>
</div>
<div className="flex-overflow-auto p-6 pt-2 bg-white rounded-md shadow">
<Tabs activeKey={activeTab} items={tabList} onChange={setActiveTab} />
<div className="h-full flex-1 overflow-auto">
<OperatorTable task={template} />
</div>
</div>
</>
);
}

View File

@@ -1,8 +1,8 @@
import type { CleansingTask } from "@/pages/DataCleansing/cleansing.model"; import {CleansingTask, TaskStatus} from "@/pages/DataCleansing/cleansing.model";
import { OperatorI } from "@/pages/OperatorMarket/operator.model"; import { Button, Card, Descriptions, Progress } from "antd";
import { Button, Card, Descriptions, Progress, Tag } from "antd";
import { Activity, AlertCircle, CheckCircle, Clock } from "lucide-react"; import { Activity, AlertCircle, CheckCircle, Clock } from "lucide-react";
import { useNavigate } from "react-router"; import { useNavigate } from "react-router";
import {formatExecutionDuration} from "@/utils/unit.ts";
export default function BasicInfo({ task }: { task: CleansingTask }) { export default function BasicInfo({ task }: { task: CleansingTask }) {
const navigate = useNavigate(); const navigate = useNavigate();
@@ -11,7 +11,7 @@ export default function BasicInfo({ task }: { task: CleansingTask }) {
{ {
key: "id", key: "id",
label: "任务ID", label: "任务ID",
children: <span className="font-mono">#{task?.id}</span>, children: <span className="font-mono">{task?.id}</span>,
}, },
{ key: "name", label: "任务名称", children: task?.name }, { key: "name", label: "任务名称", children: task?.name },
{ {
@@ -19,6 +19,7 @@ export default function BasicInfo({ task }: { task: CleansingTask }) {
label: "源数据集", label: "源数据集",
children: ( children: (
<Button <Button
style={{ paddingLeft: 0, marginLeft: 0 }}
type="link" type="link"
size="small" size="small"
onClick={() => onClick={() =>
@@ -34,6 +35,7 @@ export default function BasicInfo({ task }: { task: CleansingTask }) {
label: "目标数据集", label: "目标数据集",
children: ( children: (
<Button <Button
style={{ paddingLeft: 0, marginLeft: 0 }}
type="link" type="link"
size="small" size="small"
onClick={() => onClick={() =>
@@ -44,26 +46,12 @@ export default function BasicInfo({ task }: { task: CleansingTask }) {
</Button> </Button>
), ),
}, },
{ key: "template", label: "使用模板", children: task?.template },
{ key: "startTime", label: "开始时间", children: task?.startedAt }, { key: "startTime", label: "开始时间", children: task?.startedAt },
{ key: "estimatedTime", label: "预计用时", children: task?.estimatedTime },
{ {
key: "description", key: "description",
label: "任务描述", label: "任务描述",
children: ( children: (
<span className="text-gray-600">{task?.description || "暂无描述"}</span> <span className="text-gray-600">{task?.description || "--"}</span>
),
span: 2,
},
{
key: "rules",
label: "处理算子",
children: (
<div className="flex flex-wrap gap-1">
{task?.instance?.map?.((op: OperatorI) => (
<Tag key={op.id}>{op.name}</Tag>
))}
</div>
), ),
span: 2, span: 2,
}, },
@@ -77,28 +65,30 @@ export default function BasicInfo({ task }: { task: CleansingTask }) {
<div className="text-center p-4 bg-gradient-to-br from-blue-50 to-blue-100 rounded-lg"> <div className="text-center p-4 bg-gradient-to-br from-blue-50 to-blue-100 rounded-lg">
<Clock className="w-8 h-8 text-blue-500 mb-2 mx-auto" /> <Clock className="w-8 h-8 text-blue-500 mb-2 mx-auto" />
<div className="text-xl font-bold text-blue-500"> <div className="text-xl font-bold text-blue-500">
{task?.duration || "--"} {formatExecutionDuration(task?.startedAt, task?.finishedAt) || "--"}
</div> </div>
<div className="text-sm text-gray-600"></div> <div className="text-sm text-gray-600"></div>
</div> </div>
<div className="text-center p-4 bg-gradient-to-br from-green-50 to-green-100 rounded-lg"> <div className="text-center p-4 bg-gradient-to-br from-green-50 to-green-100 rounded-lg">
<CheckCircle className="w-8 h-8 text-green-500 mb-2 mx-auto" /> <CheckCircle className="w-8 h-8 text-green-500 mb-2 mx-auto" />
<div className="text-xl font-bold text-green-500"> <div className="text-xl font-bold text-green-500">
{task?.successFiles || "--"} {task?.progress?.succeedFileNum || "0"}
</div> </div>
<div className="text-sm text-gray-600"></div> <div className="text-sm text-gray-600"></div>
</div> </div>
<div className="text-center p-4 bg-gradient-to-br from-red-50 to-red-100 rounded-lg"> <div className="text-center p-4 bg-gradient-to-br from-red-50 to-red-100 rounded-lg">
<AlertCircle className="w-8 h-8 text-red-500 mb-2 mx-auto" /> <AlertCircle className="w-8 h-8 text-red-500 mb-2 mx-auto" />
<div className="text-xl font-bold text-red-500"> <div className="text-xl font-bold text-red-500">
{task?.failedFiles || "--"} {(task?.status.value === TaskStatus.RUNNING || task?.status.value === TaskStatus.PENDING) ?
task?.progress.failedFileNum :
task?.progress?.totalFileNum - task?.progress.succeedFileNum}
</div> </div>
<div className="text-sm text-gray-600"></div> <div className="text-sm text-gray-600"></div>
</div> </div>
<div className="text-center p-4 bg-gradient-to-br from-purple-50 to-purple-100 rounded-lg"> <div className="text-center p-4 bg-gradient-to-br from-purple-50 to-purple-100 rounded-lg">
<Activity className="w-8 h-8 text-purple-500 mb-2 mx-auto" /> <Activity className="w-8 h-8 text-purple-500 mb-2 mx-auto" />
<div className="text-xl font-bold text-purple-500"> <div className="text-xl font-bold text-purple-500">
{task?.progress || "--"} {task?.progress?.successRate ? task?.progress?.successRate + "%" : "--"}
</div> </div>
<div className="text-sm text-gray-600"></div> <div className="text-sm text-gray-600"></div>
</div> </div>
@@ -120,25 +110,22 @@ export default function BasicInfo({ task }: { task: CleansingTask }) {
{/* 处理进度 */} {/* 处理进度 */}
<div> <div>
<h3 className="text-lg font-semibold text-gray-900 mb-4"></h3> <h3 className="text-lg font-semibold text-gray-900 mb-4"></h3>
<Progress percent={task?.progress} showInfo /> <Progress percent={task?.progress?.process} showInfo />
<div className="grid grid-cols-2 gap-4 text-sm mt-4"> <div className="grid grid-cols-2 gap-4 text-sm mt-4">
<div className="flex items-center gap-2"> <div className="flex items-center gap-2">
<span className="w-3 h-3 bg-green-500 rounded-full inline-block" /> <span className="w-3 h-3 bg-green-500 rounded-full inline-block" />
<span>: {task?.processedFiles || "--"}</span> <span>: {task?.progress?.succeedFileNum || "0"}</span>
</div> </div>
<div className="flex items-center gap-2"> <div className="flex items-center gap-2">
<span className="w-3 h-3 bg-blue-500 rounded-full inline-block" /> <span className="w-3 h-3 bg-blue-500 rounded-full inline-block" />
<span>: {task?.processingFiles || "--"}</span> <span>: {(task?.status.value === TaskStatus.RUNNING || task?.status.value === TaskStatus.PENDING) ?
</div> task?.progress?.totalFileNum - task?.progress.succeedFileNum : 0}</span>
<div className="flex items-center gap-2">
<span className="w-3 h-3 bg-gray-300 rounded-full inline-block" />
<span>
: {task?.totalFiles - task?.processedFiles || "--"}
</span>
</div> </div>
<div className="flex items-center gap-2"> <div className="flex items-center gap-2">
<span className="w-3 h-3 bg-red-500 rounded-full inline-block" /> <span className="w-3 h-3 bg-red-500 rounded-full inline-block" />
<span>: {task?.failedFiles || "--"}</span> <span>: {(task?.status.value === TaskStatus.RUNNING || task?.status.value === TaskStatus.PENDING) ?
task?.progress.failedFileNum :
task?.progress?.totalFileNum - task?.progress.succeedFileNum}</span>
</div> </div>
</div> </div>
</div> </div>

View File

@@ -1,70 +1,30 @@
import { Button, Modal, Table, Badge, Input } from "antd"; import {Button, Modal, Table, Badge, Input} from "antd";
import { Download, FileText } from "lucide-react"; import { Download } from "lucide-react";
import { useState } from "react"; import {useEffect, useState} from "react";
import {useParams} from "react-router";
import {TaskStatus} from "@/pages/DataCleansing/cleansing.model.ts";
import {TaskStatusMap} from "@/pages/DataCleansing/cleansing.const.tsx";
// 模拟文件列表数据 // 模拟文件列表数据
const fileList = [ export default function FileTable({result, fetchTaskResult}) {
{ const { id = "" } = useParams();
id: 1,
fileName: "lung_cancer_001.svs",
originalSize: "15.2MB",
processedSize: "8.5MB",
status: "已完成",
duration: "2分15秒",
processedAt: "2024-01-20 09:32:40",
},
{
id: 2,
fileName: "lung_cancer_002.svs",
originalSize: "18.7MB",
processedSize: "10.2MB",
status: "已完成",
duration: "2分38秒",
processedAt: "2024-01-20 09:35:18",
},
{
id: 3,
fileName: "lung_cancer_003.svs",
originalSize: "12.3MB",
processedSize: "6.8MB",
status: "已完成",
duration: "1分52秒",
processedAt: "2024-01-20 09:37:10",
},
{
id: 4,
fileName: "lung_cancer_004.svs",
originalSize: "20.1MB",
processedSize: "-",
status: "失败",
duration: "0分45秒",
processedAt: "2024-01-20 09:38:55",
},
{
id: 5,
fileName: "lung_cancer_005.svs",
originalSize: "16.8MB",
processedSize: "9.3MB",
status: "已完成",
duration: "2分22秒",
processedAt: "2024-01-20 09:41:17",
},
];
export default function FileTable() {
const [showFileCompareDialog, setShowFileCompareDialog] = useState(false); const [showFileCompareDialog, setShowFileCompareDialog] = useState(false);
const [showFileLogDialog, setShowFileLogDialog] = useState(false);
const [selectedFile, setSelectedFile] = useState<any>(null); const [selectedFile, setSelectedFile] = useState<any>(null);
const [selectedFileIds, setSelectedFileIds] = useState<number[]>([]); const [selectedFileIds, setSelectedFileIds] = useState<string[]>([]);
useEffect(() => {
fetchTaskResult();
}, [id]);
const handleSelectAllFiles = (checked: boolean) => { const handleSelectAllFiles = (checked: boolean) => {
if (checked) { if (checked) {
setSelectedFileIds(fileList.map((file) => file.id)); setSelectedFileIds(result.map((file) => file.instanceId));
} else { } else {
setSelectedFileIds([]); setSelectedFileIds([]);
} }
}; };
const handleSelectFile = (fileId: number, checked: boolean) => { const handleSelectFile = (fileId: string, checked: boolean) => {
if (checked) { if (checked) {
setSelectedFileIds([...selectedFileIds, fileId]); setSelectedFileIds([...selectedFileIds, fileId]);
} else { } else {
@@ -79,116 +39,16 @@ export default function FileTable() {
// 实际下载逻辑 // 实际下载逻辑
}; };
const handleBatchDeleteFiles = () => { function formatFileSize(bytes: number, decimals: number = 2): string {
// 实际删除逻辑 if (bytes === 0) return '0 Bytes';
setSelectedFileIds([]);
};
const handleViewFileLog = (file: any) => {
setSelectedFile(file);
setShowFileLogDialog(true);
};
// 模拟单个文件的处理日志 const k = 1024;
const getFileProcessLog = (fileName: string) => [ const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
{
time: "09:30:18", const i = Math.floor(Math.log(bytes) / Math.log(k));
step: "开始处理",
operator: "格式转换", return parseFloat((bytes / Math.pow(k, i)).toFixed(decimals)) + ' ' + sizes[i];
status: "INFO", }
message: `开始处理文件: ${fileName}`,
},
{
time: "09:30:19",
step: "文件验证",
operator: "格式转换",
status: "INFO",
message: "验证文件格式和完整性",
},
{
time: "09:30:20",
step: "格式解析",
operator: "格式转换",
status: "INFO",
message: "解析SVS格式文件",
},
{
time: "09:30:25",
step: "格式转换",
operator: "格式转换",
status: "SUCCESS",
message: "成功转换为JPEG格式",
},
{
time: "09:30:26",
step: "噪声检测",
operator: "噪声去除",
status: "INFO",
message: "检测图像噪声水平",
},
{
time: "09:30:28",
step: "噪声去除",
operator: "噪声去除",
status: "INFO",
message: "应用高斯滤波去除噪声",
},
{
time: "09:30:31",
step: "噪声去除完成",
operator: "噪声去除",
status: "SUCCESS",
message: "噪声去除处理完成",
},
{
time: "09:30:32",
step: "尺寸检测",
operator: "尺寸标准化",
status: "INFO",
message: "检测当前图像尺寸: 2048x1536",
},
{
time: "09:30:33",
step: "尺寸调整",
operator: "尺寸标准化",
status: "INFO",
message: "调整图像尺寸至512x512",
},
{
time: "09:30:35",
step: "尺寸标准化完成",
operator: "尺寸标准化",
status: "SUCCESS",
message: "图像尺寸标准化完成",
},
{
time: "09:30:36",
step: "质量检查",
operator: "质量检查",
status: "INFO",
message: "检查图像质量指标",
},
{
time: "09:30:38",
step: "分辨率检查",
operator: "质量检查",
status: "SUCCESS",
message: "分辨率符合要求",
},
{
time: "09:30:39",
step: "清晰度检查",
operator: "质量检查",
status: "SUCCESS",
message: "图像清晰度良好",
},
{
time: "09:30:40",
step: "处理完成",
operator: "质量检查",
status: "SUCCESS",
message: `文件 ${fileName} 处理完成`,
},
];
const fileColumns = [ const fileColumns = [
{ {
@@ -196,7 +56,7 @@ export default function FileTable() {
<input <input
type="checkbox" type="checkbox"
checked={ checked={
selectedFileIds.length === fileList.length && fileList.length > 0 selectedFileIds.length === result?.length && result?.length > 0
} }
onChange={(e) => handleSelectAllFiles(e.target.checked)} onChange={(e) => handleSelectAllFiles(e.target.checked)}
className="w-4 h-4" className="w-4 h-4"
@@ -205,7 +65,7 @@ export default function FileTable() {
dataIndex: "select", dataIndex: "select",
key: "select", key: "select",
width: 50, width: 50,
render: (text: string, record: any) => ( render: (_text: string, record: any) => (
<input <input
type="checkbox" type="checkbox"
checked={selectedFileIds.includes(record.id)} checked={selectedFileIds.includes(record.id)}
@@ -216,8 +76,8 @@ export default function FileTable() {
}, },
{ {
title: "文件名", title: "文件名",
dataIndex: "fileName", dataIndex: "srcName",
key: "fileName", key: "srcName",
filterDropdown: ({ filterDropdown: ({
setSelectedKeys, setSelectedKeys,
selectedKeys, selectedKeys,
@@ -245,15 +105,87 @@ export default function FileTable() {
</div> </div>
), ),
onFilter: (value: string, record: any) => onFilter: (value: string, record: any) =>
record.fileName.toLowerCase().includes(value.toLowerCase()), record.srcName.toLowerCase().includes(value.toLowerCase()),
render: (text: string) => (
<span className="font-mono text-sm">{text?.replace(/\.[^/.]+$/, "")}</span>
),
},
{
title: "文件类型",
dataIndex: "srcType",
key: "srcType",
filterDropdown: ({
setSelectedKeys,
selectedKeys,
confirm,
clearFilters,
}: any) => (
<div className="p-4 w-64">
<Input
placeholder="搜索文件类型"
value={selectedKeys[0]}
onChange={(e) =>
setSelectedKeys(e.target.value ? [e.target.value] : [])
}
onPressEnter={() => confirm()}
className="mb-2"
/>
<div className="flex gap-2">
<Button size="small" onClick={() => confirm()}>
</Button>
<Button size="small" onClick={() => clearFilters()}>
</Button>
</div>
</div>
),
onFilter: (value: string, record: any) =>
record.srcType.toLowerCase().includes(value.toLowerCase()),
render: (text: string) => (
<span className="font-mono text-sm">{text}</span>
),
},
{
title: "清洗后文件类型",
dataIndex: "destType",
key: "destType",
filterDropdown: ({
setSelectedKeys,
selectedKeys,
confirm,
clearFilters,
}: any) => (
<div className="p-4 w-64">
<Input
placeholder="搜索文件类型"
value={selectedKeys[0]}
onChange={(e) =>
setSelectedKeys(e.target.value ? [e.target.value] : [])
}
onPressEnter={() => confirm()}
className="mb-2"
/>
<div className="flex gap-2">
<Button size="small" onClick={() => confirm()}>
</Button>
<Button size="small" onClick={() => clearFilters()}>
</Button>
</div>
</div>
),
onFilter: (value: string, record: any) =>
record.destType.toLowerCase().includes(value.toLowerCase()),
render: (text: string) => ( render: (text: string) => (
<span className="font-mono text-sm">{text}</span> <span className="font-mono text-sm">{text}</span>
), ),
}, },
{ {
title: "清洗前大小", title: "清洗前大小",
dataIndex: "originalSize", dataIndex: "srcSize",
key: "originalSize", key: "srcSize",
sorter: (a: any, b: any) => { sorter: (a: any, b: any) => {
const getSizeInBytes = (size: string) => { const getSizeInBytes = (size: string) => {
if (!size || size === "-") return 0; if (!size || size === "-") return 0;
@@ -265,11 +197,14 @@ export default function FileTable() {
}; };
return getSizeInBytes(a.originalSize) - getSizeInBytes(b.originalSize); return getSizeInBytes(a.originalSize) - getSizeInBytes(b.originalSize);
}, },
render: (number: number) => (
<span className="font-mono text-sm">{formatFileSize(number)}</span>
),
}, },
{ {
title: "清洗后大小", title: "清洗后大小",
dataIndex: "processedSize", dataIndex: "destSize",
key: "processedSize", key: "destSize",
sorter: (a: any, b: any) => { sorter: (a: any, b: any) => {
const getSizeInBytes = (size: string) => { const getSizeInBytes = (size: string) => {
if (!size || size === "-") return 0; if (!size || size === "-") return 0;
@@ -283,6 +218,9 @@ export default function FileTable() {
getSizeInBytes(a.processedSize) - getSizeInBytes(b.processedSize) getSizeInBytes(a.processedSize) - getSizeInBytes(b.processedSize)
); );
}, },
render: (number: number) => (
<span className="font-mono text-sm">{formatFileSize(number)}</span>
),
}, },
{ {
title: "状态", title: "状态",
@@ -297,43 +235,22 @@ export default function FileTable() {
render: (status: string) => ( render: (status: string) => (
<Badge <Badge
status={ status={
status === "已完成" status === "COMPLETED"
? "success" ? "success"
: status === "失败" : status === "FAILED"
? "error" ? "error"
: "processing" : "processing"
} }
text={status} text={TaskStatusMap[status as TaskStatus].label}
/> />
), ),
}, },
{
title: "执行耗时",
dataIndex: "duration",
key: "duration",
sorter: (a: any, b: any) => {
const getTimeInSeconds = (duration: string) => {
const parts = duration.split(/[分秒]/);
const minutes = Number.parseInt(parts[0]) || 0;
const seconds = Number.parseInt(parts[1]) || 0;
return minutes * 60 + seconds;
};
return getTimeInSeconds(a.duration) - getTimeInSeconds(b.duration);
},
},
{ {
title: "操作", title: "操作",
key: "action", key: "action",
render: (text: string, record: any) => ( render: (_text: string, record: any) => (
<div className="flex"> <div className="flex">
<Button {record.status === "COMPLETED" && (
type="link"
size="small"
onClick={() => handleViewFileLog(record)}
>
</Button>
{record.status === "已完成" && (
<Button <Button
type="link" type="link"
size="small" size="small"
@@ -371,53 +288,12 @@ export default function FileTable() {
)} )}
<Table <Table
columns={fileColumns} columns={fileColumns}
dataSource={fileList} dataSource={result}
pagination={{ pageSize: 10, showSizeChanger: true }} pagination={{ pageSize: 10, showSizeChanger: true }}
size="middle" size="middle"
rowKey="id" rowKey="id"
/> />
{/* 文件日志弹窗 */}
<Modal
open={showFileLogDialog}
onCancel={() => setShowFileLogDialog(false)}
footer={null}
width={700}
title={
<span>
<FileText className="w-4 h-4 mr-2 inline" />
- {selectedFile?.fileName}
</span>
}
>
<div className="py-4">
<div className="bg-gray-900 rounded-lg p-4 max-h-96 overflow-y-auto">
<div className="font-mono text-sm">
{selectedFile &&
getFileProcessLog(selectedFile.fileName).map((log, index) => (
<div key={index} className="flex gap-3">
<span className="text-gray-500 min-w-20">{log.time}</span>
<span className="text-blue-400 min-w-24">
[{log.operator}]
</span>
<span
className={`min-w-20 ${
log.status === "ERROR"
? "text-red-400"
: log.status === "SUCCESS"
? "text-green-400"
: "text-yellow-400"
}`}
>
{log.step}
</span>
<span className="text-gray-100">{log.message}</span>
</div>
))}
</div>
</div>
</div>
</Modal>
{/* 文件对比弹窗 */} {/* 文件对比弹窗 */}
<Modal <Modal
open={showFileCompareDialog} open={showFileCompareDialog}
@@ -434,22 +310,13 @@ export default function FileTable() {
<div className="w-16 h-16 bg-gray-300 rounded-lg mx-auto mb-2" /> <div className="w-16 h-16 bg-gray-300 rounded-lg mx-auto mb-2" />
<div className="text-sm"></div> <div className="text-sm"></div>
<div className="text-xs text-gray-400"> <div className="text-xs text-gray-400">
: {selectedFile?.originalSize} : {formatFileSize(selectedFile?.srcSize)}
</div> </div>
</div> </div>
</div> </div>
<div className="text-sm text-gray-600 mt-3 space-y-1"> <div className="text-sm text-gray-600 mt-3 space-y-1">
<div> <div>
<span className="font-medium">:</span> SVS <span className="font-medium">:</span> {selectedFile?.srcType}
</div>
<div>
<span className="font-medium">:</span> 2048x1536
</div>
<div>
<span className="font-medium">:</span> RGB
</div>
<div>
<span className="font-medium">:</span>
</div> </div>
</div> </div>
</div> </div>
@@ -460,22 +327,13 @@ export default function FileTable() {
<div className="w-16 h-16 bg-blue-300 rounded-lg mx-auto mb-2" /> <div className="w-16 h-16 bg-blue-300 rounded-lg mx-auto mb-2" />
<div className="text-sm"></div> <div className="text-sm"></div>
<div className="text-xs text-gray-400"> <div className="text-xs text-gray-400">
: {selectedFile?.processedSize} : {formatFileSize(selectedFile?.destSize)}
</div> </div>
</div> </div>
</div> </div>
<div className="text-sm text-gray-600 mt-3 space-y-1"> <div className="text-sm text-gray-600 mt-3 space-y-1">
<div> <div>
<span className="font-medium">:</span> JPEG <span className="font-medium">:</span> {selectedFile?.destType}
</div>
<div>
<span className="font-medium">:</span> 512x512
</div>
<div>
<span className="font-medium">:</span> RGB
</div>
<div>
<span className="font-medium">:</span> JPEG压缩
</div> </div>
</div> </div>
</div> </div>
@@ -485,15 +343,7 @@ export default function FileTable() {
<div className="grid grid-cols-3 gap-4 text-sm"> <div className="grid grid-cols-3 gap-4 text-sm">
<div className="bg-green-50 p-4 rounded-lg"> <div className="bg-green-50 p-4 rounded-lg">
<div className="font-medium text-green-700"></div> <div className="font-medium text-green-700"></div>
<div className="text-green-600"> 44.1%</div> <div className="text-green-600"> {(100 * (selectedFile?.srcSize - selectedFile?.destSize) / selectedFile?.srcSize).toFixed(2)}%</div>
</div>
<div className="bg-blue-50 p-4 rounded-lg">
<div className="font-medium text-blue-700"></div>
<div className="text-blue-600">{selectedFile?.duration}</div>
</div>
<div className="bg-purple-50 p-4 rounded-lg">
<div className="font-medium text-purple-700"></div>
<div className="text-purple-600"> (9.2/10)</div>
</div> </div>
</div> </div>
</div> </div>

View File

@@ -1,110 +1,43 @@
export default function LogsTable({ task }: { task: any }) { import {useEffect} from "react";
// 模拟运行日志 import {useParams} from "react-router";
const runLogs = [ import {FileClock} from "lucide-react";
{
time: "09:30:15",
level: "INFO",
message: "开始执行数据清洗任务: 肺癌WSI图像清洗任务",
},
{
time: "09:30:16",
level: "INFO",
message: "加载源数据集: 肺癌WSI病理图像数据集 (1250 文件)",
},
{ time: "09:30:17", level: "INFO", message: "初始化算子: 格式转换" },
{
time: "09:30:18",
level: "INFO",
message: "开始处理文件: lung_cancer_001.svs",
},
{
time: "09:30:25",
level: "SUCCESS",
message: "文件处理成功: lung_cancer_001.svs -> lung_cancer_001.jpg",
},
{
time: "09:30:26",
level: "INFO",
message: "开始处理文件: lung_cancer_002.svs",
},
{
time: "09:30:33",
level: "SUCCESS",
message: "文件处理成功: lung_cancer_002.svs -> lung_cancer_002.jpg",
},
{
time: "09:58:42",
level: "INFO",
message: "格式转换完成,成功处理 1250/1250 文件",
},
{ time: "09:58:43", level: "INFO", message: "初始化算子: 噪声去除" },
{
time: "09:58:44",
level: "INFO",
message: "开始处理文件: lung_cancer_001.jpg",
},
{
time: "09:58:51",
level: "SUCCESS",
message: "噪声去除成功: lung_cancer_001.jpg",
},
{
time: "10:15:23",
level: "WARNING",
message: "文件质量较低,跳过处理: lung_cancer_156.jpg",
},
{
time: "10:35:18",
level: "INFO",
message: "噪声去除完成,成功处理 1228/1250 文件",
},
{ time: "10:35:19", level: "INFO", message: "初始化算子: 尺寸标准化" },
{
time: "11:12:05",
level: "INFO",
message: "尺寸标准化完成,成功处理 1222/1228 文件",
},
{ time: "11:12:06", level: "INFO", message: "初始化算子: 质量检查" },
{
time: "11:25:33",
level: "ERROR",
message: "质量检查失败: lung_cancer_089.jpg - 分辨率过低",
},
{
time: "11:45:32",
level: "INFO",
message: "质量检查完成,成功处理 1198/1222 文件",
},
{
time: "11:45:33",
level: "SUCCESS",
message: "数据清洗任务完成!总成功率: 95.8%",
},
];
return ( export default function LogsTable({taskLog, fetchTaskLog} : {taskLog: any[], fetchTaskLog: () => Promise<any>}) {
<div className="text-gray-300 p-4 border border-gray-700 bg-gray-800 rounded-lg"> const { id = "" } = useParams();
<div className="font-mono text-sm">
{runLogs?.map?.((log, index) => ( useEffect(() => {
<div key={index} className="flex gap-3"> fetchTaskLog();
<span className="text-gray-500 min-w-20">{log.time}</span> }, [id]);
<span
className={`min-w-20 ${ return taskLog?.length > 0 ? (
log.level === "ERROR" <>
? "text-red-500" <div className="text-gray-300 p-4 border border-gray-700 bg-gray-800 rounded-lg">
: log.level === "WARNING" <div className="font-mono text-sm">
? "text-yellow-500" {taskLog?.map?.((log, index) => (
: log.level === "SUCCESS" <div key={index} className="flex gap-3">
? "text-green-500" <span
: "text-blue-500" className={`min-w-20 ${
}`} log.level === "ERROR" || log.level === "FATAL"
> ? "text-red-500"
[{log.level}] : log.level === "WARNING" || log.level === "WARN"
</span> ? "text-yellow-500"
<span className="text-gray-100">{log.message}</span> : "text-green-500"
</div> }`}
))} >
[{log.level}]
</span>
<span className="text-gray-100">{log.message}</span>
</div>
))}
</div>
</div> </div>
</>
) : (
<div className="text-center py-12">
<FileClock className="w-16 h-16 text-gray-300 mx-auto mb-4" />
<h3 className="text-lg font-medium text-gray-900 mb-2">
</h3>
</div> </div>
); );
} }

View File

@@ -1,103 +1,25 @@
import { Button, Input, Table } from "antd"; import {Steps, Typography} from "antd";
import {useNavigate} from "react-router";
const operators = [
{
name: "格式转换",
startTime: "09:30:15",
endTime: "09:58:42",
duration: "28分27秒",
status: "成功",
processedFiles: 1250,
successRate: 100,
},
{
name: "噪声去除",
startTime: "09:58:42",
endTime: "10:35:18",
duration: "36分36秒",
status: "成功",
processedFiles: 1250,
successRate: 98.2,
},
{
name: "尺寸标准化",
startTime: "10:35:18",
endTime: "11:12:05",
duration: "36分47秒",
status: "成功",
processedFiles: 1228,
successRate: 99.5,
},
{
name: "质量检查",
startTime: "11:12:05",
endTime: "11:45:32",
duration: "33分27秒",
status: "成功",
processedFiles: 1222,
successRate: 97.8,
},
];
export default function OperatorTable({ task }: { task: any }) { export default function OperatorTable({ task }: { task: any }) {
const operatorColumns = [ const navigate = useNavigate();
{
title: "算子名称",
dataIndex: "name",
key: "name",
fixed: "left",
width: 200,
filterDropdown: ({
setSelectedKeys,
selectedKeys,
confirm,
clearFilters,
}: any) => (
<div className="p-4 w-64">
<Input
placeholder="搜索算子名称"
value={selectedKeys[0]}
onChange={(e) =>
setSelectedKeys(e.target.value ? [e.target.value] : [])
}
onPressEnter={() => confirm()}
className="mb-2"
/>
<div className="flex gap-2">
<Button size="small" onClick={() => confirm()}>
</Button>
<Button size="small" onClick={() => clearFilters()}>
</Button>
</div>
</div>
),
onFilter: (value: string, record: any) =>
record.name.toLowerCase().includes(value.toLowerCase()),
},
{
title: "版本",
dataIndex: "version",
key: "version",
},
{
title: "创建时间",
dataIndex: "createdAt",
key: "createdAt",
},
{
title: "更新时间",
dataIndex: "updatedAt",
key: "updatedAt",
},
];
return ( return task?.instance?.length > 0 && (
<Table <>
columns={operatorColumns} <Steps
dataSource={task?.instance || operators} progressDot
pagination={false} direction="vertical"
size="middle" items={Object.values(task?.instance).map((item) => ({
/> title: <Typography.Link
onClick={() => navigate(`/data/operator-market/plugin-detail/${item?.id}`)}
>
{item?.name}
</Typography.Link>,
description: item?.description,
status: "finish"
}))}
className="overflow-auto"
/>
</>
); );
} }

View File

@@ -43,10 +43,6 @@ export default function TaskList() {
handleFiltersChange, handleFiltersChange,
} = useFetchData(queryCleaningTasksUsingGet, mapTask); } = useFetchData(queryCleaningTasksUsingGet, mapTask);
const handleViewTask = (task: any) => {
navigate("/data/cleansing/task-detail/" + task.id);
};
const pauseTask = async (item: CleansingTask) => { const pauseTask = async (item: CleansingTask) => {
await stopCleaningTaskUsingPost(item.id); await stopCleaningTaskUsingPost(item.id);
message.success("任务已暂停"); message.success("任务已暂停");
@@ -86,8 +82,12 @@ export default function TaskList() {
onClick: startTask, // implement pause/play logic onClick: startTask, // implement pause/play logic
}; };
return [ return [
isRunning && pauseBtn, ...(isRunning
showStart && startBtn, ? [ pauseBtn ]
: []),
...(showStart
? [ startBtn ]
: []),
{ {
key: "delete", key: "delete",
label: "删除", label: "删除",
@@ -106,6 +106,18 @@ export default function TaskList() {
fixed: "left", fixed: "left",
width: 150, width: 150,
ellipsis: true, ellipsis: true,
render: (_, task: CleansingTask) => {
return (
<Button
type="link"
onClick={() =>
navigate("/data/cleansing/task-detail/" + task.id)
}
>
{task.name}
</Button>
);
},
}, },
{ {
title: "任务ID", title: "任务ID",
@@ -273,6 +285,9 @@ export default function TaskList() {
data={tableData} data={tableData}
operations={taskOperations} operations={taskOperations}
pagination={pagination} pagination={pagination}
onView={(tableData) => {
navigate("/data/cleansing/task-detail/" + tableData.id)
}}
/> />
) : ( ) : (
<Card> <Card>

View File

@@ -1,21 +1,102 @@
import { DeleteOutlined } from "@ant-design/icons"; import {DeleteOutlined, EditOutlined} from "@ant-design/icons";
import CardView from "@/components/CardView"; import CardView from "@/components/CardView";
import { import {
deleteCleaningTemplateByIdUsingDelete, deleteCleaningTemplateByIdUsingDelete, queryCleaningTemplatesUsingGet,
queryCleaningTemplatesUsingGet,
} from "../../cleansing.api"; } from "../../cleansing.api";
import useFetchData from "@/hooks/useFetchData"; import useFetchData from "@/hooks/useFetchData";
import { mapTemplate } from "../../cleansing.const"; import {mapTemplate} from "../../cleansing.const";
import { App } from "antd"; import {App, Button, Card, Table, Tooltip} from "antd";
import { CleansingTemplate } from "../../cleansing.model"; import {CleansingTemplate} from "../../cleansing.model";
import {SearchControls} from "@/components/SearchControls.tsx";
import {useNavigate} from "react-router";
import {useState} from "react";
export default function TemplateList() { export default function TemplateList() {
const navigate = useNavigate();
const { message } = App.useApp(); const { message } = App.useApp();
const [viewMode, setViewMode] = useState<"card" | "list">("list");
const { tableData, pagination, fetchData } = useFetchData( const {
queryCleaningTemplatesUsingGet, loading,
mapTemplate tableData,
); pagination,
searchParams,
setSearchParams,
fetchData,
handleFiltersChange,
} = useFetchData(queryCleaningTemplatesUsingGet, mapTemplate);
const templateOperations = () => {
return [
{
key: "update",
label: "编辑",
icon: <EditOutlined />,
onClick: (template: CleansingTemplate) => navigate(`/data/cleansing/update-template/${template.id}`)
},
{
key: "delete",
label: "删除",
danger: true,
icon: <DeleteOutlined />,
onClick: deleteTemplate, // implement delete logic
},
];
};
const templateColumns = [
{
title: "模板名称",
dataIndex: "name",
key: "name",
fixed: "left",
width: 150,
ellipsis: true,
render: (_, template: CleansingTemplate) => {
return (
<Button
type="link"
onClick={() =>
navigate("/data/cleansing/template-detail/" + template.id)
}
>
{template.name}
</Button>
);
}},
{
title: "算子数量",
dataIndex: "num",
key: "num",
width: 100,
ellipsis: true,
render: (_, template: CleansingTemplate) => {
return template.instance?.length ?? 0;
},
},
{
title: "操作",
key: "action",
fixed: "right",
width: 20,
render: (text: string, record: any) => (
<div className="flex gap-2">
{templateOperations(record).map((op) =>
op ? (
<Tooltip key={op.key} title={op.label}>
<Button
type="text"
icon={op.icon}
danger={op?.danger}
onClick={() => op.onClick(record)}
/>
</Tooltip>
) : null
)}
</div>
),
},
]
const deleteTemplate = async (template: CleansingTemplate) => { const deleteTemplate = async (template: CleansingTemplate) => {
if (!template.id) { if (!template.id) {
@@ -27,21 +108,43 @@ export default function TemplateList() {
message.success("模板删除成功"); message.success("模板删除成功");
}; };
const operations = [
{
key: "delete",
label: "删除模板",
danger: true,
icon: <DeleteOutlined />,
onClick: (template: CleansingTemplate) => deleteTemplate(template), // 可实现删除逻辑
},
];
return ( return (
<CardView <>
data={tableData} {/* Search and Filters */}
operations={operations} <SearchControls
pagination={pagination} searchTerm={searchParams.keyword}
/> onSearchChange={(keyword) =>
setSearchParams({ ...searchParams, keyword })
}
searchPlaceholder="搜索模板名称、描述"
onFiltersChange={handleFiltersChange}
viewMode={viewMode}
onViewModeChange={setViewMode}
showViewToggle={true}
onReload={fetchData}
onClearFilters={() => setSearchParams({ ...searchParams, filter: {} })}
/>
{viewMode === "card" ? (
<CardView
data={tableData}
operations={templateOperations}
pagination={pagination}
onView={(tableData) => {
navigate("/data/cleansing/template-detail/" + tableData.id)
}}
/>
) : (
<Card>
<Table
columns={templateColumns}
dataSource={tableData}
rowKey="id"
loading={loading}
scroll={{ x: "max-content", y: "calc(100vh - 35rem)" }}
pagination={pagination}
/>
</Card>
)}
</>
); );
} }

View File

@@ -13,6 +13,14 @@ export function queryCleaningTaskByIdUsingGet(taskId: string | number) {
return get(`/api/cleaning/tasks/${taskId}`); return get(`/api/cleaning/tasks/${taskId}`);
} }
export function queryCleaningTaskResultByIdUsingGet(taskId: string | number) {
return get(`/api/cleaning/tasks/${taskId}/result`);
}
export function queryCleaningTaskLogByIdUsingGet(taskId: string | number) {
return get(`/api/cleaning/tasks/${taskId}/log`);
}
export function updateCleaningTaskByIdUsingPut(taskId: string | number, data: any) { export function updateCleaningTaskByIdUsingPut(taskId: string | number, data: any) {
return put(`/api/cleaning/tasks/${taskId}`, data); return put(`/api/cleaning/tasks/${taskId}`, data);
} }

View File

@@ -98,6 +98,11 @@ export const mapTask = (task: CleansingTask) => {
createdAt, createdAt,
startedAt, startedAt,
finishedAt, finishedAt,
updatedAt: formatDateTime(
new Date(Math.max(...[
new Date(task.finishedAt).getTime(),
new Date(task.startedAt).getTime(),
new Date(task.createdAt).getTime()])).toISOString()),
icon: <BrushCleaning className="w-full h-full" />, icon: <BrushCleaning className="w-full h-full" />,
status, status,
duration, duration,

View File

@@ -18,10 +18,13 @@ export interface CleansingTask {
startedAt: string; startedAt: string;
progress: { progress: {
finishedFileNum: number; finishedFileNum: number;
process: 100, succeedFileNum: number;
failedFileNum: number;
process: 100;
totalFileNum: number; totalFileNum: number;
successRate: 100;
}; };
operators: OperatorI[]; instance: OperatorI[];
createdAt: string; createdAt: string;
updatedAt: string; updatedAt: string;
finishedAt: string; finishedAt: string;
@@ -70,3 +73,17 @@ export enum TemplateType {
AUDIO = "AUDIO", AUDIO = "AUDIO",
IMAGE2TEXT = "IMAGE2TEXT", IMAGE2TEXT = "IMAGE2TEXT",
} }
export interface CleansingResult {
instanceId: string;
srcFileId: string;
destFileId: string;
srcName: string;
destName: string;
srcType: string;
destType: string;
srcSize: number;
destSize: number;
status: string;
result: string;
}

View File

@@ -110,9 +110,6 @@ export function ListView({ operators = [], pagination, operations }) {
{operator.name} {operator.name}
</span> </span>
<Tag color="default">v{operator.version}</Tag> <Tag color="default">v{operator.version}</Tag>
<Badge color={getStatusBadge(operator.status).color}>
{getStatusBadge(operator.status).label}
</Badge>
</div> </div>
} }
description={ description={

View File

@@ -33,7 +33,7 @@ export interface OperatorI {
tags: string[]; tags: string[];
isStar?: boolean; isStar?: boolean;
originalId?: string; // 用于标识原始算子ID,便于去重 originalId?: string; // 用于标识原始算子ID,便于去重
categories: number[]; // 分类列表 categories: string[]; // 分类列表
settings: string; settings: string;
overrides?: { [key: string]: any }; // 用户配置的参数 overrides?: { [key: string]: any }; // 用户配置的参数
defaultParams?: { [key: string]: any }; // 默认参数 defaultParams?: { [key: string]: any }; // 默认参数
@@ -50,6 +50,8 @@ export interface CategoryI {
count: number; // 该分类下的算子数量 count: number; // 该分类下的算子数量
type: string; // e.g., "数据源", "数据清洗", "数据分析", "数据可视化" type: string; // e.g., "数据源", "数据清洗", "数据分析", "数据可视化"
parentId?: number; // 父分类ID,若无父分类则为null parentId?: number; // 父分类ID,若无父分类则为null
value: string;
createdAt: string;
} }
export interface CategoryTreeI { export interface CategoryTreeI {

View File

@@ -12,7 +12,7 @@ import DatasetDetail from "@/pages/DataManagement/Detail/DatasetDetail";
import DataCleansing from "@/pages/DataCleansing/Home/DataCleansing"; import DataCleansing from "@/pages/DataCleansing/Home/DataCleansing";
import CleansingTaskCreate from "@/pages/DataCleansing/Create/CreateTask"; import CleansingTaskCreate from "@/pages/DataCleansing/Create/CreateTask";
import CleansingTaskDetail from "@/pages/DataCleansing/Detail/TaskDetail"; import CleansingTaskDetail from "@/pages/DataCleansing/Detail/TaskDetail";
import CleansingTemplateCreate from "@/pages/DataCleansing/Create/CreateTempate"; import CleansingTemplateCreate from "@/pages/DataCleansing/Create/CreateTemplate";
import DataAnnotation from "@/pages/DataAnnotation/Home/DataAnnotation"; import DataAnnotation from "@/pages/DataAnnotation/Home/DataAnnotation";
import AnnotationTaskCreate from "@/pages/DataAnnotation/Create/CreateTask"; import AnnotationTaskCreate from "@/pages/DataAnnotation/Create/CreateTask";
@@ -39,6 +39,7 @@ import OrchestrationPage from "@/pages/Orchestration/Orchestration";
import WorkflowEditor from "@/pages/Orchestration/WorkflowEditor"; import WorkflowEditor from "@/pages/Orchestration/WorkflowEditor";
import { withErrorBoundary } from "@/components/ErrorBoundary"; import { withErrorBoundary } from "@/components/ErrorBoundary";
import AgentPage from "@/pages/Agent/Agent.tsx"; import AgentPage from "@/pages/Agent/Agent.tsx";
import CleansingTemplateDetail from "@/pages/DataCleansing/Detail/TemplateDetail.tsx";
import RatioTaskDetail from "@/pages/RatioTask/Detail/RatioTaskDetail"; import RatioTaskDetail from "@/pages/RatioTask/Detail/RatioTaskDetail";
const router = createBrowserRouter([ const router = createBrowserRouter([
@@ -120,6 +121,14 @@ const router = createBrowserRouter([
path: "create-template", path: "create-template",
Component: CleansingTemplateCreate, Component: CleansingTemplateCreate,
}, },
{
path: "template-detail/:id",
Component: CleansingTemplateDetail,
},
{
path: "update-template/:id",
Component: CleansingTemplateCreate,
},
], ],
}, },
{ {

View File

@@ -160,7 +160,7 @@ class Mapper(BaseOp):
sample["execute_status"] = execute_status sample["execute_status"] = execute_status
task_info = TaskInfoPersistence() task_info = TaskInfoPersistence()
task_info.persistence_task_info(sample) task_info.persistence_task_info(sample)
return sample raise e
sample["execute_status"] = execute_status sample["execute_status"] = execute_status
# 加载文件成功执行信息到数据库 # 加载文件成功执行信息到数据库

View File

@@ -10,14 +10,13 @@ from .scheduler import Task, TaskStatus, TaskResult, TaskScheduler
class CommandTask(Task): class CommandTask(Task):
"""命令任务包装类""" """命令任务包装类"""
def __init__(self, task_id: str, command: str, shell: bool = True, def __init__(self, task_id: str, command: str, log_path = None, shell: bool = True,
timeout: Optional[int] = None, *args, **kwargs): timeout: Optional[int] = None, *args, **kwargs):
super().__init__(task_id, *args, **kwargs) super().__init__(task_id, *args, **kwargs)
self.log_path = log_path
self.command = command self.command = command
self.shell = shell self.shell = shell
self.timeout = timeout self.timeout = timeout
self.stdout = None
self.stderr = None
self.return_code = None self.return_code = None
self._process = None self._process = None
@@ -35,56 +34,54 @@ class CommandTask(Task):
self.status = TaskStatus.RUNNING self.status = TaskStatus.RUNNING
self.started_at = datetime.now() self.started_at = datetime.now()
# 使用 asyncio.create_subprocess_shell 或 create_subprocess_exec with open(self.log_path, 'a') as f:
if self.shell: # 使用 asyncio.create_subprocess_shell 或 create_subprocess_exec
process = await asyncio.create_subprocess_shell( if self.shell:
self.command, process = await asyncio.create_subprocess_shell(
stdout=asyncio.subprocess.PIPE, self.command,
stderr=asyncio.subprocess.PIPE, stdout=f,
**self.kwargs stderr=asyncio.subprocess.STDOUT,
) **self.kwargs
else:
process = await asyncio.create_subprocess_exec(
*self.command.split(),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
**self.kwargs
)
self._process = process
# 等待进程完成(带超时)
try:
if self.timeout:
stdout, stderr = await asyncio.wait_for(
process.communicate(),
timeout=self.timeout
) )
else: else:
stdout, stderr = await process.communicate() process = await asyncio.create_subprocess_exec(
*self.command.split(),
stdout=f,
stderr=asyncio.subprocess.STDOUT,
**self.kwargs
)
self.stdout = stdout.decode() if stdout else "" self._process = process
self.stderr = stderr.decode() if stderr else ""
self.return_code = process.returncode
if self._cancelled: # 等待进程完成(带超时)
self.status = TaskStatus.CANCELLED
elif process.returncode == 0:
self.status = TaskStatus.COMPLETED
else:
self.status = TaskStatus.FAILED
except asyncio.TimeoutError:
# 超时处理
self._process.terminate()
try: try:
await asyncio.wait_for(self._process.wait(), timeout=5.0) if self.timeout:
except asyncio.TimeoutError: await asyncio.wait_for(
self._process.kill() process.wait(),
await self._process.wait() timeout=self.timeout
)
else:
await process.wait()
self.return_code = process.returncode
self.status = TaskStatus.FAILED if self._cancelled:
self.stderr = f"Command timed out after {self.timeout} seconds" self.status = TaskStatus.CANCELLED
elif process.returncode == 0:
self.status = TaskStatus.COMPLETED
else:
self.status = TaskStatus.FAILED
except asyncio.TimeoutError:
# 超时处理
self._process.terminate()
try:
await asyncio.wait_for(self._process.wait(), timeout=5.0)
except asyncio.TimeoutError:
self._process.kill()
await self._process.wait()
self.status = TaskStatus.FAILED
f.write(f"\nCommand timed out after {self.timeout} seconds\n")
except asyncio.CancelledError: except asyncio.CancelledError:
# 任务被取消 # 任务被取消
@@ -101,7 +98,7 @@ class CommandTask(Task):
except Exception as e: except Exception as e:
self.status = TaskStatus.FAILED self.status = TaskStatus.FAILED
self.stderr = str(e) logger.error(f"Task(id: {self.task_id}) run failed. Cause: {e}")
finally: finally:
self.completed_at = datetime.now() self.completed_at = datetime.now()
@@ -127,8 +124,6 @@ class CommandTask(Task):
"""转换为结果对象""" """转换为结果对象"""
self.result = { self.result = {
"command": self.command, "command": self.command,
"stdout": self.stdout,
"stderr": self.stderr,
"return_code": self.return_code, "return_code": self.return_code,
} }
return super().to_result() return super().to_result()
@@ -140,10 +135,13 @@ class CommandScheduler(TaskScheduler):
def __init__(self, max_concurrent: int = 5): def __init__(self, max_concurrent: int = 5):
super().__init__(max_concurrent) super().__init__(max_concurrent)
async def submit(self, task_id, command: str, shell: bool = True, async def submit(self, task_id, command: str, log_path = None, shell: bool = True,
timeout: Optional[int] = None, **kwargs) -> str: timeout: Optional[int] = None, **kwargs) -> str:
if log_path is None:
log_path = f"/flow/{task_id}/output.log"
"""提交命令任务""" """提交命令任务"""
task = CommandTask(task_id, command, shell, timeout, **kwargs) task = CommandTask(task_id, command, log_path, shell, timeout, **kwargs)
self.tasks[task_id] = task self.tasks[task_id] = task
# 使用信号量限制并发 # 使用信号量限制并发

View File

@@ -1,5 +1,10 @@
FROM ghcr.io/astral-sh/uv:python3.11-bookworm FROM ghcr.io/astral-sh/uv:python3.11-bookworm
RUN --mount=type=cache,target=/var/cache/apt \
--mount=type=cache,target=/var/lib/apt \
apt update \
&& apt install -y libgl1 libglib2.0-0 vim libmagic1 libreoffice dos2unix
COPY runtime/python-executor /opt/runtime COPY runtime/python-executor /opt/runtime
COPY runtime/ops /opt/runtime/datamate/ops COPY runtime/ops /opt/runtime/datamate/ops
COPY runtime/ops/user /opt/runtime/user COPY runtime/ops/user /opt/runtime/user
@@ -7,11 +12,6 @@ COPY scripts/images/runtime/start.sh /opt/runtime/start.sh
ENV PYTHONPATH=/opt/runtime/datamate/ ENV PYTHONPATH=/opt/runtime/datamate/
RUN --mount=type=cache,target=/var/cache/apt \
--mount=type=cache,target=/var/lib/apt \
apt update \
&& apt install -y libgl1 libglib2.0-0 vim libmagic1 libreoffice dos2unix
WORKDIR /opt/runtime WORKDIR /opt/runtime
RUN --mount=type=cache,target=/root/.cache/uv \ RUN --mount=type=cache,target=/root/.cache/uv \