q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Feature] Refactor project to use 'datamate' naming convention for services and configurations (#14)

Browse Source 
* Enhance CleaningTaskService to track cleaning process progress and update ExecutorType to DATAMATE

* Refactor project to use 'datamate' naming convention for services and configurations
This commit is contained in:
hhhhsc701
2025-10-22 17:53:16 +08:00
committed by GitHub
parent 175d9ded93
commit 31ef8bc265
39 changed files with 312 additions and 737 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/httpclient/RuntimeClient.java
View File

@@ -14,7 +14,7 @@ import java.time.Duration;
@Slf4j
public class RuntimeClient {
private static final String BASE_URL = "http://runtime:8081/api";
private static final String BASE_URL = "http://datamate-runtime:8081/api";
private static final String CREATE_TASK_URL = BASE_URL + "/task/{0}/submit";

17
backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/service/CleaningTaskService.java
View File

@@ -12,6 +12,7 @@ import com.datamate.cleaning.domain.model.TaskProcess;
import com.datamate.cleaning.infrastructure.persistence.mapper.CleaningResultMapper;
import com.datamate.cleaning.infrastructure.persistence.mapper.CleaningTaskMapper;
import com.datamate.cleaning.infrastructure.persistence.mapper.OperatorInstanceMapper;
import com.datamate.cleaning.interfaces.dto.CleaningProcess;
import com.datamate.cleaning.interfaces.dto.CleaningTask;
import com.datamate.cleaning.interfaces.dto.CreateCleaningTaskRequest;
import com.datamate.cleaning.interfaces.dto.OperatorInstance;
@@ -55,7 +56,14 @@ public class CleaningTaskService {
public List<CleaningTask> getTasks(String status, String keywords, Integer page, Integer size) {
Integer offset = page * size;
return cleaningTaskMapper.findTasks(status, keywords, size, offset);
List<CleaningTask> tasks = cleaningTaskMapper.findTasks(status, keywords, size, offset);
tasks.forEach(this::setProcess);
return tasks;
}
private void setProcess(CleaningTask task) {
int count = cleaningResultMapper.countByInstanceId(task.getId());
task.setProgress(CleaningProcess.of(task.getFileCount(), count));
}
public int countTasks(String status, String keywords) {
@@ -80,6 +88,7 @@ public class CleaningTaskService {
task.setDestDatasetId(destDataset.getId());
task.setDestDatasetName(destDataset.getName());
task.setBeforeSize(srcDataset.getTotalSize());
task.setFileCount(srcDataset.getFileCount());
cleaningTaskMapper.insertTask(task);
List<OperatorInstancePo> instancePos = request.getInstance().stream()
@@ -93,7 +102,9 @@ public class CleaningTaskService {
}
public CleaningTask getTask(String taskId) {
return cleaningTaskMapper.findTaskById(taskId);
CleaningTask task = cleaningTaskMapper.findTaskById(taskId);
setProcess(task);
return task;
}
@Transactional
@@ -113,7 +124,7 @@ public class CleaningTaskService {
process.setDatasetId(task.getDestDatasetId());
process.setDatasetPath(FLOW_PATH + "/" + task.getId() + "/dataset.jsonl");
process.setExportPath(DATASET_PATH + "/" + task.getDestDatasetId());
process.setExecutorType(ExecutorType.DATA_PLATFORM.getValue());
process.setExecutorType(ExecutorType.DATAMATE.getValue());
process.setProcess(instances.stream()
.map(instance -> Map.of(instance.getId(), instance.getOverrides()))
.toList());

2
backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/ExecutorType.java
View File

@@ -4,7 +4,7 @@ import lombok.Getter;
@Getter
public enum ExecutorType {
DATA_PLATFORM("data_platform"),
DATAMATE("datamate"),
DATA_JUICER_RAY("ray"),
DATA_JUICER_DEFAULT("default");

2
backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/mapper/CleaningResultMapper.java
View File

@@ -6,4 +6,6 @@ import org.apache.ibatis.annotations.Param;
@Mapper
public interface CleaningResultMapper {
void deleteByInstanceId(@Param("instanceId") String instanceId);
int countByInstanceId(@Param("instanceId") String instanceId);
}

18
backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/CleaningProcess.java
View File

@@ -4,6 +4,9 @@ package com.datamate.cleaning.interfaces.dto;
import lombok.Getter;
import lombok.Setter;
import java.math.BigDecimal;
import java.math.RoundingMode;
/**
* CleaningProcess
*/
@@ -16,5 +19,20 @@ public class CleaningProcess {
private Integer totalFileNum;
private Integer finishedFileNum;
public CleaningProcess(int totalFileNum, int finishedFileNum) {
this.totalFileNum = totalFileNum;
this.finishedFileNum = finishedFileNum;
if (totalFileNum == 0) {
this.process = 0.0f;
} else {
this.process = BigDecimal.valueOf(finishedFileNum * 100L)
.divide(BigDecimal.valueOf(totalFileNum), 2, RoundingMode.HALF_UP).floatValue();
}
}
public static CleaningProcess of(int totalFileNum, int finishedFileNum) {
return new CleaningProcess(totalFileNum, finishedFileNum);
}
}

2
backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/CleaningTask.java
View File

@@ -36,6 +36,8 @@ public class CleaningTask {
private long afterSize;
private int fileCount;
/**
* 任务当前状态
*/

4
backend/services/data-cleaning-service/src/main/resources/mappers/CleaningResultMapper.xml
View File

@@ -5,4 +5,8 @@
DELETE FROM t_clean_result WHERE instance_id = #{instanceId}
</delete>
<select id="countByInstanceId" resultType="java.lang.Integer">
SELECT COUNT(1) FROM t_clean_result WHERE instance_id = #{instanceId}
</select>
</mapper>

6
backend/services/data-cleaning-service/src/main/resources/mappers/CleaningTaskMapper.xml
View File

@@ -3,7 +3,7 @@
<mapper namespace="com.datamate.cleaning.infrastructure.persistence.mapper.CleaningTaskMapper">
<sql id="Base_Column_List">
id, name, description, src_dataset_id, src_dataset_name, dest_dataset_id, dest_dataset_name, before_size,
after_size, status, created_at, started_at, finished_at
after_size, file_count, status, created_at, started_at, finished_at
</sql>
<select id="findTasks" resultType="com.datamate.cleaning.interfaces.dto.CleaningTask">
@@ -28,9 +28,9 @@
<insert id="insertTask">
INSERT INTO t_clean_task (id, name, description, status, src_dataset_id, src_dataset_name, dest_dataset_id,
dest_dataset_name, before_size, after_size, created_at)
dest_dataset_name, before_size, after_size, file_count, created_at)
VALUES (#{id}, #{name}, #{description}, #{status}, #{srcDatasetId}, #{srcDatasetName}, #{destDatasetId},
#{destDatasetName}, ${beforeSize}, ${afterSize}, NOW())
#{destDatasetName}, #{beforeSize}, #{afterSize}, #{fileCount}, NOW())
</insert>
<update id="updateTask">

2
backend/services/main-application/src/main/resources/application.yml
View File

@@ -12,7 +12,7 @@ spring:
# 数据源配置
datasource:
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://mysql:3306/datamate?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true
url: jdbc:mysql://datamate-database:3306/datamate?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true
username: ${DB_USERNAME:root}
password: ${DB_PASSWORD:Huawei@123}
hikari:

2
backend/services/main-application/src/main/resources/log4j2.xml
View File

@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="WARN" monitorInterval="30">
<Properties>
<Property name="LOG_PATH">/var/log/data-mate/backend</Property>
<Property name="LOG_PATH">/var/log/datamate/backend</Property>
<Property name="LOG_PATTERN">%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{50} - %msg%n</Property>
<Property name="MAX_FILE_SIZE">100MB</Property>
<Property name="MAX_HISTORY">30</Property>