init datamate

backend/services/data-collection-service/src/main/java/com/datamate/collection/DataCollectionServiceConfiguration.java

@@ -0,0 +1,24 @@
+package com.datamate.collection;
+
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.scheduling.annotation.EnableAsync;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.transaction.annotation.EnableTransactionManagement;
+
+/**
+ * 数据归集服务配置类
+ *
+ * 基于DataX的数据归集和同步服务，支持多种数据源的数据采集和归集
+ */
+@SpringBootApplication
+@EnableAsync
+@EnableScheduling
+@EnableTransactionManagement
+@ComponentScan(basePackages = {
+        "com.datamate.collection",
+        "com.datamate.shared"
+})
+public class DataCollectionServiceConfiguration {
+    // Configuration class for JAR packaging - no main method needed
+}

backend/services/data-collection-service/src/main/java/com/datamate/collection/application/scheduler/TaskSchedulerInitializer.java

@@ -0,0 +1,66 @@
+package com.datamate.collection.application.scheduler;
+
+import com.datamate.collection.application.service.DataxExecutionService;
+import com.datamate.collection.domain.model.CollectionTask;
+import com.datamate.collection.domain.model.TaskStatus;
+import com.datamate.collection.domain.model.TaskExecution;
+import com.datamate.collection.infrastructure.persistence.mapper.CollectionTaskMapper;
+import com.datamate.collection.infrastructure.persistence.mapper.TaskExecutionMapper;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.scheduling.support.CronExpression;
+import org.springframework.stereotype.Component;
+import org.springframework.util.StringUtils;
+
+import java.time.LocalDateTime;
+import java.util.List;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class TaskSchedulerInitializer {
+
+    private final CollectionTaskMapper taskMapper;
+    private final TaskExecutionMapper executionMapper;
+    private final DataxExecutionService dataxExecutionService;
+
+    // 定期扫描激活的采集任务，根据 Cron 判断是否到期执行
+    @Scheduled(fixedDelayString = "${datamate.data-collection.scheduler.scan-interval-ms:10000}")
+    public void scanAndTrigger() {
+        List<CollectionTask> tasks = taskMapper.selectActiveTasks();
+        if (tasks == null || tasks.isEmpty()) {
+            return;
+        }
+        LocalDateTime now = LocalDateTime.now();
+        for (CollectionTask task : tasks) {
+            String cronExpr = task.getScheduleExpression();
+            if (!StringUtils.hasText(cronExpr)) {
+                continue;
+            }
+            try {
+                // 如果最近一次执行仍在运行，则跳过
+                TaskExecution latest = executionMapper.selectLatestByTaskId(task.getId());
+                if (latest != null && latest.getStatus() == TaskStatus.RUNNING) {
+                    continue;
+                }
+
+                CronExpression cron = CronExpression.parse(cronExpr);
+                LocalDateTime base = latest != null && latest.getStartedAt() != null
+                        ? latest.getStartedAt()
+                        : now.minusYears(1); // 没有历史记录时，拉长基准时间确保到期判定
+                LocalDateTime nextTime = cron.next(base);
+
+                if (nextTime != null && !nextTime.isAfter(now)) {
+                    // 到期，触发一次执行
+                    TaskExecution exec = dataxExecutionService.createExecution(task);
+                    int timeout = task.getTimeoutSeconds() == null ? 3600 : task.getTimeoutSeconds();
+                    dataxExecutionService.runAsync(task, exec.getId(), timeout);
+                    log.info("Triggered DataX execution for task {} at {}, execId={}", task.getId(), now, exec.getId());
+                }
+            } catch (Exception ex) {
+                log.warn("Skip task {} due to invalid cron or scheduling error: {}", task.getId(), ex.getMessage());
+            }
+        }
+    }
+}

backend/services/data-collection-service/src/main/java/com/datamate/collection/application/service/CollectionTaskService.java

@@ -0,0 +1,85 @@
+package com.datamate.collection.application.service;
+
+import com.datamate.collection.domain.model.CollectionTask;
+import com.datamate.collection.domain.model.TaskExecution;
+import com.datamate.collection.domain.model.TaskStatus;
+import com.datamate.collection.domain.model.DataxTemplate;
+import com.datamate.collection.infrastructure.persistence.mapper.CollectionTaskMapper;
+import com.datamate.collection.infrastructure.persistence.mapper.TaskExecutionMapper;
+import com.datamate.collection.interfaces.dto.SyncMode;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.time.LocalDateTime;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class CollectionTaskService {
+    private final CollectionTaskMapper taskMapper;
+    private final TaskExecutionMapper executionMapper;
+    private final DataxExecutionService dataxExecutionService;
+
+    @Transactional
+    public CollectionTask create(CollectionTask task) {
+        task.setStatus(TaskStatus.READY);
+        task.setCreatedAt(LocalDateTime.now());
+        task.setUpdatedAt(LocalDateTime.now());
+        taskMapper.insert(task);
+        executeTaskNow(task);
+        return task;
+    }
+
+    private void executeTaskNow(CollectionTask task) {
+        if (Objects.equals(task.getSyncMode(), SyncMode.ONCE.getValue())) {
+            TaskExecution exec = dataxExecutionService.createExecution(task);
+            int timeout = task.getTimeoutSeconds() == null ? 3600 : task.getTimeoutSeconds();
+            dataxExecutionService.runAsync(task, exec.getId(), timeout);
+            log.info("Triggered DataX execution for task {} at {}, execId={}", task.getId(), LocalDateTime.now(), exec.getId());
+        }
+    }
+
+    @Transactional
+    public CollectionTask update(CollectionTask task) {
+        task.setUpdatedAt(LocalDateTime.now());
+        taskMapper.update(task);
+        return task;
+    }
+
+    @Transactional
+    public void delete(String id) { taskMapper.deleteById(id); }
+
+    public CollectionTask get(String id) { return taskMapper.selectById(id); }
+
+    public List<CollectionTask> list(Integer page, Integer size, String status, String name) {
+        Map<String, Object> p = new HashMap<>();
+        p.put("status", status);
+        p.put("name", name);
+        if (page != null && size != null) {
+            p.put("offset", page * size);
+            p.put("limit", size);
+        }
+        return taskMapper.selectAll(p);
+    }
+
+    @Transactional
+    public TaskExecution startExecution(CollectionTask task) {
+        return dataxExecutionService.createExecution(task);
+    }
+
+    // ---- Template related merged methods ----
+    public List<DataxTemplate> listTemplates(String sourceType, String targetType, int page, int size) {
+        int offset = page * size;
+        return taskMapper.selectList(sourceType, targetType, offset, size);
+    }
+
+    public int countTemplates(String sourceType, String targetType) {
+        return taskMapper.countTemplates(sourceType, targetType);
+    }
+}

backend/services/data-collection-service/src/main/java/com/datamate/collection/application/service/DataxExecutionService.java

@@ -0,0 +1,60 @@
+package com.datamate.collection.application.service;
+
+import com.datamate.collection.domain.model.CollectionTask;
+import com.datamate.collection.domain.model.TaskExecution;
+import com.datamate.collection.domain.model.TaskStatus;
+import com.datamate.collection.infrastructure.persistence.mapper.CollectionTaskMapper;
+import com.datamate.collection.infrastructure.persistence.mapper.TaskExecutionMapper;
+import com.datamate.collection.infrastructure.runtime.datax.DataxJobBuilder;
+import com.datamate.collection.infrastructure.runtime.datax.DataxProcessRunner;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.LocalDateTime;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class DataxExecutionService {
+
+    private final DataxJobBuilder jobBuilder;
+    private final DataxProcessRunner processRunner;
+    private final TaskExecutionMapper executionMapper;
+    private final CollectionTaskMapper taskMapper;
+
+
+    @Transactional
+    public TaskExecution createExecution(CollectionTask task) {
+        TaskExecution exec = TaskExecution.initTaskExecution();
+        exec.setTaskId(task.getId());
+        exec.setTaskName(task.getName());
+        executionMapper.insert(exec);
+        taskMapper.updateLastExecution(task.getId(), exec.getId());
+        taskMapper.updateStatus(task.getId(), TaskStatus.RUNNING.name());
+        return exec;
+    }
+
+    @Async
+    public void runAsync(CollectionTask task, String executionId, int timeoutSeconds) {
+        try {
+            Path job = jobBuilder.buildJobFile(task);
+
+            int code = processRunner.runJob(job.toFile(), executionId, Duration.ofSeconds(timeoutSeconds));
+            log.info("DataX finished with code {} for execution {}", code, executionId);
+            // 简化：成功即完成
+            executionMapper.completeExecution(executionId, TaskStatus.SUCCESS.name(), LocalDateTime.now(),
+                    0, 0L, 0L, 0L, null, null);
+            taskMapper.updateStatus(task.getId(), TaskStatus.SUCCESS.name());
+        } catch (Exception e) {
+            log.error("DataX execution failed", e);
+            executionMapper.completeExecution(executionId, TaskStatus.FAILED.name(), LocalDateTime.now(),
+                    0, 0L, 0L, 0L, e.getMessage(), null);
+            taskMapper.updateStatus(task.getId(), TaskStatus.FAILED.name());
+        }
+    }
+}

backend/services/data-collection-service/src/main/java/com/datamate/collection/application/service/TaskExecutionService.java

@@ -0,0 +1,83 @@
+package com.datamate.collection.application.service;
+
+import com.datamate.collection.domain.model.CollectionTask;
+import com.datamate.collection.domain.model.TaskExecution;
+import com.datamate.collection.domain.model.TaskStatus;
+import com.datamate.collection.infrastructure.persistence.mapper.CollectionTaskMapper;
+import com.datamate.collection.infrastructure.persistence.mapper.TaskExecutionMapper;
+import lombok.RequiredArgsConstructor;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Service
+@RequiredArgsConstructor
+public class TaskExecutionService {
+    private final TaskExecutionMapper executionMapper;
+    private final CollectionTaskMapper taskMapper;
+
+    public List<TaskExecution> list(String taskId, String status, LocalDateTime startDate,
+                                    LocalDateTime endDate, Integer page, Integer size) {
+        Map<String, Object> p = new HashMap<>();
+        p.put("taskId", taskId);
+        p.put("status", status);
+        p.put("startDate", startDate);
+        p.put("endDate", endDate);
+        if (page != null && size != null) {
+            p.put("offset", page * size);
+            p.put("limit", size);
+        }
+        return executionMapper.selectAll(p);
+    }
+
+    public long count(String taskId, String status, LocalDateTime startDate, LocalDateTime endDate) {
+        Map<String, Object> p = new HashMap<>();
+        p.put("taskId", taskId);
+        p.put("status", status);
+        p.put("startDate", startDate);
+        p.put("endDate", endDate);
+        return executionMapper.count(p);
+    }
+
+    // --- Added convenience methods ---
+    public TaskExecution get(String id) { return executionMapper.selectById(id); }
+    public TaskExecution getLatestByTaskId(String taskId) { return executionMapper.selectLatestByTaskId(taskId); }
+
+    @Transactional
+    public void complete(String executionId, boolean success, long successCount, long failedCount,
+                         long dataSizeBytes, String errorMessage, String resultJson) {
+        LocalDateTime now = LocalDateTime.now();
+        TaskExecution exec = executionMapper.selectById(executionId);
+        if (exec == null) { return; }
+        int duration = (int) Duration.between(exec.getStartedAt(), now).getSeconds();
+        executionMapper.completeExecution(executionId, success ? TaskStatus.SUCCESS.name() : TaskStatus.FAILED.name(),
+                now, duration, successCount, failedCount, dataSizeBytes, errorMessage, resultJson);
+        CollectionTask task = taskMapper.selectById(exec.getTaskId());
+        if (task != null) {
+            taskMapper.updateStatus(task.getId(), success ? TaskStatus.SUCCESS.name() : TaskStatus.FAILED.name());
+        }
+    }
+
+    @Transactional
+    public void stop(String executionId) {
+        TaskExecution exec = executionMapper.selectById(executionId);
+        if (exec == null || exec.getStatus() != TaskStatus.RUNNING) { return; }
+        LocalDateTime now = LocalDateTime.now();
+        int duration = (int) Duration.between(exec.getStartedAt(), now).getSeconds();
+        // Reuse completeExecution to persist STOPPED status and timing info
+        executionMapper.completeExecution(exec.getId(), TaskStatus.STOPPED.name(), now, duration,
+                exec.getRecordsSuccess(), exec.getRecordsFailed(), exec.getDataSizeBytes(), null, exec.getResult());
+        taskMapper.updateStatus(exec.getTaskId(), TaskStatus.STOPPED.name());
+    }
+
+    @Transactional
+    public void stopLatestByTaskId(String taskId) {
+        TaskExecution latest = executionMapper.selectLatestByTaskId(taskId);
+        if (latest != null) { stop(latest.getId()); }
+    }
+}

backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/CollectionTask.java

@@ -0,0 +1,45 @@
+package com.datamate.collection.domain.model;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.Data;
+
+import java.time.LocalDateTime;
+import java.util.Collections;
+import java.util.Map;
+
+@Data
+public class CollectionTask {
+    private String id;
+    private String name;
+    private String description;
+    private String config; // DataX JSON 配置，包含源端和目标端配置信息
+    private TaskStatus status;
+    private String syncMode; // ONCE / SCHEDULED
+    private String scheduleExpression;
+    private Integer retryCount;
+    private Integer timeoutSeconds;
+    private Long maxRecords;
+    private String sortField;
+    private String lastExecutionId;
+    private LocalDateTime createdAt;
+    private LocalDateTime updatedAt;
+    private String createdBy;
+    private String updatedBy;
+
+    public void addPath() {
+        try {
+            ObjectMapper objectMapper = new ObjectMapper();
+            Map<String, Object> parameter = objectMapper.readValue(
+                config,
+                new TypeReference<>() {}
+            );
+            parameter.put("destPath", "/dataset/local/" + id);
+            parameter.put("filePaths", Collections.singletonList(parameter.get("destPath")));
+            config = objectMapper.writeValueAsString(parameter);
+        } catch (JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/DataxTemplate.java

@@ -0,0 +1,71 @@
+package com.datamate.collection.domain.model;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+import java.time.LocalDateTime;
+
+@Data
+@EqualsAndHashCode(callSuper = false)
+public class DataxTemplate {
+
+    /**
+     * 模板ID（UUID）
+     */
+    private String id;
+
+    /**
+     * 模板名称
+     */
+    private String name;
+
+    /**
+     * 源数据源类型
+     */
+    private String sourceType;
+
+    /**
+     * 目标数据源类型
+     */
+    private String targetType;
+
+    /**
+     * 模板内容（JSON格式）
+     */
+    private String templateContent;
+
+    /**
+     * 模板描述
+     */
+    private String description;
+
+    /**
+     * 版本号
+     */
+    private String version;
+
+    /**
+     * 是否为系统模板
+     */
+    private Boolean isSystem;
+
+    /**
+     * 创建时间
+     */
+    private LocalDateTime createdAt;
+
+    /**
+     * 更新时间
+     */
+    private LocalDateTime updatedAt;
+
+    /**
+     * 创建者
+     */
+    private String createdBy;
+
+    /**
+     * 更新者
+     */
+    private String updatedBy;
+}

backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/TaskExecution.java

@@ -0,0 +1,39 @@
+package com.datamate.collection.domain.model;
+
+import lombok.Data;
+
+import java.time.LocalDateTime;
+import java.util.UUID;
+
+@Data
+public class TaskExecution {
+    private String id;
+    private String taskId;
+    private String taskName;
+    private TaskStatus status;
+    private Double progress;
+    private Long recordsTotal;
+    private Long recordsProcessed;
+    private Long recordsSuccess;
+    private Long recordsFailed;
+    private Double throughput;
+    private Long dataSizeBytes;
+    private LocalDateTime startedAt;
+    private LocalDateTime completedAt;
+    private Integer durationSeconds;
+    private String errorMessage;
+    private String dataxJobId;
+    private String config;
+    private String result;
+    private LocalDateTime createdAt;
+
+    public static TaskExecution initTaskExecution() {
+        TaskExecution exec = new TaskExecution();
+        exec.setId(UUID.randomUUID().toString());
+        exec.setStatus(TaskStatus.RUNNING);
+        exec.setProgress(0.0);
+        exec.setStartedAt(LocalDateTime.now());
+        exec.setCreatedAt(LocalDateTime.now());
+        return exec;
+    }
+}

backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/TaskStatus.java

@@ -0,0 +1,21 @@
+package com.datamate.collection.domain.model;
+
+/**
+ * 统一的任务和执行状态枚举
+ *
+ * @author Data Mate Platform Team
+ */
+public enum TaskStatus {
+    /** 草稿状态 */
+    DRAFT,
+    /** 就绪状态 */
+    READY,
+    /** 运行中 */
+    RUNNING,
+    /** 执行成功（对应原来的COMPLETED） */
+    SUCCESS,
+    /** 执行失败 */
+    FAILED,
+    /** 已停止 */
+    STOPPED
+}

backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/persistence/mapper/CollectionTaskMapper.java

@@ -0,0 +1,47 @@
+package com.datamate.collection.infrastructure.persistence.mapper;
+
+import com.datamate.collection.domain.model.CollectionTask;
+import com.datamate.collection.domain.model.DataxTemplate;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+import java.util.Map;
+
+@Mapper
+public interface CollectionTaskMapper {
+    int insert(CollectionTask entity);
+    int update(CollectionTask entity);
+    int deleteById(@Param("id") String id);
+    CollectionTask selectById(@Param("id") String id);
+    CollectionTask selectByName(@Param("name") String name);
+    List<CollectionTask> selectByStatus(@Param("status") String status);
+    List<CollectionTask> selectAll(Map<String, Object> params);
+    int updateStatus(@Param("id") String id, @Param("status") String status);
+    int updateLastExecution(@Param("id") String id, @Param("lastExecutionId") String lastExecutionId);
+    List<CollectionTask> selectActiveTasks();
+
+    /**
+     * 查询模板列表
+     *
+     * @param sourceType 源数据源类型（可选）
+     * @param targetType 目标数据源类型（可选）
+     * @param offset 偏移量
+     * @param limit 限制数量
+     * @return 模板列表
+     */
+    List<DataxTemplate> selectList(@Param("sourceType") String sourceType,
+                                   @Param("targetType") String targetType,
+                                   @Param("offset") int offset,
+                                   @Param("limit") int limit);
+
+    /**
+     * 统计模板数量
+     *
+     * @param sourceType 源数据源类型（可选）
+     * @param targetType 目标数据源类型（可选）
+     * @return 模板总数
+     */
+    int countTemplates(@Param("sourceType") String sourceType,
+                       @Param("targetType") String targetType);
+}

backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/persistence/mapper/TaskExecutionMapper.java

@@ -0,0 +1,38 @@
+package com.datamate.collection.infrastructure.persistence.mapper;
+
+import com.datamate.collection.domain.model.TaskExecution;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+
+import java.time.LocalDateTime;
+import java.util.List;
+import java.util.Map;
+
+@Mapper
+public interface TaskExecutionMapper {
+    int insert(TaskExecution entity);
+    int update(TaskExecution entity);
+    int deleteById(@Param("id") String id);
+    TaskExecution selectById(@Param("id") String id);
+    List<TaskExecution> selectByTaskId(@Param("taskId") String taskId, @Param("limit") Integer limit);
+    List<TaskExecution> selectByStatus(@Param("status") String status);
+    List<TaskExecution> selectAll(Map<String, Object> params);
+    long count(Map<String, Object> params);
+    int updateProgress(@Param("id") String id,
+                       @Param("status") String status,
+                       @Param("progress") Double progress,
+                       @Param("recordsProcessed") Long recordsProcessed,
+                       @Param("throughput") Double throughput);
+    int completeExecution(@Param("id") String id,
+                          @Param("status") String status,
+                          @Param("completedAt") LocalDateTime completedAt,
+                          @Param("durationSeconds") Integer durationSeconds,
+                          @Param("recordsSuccess") Long recordsSuccess,
+                          @Param("recordsFailed") Long recordsFailed,
+                          @Param("dataSizeBytes") Long dataSizeBytes,
+                          @Param("errorMessage") String errorMessage,
+                          @Param("result") String result);
+    List<TaskExecution> selectRunningExecutions();
+    TaskExecution selectLatestByTaskId(@Param("taskId") String taskId);
+    int deleteOldExecutions(@Param("beforeDate") LocalDateTime beforeDate);
+}

backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/runtime/datax/DataxJobBuilder.java

@@ -0,0 +1,83 @@
+package com.datamate.collection.infrastructure.runtime.datax;
+
+import com.datamate.collection.domain.model.CollectionTask;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Component;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * 根据任务配置拼装 DataX 作业 JSON 文件
+ */
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class DataxJobBuilder {
+
+    private final DataxProperties props;
+
+    public Path buildJobFile(CollectionTask task) throws IOException {
+        Files.createDirectories(Paths.get(props.getJobConfigPath()));
+        String fileName = String.format("datax-job-%s.json", task.getId());
+        Path path = Paths.get(props.getJobConfigPath(), fileName);
+        // 简化：直接将任务中的 config 字段作为 DataX 作业 JSON
+        try (FileWriter fw = new FileWriter(path.toFile())) {
+            String json = task.getConfig() == null || task.getConfig().isEmpty() ?
+                    defaultJobJson() : task.getConfig();
+            if (StringUtils.isNotBlank(task.getConfig())) {
+                json = getJobConfig(task);
+            }
+            log.info("Job config: {}", json);
+            fw.write(json);
+        }
+        return path;
+    }
+
+    private String getJobConfig(CollectionTask task) {
+        try {
+            ObjectMapper objectMapper = new ObjectMapper();
+            Map<String, Object> parameter = objectMapper.readValue(
+                task.getConfig(),
+                new TypeReference<>() {}
+            );
+            Map<String, Object> job = new HashMap<>();
+            Map<String, Object> content = new HashMap<>();
+            Map<String, Object> reader = new HashMap<>();
+            reader.put("name", "nfsreader");
+            reader.put("parameter", parameter);
+            content.put("reader", reader);
+            Map<String, Object> writer = new HashMap<>();
+            writer.put("name", "nfswriter");
+            writer.put("parameter", parameter);
+            content.put("writer", writer);
+            job.put("content", List.of(content));
+            Map<String, Object> setting = new HashMap<>();
+            Map<String, Object> channel = new HashMap<>();
+            channel.put("channel", 2);
+            setting.put("speed", channel);
+            job.put("setting", setting);
+            Map<String, Object> jobConfig = new HashMap<>();
+            jobConfig.put("job", job);
+            return objectMapper.writeValueAsString(jobConfig);
+        } catch (Exception e) {
+            log.error("Failed to parse task config", e);
+            throw new RuntimeException("Failed to parse task config", e);
+        }
+    }
+
+    private String defaultJobJson() {
+        // 提供一个最小可运行的空 job，实际会被具体任务覆盖
+        return "{\n  \"job\": {\n    \"setting\": {\n      \"speed\": {\n        \"channel\": 1\n      }\n    },\n    \"content\": []\n  }\n}";
+    }
+}

backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/runtime/datax/DataxProcessRunner.java

@@ -0,0 +1,46 @@
+package com.datamate.collection.infrastructure.runtime.datax;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.exec.*;
+import org.springframework.stereotype.Component;
+
+import java.io.File;
+import java.time.Duration;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class DataxProcessRunner {
+
+    private final DataxProperties props;
+
+    public int runJob(File jobFile, String executionId, Duration timeout) throws Exception {
+        File logFile = new File(props.getLogPath(), String.format("datax-%s.log", executionId));
+        String python = props.getPythonPath();
+        String dataxPy = props.getHomePath() + File.separator + "bin" + File.separator + "datax.py";
+        String cmd = String.format("%s %s %s", python, dataxPy, jobFile.getAbsolutePath());
+
+        log.info("Execute DataX: {}", cmd);
+
+        CommandLine cl = CommandLine.parse(cmd);
+        DefaultExecutor executor = new DefaultExecutor();
+
+        // 将日志追加输出到文件
+        File parent = logFile.getParentFile();
+        if (!parent.exists()) parent.mkdirs();
+
+        ExecuteStreamHandler streamHandler = new PumpStreamHandler(
+                new org.apache.commons.io.output.TeeOutputStream(
+                        new java.io.FileOutputStream(logFile, true), System.out),
+                new org.apache.commons.io.output.TeeOutputStream(
+                        new java.io.FileOutputStream(logFile, true), System.err)
+        );
+        executor.setStreamHandler(streamHandler);
+
+        ExecuteWatchdog watchdog = new ExecuteWatchdog(timeout.toMillis());
+        executor.setWatchdog(watchdog);
+
+        return executor.execute(cl);
+    }
+}

backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/runtime/datax/DataxProperties.java

@@ -0,0 +1,17 @@
+package com.datamate.collection.infrastructure.runtime.datax;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Data
+@Configuration
+@ConfigurationProperties(prefix = "datamate.data-collection.datax")
+public class DataxProperties {
+    private String homePath;        // DATAX_HOME
+    private String pythonPath;      // python 可执行文件
+    private String jobConfigPath;   // 生成的作业文件目录
+    private String logPath;         // 运行日志目录
+    private Integer maxMemory = 2048;
+    private Integer channelCount = 5;
+}

backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/converter/CollectionTaskConverter.java

@@ -0,0 +1,52 @@
+package com.datamate.collection.interfaces.converter;
+
+import com.datamate.collection.domain.model.CollectionTask;
+import com.datamate.collection.domain.model.DataxTemplate;
+import com.datamate.collection.interfaces.dto.*;
+import com.datamate.common.infrastructure.exception.BusinessException;
+import com.datamate.common.infrastructure.exception.SystemErrorCode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.mapstruct.Mapper;
+import org.mapstruct.Mapping;
+import org.mapstruct.Named;
+import org.mapstruct.factory.Mappers;
+
+import java.util.Map;
+
+@Mapper
+public interface CollectionTaskConverter {
+    CollectionTaskConverter INSTANCE = Mappers.getMapper(CollectionTaskConverter.class);
+
+    @Mapping(source = "config", target = "config", qualifiedByName = "parseJsonToMap")
+    CollectionTaskResponse toResponse(CollectionTask task);
+
+    CollectionTaskSummary toSummary(CollectionTask task);
+
+    DataxTemplateSummary toTemplateSummary(DataxTemplate template);
+
+    @Mapping(source = "config", target = "config", qualifiedByName = "mapToJsonString")
+    CollectionTask toCollectionTask(CreateCollectionTaskRequest request);
+
+    @Mapping(source = "config", target = "config", qualifiedByName = "mapToJsonString")
+    CollectionTask toCollectionTask(UpdateCollectionTaskRequest request);
+
+    @Named("parseJsonToMap")
+    default Map<String, Object> parseJsonToMap(String json) {
+        try {
+            ObjectMapper objectMapper = new ObjectMapper();
+            return objectMapper.readValue(json, Map.class);
+        } catch (Exception e) {
+            throw BusinessException.of(SystemErrorCode.INVALID_PARAMETER);
+        }
+    }
+
+    @Named("mapToJsonString")
+    default String mapToJsonString(Map<String, Object> map) {
+        try {
+            ObjectMapper objectMapper = new ObjectMapper();
+            return objectMapper.writeValueAsString(map != null ? map : Map.of());
+        } catch (Exception e) {
+            throw BusinessException.of(SystemErrorCode.INVALID_PARAMETER);
+        }
+    }
+}

backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/rest/CollectionTaskController.java

@@ -0,0 +1,83 @@
+package com.datamate.collection.interfaces.rest;
+
+import com.datamate.collection.application.service.CollectionTaskService;
+import com.datamate.collection.domain.model.CollectionTask;
+import com.datamate.collection.domain.model.DataxTemplate;
+import com.datamate.collection.interfaces.api.CollectionTaskApi;
+import com.datamate.collection.interfaces.converter.CollectionTaskConverter;
+import com.datamate.collection.interfaces.dto.*;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.http.ResponseEntity;
+import org.springframework.validation.annotation.Validated;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Slf4j
+@RestController
+@RequiredArgsConstructor
+@Validated
+public class CollectionTaskController implements CollectionTaskApi {
+
+    private final CollectionTaskService taskService;
+
+    @Override
+    public ResponseEntity<CollectionTaskResponse> createTask(CreateCollectionTaskRequest request) {
+        CollectionTask task = CollectionTaskConverter.INSTANCE.toCollectionTask(request);
+        task.setId(UUID.randomUUID().toString());
+        task.addPath();
+        return ResponseEntity.ok().body(CollectionTaskConverter.INSTANCE.toResponse(taskService.create(task)));
+    }
+
+    @Override
+    public ResponseEntity<CollectionTaskResponse> updateTask(String id, UpdateCollectionTaskRequest request) {
+        if (taskService.get(id) == null) {
+            return ResponseEntity.notFound().build();
+        }
+        CollectionTask task = CollectionTaskConverter.INSTANCE.toCollectionTask(request);
+        task.setId(id);
+        return ResponseEntity.ok(CollectionTaskConverter.INSTANCE.toResponse(taskService.update(task)));
+    }
+
+    @Override
+    public ResponseEntity<Void> deleteTask(String id) {
+        taskService.delete(id);
+        return ResponseEntity.ok().build();
+    }
+
+    @Override
+    public ResponseEntity<CollectionTaskResponse> getTaskDetail(String id) {
+        CollectionTask task = taskService.get(id);
+        return task == null ? ResponseEntity.notFound().build() : ResponseEntity.ok(CollectionTaskConverter.INSTANCE.toResponse(task));
+    }
+
+    @Override
+    public ResponseEntity<PagedCollectionTaskSummary> getTasks(Integer page, Integer size, TaskStatus status, String name) {
+        var list = taskService.list(page, size, status == null ? null : status.getValue(), name);
+        PagedCollectionTaskSummary response = new PagedCollectionTaskSummary();
+        response.setContent(list.stream().map(CollectionTaskConverter.INSTANCE::toSummary).collect(Collectors.toList()));
+        response.setNumber(page);
+        response.setSize(size);
+        response.setTotalElements(list.size()); // 简化处理，实际项目中应该有单独的count查询
+        response.setTotalPages(size == null || size == 0 ? 1 : (int) Math.ceil(list.size() * 1.0 / size));
+        return ResponseEntity.ok(response);
+    }
+
+    @Override
+    public ResponseEntity<PagedDataxTemplates> templatesGet(String sourceType, String targetType,
+                                                           Integer page, Integer size) {
+        int pageNum = page != null ? page : 0;
+        int pageSize = size != null ? size : 20;
+        List<DataxTemplate> templates = taskService.listTemplates(sourceType, targetType, pageNum, pageSize);
+        int totalElements = taskService.countTemplates(sourceType, targetType);
+        PagedDataxTemplates response = new PagedDataxTemplates();
+        response.setContent(templates.stream().map(CollectionTaskConverter.INSTANCE::toTemplateSummary).collect(Collectors.toList()));
+        response.setNumber(pageNum);
+        response.setSize(pageSize);
+        response.setTotalElements(totalElements);
+        response.setTotalPages(pageSize > 0 ? (int) Math.ceil(totalElements * 1.0 / pageSize) : 1);
+        return ResponseEntity.ok(response);
+    }
+}

backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/rest/TaskExecutionController.java

@@ -0,0 +1,101 @@
+package com.datamate.collection.interfaces.rest;
+
+import com.datamate.collection.application.service.CollectionTaskService;
+import com.datamate.collection.application.service.TaskExecutionService;
+import com.datamate.collection.domain.model.TaskExecution;
+import com.datamate.collection.interfaces.api.TaskExecutionApi;
+import com.datamate.collection.interfaces.dto.PagedTaskExecutions;
+import com.datamate.collection.interfaces.dto.TaskExecutionDetail;
+import com.datamate.collection.interfaces.dto.TaskExecutionResponse;
+import com.datamate.collection.interfaces.dto.TaskStatus; // DTO enum
+import lombok.RequiredArgsConstructor;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.validation.annotation.Validated;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.stream.Collectors;
+
+@RestController
+@RequiredArgsConstructor
+@Validated
+public class TaskExecutionController implements TaskExecutionApi {
+
+    private final TaskExecutionService executionService;
+    private final CollectionTaskService taskService;
+
+    private TaskExecutionDetail toDetail(TaskExecution e) {
+        TaskExecutionDetail d = new TaskExecutionDetail();
+        d.setId(e.getId());
+        d.setTaskId(e.getTaskId());
+        d.setTaskName(e.getTaskName());
+        if (e.getStatus() != null) { d.setStatus(TaskStatus.fromValue(e.getStatus().name())); }
+        d.setProgress(e.getProgress());
+        d.setRecordsTotal(e.getRecordsTotal() != null ? e.getRecordsTotal().intValue() : null);
+        d.setRecordsProcessed(e.getRecordsProcessed() != null ? e.getRecordsProcessed().intValue() : null);
+        d.setRecordsSuccess(e.getRecordsSuccess() != null ? e.getRecordsSuccess().intValue() : null);
+        d.setRecordsFailed(e.getRecordsFailed() != null ? e.getRecordsFailed().intValue() : null);
+        d.setThroughput(e.getThroughput());
+        d.setDataSizeBytes(e.getDataSizeBytes() != null ? e.getDataSizeBytes().intValue() : null);
+        d.setStartedAt(e.getStartedAt());
+        d.setCompletedAt(e.getCompletedAt());
+        d.setDurationSeconds(e.getDurationSeconds());
+        d.setErrorMessage(e.getErrorMessage());
+        return d;
+    }
+
+    // GET /executions/{id}
+    @Override
+    public ResponseEntity<TaskExecutionDetail> executionsIdGet(String id) {
+        var exec = executionService.get(id);
+        return exec == null ? ResponseEntity.notFound().build() : ResponseEntity.ok(toDetail(exec));
+    }
+
+    // DELETE /executions/{id}
+    @Override
+    public ResponseEntity<Void> executionsIdDelete(String id) {
+        executionService.stop(id); // 幂等处理，在service内部判断状态
+        return ResponseEntity.noContent().build();
+    }
+
+    // POST /tasks/{id}/execute  -> 201
+    @Override
+    public ResponseEntity<TaskExecutionResponse> tasksIdExecutePost(String id) {
+        var task = taskService.get(id);
+        if (task == null) { return ResponseEntity.notFound().build(); }
+        var latestExec = executionService.getLatestByTaskId(id);
+        if (latestExec != null && latestExec.getStatus() == com.datamate.collection.domain.model.TaskStatus.RUNNING) {
+            TaskExecutionResponse r = new TaskExecutionResponse();
+            r.setId(latestExec.getId());
+            r.setTaskId(latestExec.getTaskId());
+            r.setTaskName(latestExec.getTaskName());
+            r.setStatus(TaskStatus.fromValue(latestExec.getStatus().name()));
+            r.setStartedAt(latestExec.getStartedAt());
+            return ResponseEntity.status(HttpStatus.CREATED).body(r); // 返回已有运行实例
+        }
+        var exec = taskService.startExecution(task);
+        TaskExecutionResponse r = new TaskExecutionResponse();
+        r.setId(exec.getId());
+        r.setTaskId(exec.getTaskId());
+        r.setTaskName(exec.getTaskName());
+        r.setStatus(TaskStatus.fromValue(exec.getStatus().name()));
+        r.setStartedAt(exec.getStartedAt());
+        return ResponseEntity.status(HttpStatus.CREATED).body(r);
+    }
+
+    // GET /tasks/{id}/executions  -> 分页
+    @Override
+    public ResponseEntity<PagedTaskExecutions> tasksIdExecutionsGet(String id, Integer page, Integer size) {
+        if (page == null || page < 0) { page = 0; }
+        if (size == null || size <= 0) { size = 20; }
+        var list = executionService.list(id, null, null, null, page, size);
+        long total = executionService.count(id, null, null, null);
+        PagedTaskExecutions p = new PagedTaskExecutions();
+        p.setContent(list.stream().map(this::toDetail).collect(Collectors.toList()));
+        p.setNumber(page);
+        p.setSize(size);
+        p.setTotalElements((int) total);
+        p.setTotalPages(size == 0 ? 1 : (int) Math.ceil(total * 1.0 / size));
+        return ResponseEntity.ok(p);
+    }
+}