refactor：重构数据归集部分代码 (#75)

* fix：配比任务需要能够跳转到目标数据集

* feature：增加配比任务详情接口

* fix：删除不存在的配比详情页面

* fix：使用正式的逻辑来展示标签

* fix：参数默认值去掉多余的-

* fix：修复配比任务相关操作

* fix：去除不需要的日志打印和import

* feature：数据归集创建时将obs、mysql归集也放出

* refactor：重构数据归集的代码

* refactor：重构数据归集的代码

backend/services/data-collection-service/src/main/java/com/datamate/collection/application/CollectionTaskService.java

@@ -5,12 +5,9 @@ import com.baomidou.mybatisplus.core.metadata.IPage;
 import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import com.datamate.collection.domain.model.entity.CollectionTask;
 import com.datamate.collection.domain.model.entity.TaskExecution;
-import com.datamate.collection.common.enums.TaskStatus;
 import com.datamate.collection.domain.repository.CollectionTaskRepository;
 import com.datamate.collection.common.enums.SyncMode;
 import com.datamate.common.domain.utils.ChunksSaver;
-import com.datamate.datamanagement.application.DatasetApplicationService;
-import com.datamate.datamanagement.domain.model.dataset.Dataset;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
@@ -25,7 +22,6 @@ import java.util.Objects;
 @RequiredArgsConstructor
 public class CollectionTaskService {
     private final TaskExecutionService taskExecutionService;
-    private final DatasetApplicationService datasetApplicationService;
     private final CollectionTaskRepository collectionTaskRepository;
 
     @Transactional

backend/services/data-collection-service/src/main/java/com/datamate/collection/common/enums/SyncMode.java

@@ -7,6 +7,6 @@ public enum SyncMode {
     /** 一次性(ONCE) */
     ONCE,
     /// 定时(SCHEDULED)
-    SCHEDULED;
+    SCHEDULED
 }
 

backend/services/data-collection-service/src/main/java/com/datamate/collection/common/enums/TemplateType.java

@@ -0,0 +1,11 @@
+package com.datamate.collection.common.enums;
+
+/**
+ * 模板类型枚举
+ *
+ */
+public enum TemplateType {
+    NAS,
+    OBS,
+    MYSQL
+}

backend/services/data-collection-service/src/main/java/com/datamate/collection/domain/model/entity/CollectionTask.java

@@ -3,6 +3,7 @@ package com.datamate.collection.domain.model.entity;
 import com.baomidou.mybatisplus.annotation.TableName;
 import com.datamate.collection.common.enums.SyncMode;
 import com.datamate.collection.common.enums.TaskStatus;
+import com.datamate.collection.common.enums.TemplateType;
 import com.datamate.common.domain.model.base.BaseEntity;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
@@ -24,6 +25,8 @@ import java.util.UUID;
 public class CollectionTask extends BaseEntity<String> {
     private String name;
     private String description;
+    private TemplateType taskType; // 任务类型
+    private String targetPath; // 目标存储路径
     private String config; // DataX JSON 配置，包含源端和目标端配置信息
     private TaskStatus status;
     private SyncMode syncMode; // ONCE / SCHEDULED
@@ -51,7 +54,7 @@ public class CollectionTask extends BaseEntity<String> {
 
     public void initCreateParam() {
         this.id = UUID.randomUUID().toString();
-        this.addPath();
+        this.targetPath = "/dataset/local/" + id;
         this.status = TaskStatus.READY;
         this.createdAt = LocalDateTime.now();
         this.updatedAt = LocalDateTime.now();

backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/datax/DataxProcessRunner.java

@@ -1,28 +1,24 @@
 package com.datamate.collection.infrastructure.datax;
 
+import com.datamate.collection.common.enums.TemplateType;
 import com.datamate.collection.domain.model.entity.CollectionTask;
 import com.datamate.collection.domain.process.ProcessRunner;
+import com.datamate.collection.infrastructure.datax.config.NasConfig;
 import com.datamate.common.infrastructure.exception.BusinessException;
 import com.datamate.common.infrastructure.exception.SystemErrorCode;
-import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.exec.*;
+import org.apache.commons.io.output.TeeOutputStream;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Component;
 
-import java.io.File;
+import java.io.*;
-import java.io.FileNotFoundException;
-import java.io.FileWriter;
-import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Duration;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 
 @Slf4j
 @Component
@@ -61,10 +57,8 @@ public class DataxProcessRunner implements ProcessRunner {
         }
 
         ExecuteStreamHandler streamHandler = new PumpStreamHandler(
-            new org.apache.commons.io.output.TeeOutputStream(
+            new TeeOutputStream(new FileOutputStream(logFile, true), System.out),
-                new java.io.FileOutputStream(logFile, true), System.out),
+            new TeeOutputStream(new FileOutputStream(logFile, true), System.err)
-            new org.apache.commons.io.output.TeeOutputStream(
-                new java.io.FileOutputStream(logFile, true), System.err)
         );
         executor.setStreamHandler(streamHandler);
 
@@ -92,30 +86,18 @@ public class DataxProcessRunner implements ProcessRunner {
     private String getJobConfig(CollectionTask task) {
         try {
             ObjectMapper objectMapper = new ObjectMapper();
-            Map<String, Object> parameter = objectMapper.readValue(
+            TemplateType templateType = task.getTaskType();
-                task.getConfig(),
+            switch (templateType) {
-                new TypeReference<>() {
+                case NAS:
+                    // NAS 特殊处理
+                    // 移除 templateType 字段
+                    NasConfig nasConfig = objectMapper.readValue(task.getConfig(), NasConfig.class);
+                    return nasConfig.toJobConfig(objectMapper, task);
+                case OBS:
+                case MYSQL:
+                default:
+                    throw BusinessException.of(SystemErrorCode.UNKNOWN_ERROR, "Unsupported template type: " + templateType);
             }
-            );
-            Map<String, Object> job = new HashMap<>();
-            Map<String, Object> content = new HashMap<>();
-            Map<String, Object> reader = new HashMap<>();
-            reader.put("name", "nfsreader");
-            reader.put("parameter", parameter);
-            content.put("reader", reader);
-            Map<String, Object> writer = new HashMap<>();
-            writer.put("name", "nfswriter");
-            writer.put("parameter", parameter);
-            content.put("writer", writer);
-            job.put("content", List.of(content));
-            Map<String, Object> setting = new HashMap<>();
-            Map<String, Object> channel = new HashMap<>();
-            channel.put("channel", 2);
-            setting.put("speed", channel);
-            job.put("setting", setting);
-            Map<String, Object> jobConfig = new HashMap<>();
-            jobConfig.put("job", job);
-            return objectMapper.writeValueAsString(jobConfig);
         } catch (Exception e) {
             log.error("Failed to parse task config", e);
             throw new RuntimeException("Failed to parse task config", e);

backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/datax/config/BaseConfig.java

@@ -0,0 +1,4 @@
+package com.datamate.collection.infrastructure.datax.config;
+
+public interface BaseConfig {
+}

backend/services/data-collection-service/src/main/java/com/datamate/collection/infrastructure/datax/config/NasConfig.java

@@ -0,0 +1,54 @@
+package com.datamate.collection.infrastructure.datax.config;
+
+import com.datamate.collection.domain.model.entity.CollectionTask;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.Getter;
+import lombok.Setter;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Getter
+@Setter
+public class NasConfig implements BaseConfig{
+    private String ip;
+
+    private String path;
+
+    private List<String> files;
+
+    /**
+     * 将当前 NAS 配置构造成 DataX 所需的 job JSON 字符串。
+     */
+    public String toJobConfig(ObjectMapper objectMapper, CollectionTask task) throws Exception {
+        Map<String, Object> parameter = new HashMap<>();
+        if (ip != null) parameter.put("ip", ip);
+        if (path != null) parameter.put("path", path);
+        if (files != null) parameter.put("files", files);
+        parameter.put("destPath", task.getTargetPath());
+
+        Map<String, Object> job = new HashMap<>();
+        Map<String, Object> content = new HashMap<>();
+        Map<String, Object> reader = new HashMap<>();
+        reader.put("name", "nfsreader");
+        reader.put("parameter", parameter);
+        content.put("reader", reader);
+
+        Map<String, Object> writer = new HashMap<>();
+        writer.put("name", "nfswriter");
+        writer.put("parameter", parameter);
+        content.put("writer", writer);
+
+        job.put("content", List.of(content));
+        Map<String, Object> setting = new HashMap<>();
+        Map<String, Object> channel = new HashMap<>();
+        channel.put("channel", 2);
+        setting.put("speed", channel);
+        job.put("setting", setting);
+
+        Map<String, Object> jobConfig = new HashMap<>();
+        jobConfig.put("job", job);
+        return objectMapper.writeValueAsString(jobConfig);
+    }
+}

backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/dto/CollectionTaskResponse.java

@@ -29,7 +29,8 @@ public class CollectionTaskResponse {
 
   private String description;
 
-  @Valid
+  private String targetPath;
+
   private Map<String, Object> config = new HashMap<>();
 
   private TaskStatus status;

backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/dto/CreateCollectionTaskRequest.java

@@ -1,6 +1,7 @@
 package com.datamate.collection.interfaces.dto;
 
 import com.datamate.collection.common.enums.SyncMode;
+import com.datamate.collection.common.enums.TemplateType;
 import com.datamate.datamanagement.interfaces.dto.CreateDatasetRequest;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
@@ -35,6 +36,11 @@ public class CreateCollectionTaskRequest {
     @JsonProperty("description")
     private String description;
 
+    @NotNull
+    @Schema(name = "taskType", description = "任务类型", requiredMode = Schema.RequiredMode.REQUIRED)
+    @JsonProperty("taskType")
+    private TemplateType taskType;
+
     @Valid
     @NotNull
     @Schema(name = "config", description = "归集配置，包含源端和目标端配置信息", requiredMode = Schema.RequiredMode.REQUIRED)

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java

@@ -285,12 +285,7 @@ public class DatasetApplicationService {
             return Collections.emptyList();
         }
         log.info("获取到归集任务详情: {}", taskDetail);
-        LocalCollectionConfig config = parseTaskConfig(taskDetail.getConfig());
+        return Collections.singletonList(taskDetail.getTargetPath());
-        if (config == null) {
-            log.warn("解析任务配置失败，任务ID: {}", dataSourceId);
-            return Collections.emptyList();
-        }
-        return config.getFilePaths();
     }
 
     /**

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/domain/model/dataset/DatasetFile.java

@@ -6,10 +6,8 @@ import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.*;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
 
 import java.time.LocalDateTime;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
@@ -50,7 +48,6 @@ public class DatasetFile {
             ObjectMapper mapper = new ObjectMapper();
             return mapper.readValue(tags, new TypeReference<List<FileTag>>() {});
         } catch (Exception e) {
-            log.error(e.getMessage(), e);
             return Collections.emptyList();
         }
     }

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/client/dto/CollectionTaskDetailResponse.java

@@ -13,6 +13,7 @@ public class CollectionTaskDetailResponse {
     private String id;
     private String name;
     private String description;
+    private String targetPath;
     private Map<String, Object> config;
     private String status;
     private String syncMode;

frontend/src/pages/DataCollection/Create/CreateTask.tsx

@@ -14,7 +14,7 @@ const { TextArea } = Input;
 
 const defaultTemplates = [
   {
-    id: "nas",
+    id: "NAS",
     name: "NAS到本地",
     description: "从NAS文件系统导入数据到本地文件系统",
     config: {
@@ -23,7 +23,7 @@ const defaultTemplates = [
     },
   },
   {
-    id: "obs",
+    id: "OBS",
     name: "OBS到本地",
     description: "从OBS文件系统导入数据到本地文件系统",
     config: {
@@ -32,11 +32,11 @@ const defaultTemplates = [
     },
   },
   {
-    id: "web",
+    id: "MYSQL",
-    name: "Web到本地",
+    name: "Mysql到本地",
-    description: "从Web URL导入数据到本地文件系统",
+    description: "从Mysql中导入数据到本地文件系统",
     config: {
-      reader: "webreader",
+      reader: "mysqlreader",
       writer: "localwriter",
     },
   },
@@ -45,9 +45,9 @@ const defaultTemplates = [
 const syncModeOptions = Object.values(SyncModeMap);
 
 enum TemplateType {
-  NAS = "nas",
+  NAS = "NAS",
-  OBS = "obs",
+  OBS = "OBS",
-  WEB = "web",
+  MYSQL = "MYSQL",
 }
 
 export default function CollectionTaskCreate() {
@@ -58,16 +58,22 @@ export default function CollectionTaskCreate() {
   const [templateType, setTemplateType] = useState<"default" | "custom">(
     "default"
   );
-  const [selectedTemplate, setSelectedTemplate] = useState("nas");
+  // 默认模板类型设为 NAS
+  const [selectedTemplate, setSelectedTemplate] = useState<TemplateType>(
+    TemplateType.NAS
+  );
   const [customConfig, setCustomConfig] = useState("");
 
-  const [newTask, setNewTask] = useState({
+  // 将 newTask 设为 any，并初始化 config.templateType 为 NAS
+  const [newTask, setNewTask] = useState<any>({
     name: "",
     description: "",
     syncMode: SyncMode.ONCE,
     cronExpression: "",
     maxRetries: 10,
-    dataset: {},
+    dataset: null,
+    config: { templateType: TemplateType.NAS },
+    createDataset: false,
   });
   const [scheduleExpression, setScheduleExpression] = useState({
     type: SyncMode.SCHEDULED,
@@ -79,7 +85,7 @@ export default function CollectionTaskCreate() {
 
   const handleSubmit = async () => {
     try {
-      const formData = await form.validateFields();
+      await form.validateFields();
       if (templateType === "default" && !selectedTemplate) {
         window.alert("请选择默认模板");
         return;
@@ -88,8 +94,21 @@ export default function CollectionTaskCreate() {
         window.alert("请填写自定义配置");
         return;
       }
-      // Create task logic here
+
-      await createTaskUsingPost(newTask);
+      // 构建最终 payload，不依赖异步 setState
+      const payload = {
+        ...newTask,
+        taskType:
+          templateType === "default" ? selectedTemplate : "CUSTOM",
+        config: {
+          ...((newTask && newTask.config) || {}),
+          ...(templateType === "custom" ? { dataxJson: customConfig } : {}),
+        },
+      };
+
+      console.log("创建任务 payload:", payload);
+
+      await createTaskUsingPost(payload);
       message.success("任务创建成功");
       navigate("/data/collection");
     } catch (error) {
@@ -192,7 +211,8 @@ export default function CollectionTaskCreate() {
             </Form.Item> */}
             {templateType === "default" && (
               <>
-                {/* <Form.Item label="选择模板">
+                {
+                  <Form.Item label="选择模板">
                     <div className="grid grid-cols-1 sm:grid-cols-2 md:grid-cols-3 gap-4">
                       {defaultTemplates.map((template) => (
                         <div
@@ -202,7 +222,20 @@ export default function CollectionTaskCreate() {
                               ? "border-blue-500"
                               : "border-gray-300"
                           }`}
-                        onClick={() => setSelectedTemplate(template.id)}
+                          onClick={() => {
+                            setSelectedTemplate(template.id as TemplateType);
+                            // 使用函数式更新，合并之前的 config
+                            setNewTask((prev: any) => ({
+                              ...prev,
+                              config: {
+                                templateType: template.id,
+                              },
+                            }));
+                            // 同步表单显示
+                            form.setFieldsValue({
+                              config: { templateType: template.id },
+                            });
+                          }}
                         >
                           <div className="font-medium">{template.name}</div>
                           <div className="text-gray-500">
@@ -214,10 +247,11 @@ export default function CollectionTaskCreate() {
                         </div>
                       ))}
                     </div>
-                </Form.Item> */}
+                  </Form.Item>
+                }
                 {/* nas import */}
                 {selectedTemplate === TemplateType.NAS && (
-                  <div className="grid grid-cols-2 gap-3 px-2 rounded">
+                  <div className="grid grid-cols-2 gap-3 px-2 bg-blue-50 rounded">
                     <Form.Item
                       name={["config", "ip"]}
                       rules={[{ required: true, message: "请输入NAS地址" }]}
@@ -246,7 +280,7 @@ export default function CollectionTaskCreate() {
                 {selectedTemplate === TemplateType.OBS && (
                   <div className="grid grid-cols-2 gap-3 p-4 bg-blue-50 rounded-lg">
                     <Form.Item
-                      name="endpoint"
+                      name={["config", "endpoint"]}
                       rules={[{ required: true }]}
                       label="Endpoint"
                     >
@@ -256,21 +290,21 @@ export default function CollectionTaskCreate() {
                       />
                     </Form.Item>
                     <Form.Item
-                      name="bucket"
+                      name={["config", "bucket"]}
                       rules={[{ required: true }]}
                       label="Bucket"
                     >
                       <Input className="h-8 text-xs" placeholder="my-bucket" />
                     </Form.Item>
                     <Form.Item
-                      name="accessKey"
+                      name={["config", "accessKey"]}
                       rules={[{ required: true }]}
                       label="Access Key"
                     >
                       <Input className="h-8 text-xs" placeholder="Access Key" />
                     </Form.Item>
                     <Form.Item
-                      name="secretKey"
+                      name={["config", "secretKey"]}
                       rules={[{ required: true }]}
                       label="Secret Key"
                     >
@@ -282,6 +316,54 @@ export default function CollectionTaskCreate() {
                     </Form.Item>
                   </div>
                 )}
+
+                {/* mysql import */}
+                {selectedTemplate === TemplateType.MYSQL && (
+                  <div className="grid grid-cols-2 gap-3 px-2 bg-blue-50 rounded">
+                    <Form.Item
+                      name={["config", "host"]}
+                      rules={[{ required: true, message: "请输入MYSQL主机名" }]}
+                      label="MYSQL主机名"
+                    >
+                      <Input placeholder="192.168.1.100" />
+                    </Form.Item>
+                    <Form.Item
+                      name={["config", "port"]}
+                      rules={[{ required: true, message: "请输入端口号" }]}
+                      label="端口号"
+                    >
+                      <Input placeholder="3306" />
+                    </Form.Item>
+                    <Form.Item
+                      name={["config", "user"]}
+                      rules={[{ required: true, message: "请输入用户名" }]}
+                      label="用户名"
+                    >
+                      <Input placeholder="mysql" />
+                    </Form.Item>
+                    <Form.Item
+                      name={["config", "password"]}
+                      rules={[{ required: true, message: "请输入密码" }]}
+                      label="密码"
+                    >
+                      <Input placeholder="" />
+                    </Form.Item>
+                    <Form.Item
+                      name={["config", "schema"]}
+                      rules={[{ required: true, message: "请输入数据库" }]}
+                      label="数据库"
+                    >
+                      <Input placeholder="public" />
+                    </Form.Item>
+                    <Form.Item
+                      name={["config", "sql"]}
+                      rules={[{ required: true, message: "请输入查询语句" }]}
+                      label="查询语句"
+                    >
+                      <Input placeholder="select * from your_table" />
+                    </Form.Item>
+                  </div>
+                )}
               </>
             )}
 
@@ -313,15 +395,17 @@ export default function CollectionTaskCreate() {
                     value={isCreateDataset}
                     onChange={(e) => {
                       const value = e.target.value;
-                      if (value === false) {
+                      let datasetInit = null;
-                        form.setFieldsValue({
+                      if (value === true) {
-                          dataset: {},
+                        datasetInit = {};
-                        });
-                        setNewTask({
-                          ...newTask,
-                          dataset: {},
-                        });
                       }
+                      form.setFieldsValue({
+                        dataset: datasetInit,
+                      });
+                      setNewTask((prev: any) => ({
+                        ...prev,
+                        dataset: datasetInit,
+                      }));
                       setIsCreateDataset(e.target.value);
                     }}
                   >
@@ -339,13 +423,13 @@ export default function CollectionTaskCreate() {
                       <Input
                         placeholder="输入数据集名称"
                         onChange={(e) => {
-                          setNewTask({
+                          setNewTask((prev: any) => ({
-                            ...newTask,
+                            ...prev,
                             dataset: {
-                              ...newTask.dataset,
+                              ...(prev.dataset || {}),
                               name: e.target.value,
                             },
-                          });
+                          }));
                         }}
                       />
                     </Form.Item>
@@ -356,15 +440,16 @@ export default function CollectionTaskCreate() {
                     >
                       <RadioCard
                         options={datasetTypes}
-                        value={newTask.dataset.datasetType}
+                        value={newTask.dataset?.datasetType}
                         onChange={(type) => {
                           form.setFieldValue(["dataset", "datasetType"], type);
-                          setNewTask({
+                          setNewTask((prev: any) => ({
-                            ...newTask,
+                            ...prev,
                             dataset: {
+                              ...(prev.dataset || {}),
                               datasetType: type as DatasetSubType,
                             },
-                          });
+                          }));
                         }}
                       />
                     </Form.Item>

scripts/db/data-collection-init.sql

@@ -46,6 +46,8 @@ CREATE TABLE t_dc_collection_tasks (
     name VARCHAR(255) NOT NULL COMMENT '任务名称',
     description TEXT COMMENT '任务描述',
     sync_mode VARCHAR(20) DEFAULT 'ONCE' COMMENT '同步模式：ONCE/SCHEDULED',
+    task_type VARCHAR(20) DEFAULT 'NAS' COMMENT '任务类型：NAS/OBS/MYSQL/CUSTOM',
+    target_path VARCHAR(1000) DEFAULT '' COMMENT '目标存储路径',
     config TEXT  NOT NULL COMMENT '归集配置（DataX配置），包含源端和目标端配置信息',
     schedule_expression VARCHAR(255) COMMENT 'Cron调度表达式',
     status VARCHAR(20) DEFAULT 'DRAFT' COMMENT '任务状态：DRAFT/READY/RUNNING/SUCCESS/FAILED/STOPPED',