add operator create page (#38)

* feat: Update site name to DataMate and refine text for AI data processing

* feat: Refactor settings page and implement model access functionality

- Created a new ModelAccess component for managing model configurations.
- Removed the old Settings component and replaced it with a new SettingsPage component that integrates ModelAccess, SystemConfig, and WebhookConfig.
- Added SystemConfig component for managing system settings.
- Implemented WebhookConfig component for managing webhook configurations.
- Updated API functions for model management in settings.apis.ts.
- Adjusted routing to point to the new SettingsPage component.

* feat: Implement Data Collection Page with Task Management and Execution Log

- Created DataCollectionPage component to manage data collection tasks.
- Added TaskManagement and ExecutionLog components for task handling and logging.
- Integrated task operations including start, stop, edit, and delete functionalities.
- Implemented filtering and searching capabilities in task management.
- Introduced SimpleCronScheduler for scheduling tasks with cron expressions.
- Updated CreateTask component to utilize new scheduling and template features.
- Enhanced BasicInformation component to conditionally render fields based on visibility settings.
- Refactored ImportConfiguration component to remove NAS import section.

* feat: Update task creation API endpoint and enhance task creation form with new fields and validation

* Refactor file upload and operator management components

- Removed unnecessary console logs from file download and export functions.
- Added size property to TaskItem interface for better task management.
- Simplified TaskUpload component by utilizing useFileSliceUpload hook for file upload logic.
- Enhanced OperatorPluginCreate component to handle file uploads and parsing more efficiently.
- Updated ConfigureStep component to use Ant Design Form for better data handling and validation.
- Improved PreviewStep component to navigate back to the operator market.
- Added support for additional file types in UploadStep component.
- Implemented delete operator functionality in OperatorMarketPage with confirmation prompts.
- Cleaned up unused API functions in operator.api.ts to streamline the codebase.
- Fixed number formatting utility to handle zero values correctly.

frontend/src/hooks/useFetchData.ts

@@ -16,11 +16,10 @@ import { useState, useRef, useEffect, useCallback } from "react";
 import { useDebouncedEffect } from "./useDebouncedEffect";
 import Loading from "@/utils/loading";
 import { App } from "antd";
-import { AnyObject } from "antd/es/_util/type";
 
 export default function useFetchData<T>(
   fetchFunc: (params?: any) => Promise<any>,
-  mapDataFunc: (data: AnyObject) => T = (data) => data as T,
+  mapDataFunc: (data: Partial<T>) => T = (data) => data as T,
   pollingInterval: number = 30000, // 默认30秒轮询一次
   autoRefresh: boolean = true,
   additionalPollingFuncs: (() => Promise<any>)[] = [], // 额外的轮询函数

frontend/src/hooks/useSliceUpload.tsx

@@ -0,0 +1,185 @@
+import { TaskItem } from "@/pages/DataManagement/dataset.model";
+import { calculateSHA256, checkIsFilesExist } from "@/utils/file.util";
+import { App } from "antd";
+import { useRef, useState } from "react";
+
+export function useFileSliceUpload(
+  {
+    preUpload,
+    uploadChunk,
+    cancelUpload,
+  }: {
+    preUpload: (id: string, params: any) => Promise<{ data: number }>;
+    uploadChunk: (id: string, formData: FormData, config: any) => Promise<any>;
+    cancelUpload: ((reqId: number) => Promise<any>) | null;
+  },
+  showTaskCenter = true // 上传时是否显示任务中心
+) {
+  const { message } = App.useApp();
+  const [taskList, setTaskList] = useState<TaskItem[]>([]);
+  const taskListRef = useRef<TaskItem[]>([]); // 用于固定任务顺序
+
+  const createTask = (detail: any = {}) => {
+    const { dataset } = detail;
+    const title = `上传数据集: ${dataset.name} `;
+    const controller = new AbortController();
+    const task: TaskItem = {
+      key: dataset.id,
+      title,
+      percent: 0,
+      reqId: -1,
+      controller,
+      size: 0,
+      updateEvent: detail.updateEvent,
+    };
+    taskListRef.current = [task, ...taskListRef.current];
+
+    setTaskList(taskListRef.current);
+    return task;
+  };
+
+  const updateTaskList = (task: TaskItem) => {
+    taskListRef.current = taskListRef.current.map((item) =>
+      item.key === task.key ? task : item
+    );
+    setTaskList(taskListRef.current);
+  };
+
+  const removeTask = (task: TaskItem) => {
+    const { key } = task;
+    taskListRef.current = taskListRef.current.filter(
+      (item) => item.key !== key
+    );
+    setTaskList(taskListRef.current);
+    if (task.isCancel && task.cancelFn) {
+      task.cancelFn();
+    }
+    if (task.updateEvent) window.dispatchEvent(new Event(task.updateEvent));
+    if (showTaskCenter) {
+      window.dispatchEvent(
+        new CustomEvent("show:task-popover", { detail: { show: false } })
+      );
+    }
+  };
+
+  async function buildFormData({ file, reqId, i, j }) {
+    const formData = new FormData();
+    const { slices, name, size } = file;
+    const checkSum = await calculateSHA256(slices[j]);
+    formData.append("file", slices[j]);
+    formData.append("reqId", reqId.toString());
+    formData.append("fileNo", (i + 1).toString());
+    formData.append("chunkNo", (j + 1).toString());
+    formData.append("fileName", name);
+    formData.append("fileSize", size.toString());
+    formData.append("totalChunkNum", slices.length.toString());
+    formData.append("checkSumHex", checkSum);
+    return formData;
+  }
+
+  async function uploadSlice(task: TaskItem, fileInfo) {
+    if (!task) {
+      return;
+    }
+    const { reqId, key } = task;
+    const { loaded, i, j, files, totalSize } = fileInfo;
+    const formData = await buildFormData({
+      file: files[i],
+      i,
+      j,
+      reqId,
+    });
+
+    let newTask = { ...task };
+    await uploadChunk(key, formData, {
+      onUploadProgress: (e) => {
+        const loadedSize = loaded + e.loaded;
+        const curPercent = Number((loadedSize / totalSize) * 100).toFixed(2);
+
+        newTask = {
+          ...newTask,
+          ...taskListRef.current.find((item) => item.key === key),
+          size: loadedSize,
+          percent: curPercent >= 100 ? 99.99 : curPercent,
+        };
+        updateTaskList(newTask);
+      },
+    });
+  }
+
+  async function uploadFile({ task, files, totalSize }) {
+    const { data: reqId } = await preUpload(task.key, {
+      totalFileNum: files.length,
+      totalSize,
+      datasetId: task.key,
+    });
+
+    const newTask: TaskItem = {
+      ...task,
+      reqId,
+      isCancel: false,
+      cancelFn: () => {
+        task.controller.abort();
+        cancelUpload?.(reqId);
+        if (task.updateEvent) window.dispatchEvent(new Event(task.updateEvent));
+      },
+    };
+    updateTaskList(newTask);
+    if (showTaskCenter) {
+      window.dispatchEvent(
+        new CustomEvent("show:task-popover", { detail: { show: true } })
+      );
+    }
+    // // 更新数据状态
+    if (task.updateEvent) window.dispatchEvent(new Event(task.updateEvent));
+
+    let loaded = 0;
+    for (let i = 0; i < files.length; i++) {
+      const { slices } = files[i];
+      for (let j = 0; j < slices.length; j++) {
+        await uploadSlice(newTask, {
+          loaded,
+          i,
+          j,
+          files,
+          totalSize,
+        });
+        loaded += slices[j].size;
+      }
+    }
+    removeTask(newTask);
+  }
+
+  const handleUpload = async ({ task, files }) => {
+    const isErrorFile = await checkIsFilesExist(files);
+    if (isErrorFile) {
+      message.error("文件被修改或删除，请重新选择文件上传");
+      removeTask({
+        ...task,
+        isCancel: false,
+        ...taskListRef.current.find((item) => item.key === task.key),
+      });
+      return;
+    }
+
+    try {
+      const totalSize = files.reduce((acc, file) => acc + file.size, 0);
+      await uploadFile({ task, files, totalSize });
+    } catch (err) {
+      console.error(err);
+      message.error("文件上传失败，请稍后重试");
+      removeTask({
+        ...task,
+        isCancel: true,
+        ...taskListRef.current.find((item) => item.key === task.key),
+      });
+    }
+  };
+
+  return {
+    taskList,
+    createTask,
+    removeTask,
+    handleUpload,
+  };
+}