q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add operator create page (#38)

Browse Source 
* feat: Update site name to DataMate and refine text for AI data processing

* feat: Refactor settings page and implement model access functionality

- Created a new ModelAccess component for managing model configurations.
- Removed the old Settings component and replaced it with a new SettingsPage component that integrates ModelAccess, SystemConfig, and WebhookConfig.
- Added SystemConfig component for managing system settings.
- Implemented WebhookConfig component for managing webhook configurations.
- Updated API functions for model management in settings.apis.ts.
- Adjusted routing to point to the new SettingsPage component.

* feat: Implement Data Collection Page with Task Management and Execution Log

- Created DataCollectionPage component to manage data collection tasks.
- Added TaskManagement and ExecutionLog components for task handling and logging.
- Integrated task operations including start, stop, edit, and delete functionalities.
- Implemented filtering and searching capabilities in task management.
- Introduced SimpleCronScheduler for scheduling tasks with cron expressions.
- Updated CreateTask component to utilize new scheduling and template features.
- Enhanced BasicInformation component to conditionally render fields based on visibility settings.
- Refactored ImportConfiguration component to remove NAS import section.

* feat: Update task creation API endpoint and enhance task creation form with new fields and validation

* Refactor file upload and operator management components

- Removed unnecessary console logs from file download and export functions.
- Added size property to TaskItem interface for better task management.
- Simplified TaskUpload component by utilizing useFileSliceUpload hook for file upload logic.
- Enhanced OperatorPluginCreate component to handle file uploads and parsing more efficiently.
- Updated ConfigureStep component to use Ant Design Form for better data handling and validation.
- Improved PreviewStep component to navigate back to the operator market.
- Added support for additional file types in UploadStep component.
- Implemented delete operator functionality in OperatorMarketPage with confirmation prompts.
- Cleaned up unused API functions in operator.api.ts to streamline the codebase.
- Fixed number formatting utility to handle zero values correctly.
This commit is contained in:
chenghh-9609
2025-10-30 16:30:01 +08:00
committed by GitHub
parent e0884ab048
commit 5612c7cd91
22 changed files with 640 additions and 979 deletions
Download Patch File Download Diff File Expand all files Collapse all files

169
frontend/src/pages/Layout/TaskUpload.tsx
View File

@@ -3,167 +3,19 @@ import {
preUploadUsingPost,
uploadFileChunkUsingPost,
} from "@/pages/DataManagement/dataset.api";
import { TaskItem } from "@/pages/DataManagement/dataset.model";
import { calculateSHA256, checkIsFilesExist } from "@/utils/file.util";
import { App, Button, Empty, Progress } from "antd";
import { Button, Empty, Progress } from "antd";
import { DeleteOutlined } from "@ant-design/icons";
import { useState, useRef, useEffect } from "react";
import { useEffect } from "react";
import { useFileSliceUpload } from "@/hooks/useSliceUpload";
export default function TaskUpload() {
const { message } = App.useApp();
const [taskList, setTaskList] = useState<TaskItem[]>([]);
const taskListRef = useRef<TaskItem[]>([]); // 用于固定任务顺序
const createTask = (detail: any = {}) => {
const { dataset } = detail;
const title = `上传数据集: ${dataset.name} `;
const controller = new AbortController();
const task: TaskItem = {
key: dataset.id,
title,
percent: 0,
reqId: -1,
controller,
updateEvent: detail.updateEvent || "update:dataset",
};
taskListRef.current = [task, ...taskListRef.current];
setTaskList(taskListRef.current);
return task;
};
const updateTaskList = (task: TaskItem) => {
taskListRef.current = taskListRef.current.map((item) =>
item.key === task.key ? task : item
);
setTaskList(taskListRef.current);
};
const removeTask = (task: TaskItem) => {
const { key } = task;
taskListRef.current = taskListRef.current.filter(
(item) => item.key !== key
);
setTaskList(taskListRef.current);
if (task.isCancel && task.cancelFn) {
task.cancelFn();
const { createTask, taskList, removeTask, handleUpload } = useFileSliceUpload(
{
preUpload: preUploadUsingPost,
uploadChunk: uploadFileChunkUsingPost,
cancelUpload: cancelUploadUsingPut,
}
window.dispatchEvent(new Event(task.updateEvent || "update:dataset"));
window.dispatchEvent(
new CustomEvent("show:task-popover", { detail: { show: false } })
);
};
async function buildFormData({ file, reqId, i, j }) {
const formData = new FormData();
const { slices, name, size } = file;
const checkSum = await calculateSHA256(slices[j]);
formData.append("file", slices[j]);
formData.append("reqId", reqId.toString());
formData.append("fileNo", (i + 1).toString());
formData.append("chunkNo", (j + 1).toString());
formData.append("fileName", name);
formData.append("fileSize", size.toString());
formData.append("totalChunkNum", slices.length.toString());
formData.append("checkSumHex", checkSum);
return formData;
}
async function uploadSlice(task: TaskItem, fileInfo) {
if (!task) {
return;
}
const { reqId, key, signal } = task;
const { loaded, i, j, files, totalSize } = fileInfo;
const formData = await buildFormData({
file: files[i],
i,
j,
reqId,
});
let newTask = { ...task };
await uploadFileChunkUsingPost(key, formData, {
onUploadProgress: (e) => {
const loadedSize = loaded + e.loaded;
const curPercent = Math.round(loadedSize / totalSize) * 100;
newTask = {
...newTask,
...taskListRef.current.find((item) => item.key === key),
percent: curPercent >= 100 ? 99.99 : curPercent,
};
updateTaskList(newTask);
},
signal,
});
}
async function uploadFile({ task, files, totalSize }) {
const { data: reqId } = await preUploadUsingPost(task.key, {
totalFileNum: files.length,
totalSize,
datasetId: task.key,
});
const newTask: TaskItem = {
...task,
reqId,
isCancel: false,
cancelFn: () => {
task.controller.abort();
cancelUploadUsingPut(reqId);
window.dispatchEvent(new Event(task.updateEvent || "update:dataset"));
},
};
updateTaskList(newTask);
window.dispatchEvent(
new CustomEvent("show:task-popover", { detail: { show: true } })
);
// 更新数据状态
window.dispatchEvent(new Event("update:dataset-status"));
let loaded = 0;
for (let i = 0; i < files.length; i++) {
const { slices } = files[i];
for (let j = 0; j < slices.length; j++) {
await uploadSlice(newTask, {
loaded,
i,
j,
files,
totalSize,
});
loaded += slices[j].size;
}
}
removeTask(newTask);
}
const handleUpload = async ({ task, files }) => {
const isErrorFile = await checkIsFilesExist(files);
if (isErrorFile) {
message.error("文件被修改或删除，请重新选择文件上传");
removeTask({
...task,
isCancel: false,
...taskListRef.current.find((item) => item.key === task.key),
});
return;
}
try {
const totalSize = files.reduce((acc, file) => acc + file.size, 0);
await uploadFile({ task, files, totalSize });
} catch (err) {
console.error(err);
message.error("文件上传失败，请稍后重试");
removeTask({
...task,
isCancel: true,
...taskListRef.current.find((item) => item.key === task.key),
});
}
};
);
useEffect(() => {
const uploadHandler = (e: any) => {
@@ -195,9 +47,6 @@ export default function TaskUpload() {
removeTask({
...task,
isCancel: true,
...taskListRef.current.find(
(item) => item.key === task.key
),
})
}
icon={<DeleteOutlined />}