diff --git a/frontend/src/components/CardView.tsx b/frontend/src/components/CardView.tsx index 36b3a2d..11bb6d3 100644 --- a/frontend/src/components/CardView.tsx +++ b/frontend/src/components/CardView.tsx @@ -246,10 +246,10 @@ function CardView(props: CardViewProps) {
{item?.statistics?.map((stat, idx) => (
-
+
{stat?.label}:
-
+
{stat?.value}
diff --git a/frontend/src/hooks/useFetchData.ts b/frontend/src/hooks/useFetchData.ts index b0d7ff5..6146c9d 100644 --- a/frontend/src/hooks/useFetchData.ts +++ b/frontend/src/hooks/useFetchData.ts @@ -16,11 +16,10 @@ import { useState, useRef, useEffect, useCallback } from "react"; import { useDebouncedEffect } from "./useDebouncedEffect"; import Loading from "@/utils/loading"; import { App } from "antd"; -import { AnyObject } from "antd/es/_util/type"; export default function useFetchData( fetchFunc: (params?: any) => Promise, - mapDataFunc: (data: AnyObject) => T = (data) => data as T, + mapDataFunc: (data: Partial) => T = (data) => data as T, pollingInterval: number = 30000, // 默认30秒轮询一次 autoRefresh: boolean = true, additionalPollingFuncs: (() => Promise)[] = [], // 额外的轮询函数 diff --git a/frontend/src/hooks/useSliceUpload.tsx b/frontend/src/hooks/useSliceUpload.tsx new file mode 100644 index 0000000..a3f360f --- /dev/null +++ b/frontend/src/hooks/useSliceUpload.tsx @@ -0,0 +1,185 @@ +import { TaskItem } from "@/pages/DataManagement/dataset.model"; +import { calculateSHA256, checkIsFilesExist } from "@/utils/file.util"; +import { App } from "antd"; +import { useRef, useState } from "react"; + +export function useFileSliceUpload( + { + preUpload, + uploadChunk, + cancelUpload, + }: { + preUpload: (id: string, params: any) => Promise<{ data: number }>; + uploadChunk: (id: string, formData: FormData, config: any) => Promise; + cancelUpload: ((reqId: number) => Promise) | null; + }, + showTaskCenter = true // 上传时是否显示任务中心 +) { + const { message } = App.useApp(); + const [taskList, setTaskList] = useState([]); + const taskListRef = useRef([]); // 用于固定任务顺序 + + const createTask = (detail: any = {}) => { + const { dataset } = detail; + const title = `上传数据集: ${dataset.name} `; + const controller = new AbortController(); + const task: TaskItem = { + key: dataset.id, + title, + percent: 0, + reqId: -1, + controller, + size: 0, + updateEvent: detail.updateEvent, + }; + taskListRef.current = [task, ...taskListRef.current]; + + setTaskList(taskListRef.current); + return task; + }; + + const updateTaskList = (task: TaskItem) => { + taskListRef.current = taskListRef.current.map((item) => + item.key === task.key ? task : item + ); + setTaskList(taskListRef.current); + }; + + const removeTask = (task: TaskItem) => { + const { key } = task; + taskListRef.current = taskListRef.current.filter( + (item) => item.key !== key + ); + setTaskList(taskListRef.current); + if (task.isCancel && task.cancelFn) { + task.cancelFn(); + } + if (task.updateEvent) window.dispatchEvent(new Event(task.updateEvent)); + if (showTaskCenter) { + window.dispatchEvent( + new CustomEvent("show:task-popover", { detail: { show: false } }) + ); + } + }; + + async function buildFormData({ file, reqId, i, j }) { + const formData = new FormData(); + const { slices, name, size } = file; + const checkSum = await calculateSHA256(slices[j]); + formData.append("file", slices[j]); + formData.append("reqId", reqId.toString()); + formData.append("fileNo", (i + 1).toString()); + formData.append("chunkNo", (j + 1).toString()); + formData.append("fileName", name); + formData.append("fileSize", size.toString()); + formData.append("totalChunkNum", slices.length.toString()); + formData.append("checkSumHex", checkSum); + return formData; + } + + async function uploadSlice(task: TaskItem, fileInfo) { + if (!task) { + return; + } + const { reqId, key } = task; + const { loaded, i, j, files, totalSize } = fileInfo; + const formData = await buildFormData({ + file: files[i], + i, + j, + reqId, + }); + + let newTask = { ...task }; + await uploadChunk(key, formData, { + onUploadProgress: (e) => { + const loadedSize = loaded + e.loaded; + const curPercent = Number((loadedSize / totalSize) * 100).toFixed(2); + + newTask = { + ...newTask, + ...taskListRef.current.find((item) => item.key === key), + size: loadedSize, + percent: curPercent >= 100 ? 99.99 : curPercent, + }; + updateTaskList(newTask); + }, + }); + } + + async function uploadFile({ task, files, totalSize }) { + const { data: reqId } = await preUpload(task.key, { + totalFileNum: files.length, + totalSize, + datasetId: task.key, + }); + + const newTask: TaskItem = { + ...task, + reqId, + isCancel: false, + cancelFn: () => { + task.controller.abort(); + cancelUpload?.(reqId); + if (task.updateEvent) window.dispatchEvent(new Event(task.updateEvent)); + }, + }; + updateTaskList(newTask); + if (showTaskCenter) { + window.dispatchEvent( + new CustomEvent("show:task-popover", { detail: { show: true } }) + ); + } + // // 更新数据状态 + if (task.updateEvent) window.dispatchEvent(new Event(task.updateEvent)); + + let loaded = 0; + for (let i = 0; i < files.length; i++) { + const { slices } = files[i]; + for (let j = 0; j < slices.length; j++) { + await uploadSlice(newTask, { + loaded, + i, + j, + files, + totalSize, + }); + loaded += slices[j].size; + } + } + removeTask(newTask); + } + + const handleUpload = async ({ task, files }) => { + const isErrorFile = await checkIsFilesExist(files); + if (isErrorFile) { + message.error("文件被修改或删除，请重新选择文件上传"); + removeTask({ + ...task, + isCancel: false, + ...taskListRef.current.find((item) => item.key === task.key), + }); + return; + } + + try { + const totalSize = files.reduce((acc, file) => acc + file.size, 0); + await uploadFile({ task, files, totalSize }); + } catch (err) { + console.error(err); + message.error("文件上传失败，请稍后重试"); + removeTask({ + ...task, + isCancel: true, + ...taskListRef.current.find((item) => item.key === task.key), + }); + } + }; + + return { + taskList, + createTask, + removeTask, + handleUpload, + }; +} diff --git a/frontend/src/mock/mock-apis.cjs b/frontend/src/mock/mock-apis.cjs index bab1f9d..0752764 100644 --- a/frontend/src/mock/mock-apis.cjs +++ b/frontend/src/mock/mock-apis.cjs @@ -115,20 +115,15 @@ const MockAPI = { batchEvaluationUsingPost: "/evaluation/batch-evaluate", // 批量评测 // 知识生成接口 - queryKnowledgeBasesUsingPost: "/knowledge/bases", // 获取知识库列表 - createKnowledgeBaseUsingPost: "/knowledge/bases/create", // 创建知识库 - queryKnowledgeBaseByIdUsingGet: "/knowledge/bases/:baseId", // 根据ID获取知识库详情 - updateKnowledgeBaseByIdUsingPut: "/knowledge/bases/:baseId", // 更新知识库 - deleteKnowledgeBaseByIdUsingDelete: "/knowledge/bases/:baseId", // 删除知识库 - queryKnowledgeGenerationTasksUsingPost: "/knowledge/tasks", // 获取知识生成任务列表 - createKnowledgeGenerationTaskUsingPost: "/knowledge/tasks/create", // 创建知识生成任务 - queryKnowledgeGenerationTaskByIdUsingGet: "/knowledge/tasks/:taskId", // 根据ID获取知识生成任务详情 - updateKnowledgeGenerationTaskByIdUsingPut: "/knowledge/tasks/:taskId", // 更新知识生成任务 - deleteKnowledgeGenerationTaskByIdUsingDelete: "/knowledge/tasks/:taskId", // 删除知识生成任务 - executeKnowledgeGenerationTaskByIdUsingPost: - "/knowledge/tasks/:taskId/execute", // 执行知识生成任务 - stopKnowledgeGenerationTaskByIdUsingPost: "/knowledge/tasks/:taskId/stop", // 停止知识生成任务 - queryKnowledgeStatisticsUsingGet: "/knowledge/statistics", // 获取知识生成 + queryKnowledgeBasesUsingPost: "/knowledge-base/list", // 获取知识库列表 + createKnowledgeBaseUsingPost: "/knowledge-base/create", // 创建知识库 + queryKnowledgeBaseByIdUsingGet: "/knowledge-base/:baseId", // 根据ID获取知识库详情 + updateKnowledgeBaseByIdUsingPut: "/knowledge-base/:baseId", // 更新知识库 + deleteKnowledgeBaseByIdUsingDelete: "/knowledge-base/:baseId", // 删除知识库 + queryKnowledgeGenerationTasksUsingPost: "/knowledge-base/tasks", // 获取知识生成任务列表 + addKnowledgeGenerationFilesUsingPost: "/knowledge-base/:baseId/files", // 添加文件到知识库 + queryKnowledgeGenerationFilesByIdUsingGet: "/knowledge-base/:baseId/files/:fileId", // 根据ID获取知识生成文件详情 + deleteKnowledgeGenerationTaskByIdUsingDelete: "/knowledge-base/:baseId/files", // 删除知识生成文件 // 算子市场 queryOperatorsUsingPost: "/operators/list", // 获取算子列表 @@ -137,6 +132,10 @@ const MockAPI = { createOperatorUsingPost: "/operators/create", // 创建算子 updateOperatorByIdUsingPut: "/operators/:operatorId", // 更新算子 uploadOperatorUsingPost: "/operators/upload", // 上传算子 + uploadFileChunkUsingPost: "/operators/upload/chunk", // 上传切片 + preUploadOperatorUsingPost: "/operators/upload/pre-upload", // 预上传文件 + cancelUploadOperatorUsingPut: "/operators/upload/cancel-upload", // 取消上传 + createLabelUsingPost: "/operators/labels", // 创建算子标签 queryLabelsUsingGet: "/labels", // 获取算子标签列表 deleteLabelsUsingDelete: "/labels", // 删除算子标签 @@ -151,7 +150,6 @@ const MockAPI = { createModelUsingPost: "/models/create", // 创建模型 updateModelUsingPut: "/models/:id", // 更新模型 deleteModelUsingDelete: "/models/:id", // 删除模型 - }; module.exports = addMockPrefix("/api", MockAPI); diff --git a/frontend/src/mock/mock-seed/knowledge-base.cjs b/frontend/src/mock/mock-seed/knowledge-base.cjs new file mode 100644 index 0000000..923fcf0 --- /dev/null +++ b/frontend/src/mock/mock-seed/knowledge-base.cjs @@ -0,0 +1,161 @@ +const Mock = require("mockjs"); +const API = require("../mock-apis.cjs"); + +// 知识库数据 +function KnowledgeBaseItem() { + return { + id: Mock.Random.guid().replace(/[^a-zA-Z0-9]/g, ""), + name: Mock.Random.ctitle(5, 15), + description: Mock.Random.csentence(10, 30), + createdBy: Mock.Random.cname(), + updatedBy: Mock.Random.cname(), + embeddingModel: Mock.Random.pick([ + "text-embedding-ada-002", + "text-embedding-3-small", + "text-embedding-3-large", + ]), + chatModel: Mock.Random.pick(["gpt-3.5-turbo", "gpt-4", "gpt-4-32k"]), + createdAt: Mock.Random.datetime("yyyy-MM-dd HH:mm:ss"), + updatedAt: Mock.Random.datetime("yyyy-MM-dd HH:mm:ss"), + }; +} + +const knowledgeBaseList = new Array(50).fill(null).map(KnowledgeBaseItem); + +module.exports = function (router) { + // 获取知识库列表 + router.post(API.queryKnowledgeBasesUsingPost, (req, res) => { + const { page = 0, size, keyword } = req.body; + let filteredList = knowledgeBaseList; + if (keyword) { + filteredList = knowledgeBaseList.filter( + (kb) => kb.name.includes(keyword) || kb.description.includes(keyword) + ); + } + const start = page * size; + const end = start + size; + const totalElements = filteredList.length; + const paginatedList = filteredList.slice(start, end); + res.send({ + code: "0", + msg: "Success", + data: { + totalElements, + page, + size, + content: paginatedList, + }, + }); + }); + + // 创建知识库 + router.post(API.createKnowledgeBaseUsingPost, (req, res) => { + const item = KnowledgeBaseItem(); + knowledgeBaseList.unshift(item); + res.status(201).send(item); + }); + + // 获取知识库详情 + router.get( + new RegExp(API.queryKnowledgeBaseByIdUsingGet.replace(":baseId", "(\\w+)")), + (req, res) => { + const id = req.params.baseId; + const item = + knowledgeBaseList.find((kb) => kb.id === id) || KnowledgeBaseItem(); + res.send(item); + } + ); + + // 更新知识库 + router.put(API.updateKnowledgeBaseByIdUsingPut, (req, res) => { + const id = req.params.baseId; + const idx = knowledgeBaseList.findIndex((kb) => kb.id === id); + if (idx >= 0) { + knowledgeBaseList[idx] = { ...knowledgeBaseList[idx], ...req.body }; + res.status(201).send(knowledgeBaseList[idx]); + } else { + res.status(404).send({ message: "Not found" }); + } + }); + + // 删除知识库 + router.delete(API.deleteKnowledgeBaseByIdUsingDelete, (req, res) => { + const id = req.params.baseId; + const idx = knowledgeBaseList.findIndex((kb) => kb.id === id); + if (idx >= 0) { + knowledgeBaseList.splice(idx, 1); + res.status(201).send({ success: true }); + } else { + res.status(404).send({ message: "Not found" }); + } + }); + + // 获取知识生成任务列表 + router.post(API.queryKnowledgeGenerationTasksUsingPost, (req, res) => { + const tasks = Mock.mock({ + "data|10": [ + { + id: "@guid", + name: "@ctitle(5,15)", + status: '@pick(["pending","running","success","failed"])', + createdAt: "@datetime", + updatedAt: "@datetime", + progress: "@integer(0,100)", + }, + ], + total: 10, + current: 1, + pageSize: 10, + }); + res.send(tasks); + }); + + // 添加文件到知识库 + router.post( + new RegExp( + API.addKnowledgeGenerationFilesUsingPost.replace(":baseId", "(\\w+)") + ), + (req, res) => { + const file = Mock.mock({ + id: "@guid", + name: "@ctitle(5,15)", + size: "@integer(1000,1000000)", + status: "uploaded", + createdAt: "@datetime", + }); + res.status(201).send(file); + } + ); + + // 获取知识生成文件详情 + router.get( + new RegExp( + API.queryKnowledgeGenerationFilesByIdUsingGet + .replace(":baseId", "(\\w+)") + .replace(":fileId", "(\\w+)") + ), + (req, res) => { + const file = Mock.mock({ + id: req.params.fileId, + name: "@ctitle(5,15)", + size: "@integer(1000,1000000)", + status: "uploaded", + createdAt: "@datetime", + }); + res.send(file); + } + ); + + // 删除知识生成文件 + router.delete( + new RegExp( + API.deleteKnowledgeGenerationTaskByIdUsingDelete.replace( + ":baseId", + "(\\w+)" + ) + ), + (req, res) => { + res.send({ success: true }); + } + ); +}; diff --git a/frontend/src/mock/mock-seed/knowledge-generation.cjs b/frontend/src/mock/mock-seed/knowledge-generation.cjs deleted file mode 100644 index e69de29..0000000 diff --git a/frontend/src/mock/mock-seed/operator-market.cjs b/frontend/src/mock/mock-seed/operator-market.cjs index 0780e92..97c93cb 100644 --- a/frontend/src/mock/mock-seed/operator-market.cjs +++ b/frontend/src/mock/mock-seed/operator-market.cjs @@ -34,6 +34,32 @@ function labelItem() { const labelList = new Array(50).fill(null).map(labelItem); module.exports = function (router) { + router.post(API.preUploadOperatorUsingPost, (req, res) => { + res.status(201).send(Mock.Random.guid()); + }); + + // 上传切片 + router.post(API.uploadFileChunkUsingPost, (req, res) => { + // res.status(500).send({ message: "Simulated upload failure" }); + res.status(201).send({ data: "success" }); + }); + + // 取消上传 + router.put(API.cancelUploadOperatorUsingPut, (req, res) => { + res.status(201).send({ data: "success" }); + }); + + router.post(API.uploadOperatorUsingPost, (req, res) => { + res.status(201).send({ + code: "0", + msg: "Upload successful", + data: { + operatorId: Mock.Random.guid().replace(/[^a-zA-Z0-9]/g, ""), + // 其他返回数据 + }, + }); + }); + // 获取算子标签列表 router.get(API.queryLabelsUsingGet, (req, res) => { const { page = 0, size = 20, keyword = "" } = req.query; diff --git a/frontend/src/mock/operator.tsx b/frontend/src/mock/operator.tsx deleted file mode 100644 index db6ccd4..0000000 --- a/frontend/src/mock/operator.tsx +++ /dev/null @@ -1,196 +0,0 @@ -export const mockOperators: Operator[] = [ - { - id: 1, - name: "图像预处理算子", - version: "1.2.0", - description: - "支持图像缩放、裁剪、旋转、颜色空间转换等常用预处理操作，优化了内存使用和处理速度", - author: "张三", - category: "图像处理", - modality: ["image"], - type: "preprocessing", - tags: ["图像处理", "预处理", "缩放", "裁剪", "旋转"], - createdAt: "2024-01-15", - lastModified: "2024-01-23", - status: "active", - isFavorited: true, - downloads: 1247, - usage: 856, - framework: "PyTorch", - language: "Python", - size: "2.3MB", - dependencies: ["opencv-python", "pillow", "numpy"], - inputFormat: ["jpg", "png", "bmp", "tiff"], - outputFormat: ["jpg", "png", "tensor"], - performance: { - accuracy: 99.5, - speed: "50ms/image", - memory: "128MB", - }, - }, - { - id: 2, - name: "文本分词算子", - version: "2.1.3", - description: - "基于深度学习的中文分词算子，支持自定义词典，在医学文本上表现优异", - author: "李四", - category: "自然语言处理", - modality: ["text"], - type: "preprocessing", - tags: ["文本处理", "分词", "中文", "NLP", "医学"], - createdAt: "2024-01-10", - lastModified: "2024-01-20", - status: "active", - isFavorited: false, - downloads: 892, - usage: 634, - framework: "TensorFlow", - language: "Python", - size: "15.6MB", - dependencies: ["tensorflow", "jieba", "transformers"], - inputFormat: ["txt", "json", "csv"], - outputFormat: ["json", "txt"], - performance: { - accuracy: 96.8, - speed: "10ms/sentence", - memory: "256MB", - }, - }, - { - id: 3, - name: "音频特征提取", - version: "1.0.5", - description: "提取音频的MFCC、梅尔频谱、色度等特征，支持多种音频格式", - author: "王五", - category: "音频处理", - modality: ["audio"], - type: "preprocessing", - tags: ["音频处理", "特征提取", "MFCC", "频谱分析"], - createdAt: "2024-01-08", - lastModified: "2024-01-18", - status: "active", - isFavorited: true, - downloads: 456, - usage: 312, - framework: "PyTorch", - language: "Python", - size: "8.9MB", - dependencies: ["librosa", "scipy", "numpy"], - inputFormat: ["wav", "mp3", "flac", "m4a"], - outputFormat: ["npy", "json", "csv"], - performance: { - speed: "2x实时", - memory: "64MB", - }, - }, - { - id: 4, - name: "视频帧提取算子", - version: "1.3.2", - description: "高效的视频帧提取算子，支持关键帧检测和均匀采样", - author: "赵六", - category: "视频处理", - modality: ["video"], - type: "preprocessing", - tags: ["视频处理", "帧提取", "关键帧", "采样"], - createdAt: "2024-01-05", - lastModified: "2024-01-22", - status: "active", - isFavorited: false, - downloads: 723, - usage: 445, - framework: "OpenCV", - language: "Python", - size: "12.4MB", - dependencies: ["opencv-python", "ffmpeg-python"], - inputFormat: ["mp4", "avi", "mov", "mkv"], - outputFormat: ["jpg", "png", "npy"], - performance: { - speed: "30fps处理", - memory: "512MB", - }, - }, - { - id: 5, - name: "多模态融合算子", - version: "2.0.1", - description: "支持文本、图像、音频多模态数据融合的深度学习算子", - author: "孙七", - category: "多模态处理", - modality: ["text", "image", "audio"], - type: "training", - tags: ["多模态", "融合", "深度学习", "注意力机制"], - createdAt: "2024-01-12", - lastModified: "2024-01-21", - status: "beta", - isFavorited: false, - downloads: 234, - usage: 156, - framework: "PyTorch", - language: "Python", - size: "45.2MB", - dependencies: ["torch", "transformers", "torchvision", "torchaudio"], - inputFormat: ["json", "jpg", "wav"], - outputFormat: ["tensor", "json"], - performance: { - accuracy: 94.2, - speed: "100ms/sample", - memory: "2GB", - }, - }, - { - id: 6, - name: "模型推理加速", - version: "1.1.0", - description: "基于TensorRT的模型推理加速算子，支持多种深度学习框架", - author: "周八", - category: "模型优化", - modality: ["image", "text"], - type: "inference", - tags: ["推理加速", "TensorRT", "优化", "GPU"], - createdAt: "2024-01-03", - lastModified: "2024-01-19", - status: "active", - isFavorited: true, - downloads: 567, - usage: 389, - framework: "TensorRT", - language: "Python", - size: "23.7MB", - dependencies: ["tensorrt", "pycuda", "numpy"], - inputFormat: ["onnx", "pb", "pth"], - outputFormat: ["tensor", "json"], - performance: { - speed: "5x加速", - memory: "减少40%", - }, - }, - { - id: 7, - name: "数据增强算子", - version: "1.4.1", - description: "丰富的数据增强策略，包括几何变换、颜色变换、噪声添加等", - author: "吴九", - category: "数据增强", - modality: ["image"], - type: "preprocessing", - tags: ["数据增强", "几何变换", "颜色变换", "噪声"], - createdAt: "2024-01-01", - lastModified: "2024-01-17", - status: "active", - isFavorited: false, - downloads: 934, - usage: 678, - framework: "Albumentations", - language: "Python", - size: "6.8MB", - dependencies: ["albumentations", "opencv-python", "numpy"], - inputFormat: ["jpg", "png", "bmp"], - outputFormat: ["jpg", "png", "npy"], - performance: { - speed: "20ms/image", - memory: "32MB", - }, - }, -]; \ No newline at end of file diff --git a/frontend/src/pages/DataCollection/Create/CreateTask.tsx b/frontend/src/pages/DataCollection/Create/CreateTask.tsx index 8d594c4..8284e49 100644 --- a/frontend/src/pages/DataCollection/Create/CreateTask.tsx +++ b/frontend/src/pages/DataCollection/Create/CreateTask.tsx @@ -146,6 +146,7 @@ export default function CollectionTaskCreate() { const value = e.target.value; setNewTask({ ...newTask, + syncMode: value, scheduleExpression: value === SyncMode.SCHEDULED ? scheduleExpression.cronExpression diff --git a/frontend/src/pages/DataManagement/Detail/DatasetDetail.tsx b/frontend/src/pages/DataManagement/Detail/DatasetDetail.tsx index 18d140d..de03aa7 100644 --- a/frontend/src/pages/DataManagement/Detail/DatasetDetail.tsx +++ b/frontend/src/pages/DataManagement/Detail/DatasetDetail.tsx @@ -92,17 +92,12 @@ export default function DatasetDetail() { }; useEffect(() => { - const refreshDataset = () => { - fetchDataset(); - }; const refreshData = () => { handleRefresh(false); }; window.addEventListener("update:dataset", refreshData); - window.addEventListener("update:dataset-status", () => refreshDataset()); return () => { window.removeEventListener("update:dataset", refreshData); - window.removeEventListener("update:dataset-status", refreshDataset); }; }, []); diff --git a/frontend/src/pages/DataManagement/Detail/useFilesOperation.ts b/frontend/src/pages/DataManagement/Detail/useFilesOperation.ts index 1c58b19..cf8f2fe 100644 --- a/frontend/src/pages/DataManagement/Detail/useFilesOperation.ts +++ b/frontend/src/pages/DataManagement/Detail/useFilesOperation.ts @@ -49,7 +49,6 @@ export function useFilesOperation(dataset: Dataset) { }; const handleDownloadFile = async (file: DatasetFile) => { - console.log("批量下载文件:", selectedFiles); // 实际导出逻辑 await downloadFileByIdUsingGet(dataset.id, file.id, file.fileName); // 假设导出成功 @@ -88,7 +87,6 @@ export function useFilesOperation(dataset: Dataset) { return; } // 执行批量导出逻辑 - console.log("批量导出文件:", selectedFiles); exportDatasetUsingPost(dataset.id, { fileIds: selectedFiles }) .then(() => { message.success({ diff --git a/frontend/src/pages/DataManagement/dataset.model.ts b/frontend/src/pages/DataManagement/dataset.model.ts index 8dfc602..411de32 100644 --- a/frontend/src/pages/DataManagement/dataset.model.ts +++ b/frontend/src/pages/DataManagement/dataset.model.ts @@ -98,4 +98,5 @@ export interface TaskItem { controller: AbortController; cancelFn?: () => void; updateEvent?: string; + size?: number; } diff --git a/frontend/src/pages/Layout/TaskUpload.tsx b/frontend/src/pages/Layout/TaskUpload.tsx index fecd0e8..3c0e1c7 100644 --- a/frontend/src/pages/Layout/TaskUpload.tsx +++ b/frontend/src/pages/Layout/TaskUpload.tsx @@ -3,167 +3,19 @@ import { preUploadUsingPost, uploadFileChunkUsingPost, } from "@/pages/DataManagement/dataset.api"; -import { TaskItem } from "@/pages/DataManagement/dataset.model"; -import { calculateSHA256, checkIsFilesExist } from "@/utils/file.util"; -import { App, Button, Empty, Progress } from "antd"; +import { Button, Empty, Progress } from "antd"; import { DeleteOutlined } from "@ant-design/icons"; -import { useState, useRef, useEffect } from "react"; +import { useEffect } from "react"; +import { useFileSliceUpload } from "@/hooks/useSliceUpload"; export default function TaskUpload() { - const { message } = App.useApp(); - const [taskList, setTaskList] = useState([]); - const taskListRef = useRef([]); // 用于固定任务顺序 - - const createTask = (detail: any = {}) => { - const { dataset } = detail; - const title = `上传数据集: ${dataset.name} `; - const controller = new AbortController(); - const task: TaskItem = { - key: dataset.id, - title, - percent: 0, - reqId: -1, - controller, - updateEvent: detail.updateEvent || "update:dataset", - }; - taskListRef.current = [task, ...taskListRef.current]; - - setTaskList(taskListRef.current); - return task; - }; - - const updateTaskList = (task: TaskItem) => { - taskListRef.current = taskListRef.current.map((item) => - item.key === task.key ? task : item - ); - setTaskList(taskListRef.current); - }; - - const removeTask = (task: TaskItem) => { - const { key } = task; - taskListRef.current = taskListRef.current.filter( - (item) => item.key !== key - ); - setTaskList(taskListRef.current); - if (task.isCancel && task.cancelFn) { - task.cancelFn(); + const { createTask, taskList, removeTask, handleUpload } = useFileSliceUpload( + { + preUpload: preUploadUsingPost, + uploadChunk: uploadFileChunkUsingPost, + cancelUpload: cancelUploadUsingPut, } - window.dispatchEvent(new Event(task.updateEvent || "update:dataset")); - window.dispatchEvent( - new CustomEvent("show:task-popover", { detail: { show: false } }) - ); - }; - - async function buildFormData({ file, reqId, i, j }) { - const formData = new FormData(); - const { slices, name, size } = file; - const checkSum = await calculateSHA256(slices[j]); - formData.append("file", slices[j]); - formData.append("reqId", reqId.toString()); - formData.append("fileNo", (i + 1).toString()); - formData.append("chunkNo", (j + 1).toString()); - formData.append("fileName", name); - formData.append("fileSize", size.toString()); - formData.append("totalChunkNum", slices.length.toString()); - formData.append("checkSumHex", checkSum); - return formData; - } - - async function uploadSlice(task: TaskItem, fileInfo) { - if (!task) { - return; - } - const { reqId, key, signal } = task; - const { loaded, i, j, files, totalSize } = fileInfo; - const formData = await buildFormData({ - file: files[i], - i, - j, - reqId, - }); - - let newTask = { ...task }; - await uploadFileChunkUsingPost(key, formData, { - onUploadProgress: (e) => { - const loadedSize = loaded + e.loaded; - const curPercent = Math.round(loadedSize / totalSize) * 100; - newTask = { - ...newTask, - ...taskListRef.current.find((item) => item.key === key), - percent: curPercent >= 100 ? 99.99 : curPercent, - }; - updateTaskList(newTask); - }, - signal, - }); - } - - async function uploadFile({ task, files, totalSize }) { - const { data: reqId } = await preUploadUsingPost(task.key, { - totalFileNum: files.length, - totalSize, - datasetId: task.key, - }); - - const newTask: TaskItem = { - ...task, - reqId, - isCancel: false, - cancelFn: () => { - task.controller.abort(); - cancelUploadUsingPut(reqId); - window.dispatchEvent(new Event(task.updateEvent || "update:dataset")); - }, - }; - updateTaskList(newTask); - window.dispatchEvent( - new CustomEvent("show:task-popover", { detail: { show: true } }) - ); - // 更新数据状态 - window.dispatchEvent(new Event("update:dataset-status")); - - let loaded = 0; - for (let i = 0; i < files.length; i++) { - const { slices } = files[i]; - for (let j = 0; j < slices.length; j++) { - await uploadSlice(newTask, { - loaded, - i, - j, - files, - totalSize, - }); - loaded += slices[j].size; - } - } - removeTask(newTask); - } - - const handleUpload = async ({ task, files }) => { - const isErrorFile = await checkIsFilesExist(files); - if (isErrorFile) { - message.error("文件被修改或删除，请重新选择文件上传"); - removeTask({ - ...task, - isCancel: false, - ...taskListRef.current.find((item) => item.key === task.key), - }); - return; - } - - try { - const totalSize = files.reduce((acc, file) => acc + file.size, 0); - await uploadFile({ task, files, totalSize }); - } catch (err) { - console.error(err); - message.error("文件上传失败，请稍后重试"); - removeTask({ - ...task, - isCancel: true, - ...taskListRef.current.find((item) => item.key === task.key), - }); - } - }; + ); useEffect(() => { const uploadHandler = (e: any) => { @@ -195,9 +47,6 @@ export default function TaskUpload() { removeTask({ ...task, isCancel: true, - ...taskListRef.current.find( - (item) => item.key === task.key - ), }) } icon={} diff --git a/frontend/src/pages/OperatorMarket/Create/OperatorPluginCreate.tsx b/frontend/src/pages/OperatorMarket/Create/OperatorPluginCreate.tsx index c94b125..19509ee 100644 --- a/frontend/src/pages/OperatorMarket/Create/OperatorPluginCreate.tsx +++ b/frontend/src/pages/OperatorMarket/Create/OperatorPluginCreate.tsx @@ -1,4 +1,4 @@ -import { Button, Steps } from "antd"; +import { Button, App, Steps } from "antd"; import { ArrowLeft, CheckCircle, @@ -6,122 +6,102 @@ import { TagIcon, Upload, } from "lucide-react"; -import { useNavigate } from "react-router"; -import { useCallback, useState } from "react"; +import { useNavigate, useParams } from "react-router"; +import { useEffect, useState } from "react"; import UploadStep from "./components/UploadStep"; import ParsingStep from "./components/ParsingStep"; import ConfigureStep from "./components/ConfigureStep"; import PreviewStep from "./components/PreviewStep"; - -interface ParsedOperatorInfo { - name: string; - version: string; - description: string; - author: string; - category: string; - modality: string[]; - type: "preprocessing" | "training" | "inference" | "postprocessing"; - framework: string; - language: string; - size: string; - dependencies: string[]; - inputFormat: string[]; - outputFormat: string[]; - performance: { - accuracy?: number; - speed: string; - memory: string; - }; - documentation?: string; - examples?: string[]; -} +import { useFileSliceUpload } from "@/hooks/useSliceUpload"; +import { + createOperatorUsingPost, + preUploadOperatorUsingPost, + queryOperatorByIdUsingGet, + updateOperatorByIdUsingPut, + uploadOperatorChunkUsingPost, + uploadOperatorUsingPost, +} from "../operator.api"; +import { sliceFile } from "@/utils/file.util"; export default function OperatorPluginCreate() { const navigate = useNavigate(); + const { id } = useParams(); + const { message } = App.useApp(); const [uploadStep, setUploadStep] = useState< "upload" | "parsing" | "configure" | "preview" >("upload"); const [isUploading, setIsUploading] = useState(false); - const [uploadedFiles, setUploadedFiles] = useState([]); - const [parseProgress, setParseProgress] = useState(0); - const [parsedInfo, setParsedInfo] = useState(null); + const [parsedInfo, setParsedInfo] = useState({}); const [parseError, setParseError] = useState(null); + + const { handleUpload, createTask, taskList } = useFileSliceUpload( + { + preUpload: preUploadOperatorUsingPost, + uploadChunk: uploadOperatorChunkUsingPost, + cancelUpload: null, + }, + false + ); + // 模拟文件上传 - const handleFileUpload = useCallback((files: FileList) => { + const handleFileUpload = async (files: FileList) => { setIsUploading(true); setParseError(null); - - // 模拟文件上传过程 - setTimeout(() => { - const fileArray = Array.from(files).map((file) => ({ - name: file.name, - size: file.size, - type: file.type, - })); - setUploadedFiles(fileArray); - setIsUploading(false); - setUploadStep("parsing"); - startParsing(); - }, 1000); - }, []); - - // 模拟解析过程 - const startParsing = useCallback(() => { - setParseProgress(0); - const interval = setInterval(() => { - setParseProgress((prev) => { - if (prev >= 100) { - clearInterval(interval); - // 模拟解析完成 - setTimeout(() => { - setParsedInfo({ - name: "图像预处理算子", - version: "1.2.0", - description: - "支持图像缩放、裁剪、旋转、颜色空间转换等常用预处理操作，优化了内存使用和处理速度", - author: "当前用户", - category: "图像处理", - modality: ["image"], - type: "preprocessing", - framework: "PyTorch", - language: "Python", - size: "2.3MB", - dependencies: [ - "opencv-python>=4.5.0", - "pillow>=8.0.0", - "numpy>=1.20.0", - ], - inputFormat: ["jpg", "png", "bmp", "tiff"], - outputFormat: ["jpg", "png", "tensor"], - performance: { - accuracy: 99.5, - speed: "50ms/image", - memory: "128MB", - }, - documentation: - "# 图像预处理算子\n\n这是一个高效的图像预处理算子...", - examples: [ - "from operator import ImagePreprocessor\nprocessor = ImagePreprocessor()\nresult = processor.process(image)", - ], - }); - setUploadStep("configure"); - }, 500); - return 100; - } - return prev + 10; + setUploadStep("parsing"); + try { + const fileName = files[0].name; + await handleUpload({ + task: createTask({ + dataset: { id: "operator-upload", name: "上传算子" }, + }), + files: [ + { + originFile: files[0], + slices: sliceFile(files[0]), + name: fileName, + size: files[0].size, + }, + ], // 假设只上传一个文件 }); - }, 200); - }, []); - - const handlePublish = () => { - // 模拟发布过程 - setUploadStep("preview"); - setTimeout(() => { - alert("算子发布成功！"); - // 这里可以重置状态或跳转到其他页面 - }, 2000); + setParsedInfo({ ...parsedInfo, fileName, percent: 100 }); // 上传完成，进度100% + // 解析文件过程 + const res = await uploadOperatorUsingPost({ fileName }); + setParsedInfo({ ...parsedInfo, ...res.data }); + } catch (err) { + setParseError("文件解析失败，" + err.data.message); + } finally { + setIsUploading(false); + setUploadStep("configure"); + } }; + const handlePublish = async () => { + try { + if (id) { + await updateOperatorByIdUsingPut(id, parsedInfo!); + } else { + await createOperatorUsingPost(parsedInfo); + } + setUploadStep("preview"); + } catch (err) { + message.error("算子发布失败，" + err.data.message); + } + }; + + const onFetchOperator = async (operatorId: string) => { + // 编辑模式，加载已有算子信息逻辑待实现 + const { data } = await queryOperatorByIdUsingGet(operatorId); + setParsedInfo(data); + setUploadStep("configure"); + }; + + useEffect(() => { + if (id) { + // 编辑模式，加载已有算子信息逻辑待实现 + onFetchOperator(id); + } + }, [id]); + return (
{/* Header */} @@ -174,13 +154,13 @@ export default function OperatorPluginCreate() { )} {uploadStep === "parsing" && ( )} {uploadStep === "configure" && ( @@ -192,7 +172,6 @@ export default function OperatorPluginCreate() { {uploadStep === "configure" && (
- diff --git a/frontend/src/pages/OperatorMarket/Create/components/ConfigureStep.tsx b/frontend/src/pages/OperatorMarket/Create/components/ConfigureStep.tsx index 21add01..e317d1a 100644 --- a/frontend/src/pages/OperatorMarket/Create/components/ConfigureStep.tsx +++ b/frontend/src/pages/OperatorMarket/Create/components/ConfigureStep.tsx @@ -1,274 +1,75 @@ -import { Alert, Input, Button } from "antd"; -import { CheckCircle, Plus, TagIcon, X } from "lucide-react"; -import { useState } from "react"; - -export default function ConfigureStep({ parsedInfo, parseError }) { - const [selectedTags, setSelectedTags] = useState([]); - const [customTag, setCustomTag] = useState(""); - const availableTags = [ - "图像处理", - "预处理", - "缩放", - "裁剪", - "旋转", - "文本处理", - "分词", - "中文", - "NLP", - "医学", - "音频处理", - "特征提取", - "MFCC", - "频谱分析", - "视频处理", - "帧提取", - "关键帧", - "采样", - "多模态", - "融合", - "深度学习", - "注意力机制", - "推理加速", - "TensorRT", - "优化", - "GPU", - "数据增强", - "几何变换", - "颜色变换", - "噪声", - ]; - - const handleAddCustomTag = () => { - if (customTag.trim() && !selectedTags.includes(customTag.trim())) { - setSelectedTags([...selectedTags, customTag.trim()]); - setCustomTag(""); - } - }; - - const handleRemoveTag = (tagToRemove: string) => { - setSelectedTags(selectedTags.filter((tag) => tag !== tagToRemove)); - }; +import { Alert, Input, Form } from "antd"; +import TextArea from "antd/es/input/TextArea"; +export default function ConfigureStep({ + parsedInfo, + parseError, + setParsedInfo, +}) { return ( <> {/* 解析结果 */} -
- -

解析完成

-
- {parseError && ( )} {parsedInfo && ( -
+
{ + setParsedInfo({ ...parsedInfo, ...allValues }); + }} + > {/* 基本信息 */} -
-

基本信息

-
-
- -
- {parsedInfo.name} -
-
-
- -
- {parsedInfo.version} -
-
-
- -
- {parsedInfo.author} -
-
-
- -
- {parsedInfo.category} -
-
-
-
+

基本信息

+ + + + + + + + + + +