diff --git a/frontend/src/hooks/useSliceUpload.tsx b/frontend/src/hooks/useSliceUpload.tsx index 56b9517..a43385a 100644 --- a/frontend/src/hooks/useSliceUpload.tsx +++ b/frontend/src/hooks/useSliceUpload.tsx @@ -1,198 +1,202 @@ -import { TaskItem } from "@/pages/DataManagement/dataset.model"; -import { calculateSHA256, checkIsFilesExist } from "@/utils/file.util"; -import { App } from "antd"; -import { useRef, useState } from "react"; - -export function useFileSliceUpload( - { - preUpload, - uploadChunk, - cancelUpload, - }: { - preUpload: (id: string, params: any) => Promise<{ data: number }>; - uploadChunk: (id: string, formData: FormData, config: any) => Promise; - cancelUpload: ((reqId: number) => Promise) | null; - }, - showTaskCenter = true // 上传时是否显示任务中心 -) { - const { message } = App.useApp(); - const [taskList, setTaskList] = useState([]); - const taskListRef = useRef([]); // 用于固定任务顺序 - - const createTask = (detail: any = {}) => { - const { dataset } = detail; - const title = `上传数据集: ${dataset.name} `; - const controller = new AbortController(); - const task: TaskItem = { - key: dataset.id, - title, - percent: 0, - reqId: -1, - controller, - size: 0, - updateEvent: detail.updateEvent, - hasArchive: detail.hasArchive, - prefix: detail.prefix, - }; - taskListRef.current = [task, ...taskListRef.current]; - - setTaskList(taskListRef.current); - return task; - }; - - const updateTaskList = (task: TaskItem) => { - taskListRef.current = taskListRef.current.map((item) => - item.key === task.key ? task : item - ); - setTaskList(taskListRef.current); - }; - - const removeTask = (task: TaskItem) => { - const { key } = task; - taskListRef.current = taskListRef.current.filter( - (item) => item.key !== key - ); - setTaskList(taskListRef.current); - if (task.isCancel && task.cancelFn) { - task.cancelFn(); - } - if (task.updateEvent) { - // 携带前缀信息,便于刷新后仍停留在当前目录 - window.dispatchEvent( - new CustomEvent(task.updateEvent, { - detail: { prefix: (task as any).prefix }, - }) - ); - } - if (showTaskCenter) { - window.dispatchEvent( - new CustomEvent("show:task-popover", { detail: { show: false } }) - ); - } - }; - - async function buildFormData({ file, reqId, i, j }) { - const formData = new FormData(); - const { slices, name, size } = file; - const checkSum = await calculateSHA256(slices[j]); - formData.append("file", slices[j]); - formData.append("reqId", reqId.toString()); - formData.append("fileNo", (i + 1).toString()); - formData.append("chunkNo", (j + 1).toString()); - formData.append("fileName", name); - formData.append("fileSize", size.toString()); - formData.append("totalChunkNum", slices.length.toString()); - formData.append("checkSumHex", checkSum); - return formData; - } - - async function uploadSlice(task: TaskItem, fileInfo) { - if (!task) { - return; - } - const { reqId, key } = task; - const { loaded, i, j, files, totalSize } = fileInfo; - const formData = await buildFormData({ - file: files[i], - i, - j, - reqId, - }); - - let newTask = { ...task }; - await uploadChunk(key, formData, { - onUploadProgress: (e) => { - const loadedSize = loaded + e.loaded; - const curPercent = Number((loadedSize / totalSize) * 100).toFixed(2); - - newTask = { - ...newTask, - ...taskListRef.current.find((item) => item.key === key), - size: loadedSize, - percent: curPercent >= 100 ? 99.99 : curPercent, - }; - updateTaskList(newTask); - }, - }); - } - - async function uploadFile({ task, files, totalSize }) { - console.log('[useSliceUpload] Calling preUpload with prefix:', task.prefix); - const { data: reqId } = await preUpload(task.key, { - totalFileNum: files.length, - totalSize, - datasetId: task.key, - hasArchive: task.hasArchive, - prefix: task.prefix, - }); - console.log('[useSliceUpload] PreUpload response reqId:', reqId); - - const newTask: TaskItem = { - ...task, - reqId, - isCancel: false, - cancelFn: () => { - task.controller.abort(); - cancelUpload?.(reqId); - if (task.updateEvent) window.dispatchEvent(new Event(task.updateEvent)); - }, - }; - updateTaskList(newTask); - if (showTaskCenter) { - window.dispatchEvent( - new CustomEvent("show:task-popover", { detail: { show: true } }) - ); - } - // // 更新数据状态 - if (task.updateEvent) window.dispatchEvent(new Event(task.updateEvent)); - - let loaded = 0; - for (let i = 0; i < files.length; i++) { - const { slices } = files[i]; - for (let j = 0; j < slices.length; j++) { - await uploadSlice(newTask, { - loaded, - i, - j, - files, - totalSize, - }); - loaded += slices[j].size; - } - } - removeTask(newTask); - } - - const handleUpload = async ({ task, files }) => { - const isErrorFile = await checkIsFilesExist(files); - if (isErrorFile) { - message.error("文件被修改或删除,请重新选择文件上传"); - removeTask({ - ...task, - isCancel: false, - ...taskListRef.current.find((item) => item.key === task.key), - }); - return; - } - - try { - const totalSize = files.reduce((acc, file) => acc + file.size, 0); - await uploadFile({ task, files, totalSize }); - } catch (err) { - console.error(err); - message.error("文件上传失败,请稍后重试"); - removeTask({ - ...task, - isCancel: true, - ...taskListRef.current.find((item) => item.key === task.key), - }); - } - }; - - return { - taskList, - createTask, - removeTask, - handleUpload, - }; -} +import { TaskItem } from "@/pages/DataManagement/dataset.model"; +import { calculateSHA256, checkIsFilesExist } from "@/utils/file.util"; +import { App } from "antd"; +import { useRef, useState } from "react"; + +export function useFileSliceUpload( + { + preUpload, + uploadChunk, + cancelUpload, + }: { + preUpload: (id: string, params: any) => Promise<{ data: number }>; + uploadChunk: (id: string, formData: FormData, config: any) => Promise; + cancelUpload: ((reqId: number) => Promise) | null; + }, + showTaskCenter = true // 上传时是否显示任务中心 +) { + const { message } = App.useApp(); + const [taskList, setTaskList] = useState([]); + const taskListRef = useRef([]); // 用于固定任务顺序 + + const createTask = (detail: any = {}) => { + const { dataset } = detail; + const title = `上传数据集: ${dataset.name} `; + const controller = new AbortController(); + const task: TaskItem = { + key: dataset.id, + title, + percent: 0, + reqId: -1, + controller, + size: 0, + updateEvent: detail.updateEvent, + hasArchive: detail.hasArchive, + prefix: detail.prefix, + }; + taskListRef.current = [task, ...taskListRef.current]; + + setTaskList(taskListRef.current); + + // 立即显示任务中心,让用户感知上传已开始 + if (showTaskCenter) { + window.dispatchEvent( + new CustomEvent("show:task-popover", { detail: { show: true } }) + ); + } + + return task; + }; + + const updateTaskList = (task: TaskItem) => { + taskListRef.current = taskListRef.current.map((item) => + item.key === task.key ? task : item + ); + setTaskList(taskListRef.current); + }; + + const removeTask = (task: TaskItem) => { + const { key } = task; + taskListRef.current = taskListRef.current.filter( + (item) => item.key !== key + ); + setTaskList(taskListRef.current); + if (task.isCancel && task.cancelFn) { + task.cancelFn(); + } + if (task.updateEvent) { + // 携带前缀信息,便于刷新后仍停留在当前目录 + window.dispatchEvent( + new CustomEvent(task.updateEvent, { + detail: { prefix: (task as any).prefix }, + }) + ); + } + if (showTaskCenter) { + window.dispatchEvent( + new CustomEvent("show:task-popover", { detail: { show: false } }) + ); + } + }; + + async function buildFormData({ file, reqId, i, j }) { + const formData = new FormData(); + const { slices, name, size } = file; + const checkSum = await calculateSHA256(slices[j]); + formData.append("file", slices[j]); + formData.append("reqId", reqId.toString()); + formData.append("fileNo", (i + 1).toString()); + formData.append("chunkNo", (j + 1).toString()); + formData.append("fileName", name); + formData.append("fileSize", size.toString()); + formData.append("totalChunkNum", slices.length.toString()); + formData.append("checkSumHex", checkSum); + return formData; + } + + async function uploadSlice(task: TaskItem, fileInfo) { + if (!task) { + return; + } + const { reqId, key } = task; + const { loaded, i, j, files, totalSize } = fileInfo; + const formData = await buildFormData({ + file: files[i], + i, + j, + reqId, + }); + + let newTask = { ...task }; + await uploadChunk(key, formData, { + onUploadProgress: (e) => { + const loadedSize = loaded + e.loaded; + const curPercent = Number((loadedSize / totalSize) * 100).toFixed(2); + + newTask = { + ...newTask, + ...taskListRef.current.find((item) => item.key === key), + size: loadedSize, + percent: curPercent >= 100 ? 99.99 : curPercent, + }; + updateTaskList(newTask); + }, + }); + } + + async function uploadFile({ task, files, totalSize }) { + console.log('[useSliceUpload] Calling preUpload with prefix:', task.prefix); + const { data: reqId } = await preUpload(task.key, { + totalFileNum: files.length, + totalSize, + datasetId: task.key, + hasArchive: task.hasArchive, + prefix: task.prefix, + }); + console.log('[useSliceUpload] PreUpload response reqId:', reqId); + + const newTask: TaskItem = { + ...task, + reqId, + isCancel: false, + cancelFn: () => { + task.controller.abort(); + cancelUpload?.(reqId); + if (task.updateEvent) window.dispatchEvent(new Event(task.updateEvent)); + }, + }; + updateTaskList(newTask); + // 注意:show:task-popover 事件已在 createTask 中触发,此处不再重复触发 + // // 更新数据状态 + if (task.updateEvent) window.dispatchEvent(new Event(task.updateEvent)); + + let loaded = 0; + for (let i = 0; i < files.length; i++) { + const { slices } = files[i]; + for (let j = 0; j < slices.length; j++) { + await uploadSlice(newTask, { + loaded, + i, + j, + files, + totalSize, + }); + loaded += slices[j].size; + } + } + removeTask(newTask); + } + + const handleUpload = async ({ task, files }) => { + const isErrorFile = await checkIsFilesExist(files); + if (isErrorFile) { + message.error("文件被修改或删除,请重新选择文件上传"); + removeTask({ + ...task, + isCancel: false, + ...taskListRef.current.find((item) => item.key === task.key), + }); + return; + } + + try { + const totalSize = files.reduce((acc, file) => acc + file.size, 0); + await uploadFile({ task, files, totalSize }); + } catch (err) { + console.error(err); + message.error("文件上传失败,请稍后重试"); + removeTask({ + ...task, + isCancel: true, + ...taskListRef.current.find((item) => item.key === task.key), + }); + } + }; + + return { + taskList, + createTask, + removeTask, + handleUpload, + }; +} diff --git a/frontend/src/pages/DataManagement/Detail/components/ImportConfiguration.tsx b/frontend/src/pages/DataManagement/Detail/components/ImportConfiguration.tsx index 8ec1278..fee9399 100644 --- a/frontend/src/pages/DataManagement/Detail/components/ImportConfiguration.tsx +++ b/frontend/src/pages/DataManagement/Detail/components/ImportConfiguration.tsx @@ -1,13 +1,13 @@ -import { Select, Input, Form, Radio, Modal, Button, UploadFile, Switch, Tooltip } from "antd"; -import { InboxOutlined, QuestionCircleOutlined } from "@ant-design/icons"; -import { dataSourceOptions } from "../../dataset.const"; +import { Select, Input, Form, Radio, Modal, Button, UploadFile, Switch, Tooltip } from "antd"; +import { InboxOutlined, QuestionCircleOutlined } from "@ant-design/icons"; +import { dataSourceOptions } from "../../dataset.const"; import { Dataset, DatasetType, DataSource } from "../../dataset.model"; import { useCallback, useEffect, useMemo, useState } from "react"; -import { queryTasksUsingGet } from "@/pages/DataCollection/collection.apis"; -import { updateDatasetByIdUsingPut } from "../../dataset.api"; -import { sliceFile } from "@/utils/file.util"; -import Dragger from "antd/es/upload/Dragger"; - +import { queryTasksUsingGet } from "@/pages/DataCollection/collection.apis"; +import { updateDatasetByIdUsingPut } from "../../dataset.api"; +import { sliceFile } from "@/utils/file.util"; +import Dragger from "antd/es/upload/Dragger"; + const TEXT_FILE_MIME_PREFIX = "text/"; const TEXT_FILE_MIME_TYPES = new Set([ "application/json", @@ -131,18 +131,18 @@ type ImportConfig = { }; export default function ImportConfiguration({ - data, - open, - onClose, - updateEvent = "update:dataset", - prefix, -}: { - data: Dataset | null; - open: boolean; - onClose: () => void; - updateEvent?: string; - prefix?: string; -}) { + data, + open, + onClose, + updateEvent = "update:dataset", + prefix, +}: { + data: Dataset | null; + open: boolean; + onClose: () => void; + updateEvent?: string; + prefix?: string; +}) { const [form] = Form.useForm(); const [collectionOptions, setCollectionOptions] = useState([]); const availableSourceOptions = dataSourceOptions.filter( @@ -160,13 +160,13 @@ export default function ImportConfiguration({ return files.some((file) => !isTextUploadFile(file)); }, [importConfig.files]); const isTextDataset = data?.datasetType === DatasetType.TEXT; - - // 本地上传文件相关逻辑 - - const handleUpload = async (dataset: Dataset) => { + + // 本地上传文件相关逻辑 + + const handleUpload = async (dataset: Dataset) => { let filesToUpload = (form.getFieldValue("files") as UploadFile[] | undefined) || []; - + // 如果启用分行分割,处理文件 if (importConfig.splitByLine && !hasNonTextFile) { const splitResults = await Promise.all( @@ -174,9 +174,9 @@ export default function ImportConfiguration({ ); filesToUpload = splitResults.flat(); } - - // 计算分片列表 - const sliceList = filesToUpload.map((file) => { + + // 计算分片列表 + const sliceList = filesToUpload.map((file) => { const originFile = (file.originFileObj ?? file) as Blob; const slices = sliceFile(originFile); return { @@ -185,22 +185,22 @@ export default function ImportConfiguration({ name: file.name, size: originFile.size || 0, }; - }); - - console.log("[ImportConfiguration] Uploading with currentPrefix:", currentPrefix); - window.dispatchEvent( - new CustomEvent("upload:dataset", { - detail: { - dataset, - files: sliceList, - updateEvent, - hasArchive: importConfig.hasArchive, - prefix: currentPrefix, - }, - }) - ); - }; - + }); + + console.log("[ImportConfiguration] Uploading with currentPrefix:", currentPrefix); + window.dispatchEvent( + new CustomEvent("upload:dataset", { + detail: { + dataset, + files: sliceList, + updateEvent, + hasArchive: importConfig.hasArchive, + prefix: currentPrefix, + }, + }) + ); + }; + const fetchCollectionTasks = useCallback(async () => { if (importConfig.source !== DataSource.COLLECTION) return; try { @@ -212,7 +212,7 @@ export default function ImportConfiguration({ label: task.name, value: task.id, })); - setCollectionOptions(options); + setCollectionOptions(options); } catch (error) { console.error("Error fetching collection tasks:", error); } @@ -229,27 +229,31 @@ export default function ImportConfiguration({ }); console.log('[ImportConfiguration] resetState done, currentPrefix still:', currentPrefix); }, [currentPrefix, form]); - - const handleImportData = async () => { - if (!data) return; - console.log('[ImportConfiguration] handleImportData called, currentPrefix:', currentPrefix); - if (importConfig.source === DataSource.UPLOAD) { - await handleUpload(data); - } else if (importConfig.source === DataSource.COLLECTION) { - await updateDatasetByIdUsingPut(data.id, { - ...importConfig, - }); - } - onClose(); - }; - + + const handleImportData = async () => { + if (!data) return; + console.log('[ImportConfiguration] handleImportData called, currentPrefix:', currentPrefix); + if (importConfig.source === DataSource.UPLOAD) { + // 立即显示任务中心,让用户感知上传已开始(在文件分割等耗时操作之前) + window.dispatchEvent( + new CustomEvent("show:task-popover", { detail: { show: true } }) + ); + await handleUpload(data); + } else if (importConfig.source === DataSource.COLLECTION) { + await updateDatasetByIdUsingPut(data.id, { + ...importConfig, + }); + } + onClose(); + }; + useEffect(() => { if (open) { setCurrentPrefix(prefix || ""); - console.log('[ImportConfiguration] Modal opened with prefix:', prefix); - resetState(); - fetchCollectionTasks(); - } + console.log('[ImportConfiguration] Modal opened with prefix:', prefix); + resetState(); + fetchCollectionTasks(); + } }, [fetchCollectionTasks, open, prefix, resetState]); useEffect(() => { @@ -259,111 +263,111 @@ export default function ImportConfiguration({ form.setFieldsValue({ splitByLine: false }); setImportConfig((prev) => ({ ...prev, splitByLine: false })); }, [form, hasNonTextFile, importConfig.files, importConfig.splitByLine]); - - // Separate effect for fetching collection tasks when source changes - useEffect(() => { - if (open && importConfig.source === DataSource.COLLECTION) { - fetchCollectionTasks(); - } + + // Separate effect for fetching collection tasks when source changes + useEffect(() => { + if (open && importConfig.source === DataSource.COLLECTION) { + fetchCollectionTasks(); + } }, [fetchCollectionTasks, importConfig.source, open]); - - return ( - { - onClose(); - resetState(); - }} - maskClosable={false} - footer={ - <> - - - - } - > -
setImportConfig(allValues)} - > - + + return ( + { + onClose(); + resetState(); + }} + maskClosable={false} + footer={ + <> + + + + } + > + setImportConfig(allValues)} + > + - {importConfig?.source === DataSource.COLLECTION && ( - - - - - - - - - - - - - - )} - - {/* Local Upload Component */} - {importConfig?.source === DataSource.UPLOAD && ( - <> - - - + {importConfig?.source === DataSource.COLLECTION && ( + + + + + + + + + + + + + + )} + + {/* Local Upload Component */} + {importConfig?.source === DataSource.UPLOAD && ( + <> + + + {isTextDataset && ( )} - { @@ -398,69 +402,69 @@ export default function ImportConfiguration({ } return event?.fileList; }} - rules={[ - { - required: true, - message: "请上传文件", - }, - ]} - > - false} - multiple - > -

- -

-

本地文件上传

-

拖拽文件到此处或点击选择文件

-
-
- - )} - - {/* Target Configuration */} - {importConfig?.target && importConfig?.target !== DataSource.UPLOAD && ( -
- {importConfig?.target === DataSource.DATABASE && ( -
- - - - - - - - - -
- )} -
- )} - -
- ); -} + rules={[ + { + required: true, + message: "请上传文件", + }, + ]} + > + false} + multiple + > +

+ +

+

本地文件上传

+

拖拽文件到此处或点击选择文件

+
+
+ + )} + + {/* Target Configuration */} + {importConfig?.target && importConfig?.target !== DataSource.UPLOAD && ( +
+ {importConfig?.target === DataSource.DATABASE && ( +
+ + + + + + + + + +
+ )} +
+ )} + +
+ ); +}