From 8415166949eb3d64bca534d98a1a5a63c47949a5 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Wed, 4 Feb 2026 15:58:58 +0800 Subject: [PATCH] =?UTF-8?q?refactor(upload):=20=E9=87=8D=E6=9E=84=E5=88=87?= =?UTF-8?q?=E7=89=87=E4=B8=8A=E4=BC=A0=E9=80=BB=E8=BE=91=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E5=8A=A8=E6=80=81=E8=AF=B7=E6=B1=82ID=E8=A7=A3=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 移除预先批量获取reqId的方式，改为按需解析 - 新增resolveReqId函数支持动态获取请求ID - 添加onReqIdResolved回调处理ID解析完成事件 - 改进文件按行切片上传，每行作为独立文件处理 - 优化空行跳过逻辑，统计跳过的空行数量 - 修复fileNo和chunkNo的对应关系 - 更新streamSplitAndUpload参数结构 --- frontend/src/hooks/useSliceUpload.tsx | 27 ++++++----- frontend/src/utils/file.util.ts | 66 +++++++++++++++++++++------ 2 files changed, 66 insertions(+), 27 deletions(-) diff --git a/frontend/src/hooks/useSliceUpload.tsx b/frontend/src/hooks/useSliceUpload.tsx index 6fe76b0..848d9b3 100644 --- a/frontend/src/hooks/useSliceUpload.tsx +++ b/frontend/src/hooks/useSliceUpload.tsx @@ -251,18 +251,6 @@ export function useFileSliceUpload( const file = files[i]; console.log(`[useSliceUpload] Processing file ${i + 1}/${files.length}: ${file.name}`); - // 为每个文件单独调用 preUpload，获取独立的 reqId - const { data: reqId } = await preUpload(task.key, { - totalFileNum: 1, - totalSize: file.size, - datasetId: task.key, - hasArchive: task.hasArchive, - prefix: task.prefix, - }); - - console.log(`[useSliceUpload] File ${file.name} preUpload response reqId:`, reqId); - reqIds.push(reqId); - const result = await streamSplitAndUpload( file, (formData, config) => uploadChunk(task.key, formData, { @@ -292,10 +280,21 @@ export function useFileSliceUpload( }, }; updateTaskList(updatedTask); - }, + }, 1024 * 1024, // 1MB chunk size { - reqId, + resolveReqId: async ({ totalFileNum, totalSize }) => { + const { data: reqId } = await preUpload(task.key, { + totalFileNum, + totalSize, + datasetId: task.key, + hasArchive: task.hasArchive, + prefix: task.prefix, + }); + console.log(`[useSliceUpload] File ${file.name} preUpload response reqId:`, reqId); + reqIds.push(reqId); + return reqId; + }, hasArchive: newTask.hasArchive, prefix: newTask.prefix, signal: newTask.controller.signal, diff --git a/frontend/src/utils/file.util.ts b/frontend/src/utils/file.util.ts index 9606ddc..883f3f7 100644 --- a/frontend/src/utils/file.util.ts +++ b/frontend/src/utils/file.util.ts @@ -401,7 +401,9 @@ export function readFileAsText( * @returns 上传结果统计 */ export interface StreamUploadOptions { - reqId: number; + reqId?: number; + resolveReqId?: (params: { totalFileNum: number; totalSize: number }) => Promise; + onReqIdResolved?: (reqId: number) => void; fileNamePrefix?: string; hasArchive?: boolean; prefix?: string; @@ -422,8 +424,16 @@ export async function streamSplitAndUpload( chunkSize: number = 1024 * 1024, // 1MB options: StreamUploadOptions ): Promise { - const { reqId, fileNamePrefix, prefix, signal, maxConcurrency = 3 } = options; - + const { + reqId: initialReqId, + resolveReqId, + onReqIdResolved, + fileNamePrefix, + prefix, + signal, + maxConcurrency = 3, + } = options; + const fileSize = file.size; let offset = 0; let buffer = ""; @@ -441,7 +451,7 @@ export async function streamSplitAndUpload( const pendingLines: { line: string; index: number }[] = []; let lineIndex = 0; - // 逐块读取文件并收集行 + // 逐块读取文件并收集非空行 while (offset < fileSize) { // 检查是否已取消 if (signal?.aborted) { @@ -461,11 +471,15 @@ export async function streamSplitAndUpload( // 保留最后一行（可能不完整） buffer = lines.pop() || ""; - // 收集完整行 + // 收集完整行（跳过空行） for (const line of lines) { if (signal?.aborted) { throw new Error("Upload cancelled"); } + if (!line.trim()) { + skippedEmptyCount++; + continue; + } pendingLines.push({ line, index: lineIndex++ }); } @@ -479,12 +493,38 @@ export async function streamSplitAndUpload( // 处理最后剩余的 buffer（如果文件不以换行符结尾） if (buffer.trim()) { pendingLines.push({ line: buffer, index: lineIndex++ }); + } else if (buffer.length > 0) { + skippedEmptyCount++; + } + + const totalFileNum = pendingLines.length; + if (totalFileNum === 0) { + return { + uploadedCount: 0, + totalBytes: fileSize, + skippedEmptyCount, + }; + } + + if (signal?.aborted) { + throw new Error("Upload cancelled"); + } + + let resolvedReqId = initialReqId; + if (!resolvedReqId) { + if (!resolveReqId) { + throw new Error("Missing pre-upload request id"); + } + resolvedReqId = await resolveReqId({ totalFileNum, totalSize: fileSize }); + if (!resolvedReqId) { + throw new Error("Failed to resolve pre-upload request id"); + } + onReqIdResolved?.(resolvedReqId); } /** * 上传单行内容 - * fileNo 固定为 1（因为所有行都属于同一个原始文件，只是不同的分片/行） - * chunkNo 用于标识是第几行 + * 每行作为独立文件上传，fileNo 对应行序号，chunkNo 固定为 1 */ async function uploadLine(line: string, index: number): Promise { // 检查是否已取消 @@ -498,7 +538,8 @@ export async function streamSplitAndUpload( } // 保留原始文件扩展名 - const newFileName = `${baseName}_${String(index + 1).padStart(6, "0")}${fileExtension}`; + const fileIndex = index + 1; + const newFileName = `${baseName}_${String(fileIndex).padStart(6, "0")}${fileExtension}`; const blob = new Blob([line], { type: "text/plain" }); const lineFile = new File([blob], newFileName, { type: "text/plain" }); @@ -513,11 +554,10 @@ export async function streamSplitAndUpload( const formData = new FormData(); formData.append("file", slices[0]); - formData.append("reqId", reqId.toString()); - // 所有行使用相同的 fileNo=1，因为它们属于同一个预上传请求 - // chunkNo 表示这是第几行数据 - formData.append("fileNo", "1"); - formData.append("chunkNo", (index + 1).toString()); + formData.append("reqId", resolvedReqId.toString()); + // 每行作为独立文件上传 + formData.append("fileNo", fileIndex.toString()); + formData.append("chunkNo", "1"); formData.append("fileName", newFileName); formData.append("fileSize", lineFile.size.toString()); formData.append("totalChunkNum", "1");