diff --git a/frontend/src/hooks/useSliceUpload.tsx b/frontend/src/hooks/useSliceUpload.tsx index 6fe76b0..848d9b3 100644 --- a/frontend/src/hooks/useSliceUpload.tsx +++ b/frontend/src/hooks/useSliceUpload.tsx @@ -251,18 +251,6 @@ export function useFileSliceUpload( const file = files[i]; console.log(`[useSliceUpload] Processing file ${i + 1}/${files.length}: ${file.name}`); - // 为每个文件单独调用 preUpload,获取独立的 reqId - const { data: reqId } = await preUpload(task.key, { - totalFileNum: 1, - totalSize: file.size, - datasetId: task.key, - hasArchive: task.hasArchive, - prefix: task.prefix, - }); - - console.log(`[useSliceUpload] File ${file.name} preUpload response reqId:`, reqId); - reqIds.push(reqId); - const result = await streamSplitAndUpload( file, (formData, config) => uploadChunk(task.key, formData, { @@ -292,10 +280,21 @@ export function useFileSliceUpload( }, }; updateTaskList(updatedTask); - }, + }, 1024 * 1024, // 1MB chunk size { - reqId, + resolveReqId: async ({ totalFileNum, totalSize }) => { + const { data: reqId } = await preUpload(task.key, { + totalFileNum, + totalSize, + datasetId: task.key, + hasArchive: task.hasArchive, + prefix: task.prefix, + }); + console.log(`[useSliceUpload] File ${file.name} preUpload response reqId:`, reqId); + reqIds.push(reqId); + return reqId; + }, hasArchive: newTask.hasArchive, prefix: newTask.prefix, signal: newTask.controller.signal, diff --git a/frontend/src/utils/file.util.ts b/frontend/src/utils/file.util.ts index 9606ddc..883f3f7 100644 --- a/frontend/src/utils/file.util.ts +++ b/frontend/src/utils/file.util.ts @@ -401,7 +401,9 @@ export function readFileAsText( * @returns 上传结果统计 */ export interface StreamUploadOptions { - reqId: number; + reqId?: number; + resolveReqId?: (params: { totalFileNum: number; totalSize: number }) => Promise; + onReqIdResolved?: (reqId: number) => void; fileNamePrefix?: string; hasArchive?: boolean; prefix?: string; @@ -422,8 +424,16 @@ export async function streamSplitAndUpload( chunkSize: number = 1024 * 1024, // 1MB options: StreamUploadOptions ): Promise { - const { reqId, fileNamePrefix, prefix, signal, maxConcurrency = 3 } = options; - + const { + reqId: initialReqId, + resolveReqId, + onReqIdResolved, + fileNamePrefix, + prefix, + signal, + maxConcurrency = 3, + } = options; + const fileSize = file.size; let offset = 0; let buffer = ""; @@ -441,7 +451,7 @@ export async function streamSplitAndUpload( const pendingLines: { line: string; index: number }[] = []; let lineIndex = 0; - // 逐块读取文件并收集行 + // 逐块读取文件并收集非空行 while (offset < fileSize) { // 检查是否已取消 if (signal?.aborted) { @@ -461,11 +471,15 @@ export async function streamSplitAndUpload( // 保留最后一行(可能不完整) buffer = lines.pop() || ""; - // 收集完整行 + // 收集完整行(跳过空行) for (const line of lines) { if (signal?.aborted) { throw new Error("Upload cancelled"); } + if (!line.trim()) { + skippedEmptyCount++; + continue; + } pendingLines.push({ line, index: lineIndex++ }); } @@ -479,12 +493,38 @@ export async function streamSplitAndUpload( // 处理最后剩余的 buffer(如果文件不以换行符结尾) if (buffer.trim()) { pendingLines.push({ line: buffer, index: lineIndex++ }); + } else if (buffer.length > 0) { + skippedEmptyCount++; + } + + const totalFileNum = pendingLines.length; + if (totalFileNum === 0) { + return { + uploadedCount: 0, + totalBytes: fileSize, + skippedEmptyCount, + }; + } + + if (signal?.aborted) { + throw new Error("Upload cancelled"); + } + + let resolvedReqId = initialReqId; + if (!resolvedReqId) { + if (!resolveReqId) { + throw new Error("Missing pre-upload request id"); + } + resolvedReqId = await resolveReqId({ totalFileNum, totalSize: fileSize }); + if (!resolvedReqId) { + throw new Error("Failed to resolve pre-upload request id"); + } + onReqIdResolved?.(resolvedReqId); } /** * 上传单行内容 - * fileNo 固定为 1(因为所有行都属于同一个原始文件,只是不同的分片/行) - * chunkNo 用于标识是第几行 + * 每行作为独立文件上传,fileNo 对应行序号,chunkNo 固定为 1 */ async function uploadLine(line: string, index: number): Promise { // 检查是否已取消 @@ -498,7 +538,8 @@ export async function streamSplitAndUpload( } // 保留原始文件扩展名 - const newFileName = `${baseName}_${String(index + 1).padStart(6, "0")}${fileExtension}`; + const fileIndex = index + 1; + const newFileName = `${baseName}_${String(fileIndex).padStart(6, "0")}${fileExtension}`; const blob = new Blob([line], { type: "text/plain" }); const lineFile = new File([blob], newFileName, { type: "text/plain" }); @@ -513,11 +554,10 @@ export async function streamSplitAndUpload( const formData = new FormData(); formData.append("file", slices[0]); - formData.append("reqId", reqId.toString()); - // 所有行使用相同的 fileNo=1,因为它们属于同一个预上传请求 - // chunkNo 表示这是第几行数据 - formData.append("fileNo", "1"); - formData.append("chunkNo", (index + 1).toString()); + formData.append("reqId", resolvedReqId.toString()); + // 每行作为独立文件上传 + formData.append("fileNo", fileIndex.toString()); + formData.append("chunkNo", "1"); formData.append("fileName", newFileName); formData.append("fileSize", lineFile.size.toString()); formData.append("totalChunkNum", "1");