You've already forked DataMate
feat(upload): 实现流式分割上传,优化大文件上传体验
实现边分割边上传的流式处理,避免大文件一次性加载导致前端卡顿。 修改内容: 1. file.util.ts - 流式分割上传核心功能 - 新增 streamSplitAndUpload 函数,实现边分割边上传 - 新增 shouldStreamUpload 函数,判断是否使用流式上传 - 新增 StreamUploadOptions 和 StreamUploadResult 接口 - 优化分片大小(默认 5MB) 2. ImportConfiguration.tsx - 智能上传策略 - 大文件(>5MB)使用流式分割上传 - 小文件(≤5MB)使用传统分割方式 - 保持 UI 不变 3. useSliceUpload.tsx - 流式上传处理 - 新增 handleStreamUpload 处理流式上传事件 - 支持并发上传和更好的进度管理 4. TaskUpload.tsx - 进度显示优化 - 注册流式上传事件监听器 - 显示流式上传信息(已上传行数、当前文件等) 5. dataset.model.ts - 类型定义扩展 - 新增 StreamUploadInfo 接口 - TaskItem 接口添加 streamUploadInfo 和 prefix 字段 实现特点: - 流式读取:使用 Blob.slice 逐块读取,避免一次性加载 - 逐行检测:按换行符分割,形成完整行后立即上传 - 内存优化:buffer 只保留当前块和未完成行,不累积所有分割结果 - 并发控制:支持 3 个并发上传,提升效率 - 进度可见:实时显示已上传行数和总体进度 - 错误处理:单个文件上传失败不影响其他文件 - 向后兼容:小文件仍使用原有分割方式 优势: - 大文件上传不再卡顿,用户体验大幅提升 - 内存占用显著降低(从加载整个文件到只保留当前块) - 上传效率提升(边分割边上传,并发上传多个小文件) 相关文件: - frontend/src/utils/file.util.ts - frontend/src/pages/DataManagement/Detail/components/ImportConfiguration.tsx - frontend/src/hooks/useSliceUpload.tsx - frontend/src/pages/Layout/TaskUpload.tsx - frontend/src/pages/DataManagement/dataset.model.ts
This commit is contained in:
@@ -5,7 +5,7 @@ import { Dataset, DatasetType, DataSource } from "../../dataset.model";
|
||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
import { queryTasksUsingGet } from "@/pages/DataCollection/collection.apis";
|
||||
import { updateDatasetByIdUsingPut } from "../../dataset.api";
|
||||
import { sliceFile } from "@/utils/file.util";
|
||||
import { sliceFile, shouldStreamUpload } from "@/utils/file.util";
|
||||
import Dragger from "antd/es/upload/Dragger";
|
||||
|
||||
const TEXT_FILE_MIME_PREFIX = "text/";
|
||||
@@ -164,17 +164,75 @@ export default function ImportConfiguration({
|
||||
// 本地上传文件相关逻辑
|
||||
|
||||
const handleUpload = async (dataset: Dataset) => {
|
||||
let filesToUpload =
|
||||
const filesToUpload =
|
||||
(form.getFieldValue("files") as UploadFile[] | undefined) || [];
|
||||
|
||||
// 如果启用分行分割,处理文件
|
||||
// 如果启用分行分割,对大文件使用流式处理
|
||||
if (importConfig.splitByLine && !hasNonTextFile) {
|
||||
const splitResults = await Promise.all(
|
||||
filesToUpload.map((file) => splitFileByLines(file))
|
||||
);
|
||||
filesToUpload = splitResults.flat();
|
||||
// 检查是否有大文件需要流式分割上传
|
||||
const filesForStreamUpload: File[] = [];
|
||||
const filesForNormalUpload: UploadFile[] = [];
|
||||
|
||||
for (const file of filesToUpload) {
|
||||
const originFile = file.originFileObj ?? file;
|
||||
if (originFile instanceof File && shouldStreamUpload(originFile)) {
|
||||
filesForStreamUpload.push(originFile);
|
||||
} else {
|
||||
filesForNormalUpload.push(file);
|
||||
}
|
||||
}
|
||||
|
||||
// 大文件使用流式分割上传
|
||||
if (filesForStreamUpload.length > 0) {
|
||||
window.dispatchEvent(
|
||||
new CustomEvent("upload:dataset-stream", {
|
||||
detail: {
|
||||
dataset,
|
||||
files: filesForStreamUpload,
|
||||
updateEvent,
|
||||
hasArchive: importConfig.hasArchive,
|
||||
prefix: currentPrefix,
|
||||
},
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
// 小文件使用传统分割方式
|
||||
if (filesForNormalUpload.length > 0) {
|
||||
const splitResults = await Promise.all(
|
||||
filesForNormalUpload.map((file) => splitFileByLines(file))
|
||||
);
|
||||
const smallFilesToUpload = splitResults.flat();
|
||||
|
||||
// 计算分片列表
|
||||
const sliceList = smallFilesToUpload.map((file) => {
|
||||
const originFile = (file.originFileObj ?? file) as Blob;
|
||||
const slices = sliceFile(originFile);
|
||||
return {
|
||||
originFile: originFile,
|
||||
slices,
|
||||
name: file.name,
|
||||
size: originFile.size || 0,
|
||||
};
|
||||
});
|
||||
|
||||
console.log("[ImportConfiguration] Uploading small files with currentPrefix:", currentPrefix);
|
||||
window.dispatchEvent(
|
||||
new CustomEvent("upload:dataset", {
|
||||
detail: {
|
||||
dataset,
|
||||
files: sliceList,
|
||||
updateEvent,
|
||||
hasArchive: importConfig.hasArchive,
|
||||
prefix: currentPrefix,
|
||||
},
|
||||
})
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// 未启用分行分割,使用普通上传
|
||||
// 计算分片列表
|
||||
const sliceList = filesToUpload.map((file) => {
|
||||
const originFile = (file.originFileObj ?? file) as Blob;
|
||||
|
||||
Reference in New Issue
Block a user