You've already forked DataMate
feat(upload): 实现流式分割上传,优化大文件上传体验
实现边分割边上传的流式处理,避免大文件一次性加载导致前端卡顿。 修改内容: 1. file.util.ts - 流式分割上传核心功能 - 新增 streamSplitAndUpload 函数,实现边分割边上传 - 新增 shouldStreamUpload 函数,判断是否使用流式上传 - 新增 StreamUploadOptions 和 StreamUploadResult 接口 - 优化分片大小(默认 5MB) 2. ImportConfiguration.tsx - 智能上传策略 - 大文件(>5MB)使用流式分割上传 - 小文件(≤5MB)使用传统分割方式 - 保持 UI 不变 3. useSliceUpload.tsx - 流式上传处理 - 新增 handleStreamUpload 处理流式上传事件 - 支持并发上传和更好的进度管理 4. TaskUpload.tsx - 进度显示优化 - 注册流式上传事件监听器 - 显示流式上传信息(已上传行数、当前文件等) 5. dataset.model.ts - 类型定义扩展 - 新增 StreamUploadInfo 接口 - TaskItem 接口添加 streamUploadInfo 和 prefix 字段 实现特点: - 流式读取:使用 Blob.slice 逐块读取,避免一次性加载 - 逐行检测:按换行符分割,形成完整行后立即上传 - 内存优化:buffer 只保留当前块和未完成行,不累积所有分割结果 - 并发控制:支持 3 个并发上传,提升效率 - 进度可见:实时显示已上传行数和总体进度 - 错误处理:单个文件上传失败不影响其他文件 - 向后兼容:小文件仍使用原有分割方式 优势: - 大文件上传不再卡顿,用户体验大幅提升 - 内存占用显著降低(从加载整个文件到只保留当前块) - 上传效率提升(边分割边上传,并发上传多个小文件) 相关文件: - frontend/src/utils/file.util.ts - frontend/src/pages/DataManagement/Detail/components/ImportConfiguration.tsx - frontend/src/hooks/useSliceUpload.tsx - frontend/src/pages/Layout/TaskUpload.tsx - frontend/src/pages/DataManagement/dataset.model.ts
This commit is contained in:
@@ -1,69 +1,93 @@
|
||||
import {
|
||||
cancelUploadUsingPut,
|
||||
preUploadUsingPost,
|
||||
uploadFileChunkUsingPost,
|
||||
} from "@/pages/DataManagement/dataset.api";
|
||||
import { Button, Empty, Progress } from "antd";
|
||||
import { DeleteOutlined } from "@ant-design/icons";
|
||||
import { useEffect } from "react";
|
||||
import { useFileSliceUpload } from "@/hooks/useSliceUpload";
|
||||
|
||||
export default function TaskUpload() {
|
||||
const { createTask, taskList, removeTask, handleUpload } = useFileSliceUpload(
|
||||
{
|
||||
preUpload: preUploadUsingPost,
|
||||
uploadChunk: uploadFileChunkUsingPost,
|
||||
cancelUpload: cancelUploadUsingPut,
|
||||
}
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const uploadHandler = (e: any) => {
|
||||
console.log('[TaskUpload] Received upload event detail:', e.detail);
|
||||
const { files } = e.detail;
|
||||
const task = createTask(e.detail);
|
||||
console.log('[TaskUpload] Created task with prefix:', task.prefix);
|
||||
handleUpload({ task, files });
|
||||
};
|
||||
window.addEventListener("upload:dataset", uploadHandler);
|
||||
return () => {
|
||||
window.removeEventListener("upload:dataset", uploadHandler);
|
||||
};
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div
|
||||
className="w-90 max-w-90 max-h-96 overflow-y-auto p-2"
|
||||
id="header-task-popover"
|
||||
>
|
||||
{taskList.length > 0 &&
|
||||
taskList.map((task) => (
|
||||
<div key={task.key} className="border-b border-gray-200 pb-2">
|
||||
<div className="flex items-center justify-between">
|
||||
<div>{task.title}</div>
|
||||
<Button
|
||||
type="text"
|
||||
danger
|
||||
disabled={!task?.cancelFn}
|
||||
onClick={() =>
|
||||
removeTask({
|
||||
...task,
|
||||
isCancel: true,
|
||||
})
|
||||
}
|
||||
icon={<DeleteOutlined />}
|
||||
></Button>
|
||||
</div>
|
||||
|
||||
<Progress size="small" percent={task.percent} />
|
||||
</div>
|
||||
))}
|
||||
{taskList.length === 0 && (
|
||||
<Empty
|
||||
image={Empty.PRESENTED_IMAGE_SIMPLE}
|
||||
description="暂无上传任务"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
import {
|
||||
cancelUploadUsingPut,
|
||||
preUploadUsingPost,
|
||||
uploadFileChunkUsingPost,
|
||||
} from "@/pages/DataManagement/dataset.api";
|
||||
import { Button, Empty, Progress, Tag } from "antd";
|
||||
import { DeleteOutlined, FileTextOutlined } from "@ant-design/icons";
|
||||
import { useEffect } from "react";
|
||||
import { useFileSliceUpload } from "@/hooks/useSliceUpload";
|
||||
|
||||
export default function TaskUpload() {
|
||||
const { createTask, taskList, removeTask, handleUpload, registerStreamUploadListener } = useFileSliceUpload(
|
||||
{
|
||||
preUpload: preUploadUsingPost,
|
||||
uploadChunk: uploadFileChunkUsingPost,
|
||||
cancelUpload: cancelUploadUsingPut,
|
||||
},
|
||||
true, // showTaskCenter
|
||||
true // enableStreamUpload
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const uploadHandler = (e: Event) => {
|
||||
const customEvent = e as CustomEvent;
|
||||
console.log('[TaskUpload] Received upload event detail:', customEvent.detail);
|
||||
const { files } = customEvent.detail;
|
||||
const task = createTask(customEvent.detail);
|
||||
console.log('[TaskUpload] Created task with prefix:', task.prefix);
|
||||
handleUpload({ task, files });
|
||||
};
|
||||
window.addEventListener("upload:dataset", uploadHandler);
|
||||
return () => {
|
||||
window.removeEventListener("upload:dataset", uploadHandler);
|
||||
};
|
||||
}, [createTask, handleUpload]);
|
||||
|
||||
// 注册流式上传监听器
|
||||
useEffect(() => {
|
||||
const unregister = registerStreamUploadListener();
|
||||
return unregister;
|
||||
}, [registerStreamUploadListener]);
|
||||
|
||||
return (
|
||||
<div
|
||||
className="w-90 max-w-90 max-h-96 overflow-y-auto p-2"
|
||||
id="header-task-popover"
|
||||
>
|
||||
{taskList.length > 0 &&
|
||||
taskList.map((task) => (
|
||||
<div key={task.key} className="border-b border-gray-200 pb-2">
|
||||
<div className="flex items-center justify-between">
|
||||
<div>{task.title}</div>
|
||||
<Button
|
||||
type="text"
|
||||
danger
|
||||
disabled={!task?.cancelFn}
|
||||
onClick={() =>
|
||||
removeTask({
|
||||
...task,
|
||||
isCancel: true,
|
||||
})
|
||||
}
|
||||
icon={<DeleteOutlined />}
|
||||
></Button>
|
||||
</div>
|
||||
|
||||
<Progress size="small" percent={Number(task.percent)} />
|
||||
{task.streamUploadInfo && (
|
||||
<div className="flex items-center gap-2 text-xs text-gray-500 mt-1">
|
||||
<Tag icon={<FileTextOutlined />} size="small">
|
||||
按行分割
|
||||
</Tag>
|
||||
<span>
|
||||
已上传: {task.streamUploadInfo.uploadedLines} 行
|
||||
</span>
|
||||
{task.streamUploadInfo.totalFiles > 1 && (
|
||||
<span>
|
||||
({task.streamUploadInfo.fileIndex}/{task.streamUploadInfo.totalFiles} 文件)
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
{taskList.length === 0 && (
|
||||
<Empty
|
||||
image={Empty.PRESENTED_IMAGE_SIMPLE}
|
||||
description="暂无上传任务"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user