You've already forked DataMate
实现边分割边上传的流式处理,避免大文件一次性加载导致前端卡顿。 修改内容: 1. file.util.ts - 流式分割上传核心功能 - 新增 streamSplitAndUpload 函数,实现边分割边上传 - 新增 shouldStreamUpload 函数,判断是否使用流式上传 - 新增 StreamUploadOptions 和 StreamUploadResult 接口 - 优化分片大小(默认 5MB) 2. ImportConfiguration.tsx - 智能上传策略 - 大文件(>5MB)使用流式分割上传 - 小文件(≤5MB)使用传统分割方式 - 保持 UI 不变 3. useSliceUpload.tsx - 流式上传处理 - 新增 handleStreamUpload 处理流式上传事件 - 支持并发上传和更好的进度管理 4. TaskUpload.tsx - 进度显示优化 - 注册流式上传事件监听器 - 显示流式上传信息(已上传行数、当前文件等) 5. dataset.model.ts - 类型定义扩展 - 新增 StreamUploadInfo 接口 - TaskItem 接口添加 streamUploadInfo 和 prefix 字段 实现特点: - 流式读取:使用 Blob.slice 逐块读取,避免一次性加载 - 逐行检测:按换行符分割,形成完整行后立即上传 - 内存优化:buffer 只保留当前块和未完成行,不累积所有分割结果 - 并发控制:支持 3 个并发上传,提升效率 - 进度可见:实时显示已上传行数和总体进度 - 错误处理:单个文件上传失败不影响其他文件 - 向后兼容:小文件仍使用原有分割方式 优势: - 大文件上传不再卡顿,用户体验大幅提升 - 内存占用显著降低(从加载整个文件到只保留当前块) - 上传效率提升(边分割边上传,并发上传多个小文件) 相关文件: - frontend/src/utils/file.util.ts - frontend/src/pages/DataManagement/Detail/components/ImportConfiguration.tsx - frontend/src/hooks/useSliceUpload.tsx - frontend/src/pages/Layout/TaskUpload.tsx - frontend/src/pages/DataManagement/dataset.model.ts
94 lines
3.0 KiB
TypeScript
94 lines
3.0 KiB
TypeScript
import {
|
|
cancelUploadUsingPut,
|
|
preUploadUsingPost,
|
|
uploadFileChunkUsingPost,
|
|
} from "@/pages/DataManagement/dataset.api";
|
|
import { Button, Empty, Progress, Tag } from "antd";
|
|
import { DeleteOutlined, FileTextOutlined } from "@ant-design/icons";
|
|
import { useEffect } from "react";
|
|
import { useFileSliceUpload } from "@/hooks/useSliceUpload";
|
|
|
|
export default function TaskUpload() {
|
|
const { createTask, taskList, removeTask, handleUpload, registerStreamUploadListener } = useFileSliceUpload(
|
|
{
|
|
preUpload: preUploadUsingPost,
|
|
uploadChunk: uploadFileChunkUsingPost,
|
|
cancelUpload: cancelUploadUsingPut,
|
|
},
|
|
true, // showTaskCenter
|
|
true // enableStreamUpload
|
|
);
|
|
|
|
useEffect(() => {
|
|
const uploadHandler = (e: Event) => {
|
|
const customEvent = e as CustomEvent;
|
|
console.log('[TaskUpload] Received upload event detail:', customEvent.detail);
|
|
const { files } = customEvent.detail;
|
|
const task = createTask(customEvent.detail);
|
|
console.log('[TaskUpload] Created task with prefix:', task.prefix);
|
|
handleUpload({ task, files });
|
|
};
|
|
window.addEventListener("upload:dataset", uploadHandler);
|
|
return () => {
|
|
window.removeEventListener("upload:dataset", uploadHandler);
|
|
};
|
|
}, [createTask, handleUpload]);
|
|
|
|
// 注册流式上传监听器
|
|
useEffect(() => {
|
|
const unregister = registerStreamUploadListener();
|
|
return unregister;
|
|
}, [registerStreamUploadListener]);
|
|
|
|
return (
|
|
<div
|
|
className="w-90 max-w-90 max-h-96 overflow-y-auto p-2"
|
|
id="header-task-popover"
|
|
>
|
|
{taskList.length > 0 &&
|
|
taskList.map((task) => (
|
|
<div key={task.key} className="border-b border-gray-200 pb-2">
|
|
<div className="flex items-center justify-between">
|
|
<div>{task.title}</div>
|
|
<Button
|
|
type="text"
|
|
danger
|
|
disabled={!task?.cancelFn}
|
|
onClick={() =>
|
|
removeTask({
|
|
...task,
|
|
isCancel: true,
|
|
})
|
|
}
|
|
icon={<DeleteOutlined />}
|
|
></Button>
|
|
</div>
|
|
|
|
<Progress size="small" percent={Number(task.percent)} />
|
|
{task.streamUploadInfo && (
|
|
<div className="flex items-center gap-2 text-xs text-gray-500 mt-1">
|
|
<Tag icon={<FileTextOutlined />} size="small">
|
|
按行分割
|
|
</Tag>
|
|
<span>
|
|
已上传: {task.streamUploadInfo.uploadedLines} 行
|
|
</span>
|
|
{task.streamUploadInfo.totalFiles > 1 && (
|
|
<span>
|
|
({task.streamUploadInfo.fileIndex}/{task.streamUploadInfo.totalFiles} 文件)
|
|
</span>
|
|
)}
|
|
</div>
|
|
)}
|
|
</div>
|
|
))}
|
|
{taskList.length === 0 && (
|
|
<Empty
|
|
image={Empty.PRESENTED_IMAGE_SIMPLE}
|
|
description="暂无上传任务"
|
|
/>
|
|
)}
|
|
</div>
|
|
);
|
|
}
|