You've already forked DataMate
feat(data-import): 添加文本文件类型检测和按行分割功能
- 新增 TEXT_FILE_MIME_PREFIX、TEXT_FILE_MIME_TYPES 和 TEXT_FILE_EXTENSIONS 常量用于文本文件识别 - 添加 getUploadFileName、getUploadFileType 和 isTextUploadFile 工具函数 - 在 splitFileByLines 函数中集成文本文件类型检查 - 添加 hasNonTextFile useMemo 钩子来检测是否存在非文本文件 - 当存在非文本文件时禁用按行分割功能并重置开关状态 - 更新 Tooltip 提示内容以反映文件类型限制 - 使用 useCallback 优化 fetchCollectionTasks 和 resetState 函数 - 调整 useEffect 依赖数组以确保正确的重新渲染行为
This commit is contained in:
@@ -2,40 +2,104 @@ import { Select, Input, Form, Radio, Modal, Button, UploadFile, Switch, Tooltip
|
||||
import { InboxOutlined, QuestionCircleOutlined } from "@ant-design/icons";
|
||||
import { dataSourceOptions } from "../../dataset.const";
|
||||
import { Dataset, DataSource } from "../../dataset.model";
|
||||
import { useEffect, useState } from "react";
|
||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
import { queryTasksUsingGet } from "@/pages/DataCollection/collection.apis";
|
||||
import { updateDatasetByIdUsingPut } from "../../dataset.api";
|
||||
import { sliceFile } from "@/utils/file.util";
|
||||
import Dragger from "antd/es/upload/Dragger";
|
||||
|
||||
/**
|
||||
* 按行分割文件
|
||||
* @param file 原始文件
|
||||
* @returns 分割后的文件列表，每行一个文件
|
||||
*/
|
||||
const TEXT_FILE_MIME_PREFIX = "text/";
|
||||
const TEXT_FILE_MIME_TYPES = new Set([
|
||||
"application/json",
|
||||
"application/xml",
|
||||
"application/csv",
|
||||
"application/ndjson",
|
||||
"application/x-ndjson",
|
||||
"application/x-yaml",
|
||||
"application/yaml",
|
||||
"application/javascript",
|
||||
"application/x-javascript",
|
||||
"application/sql",
|
||||
]);
|
||||
const TEXT_FILE_EXTENSIONS = new Set([
|
||||
".txt",
|
||||
".md",
|
||||
".csv",
|
||||
".tsv",
|
||||
".json",
|
||||
".jsonl",
|
||||
".ndjson",
|
||||
".log",
|
||||
".xml",
|
||||
".yaml",
|
||||
".yml",
|
||||
".sql",
|
||||
]);
|
||||
|
||||
function getUploadFileName(file: UploadFile): string {
|
||||
if (file.name) return file.name;
|
||||
const originFile = file.originFileObj;
|
||||
if (originFile instanceof File && originFile.name) {
|
||||
return originFile.name;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
function getUploadFileType(file: UploadFile): string {
|
||||
if (file.type) return file.type;
|
||||
const originFile = file.originFileObj;
|
||||
if (originFile instanceof File && typeof originFile.type === "string") {
|
||||
return originFile.type;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
function isTextUploadFile(file: UploadFile): boolean {
|
||||
const mimeType = getUploadFileType(file).toLowerCase();
|
||||
if (mimeType) {
|
||||
if (mimeType.startsWith(TEXT_FILE_MIME_PREFIX)) return true;
|
||||
if (TEXT_FILE_MIME_TYPES.has(mimeType)) return true;
|
||||
}
|
||||
|
||||
const fileName = getUploadFileName(file);
|
||||
const dotIndex = fileName.lastIndexOf(".");
|
||||
if (dotIndex < 0) return false;
|
||||
const ext = fileName.slice(dotIndex).toLowerCase();
|
||||
return TEXT_FILE_EXTENSIONS.has(ext);
|
||||
}
|
||||
|
||||
/**
|
||||
* 按行分割文件
|
||||
* @param file 原始文件
|
||||
* @returns 分割后的文件列表，每行一个文件
|
||||
*/
|
||||
async function splitFileByLines(file: UploadFile): Promise<UploadFile[]> {
|
||||
if (!isTextUploadFile(file)) {
|
||||
return [file];
|
||||
}
|
||||
|
||||
const originFile = file.originFileObj ?? file;
|
||||
if (!(originFile instanceof File) || typeof originFile.text !== "function") {
|
||||
return [file];
|
||||
}
|
||||
|
||||
const text = await originFile.text();
|
||||
if (!text) return [file];
|
||||
|
||||
// 按行分割并过滤空行
|
||||
const lines = text.split(/\r?\n/).filter((line: string) => line.trim() !== "");
|
||||
if (lines.length === 0) return [];
|
||||
|
||||
// 生成文件名：原文件名_序号.扩展名
|
||||
const nameParts = file.name.split(".");
|
||||
const ext = nameParts.length > 1 ? "." + nameParts.pop() : "";
|
||||
const baseName = nameParts.join(".");
|
||||
const padLength = String(lines.length).length;
|
||||
|
||||
return lines.map((line: string, index: number) => {
|
||||
const newFileName = `${baseName}_${String(index + 1).padStart(padLength, "0")}${ext}`;
|
||||
const blob = new Blob([line], { type: "text/plain" });
|
||||
const newFile = new File([blob], newFileName, { type: "text/plain" });
|
||||
|
||||
const text = await originFile.text();
|
||||
if (!text) return [file];
|
||||
|
||||
// 按行分割并过滤空行
|
||||
const lines = text.split(/\r?\n/).filter((line: string) => line.trim() !== "");
|
||||
if (lines.length === 0) return [];
|
||||
|
||||
// 生成文件名：原文件名_序号.扩展名
|
||||
const nameParts = file.name.split(".");
|
||||
const ext = nameParts.length > 1 ? "." + nameParts.pop() : "";
|
||||
const baseName = nameParts.join(".");
|
||||
const padLength = String(lines.length).length;
|
||||
|
||||
return lines.map((line: string, index: number) => {
|
||||
const newFileName = `${baseName}_${String(index + 1).padStart(padLength, "0")}${ext}`;
|
||||
const blob = new Blob([line], { type: "text/plain" });
|
||||
const newFile = new File([blob], newFileName, { type: "text/plain" });
|
||||
return {
|
||||
uid: `${file.uid}-${index}`,
|
||||
name: newFileName,
|
||||
@@ -89,7 +153,12 @@ export default function ImportConfiguration({
|
||||
hasArchive: true,
|
||||
splitByLine: false,
|
||||
});
|
||||
const [currentPrefix, setCurrentPrefix] = useState<string>("");
|
||||
const [currentPrefix, setCurrentPrefix] = useState<string>("");
|
||||
const hasNonTextFile = useMemo(() => {
|
||||
const files = importConfig.files ?? [];
|
||||
if (files.length === 0) return false;
|
||||
return files.some((file) => !isTextUploadFile(file));
|
||||
}, [importConfig.files]);
|
||||
|
||||
// 本地上传文件相关逻辑
|
||||
|
||||
@@ -97,13 +166,13 @@ export default function ImportConfiguration({
|
||||
let filesToUpload =
|
||||
(form.getFieldValue("files") as UploadFile[] | undefined) || [];
|
||||
|
||||
// 如果启用分行分割，处理文件
|
||||
if (importConfig.splitByLine) {
|
||||
const splitResults = await Promise.all(
|
||||
filesToUpload.map((file) => splitFileByLines(file))
|
||||
);
|
||||
filesToUpload = splitResults.flat();
|
||||
}
|
||||
// 如果启用分行分割，处理文件
|
||||
if (importConfig.splitByLine && !hasNonTextFile) {
|
||||
const splitResults = await Promise.all(
|
||||
filesToUpload.map((file) => splitFileByLines(file))
|
||||
);
|
||||
filesToUpload = splitResults.flat();
|
||||
}
|
||||
|
||||
// 计算分片列表
|
||||
const sliceList = filesToUpload.map((file) => {
|
||||
@@ -131,10 +200,10 @@ export default function ImportConfiguration({
|
||||
);
|
||||
};
|
||||
|
||||
const fetchCollectionTasks = async () => {
|
||||
if (importConfig.source !== DataSource.COLLECTION) return;
|
||||
try {
|
||||
const res = await queryTasksUsingGet({ page: 0, size: 100 });
|
||||
const fetchCollectionTasks = useCallback(async () => {
|
||||
if (importConfig.source !== DataSource.COLLECTION) return;
|
||||
try {
|
||||
const res = await queryTasksUsingGet({ page: 0, size: 100 });
|
||||
const tasks = Array.isArray(res?.data?.content)
|
||||
? (res.data.content as CollectionTask[])
|
||||
: [];
|
||||
@@ -143,13 +212,13 @@ export default function ImportConfiguration({
|
||||
value: task.id,
|
||||
}));
|
||||
setCollectionOptions(options);
|
||||
} catch (error) {
|
||||
console.error("Error fetching collection tasks:", error);
|
||||
}
|
||||
};
|
||||
|
||||
const resetState = () => {
|
||||
console.log('[ImportConfiguration] resetState called, preserving currentPrefix:', currentPrefix);
|
||||
} catch (error) {
|
||||
console.error("Error fetching collection tasks:", error);
|
||||
}
|
||||
}, [importConfig.source]);
|
||||
|
||||
const resetState = useCallback(() => {
|
||||
console.log('[ImportConfiguration] resetState called, preserving currentPrefix:', currentPrefix);
|
||||
form.resetFields();
|
||||
form.setFieldsValue({ files: null });
|
||||
setImportConfig({
|
||||
@@ -157,8 +226,8 @@ export default function ImportConfiguration({
|
||||
hasArchive: true,
|
||||
splitByLine: false,
|
||||
});
|
||||
console.log('[ImportConfiguration] resetState done, currentPrefix still:', currentPrefix);
|
||||
};
|
||||
console.log('[ImportConfiguration] resetState done, currentPrefix still:', currentPrefix);
|
||||
}, [currentPrefix, form]);
|
||||
|
||||
const handleImportData = async () => {
|
||||
if (!data) return;
|
||||
@@ -173,21 +242,29 @@ export default function ImportConfiguration({
|
||||
onClose();
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (open) {
|
||||
setCurrentPrefix(prefix || "");
|
||||
useEffect(() => {
|
||||
if (open) {
|
||||
setCurrentPrefix(prefix || "");
|
||||
console.log('[ImportConfiguration] Modal opened with prefix:', prefix);
|
||||
resetState();
|
||||
fetchCollectionTasks();
|
||||
}
|
||||
}, [open]);
|
||||
}, [fetchCollectionTasks, open, prefix, resetState]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!importConfig.files?.length) return;
|
||||
if (!importConfig.splitByLine) return;
|
||||
if (!hasNonTextFile) return;
|
||||
form.setFieldsValue({ splitByLine: false });
|
||||
setImportConfig((prev) => ({ ...prev, splitByLine: false }));
|
||||
}, [form, hasNonTextFile, importConfig.files, importConfig.splitByLine]);
|
||||
|
||||
// Separate effect for fetching collection tasks when source changes
|
||||
useEffect(() => {
|
||||
if (open && importConfig.source === DataSource.COLLECTION) {
|
||||
fetchCollectionTasks();
|
||||
}
|
||||
}, [importConfig.source]);
|
||||
}, [fetchCollectionTasks, importConfig.source, open]);
|
||||
|
||||
return (
|
||||
<Modal
|
||||
@@ -286,20 +363,26 @@ export default function ImportConfiguration({
|
||||
>
|
||||
<Switch />
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label={
|
||||
<span>
|
||||
按分行分割{" "}
|
||||
<Tooltip title="选中后，文本文件的每一行将被分割成独立文件">
|
||||
<QuestionCircleOutlined style={{ color: "#999" }} />
|
||||
</Tooltip>
|
||||
</span>
|
||||
}
|
||||
name="splitByLine"
|
||||
valuePropName="checked"
|
||||
>
|
||||
<Switch />
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label={
|
||||
<span>
|
||||
按分行分割{" "}
|
||||
<Tooltip
|
||||
title={
|
||||
hasNonTextFile
|
||||
? "已选择非文本文件，无法按行分割"
|
||||
: "选中后，文本文件的每一行将被分割成独立文件"
|
||||
}
|
||||
>
|
||||
<QuestionCircleOutlined style={{ color: "#999" }} />
|
||||
</Tooltip>
|
||||
</span>
|
||||
}
|
||||
name="splitByLine"
|
||||
valuePropName="checked"
|
||||
>
|
||||
<Switch disabled={hasNonTextFile} />
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label="上传文件"
|
||||
name="files"
|
||||
|
||||
Reference in New Issue
Block a user