From 97170a90feb8c24a69ff9d8e0b18af637c82c665 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Fri, 30 Jan 2026 23:31:02 +0800 Subject: [PATCH] =?UTF-8?q?feat(data-import):=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E6=96=87=E6=9C=AC=E6=96=87=E4=BB=B6=E7=B1=BB=E5=9E=8B=E6=A3=80?= =?UTF-8?q?=E6=B5=8B=E5=92=8C=E6=8C=89=E8=A1=8C=E5=88=86=E5=89=B2=E5=8A=9F?= =?UTF-8?q?=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 TEXT_FILE_MIME_PREFIX、TEXT_FILE_MIME_TYPES 和 TEXT_FILE_EXTENSIONS 常量用于文本文件识别 - 添加 getUploadFileName、getUploadFileType 和 isTextUploadFile 工具函数 - 在 splitFileByLines 函数中集成文本文件类型检查 - 添加 hasNonTextFile useMemo 钩子来检测是否存在非文本文件 - 当存在非文本文件时禁用按行分割功能并重置开关状态 - 更新 Tooltip 提示内容以反映文件类型限制 - 使用 useCallback 优化 fetchCollectionTasks 和 resetState 函数 - 调整 useEffect 依赖数组以确保正确的重新渲染行为 --- .../Detail/components/ImportConfiguration.tsx | 211 ++++++++++++------ 1 file changed, 147 insertions(+), 64 deletions(-) diff --git a/frontend/src/pages/DataManagement/Detail/components/ImportConfiguration.tsx b/frontend/src/pages/DataManagement/Detail/components/ImportConfiguration.tsx index 1c2c56e..bb71f7e 100644 --- a/frontend/src/pages/DataManagement/Detail/components/ImportConfiguration.tsx +++ b/frontend/src/pages/DataManagement/Detail/components/ImportConfiguration.tsx @@ -2,40 +2,104 @@ import { Select, Input, Form, Radio, Modal, Button, UploadFile, Switch, Tooltip import { InboxOutlined, QuestionCircleOutlined } from "@ant-design/icons"; import { dataSourceOptions } from "../../dataset.const"; import { Dataset, DataSource } from "../../dataset.model"; -import { useEffect, useState } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; import { queryTasksUsingGet } from "@/pages/DataCollection/collection.apis"; import { updateDatasetByIdUsingPut } from "../../dataset.api"; import { sliceFile } from "@/utils/file.util"; import Dragger from "antd/es/upload/Dragger"; -/** - * 按行分割文件 - * @param file 原始文件 - * @returns 分割后的文件列表，每行一个文件 - */ +const TEXT_FILE_MIME_PREFIX = "text/"; +const TEXT_FILE_MIME_TYPES = new Set([ + "application/json", + "application/xml", + "application/csv", + "application/ndjson", + "application/x-ndjson", + "application/x-yaml", + "application/yaml", + "application/javascript", + "application/x-javascript", + "application/sql", +]); +const TEXT_FILE_EXTENSIONS = new Set([ + ".txt", + ".md", + ".csv", + ".tsv", + ".json", + ".jsonl", + ".ndjson", + ".log", + ".xml", + ".yaml", + ".yml", + ".sql", +]); + +function getUploadFileName(file: UploadFile): string { + if (file.name) return file.name; + const originFile = file.originFileObj; + if (originFile instanceof File && originFile.name) { + return originFile.name; + } + return ""; +} + +function getUploadFileType(file: UploadFile): string { + if (file.type) return file.type; + const originFile = file.originFileObj; + if (originFile instanceof File && typeof originFile.type === "string") { + return originFile.type; + } + return ""; +} + +function isTextUploadFile(file: UploadFile): boolean { + const mimeType = getUploadFileType(file).toLowerCase(); + if (mimeType) { + if (mimeType.startsWith(TEXT_FILE_MIME_PREFIX)) return true; + if (TEXT_FILE_MIME_TYPES.has(mimeType)) return true; + } + + const fileName = getUploadFileName(file); + const dotIndex = fileName.lastIndexOf("."); + if (dotIndex < 0) return false; + const ext = fileName.slice(dotIndex).toLowerCase(); + return TEXT_FILE_EXTENSIONS.has(ext); +} + +/** + * 按行分割文件 + * @param file 原始文件 + * @returns 分割后的文件列表，每行一个文件 + */ async function splitFileByLines(file: UploadFile): Promise { + if (!isTextUploadFile(file)) { + return [file]; + } + const originFile = file.originFileObj ?? file; if (!(originFile instanceof File) || typeof originFile.text !== "function") { return [file]; } - - const text = await originFile.text(); - if (!text) return [file]; - - // 按行分割并过滤空行 - const lines = text.split(/\r?\n/).filter((line: string) => line.trim() !== ""); - if (lines.length === 0) return []; - - // 生成文件名：原文件名_序号.扩展名 - const nameParts = file.name.split("."); - const ext = nameParts.length > 1 ? "." + nameParts.pop() : ""; - const baseName = nameParts.join("."); - const padLength = String(lines.length).length; - - return lines.map((line: string, index: number) => { - const newFileName = `${baseName}_${String(index + 1).padStart(padLength, "0")}${ext}`; - const blob = new Blob([line], { type: "text/plain" }); - const newFile = new File([blob], newFileName, { type: "text/plain" }); + + const text = await originFile.text(); + if (!text) return [file]; + + // 按行分割并过滤空行 + const lines = text.split(/\r?\n/).filter((line: string) => line.trim() !== ""); + if (lines.length === 0) return []; + + // 生成文件名：原文件名_序号.扩展名 + const nameParts = file.name.split("."); + const ext = nameParts.length > 1 ? "." + nameParts.pop() : ""; + const baseName = nameParts.join("."); + const padLength = String(lines.length).length; + + return lines.map((line: string, index: number) => { + const newFileName = `${baseName}_${String(index + 1).padStart(padLength, "0")}${ext}`; + const blob = new Blob([line], { type: "text/plain" }); + const newFile = new File([blob], newFileName, { type: "text/plain" }); return { uid: `${file.uid}-${index}`, name: newFileName, @@ -89,7 +153,12 @@ export default function ImportConfiguration({ hasArchive: true, splitByLine: false, }); - const [currentPrefix, setCurrentPrefix] = useState(""); + const [currentPrefix, setCurrentPrefix] = useState(""); + const hasNonTextFile = useMemo(() => { + const files = importConfig.files ?? []; + if (files.length === 0) return false; + return files.some((file) => !isTextUploadFile(file)); + }, [importConfig.files]); // 本地上传文件相关逻辑 @@ -97,13 +166,13 @@ export default function ImportConfiguration({ let filesToUpload = (form.getFieldValue("files") as UploadFile[] | undefined) || []; - // 如果启用分行分割，处理文件 - if (importConfig.splitByLine) { - const splitResults = await Promise.all( - filesToUpload.map((file) => splitFileByLines(file)) - ); - filesToUpload = splitResults.flat(); - } + // 如果启用分行分割，处理文件 + if (importConfig.splitByLine && !hasNonTextFile) { + const splitResults = await Promise.all( + filesToUpload.map((file) => splitFileByLines(file)) + ); + filesToUpload = splitResults.flat(); + } // 计算分片列表 const sliceList = filesToUpload.map((file) => { @@ -131,10 +200,10 @@ export default function ImportConfiguration({ ); }; - const fetchCollectionTasks = async () => { - if (importConfig.source !== DataSource.COLLECTION) return; - try { - const res = await queryTasksUsingGet({ page: 0, size: 100 }); + const fetchCollectionTasks = useCallback(async () => { + if (importConfig.source !== DataSource.COLLECTION) return; + try { + const res = await queryTasksUsingGet({ page: 0, size: 100 }); const tasks = Array.isArray(res?.data?.content) ? (res.data.content as CollectionTask[]) : []; @@ -143,13 +212,13 @@ export default function ImportConfiguration({ value: task.id, })); setCollectionOptions(options); - } catch (error) { - console.error("Error fetching collection tasks:", error); - } - }; - - const resetState = () => { - console.log('[ImportConfiguration] resetState called, preserving currentPrefix:', currentPrefix); + } catch (error) { + console.error("Error fetching collection tasks:", error); + } + }, [importConfig.source]); + + const resetState = useCallback(() => { + console.log('[ImportConfiguration] resetState called, preserving currentPrefix:', currentPrefix); form.resetFields(); form.setFieldsValue({ files: null }); setImportConfig({ @@ -157,8 +226,8 @@ export default function ImportConfiguration({ hasArchive: true, splitByLine: false, }); - console.log('[ImportConfiguration] resetState done, currentPrefix still:', currentPrefix); - }; + console.log('[ImportConfiguration] resetState done, currentPrefix still:', currentPrefix); + }, [currentPrefix, form]); const handleImportData = async () => { if (!data) return; @@ -173,21 +242,29 @@ export default function ImportConfiguration({ onClose(); }; - useEffect(() => { - if (open) { - setCurrentPrefix(prefix || ""); + useEffect(() => { + if (open) { + setCurrentPrefix(prefix || ""); console.log('[ImportConfiguration] Modal opened with prefix:', prefix); resetState(); fetchCollectionTasks(); } - }, [open]); + }, [fetchCollectionTasks, open, prefix, resetState]); + + useEffect(() => { + if (!importConfig.files?.length) return; + if (!importConfig.splitByLine) return; + if (!hasNonTextFile) return; + form.setFieldsValue({ splitByLine: false }); + setImportConfig((prev) => ({ ...prev, splitByLine: false })); + }, [form, hasNonTextFile, importConfig.files, importConfig.splitByLine]); // Separate effect for fetching collection tasks when source changes useEffect(() => { if (open && importConfig.source === DataSource.COLLECTION) { fetchCollectionTasks(); } - }, [importConfig.source]); + }, [fetchCollectionTasks, importConfig.source, open]); return ( - - 按分行分割{" "} - - - - - } - name="splitByLine" - valuePropName="checked" - > - - + + 按分行分割{" "} + + + + + } + name="splitByLine" + valuePropName="checked" + > + +