q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(dataset): 更新数据集导入配置接口定义

Browse Source 
- 添加 DatasetImportConfig 接口定义
- 定义 source、target、dataSource 等属性
- 支持 splitByLine 和 hasArchive 配置选项
- 将 importConfig 类型从 any 改为 DatasetImportConfig
- 增强类型安全性和代码可维护性
This commit is contained in:
Jerry Yan
2026-01-20 13:37:28 +08:00
parent 79371ba078
commit d58d026256
5 changed files with 185 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
frontend/src/pages/DataManagement/Create/EditDataset.tsx
View File

@@ -3,7 +3,7 @@ import {
queryDatasetByIdUsingGet, queryDatasetByIdUsingGet,
updateDatasetByIdUsingPut, updateDatasetByIdUsingPut,
} from "../dataset.api"; } from "../dataset.api";
import { useEffect, useState } from "react"; import { useCallback, useEffect, useState } from "react";
import { Dataset, DatasetType } from "../dataset.model"; import { Dataset, DatasetType } from "../dataset.model";
import { App, Button, Form, Modal } from "antd"; import { App, Button, Form, Modal } from "antd";
@@ -28,7 +28,7 @@ export default function EditDataset({
tags: [], tags: [],
parentDatasetId: "", parentDatasetId: "",
}); });
const fetchDataset = async () => { const fetchDataset = useCallback(async () => {
if (!open) return; if (!open) return;
// 如果有id，说明是编辑模式 // 如果有id，说明是编辑模式
if (data && data.id) { if (data && data.id) {
@@ -42,14 +42,14 @@ export default function EditDataset({
setNewDataset(updatedDataset); setNewDataset(updatedDataset);
form.setFieldsValue(updatedDataset); form.setFieldsValue(updatedDataset);
} }
}; }, [data, form, open]);
useEffect(() => { useEffect(() => {
fetchDataset(); fetchDataset();
}, [data]); }, [fetchDataset]);
const handleValuesChange = (_, allValues) => { const handleValuesChange = (_, allValues) => {
setNewDataset({ ...newDataset, ...allValues }); setNewDataset((prev) => ({ ...prev, ...allValues }));
}; };
const handleSubmit = async () => { const handleSubmit = async () => {

72
frontend/src/pages/DataManagement/Create/components/BasicInformation.tsx
View File

@@ -1,37 +1,33 @@
import RadioCard from "@/components/RadioCard"; import RadioCard from "@/components/RadioCard";
import { Input, Select, Form } from "antd"; import { Input, Select, Form } from "antd";
import { datasetTypes } from "../../dataset.const"; import { datasetTypes } from "../../dataset.const";
import { useEffect, useState } from "react"; import { useCallback, useEffect, useState } from "react";
import type { Dispatch, SetStateAction } from "react";
import { queryDatasetTagsUsingGet, queryDatasetsUsingGet } from "../../dataset.api"; import { queryDatasetTagsUsingGet, queryDatasetsUsingGet } from "../../dataset.api";
import {queryTasksUsingGet} from "@/pages/DataCollection/collection.apis.ts"; import { queryTasksUsingGet } from "@/pages/DataCollection/collection.apis.ts";
import type { Dataset, TagItem } from "../../dataset.model";
export default function BasicInformation({ export default function BasicInformation({
data, data,
setData, setData,
hidden = [], hidden = [],
}: { }: {
data: any; data: DatasetFormData;
setData: any; setData: Dispatch<SetStateAction<DatasetFormData>>;
hidden?: string[]; hidden?: string[];
}) { }) {
const [tagOptions, setTagOptions] = useState< const [tagOptions, setTagOptions] = useState<DatasetTagOption[]>([]);
{ const [collectionOptions, setCollectionOptions] = useState<SelectOption[]>([]);
label: JSX.Element;
title: string;
options: { label: JSX.Element; value: string }[];
}[]
>([]);
const [collectionOptions, setCollectionOptions] = useState([]);
const [parentDatasetOptions, setParentDatasetOptions] = useState< const [parentDatasetOptions, setParentDatasetOptions] = useState<
{ label: string; value: string }[] { label: string; value: string }[]
>([]); >([]);
// 获取标签 // 获取标签
const fetchTags = async () => { const fetchTags = useCallback(async () => {
if (hidden.includes("tags")) return; if (hidden.includes("tags")) return;
try { try {
const { data } = await queryDatasetTagsUsingGet(); const { data } = await queryDatasetTagsUsingGet();
const customTags = data.map((tag) => ({ const tags = Array.isArray(data) ? (data as TagItem[]) : [];
const customTags = tags.map((tag) => ({
label: tag.name, label: tag.name,
value: tag.name, value: tag.name,
})); }));
@@ -39,13 +35,16 @@ export default function BasicInformation({
} catch (error) { } catch (error) {
console.error("Error fetching tags: ", error); console.error("Error fetching tags: ", error);
} }
}; }, [hidden]);
// 获取归集任务 // 获取归集任务
const fetchCollectionTasks = async () => { const fetchCollectionTasks = useCallback(async () => {
try { try {
const res = await queryTasksUsingGet({ page: 0, size: 100 }); const res = await queryTasksUsingGet({ page: 0, size: 100 });
const options = res.data.content.map((task: any) => ({ const tasks = Array.isArray(res?.data?.content)
? (res.data.content as CollectionTask[])
: [];
const options = tasks.map((task) => ({
label: task.name, label: task.name,
value: task.id, value: task.id,
})); }));
@@ -53,9 +52,9 @@ export default function BasicInformation({
} catch (error) { } catch (error) {
console.error("Error fetching collection tasks:", error); console.error("Error fetching collection tasks:", error);
} }
}; }, []);
const fetchParentDatasets = async () => { const fetchParentDatasets = useCallback(async () => {
if (hidden.includes("parentDatasetId")) return; if (hidden.includes("parentDatasetId")) return;
try { try {
const { data: resData } = await queryDatasetsUsingGet({ const { data: resData } = await queryDatasetsUsingGet({
@@ -64,7 +63,9 @@ export default function BasicInformation({
size: 1000, size: 1000,
}); });
const currentDatasetId = data?.id; const currentDatasetId = data?.id;
const rootDatasets = resData?.content || []; const rootDatasets = Array.isArray(resData?.content)
? (resData.content as DatasetSummary[])
: [];
const options = rootDatasets const options = rootDatasets
.filter((dataset) => dataset.id !== currentDatasetId) .filter((dataset) => dataset.id !== currentDatasetId)
.map((dataset) => ({ .map((dataset) => ({
@@ -78,13 +79,13 @@ export default function BasicInformation({
} catch (error) { } catch (error) {
console.error("Error fetching parent datasets:", error); console.error("Error fetching parent datasets:", error);
} }
}; }, [data?.id, hidden]);
useEffect(() => { useEffect(() => {
fetchTags(); fetchTags();
fetchCollectionTasks(); fetchCollectionTasks();
fetchParentDatasets(); fetchParentDatasets();
}, [data?.id, hidden.join(",")]); }, [fetchTags, fetchCollectionTasks, fetchParentDatasets]);
return ( return (
<> <>
<Form.Item <Form.Item
@@ -141,3 +142,28 @@ export default function BasicInformation({
</> </>
); );
} }
type DatasetFormData = Partial<Dataset> & {
type?: string;
parentDatasetId?: string;
};
type DatasetTagOption = {
label: string;
value: string;
};
type SelectOption = {
label: string;
value: string;
};
type CollectionTask = {
id: string;
name: string;
};
type DatasetSummary = {
id: string;
name: string;
};

129
frontend/src/pages/DataManagement/Detail/useFilesOperation.ts
View File

@@ -3,7 +3,7 @@ import type {
DatasetFile, DatasetFile,
} from "@/pages/DataManagement/dataset.model"; } from "@/pages/DataManagement/dataset.model";
import { App } from "antd"; import { App } from "antd";
import { useState } from "react"; import { useState } from "react";
import { import {
deleteDatasetFileUsingDelete, deleteDatasetFileUsingDelete,
downloadFileByIdUsingGet, downloadFileByIdUsingGet,
@@ -34,16 +34,20 @@ export function useFilesOperation(dataset: Dataset) {
const [previewContent, setPreviewContent] = useState(""); const [previewContent, setPreviewContent] = useState("");
const [previewFileName, setPreviewFileName] = useState(""); const [previewFileName, setPreviewFileName] = useState("");
const fetchFiles = async (prefix?: string, current?, pageSize?) => { const fetchFiles = async (
// 如果明确传了 prefix（包括空字符串），使用传入的值；否则使用当前 pagination.prefix prefix?: string,
const targetPrefix = prefix !== undefined ? prefix : (pagination.prefix || ''); current?: number,
pageSize?: number
const params: any = { ) => {
page: current !== undefined ? current : pagination.current, // 如果明确传了 prefix（包括空字符串），使用传入的值；否则使用当前 pagination.prefix
size: pageSize !== undefined ? pageSize : pagination.pageSize, const targetPrefix = prefix !== undefined ? prefix : (pagination.prefix || '');
isWithDirectory: true,
prefix: targetPrefix, const params: DatasetFilesQueryParams = {
}; page: current !== undefined ? current : pagination.current,
size: pageSize !== undefined ? pageSize : pagination.pageSize,
isWithDirectory: true,
prefix: targetPrefix,
};
const { data } = await queryDatasetFilesUsingGet(id!, params); const { data } = await queryDatasetFilesUsingGet(id!, params);
setFileList(data.content || []); setFileList(data.content || []);
@@ -86,28 +90,28 @@ export function useFilesOperation(dataset: Dataset) {
setSelectedFiles([]); // 清空选中状态 setSelectedFiles([]); // 清空选中状态
}; };
const handleShowFile = (file: any) => async () => { const handleShowFile = (file: DatasetFile) => async () => {
// 请求文件内容并弹窗预览 // 请求文件内容并弹窗预览
try { try {
const res = await fetch(`/api/datasets/${dataset.id}/file/${file.id}`); const res = await fetch(`/api/datasets/${dataset.id}/file/${file.id}`);
const data = await res.text(); const data = await res.text();
setPreviewFileName(file.fileName); setPreviewFileName(file.fileName);
setPreviewContent(data); setPreviewContent(data);
setPreviewVisible(true); setPreviewVisible(true);
} catch (err) { } catch {
message.error({ content: "文件预览失败" }); message.error({ content: "文件预览失败" });
} }
}; };
const handleDeleteFile = async (file) => { const handleDeleteFile = async (file: DatasetFile) => {
try { try {
await deleteDatasetFileUsingDelete(dataset.id, file.id); await deleteDatasetFileUsingDelete(dataset.id, file.id);
fetchFiles(); // 刷新文件列表 fetchFiles(); // 刷新文件列表
message.success({ content: `文件 ${file.fileName} 已删除` }); message.success({ content: `文件 ${file.fileName} 已删除` });
} catch (error) { } catch {
message.error({ content: `文件 ${file.fileName} 删除失败` }); message.error({ content: `文件 ${file.fileName} 删除失败` });
} }
}; };
const handleBatchExport = () => { const handleBatchExport = () => {
if (selectedFiles.length === 0) { if (selectedFiles.length === 0) {
@@ -158,29 +162,36 @@ export function useFilesOperation(dataset: Dataset) {
// 创建成功后刷新当前目录，重置到第一页 // 创建成功后刷新当前目录，重置到第一页
await fetchFiles(currentPrefix, 1, pagination.pageSize); await fetchFiles(currentPrefix, 1, pagination.pageSize);
message.success({ content: `文件夹 ${directoryName} 创建成功` }); message.success({ content: `文件夹 ${directoryName} 创建成功` });
} catch (error) { } catch (caught) {
message.error({ content: `文件夹 ${directoryName} 创建失败` }); message.error({ content: `文件夹 ${directoryName} 创建失败` });
throw error; throw caught;
} }
}, },
handleDownloadDirectory: async (directoryPath: string, directoryName: string) => { handleDownloadDirectory: async (directoryPath: string, directoryName: string) => {
try { try {
await downloadDirectoryUsingGet(dataset.id, directoryPath); await downloadDirectoryUsingGet(dataset.id, directoryPath);
message.success({ content: `文件夹 ${directoryName} 下载成功` }); message.success({ content: `文件夹 ${directoryName} 下载成功` });
} catch (error) { } catch {
message.error({ content: `文件夹 ${directoryName} 下载失败` }); message.error({ content: `文件夹 ${directoryName} 下载失败` });
} }
}, },
handleDeleteDirectory: async (directoryPath: string, directoryName: string) => { handleDeleteDirectory: async (directoryPath: string, directoryName: string) => {
try { try {
await deleteDirectoryUsingDelete(dataset.id, directoryPath); await deleteDirectoryUsingDelete(dataset.id, directoryPath);
// 删除成功后刷新当前目录 // 删除成功后刷新当前目录
const currentPrefix = pagination.prefix || ""; const currentPrefix = pagination.prefix || "";
await fetchFiles(currentPrefix, 1, pagination.pageSize); await fetchFiles(currentPrefix, 1, pagination.pageSize);
message.success({ content: `文件夹 ${directoryName} 已删除` }); message.success({ content: `文件夹 ${directoryName} 已删除` });
} catch (error) { } catch {
message.error({ content: `文件夹 ${directoryName} 删除失败` }); message.error({ content: `文件夹 ${directoryName} 删除失败` });
} }
}, },
}; };
} }
interface DatasetFilesQueryParams {
page: number;
size: number;
isWithDirectory: boolean;
prefix: string;
}

84
frontend/src/pages/DataManagement/Home/DataManagement.tsx
View File

@@ -8,7 +8,8 @@ import {
} from "@ant-design/icons"; } from "@ant-design/icons";
import TagManager from "@/components/business/TagManagement"; import TagManager from "@/components/business/TagManagement";
import { Link, useNavigate } from "react-router"; import { Link, useNavigate } from "react-router";
import { useEffect, useMemo, useState } from "react"; import { useEffect, useMemo, useState } from "react";
import type { ReactNode } from "react";
import { SearchControls } from "@/components/SearchControls"; import { SearchControls } from "@/components/SearchControls";
import CardView from "@/components/CardView"; import CardView from "@/components/CardView";
import type { Dataset } from "@/pages/DataManagement/dataset.model"; import type { Dataset } from "@/pages/DataManagement/dataset.model";
@@ -35,19 +36,19 @@ export default function DatasetManagementPage() {
const [editDatasetOpen, setEditDatasetOpen] = useState(false); const [editDatasetOpen, setEditDatasetOpen] = useState(false);
const [currentDataset, setCurrentDataset] = useState<Dataset | null>(null); const [currentDataset, setCurrentDataset] = useState<Dataset | null>(null);
const [showUploadDialog, setShowUploadDialog] = useState(false); const [showUploadDialog, setShowUploadDialog] = useState(false);
const [statisticsData, setStatisticsData] = useState<any>({ const [statisticsData, setStatisticsData] = useState<StatisticsData>({
count: {}, count: [],
size: {}, size: [],
}); });
async function fetchStatistics() { async function fetchStatistics() {
const { data } = await getDatasetStatisticsUsingGet(); const { data } = await getDatasetStatisticsUsingGet();
const statistics = { const statistics: StatisticsData = {
size: [ size: [
{ {
title: "数据集总数", title: "数据集总数",
value: data?.totalDatasets || 0, value: data?.totalDatasets || 0,
}, },
{ {
title: "文件总数", title: "文件总数",
@@ -75,10 +76,10 @@ export default function DatasetManagementPage() {
title: "视频", title: "视频",
value: data?.count?.video || 0, value: data?.count?.video || 0,
}, },
], ],
}; };
setStatisticsData(statistics); setStatisticsData(statistics);
} }
const [tags, setTags] = useState<string[]>([]); const [tags, setTags] = useState<string[]>([]);
@@ -222,12 +223,12 @@ export default function DatasetManagementPage() {
title: "状态", title: "状态",
dataIndex: "status", dataIndex: "status",
key: "status", key: "status",
render: (status: any) => { render: (status: DatasetStatusMeta) => {
return ( return (
<Tag icon={status?.icon} color={status?.color}> <Tag icon={status?.icon} color={status?.color}>
{status?.label} {status?.label}
</Tag> </Tag>
); );
}, },
width: 120, width: 120,
}, },
@@ -273,10 +274,10 @@ export default function DatasetManagementPage() {
key: "actions", key: "actions",
width: 200, width: 200,
fixed: "right", fixed: "right",
render: (_: any, record: Dataset) => ( render: (_: unknown, record: Dataset) => (
<div className="flex items-center gap-2"> <div className="flex items-center gap-2">
{operations.map((op) => ( {operations.map((op) => (
<Tooltip key={op.key} title={op.label}> <Tooltip key={op.key} title={op.label}>
<Button <Button
type="text" type="text"
icon={op.icon} icon={op.icon}
@@ -352,13 +353,13 @@ export default function DatasetManagementPage() {
<div className="grid grid-cols-1 gap-4"> <div className="grid grid-cols-1 gap-4">
<Card> <Card>
<div className="grid grid-cols-3"> <div className="grid grid-cols-3">
{statisticsData.size?.map?.((item) => ( {statisticsData.size.map((item) => (
<Statistic <Statistic
title={item.title} title={item.title}
key={item.title} key={item.title}
value={`${item.value}`} value={`${item.value}`}
/> />
))} ))}
</div> </div>
</Card> </Card>
</div> </div>
@@ -395,5 +396,22 @@ export default function DatasetManagementPage() {
updateEvent="update:datasets" updateEvent="update:datasets"
/> />
</div> </div>
); );
} }
type StatisticsItem = {
title: string;
value: number | string;
};
type StatisticsData = {
count: StatisticsItem[];
size: StatisticsItem[];
};
type DatasetStatusMeta = {
label: string;
value: string;
color: string;
icon: ReactNode;
};

11
frontend/src/pages/DataManagement/dataset.model.ts
View File

@@ -58,6 +58,15 @@ export interface Dataset {
distribution?: Record<string, Record<string, number>>; distribution?: Record<string, Record<string, number>>;
} }
export interface DatasetImportConfig {
source?: DataSource | string;
target?: DataSource | string;
dataSource?: string;
splitByLine?: boolean;
hasArchive?: boolean;
[key: string]: string | number | boolean | null | undefined;
}
export interface TagItem { export interface TagItem {
id: string; id: string;
name: string; name: string;
@@ -84,7 +93,7 @@ export interface DatasetTask {
status: "importing" | "waiting" | "completed" | "failed"; status: "importing" | "waiting" | "completed" | "failed";
progress: number; progress: number;
createdAt: string; createdAt: string;
importConfig: any; importConfig: DatasetImportConfig;
scheduleConfig: ScheduleConfig; scheduleConfig: ScheduleConfig;
nextExecution?: string; nextExecution?: string;
lastExecution?: string; lastExecution?: string;