You've already forked DataMate
refactor(dataset): 更新数据集导入配置接口定义
- 添加 DatasetImportConfig 接口定义 - 定义 source、target、dataSource 等属性 - 支持 splitByLine 和 hasArchive 配置选项 - 将 importConfig 类型从 any 改为 DatasetImportConfig - 增强类型安全性和代码可维护性
This commit is contained in:
@@ -3,7 +3,7 @@ import {
|
||||
queryDatasetByIdUsingGet,
|
||||
updateDatasetByIdUsingPut,
|
||||
} from "../dataset.api";
|
||||
import { useEffect, useState } from "react";
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { Dataset, DatasetType } from "../dataset.model";
|
||||
import { App, Button, Form, Modal } from "antd";
|
||||
|
||||
@@ -28,7 +28,7 @@ export default function EditDataset({
|
||||
tags: [],
|
||||
parentDatasetId: "",
|
||||
});
|
||||
const fetchDataset = async () => {
|
||||
const fetchDataset = useCallback(async () => {
|
||||
if (!open) return;
|
||||
// 如果有id,说明是编辑模式
|
||||
if (data && data.id) {
|
||||
@@ -42,14 +42,14 @@ export default function EditDataset({
|
||||
setNewDataset(updatedDataset);
|
||||
form.setFieldsValue(updatedDataset);
|
||||
}
|
||||
};
|
||||
}, [data, form, open]);
|
||||
|
||||
useEffect(() => {
|
||||
fetchDataset();
|
||||
}, [data]);
|
||||
}, [fetchDataset]);
|
||||
|
||||
const handleValuesChange = (_, allValues) => {
|
||||
setNewDataset({ ...newDataset, ...allValues });
|
||||
setNewDataset((prev) => ({ ...prev, ...allValues }));
|
||||
};
|
||||
|
||||
const handleSubmit = async () => {
|
||||
|
||||
@@ -1,37 +1,33 @@
|
||||
import RadioCard from "@/components/RadioCard";
|
||||
import { Input, Select, Form } from "antd";
|
||||
import { datasetTypes } from "../../dataset.const";
|
||||
import { useEffect, useState } from "react";
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import type { Dispatch, SetStateAction } from "react";
|
||||
import { queryDatasetTagsUsingGet, queryDatasetsUsingGet } from "../../dataset.api";
|
||||
import {queryTasksUsingGet} from "@/pages/DataCollection/collection.apis.ts";
|
||||
import { queryTasksUsingGet } from "@/pages/DataCollection/collection.apis.ts";
|
||||
import type { Dataset, TagItem } from "../../dataset.model";
|
||||
|
||||
export default function BasicInformation({
|
||||
data,
|
||||
setData,
|
||||
hidden = [],
|
||||
}: {
|
||||
data: any;
|
||||
setData: any;
|
||||
data: DatasetFormData;
|
||||
setData: Dispatch<SetStateAction<DatasetFormData>>;
|
||||
hidden?: string[];
|
||||
}) {
|
||||
const [tagOptions, setTagOptions] = useState<
|
||||
{
|
||||
label: JSX.Element;
|
||||
title: string;
|
||||
options: { label: JSX.Element; value: string }[];
|
||||
}[]
|
||||
>([]);
|
||||
const [collectionOptions, setCollectionOptions] = useState([]);
|
||||
const [tagOptions, setTagOptions] = useState<DatasetTagOption[]>([]);
|
||||
const [collectionOptions, setCollectionOptions] = useState<SelectOption[]>([]);
|
||||
const [parentDatasetOptions, setParentDatasetOptions] = useState<
|
||||
{ label: string; value: string }[]
|
||||
>([]);
|
||||
|
||||
// 获取标签
|
||||
const fetchTags = async () => {
|
||||
const fetchTags = useCallback(async () => {
|
||||
if (hidden.includes("tags")) return;
|
||||
try {
|
||||
const { data } = await queryDatasetTagsUsingGet();
|
||||
const customTags = data.map((tag) => ({
|
||||
const tags = Array.isArray(data) ? (data as TagItem[]) : [];
|
||||
const customTags = tags.map((tag) => ({
|
||||
label: tag.name,
|
||||
value: tag.name,
|
||||
}));
|
||||
@@ -39,13 +35,16 @@ export default function BasicInformation({
|
||||
} catch (error) {
|
||||
console.error("Error fetching tags: ", error);
|
||||
}
|
||||
};
|
||||
}, [hidden]);
|
||||
|
||||
// 获取归集任务
|
||||
const fetchCollectionTasks = async () => {
|
||||
const fetchCollectionTasks = useCallback(async () => {
|
||||
try {
|
||||
const res = await queryTasksUsingGet({ page: 0, size: 100 });
|
||||
const options = res.data.content.map((task: any) => ({
|
||||
const tasks = Array.isArray(res?.data?.content)
|
||||
? (res.data.content as CollectionTask[])
|
||||
: [];
|
||||
const options = tasks.map((task) => ({
|
||||
label: task.name,
|
||||
value: task.id,
|
||||
}));
|
||||
@@ -53,9 +52,9 @@ export default function BasicInformation({
|
||||
} catch (error) {
|
||||
console.error("Error fetching collection tasks:", error);
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
const fetchParentDatasets = async () => {
|
||||
const fetchParentDatasets = useCallback(async () => {
|
||||
if (hidden.includes("parentDatasetId")) return;
|
||||
try {
|
||||
const { data: resData } = await queryDatasetsUsingGet({
|
||||
@@ -64,7 +63,9 @@ export default function BasicInformation({
|
||||
size: 1000,
|
||||
});
|
||||
const currentDatasetId = data?.id;
|
||||
const rootDatasets = resData?.content || [];
|
||||
const rootDatasets = Array.isArray(resData?.content)
|
||||
? (resData.content as DatasetSummary[])
|
||||
: [];
|
||||
const options = rootDatasets
|
||||
.filter((dataset) => dataset.id !== currentDatasetId)
|
||||
.map((dataset) => ({
|
||||
@@ -78,13 +79,13 @@ export default function BasicInformation({
|
||||
} catch (error) {
|
||||
console.error("Error fetching parent datasets:", error);
|
||||
}
|
||||
};
|
||||
}, [data?.id, hidden]);
|
||||
|
||||
useEffect(() => {
|
||||
fetchTags();
|
||||
fetchCollectionTasks();
|
||||
fetchParentDatasets();
|
||||
}, [data?.id, hidden.join(",")]);
|
||||
}, [fetchTags, fetchCollectionTasks, fetchParentDatasets]);
|
||||
return (
|
||||
<>
|
||||
<Form.Item
|
||||
@@ -141,3 +142,28 @@ export default function BasicInformation({
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
type DatasetFormData = Partial<Dataset> & {
|
||||
type?: string;
|
||||
parentDatasetId?: string;
|
||||
};
|
||||
|
||||
type DatasetTagOption = {
|
||||
label: string;
|
||||
value: string;
|
||||
};
|
||||
|
||||
type SelectOption = {
|
||||
label: string;
|
||||
value: string;
|
||||
};
|
||||
|
||||
type CollectionTask = {
|
||||
id: string;
|
||||
name: string;
|
||||
};
|
||||
|
||||
type DatasetSummary = {
|
||||
id: string;
|
||||
name: string;
|
||||
};
|
||||
|
||||
@@ -34,11 +34,15 @@ export function useFilesOperation(dataset: Dataset) {
|
||||
const [previewContent, setPreviewContent] = useState("");
|
||||
const [previewFileName, setPreviewFileName] = useState("");
|
||||
|
||||
const fetchFiles = async (prefix?: string, current?, pageSize?) => {
|
||||
const fetchFiles = async (
|
||||
prefix?: string,
|
||||
current?: number,
|
||||
pageSize?: number
|
||||
) => {
|
||||
// 如果明确传了 prefix(包括空字符串),使用传入的值;否则使用当前 pagination.prefix
|
||||
const targetPrefix = prefix !== undefined ? prefix : (pagination.prefix || '');
|
||||
|
||||
const params: any = {
|
||||
const params: DatasetFilesQueryParams = {
|
||||
page: current !== undefined ? current : pagination.current,
|
||||
size: pageSize !== undefined ? pageSize : pagination.pageSize,
|
||||
isWithDirectory: true,
|
||||
@@ -86,7 +90,7 @@ export function useFilesOperation(dataset: Dataset) {
|
||||
setSelectedFiles([]); // 清空选中状态
|
||||
};
|
||||
|
||||
const handleShowFile = (file: any) => async () => {
|
||||
const handleShowFile = (file: DatasetFile) => async () => {
|
||||
// 请求文件内容并弹窗预览
|
||||
try {
|
||||
const res = await fetch(`/api/datasets/${dataset.id}/file/${file.id}`);
|
||||
@@ -94,17 +98,17 @@ export function useFilesOperation(dataset: Dataset) {
|
||||
setPreviewFileName(file.fileName);
|
||||
setPreviewContent(data);
|
||||
setPreviewVisible(true);
|
||||
} catch (err) {
|
||||
} catch {
|
||||
message.error({ content: "文件预览失败" });
|
||||
}
|
||||
};
|
||||
|
||||
const handleDeleteFile = async (file) => {
|
||||
const handleDeleteFile = async (file: DatasetFile) => {
|
||||
try {
|
||||
await deleteDatasetFileUsingDelete(dataset.id, file.id);
|
||||
fetchFiles(); // 刷新文件列表
|
||||
message.success({ content: `文件 ${file.fileName} 已删除` });
|
||||
} catch (error) {
|
||||
} catch {
|
||||
message.error({ content: `文件 ${file.fileName} 删除失败` });
|
||||
}
|
||||
};
|
||||
@@ -158,16 +162,16 @@ export function useFilesOperation(dataset: Dataset) {
|
||||
// 创建成功后刷新当前目录,重置到第一页
|
||||
await fetchFiles(currentPrefix, 1, pagination.pageSize);
|
||||
message.success({ content: `文件夹 ${directoryName} 创建成功` });
|
||||
} catch (error) {
|
||||
} catch (caught) {
|
||||
message.error({ content: `文件夹 ${directoryName} 创建失败` });
|
||||
throw error;
|
||||
throw caught;
|
||||
}
|
||||
},
|
||||
handleDownloadDirectory: async (directoryPath: string, directoryName: string) => {
|
||||
try {
|
||||
await downloadDirectoryUsingGet(dataset.id, directoryPath);
|
||||
message.success({ content: `文件夹 ${directoryName} 下载成功` });
|
||||
} catch (error) {
|
||||
} catch {
|
||||
message.error({ content: `文件夹 ${directoryName} 下载失败` });
|
||||
}
|
||||
},
|
||||
@@ -178,9 +182,16 @@ export function useFilesOperation(dataset: Dataset) {
|
||||
const currentPrefix = pagination.prefix || "";
|
||||
await fetchFiles(currentPrefix, 1, pagination.pageSize);
|
||||
message.success({ content: `文件夹 ${directoryName} 已删除` });
|
||||
} catch (error) {
|
||||
} catch {
|
||||
message.error({ content: `文件夹 ${directoryName} 删除失败` });
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
interface DatasetFilesQueryParams {
|
||||
page: number;
|
||||
size: number;
|
||||
isWithDirectory: boolean;
|
||||
prefix: string;
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
import TagManager from "@/components/business/TagManagement";
|
||||
import { Link, useNavigate } from "react-router";
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import type { ReactNode } from "react";
|
||||
import { SearchControls } from "@/components/SearchControls";
|
||||
import CardView from "@/components/CardView";
|
||||
import type { Dataset } from "@/pages/DataManagement/dataset.model";
|
||||
@@ -35,15 +36,15 @@ export default function DatasetManagementPage() {
|
||||
const [editDatasetOpen, setEditDatasetOpen] = useState(false);
|
||||
const [currentDataset, setCurrentDataset] = useState<Dataset | null>(null);
|
||||
const [showUploadDialog, setShowUploadDialog] = useState(false);
|
||||
const [statisticsData, setStatisticsData] = useState<any>({
|
||||
count: {},
|
||||
size: {},
|
||||
const [statisticsData, setStatisticsData] = useState<StatisticsData>({
|
||||
count: [],
|
||||
size: [],
|
||||
});
|
||||
|
||||
async function fetchStatistics() {
|
||||
const { data } = await getDatasetStatisticsUsingGet();
|
||||
|
||||
const statistics = {
|
||||
const statistics: StatisticsData = {
|
||||
size: [
|
||||
{
|
||||
title: "数据集总数",
|
||||
@@ -222,7 +223,7 @@ export default function DatasetManagementPage() {
|
||||
title: "状态",
|
||||
dataIndex: "status",
|
||||
key: "status",
|
||||
render: (status: any) => {
|
||||
render: (status: DatasetStatusMeta) => {
|
||||
return (
|
||||
<Tag icon={status?.icon} color={status?.color}>
|
||||
{status?.label}
|
||||
@@ -273,7 +274,7 @@ export default function DatasetManagementPage() {
|
||||
key: "actions",
|
||||
width: 200,
|
||||
fixed: "right",
|
||||
render: (_: any, record: Dataset) => (
|
||||
render: (_: unknown, record: Dataset) => (
|
||||
<div className="flex items-center gap-2">
|
||||
{operations.map((op) => (
|
||||
<Tooltip key={op.key} title={op.label}>
|
||||
@@ -352,7 +353,7 @@ export default function DatasetManagementPage() {
|
||||
<div className="grid grid-cols-1 gap-4">
|
||||
<Card>
|
||||
<div className="grid grid-cols-3">
|
||||
{statisticsData.size?.map?.((item) => (
|
||||
{statisticsData.size.map((item) => (
|
||||
<Statistic
|
||||
title={item.title}
|
||||
key={item.title}
|
||||
@@ -397,3 +398,20 @@ export default function DatasetManagementPage() {
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
type StatisticsItem = {
|
||||
title: string;
|
||||
value: number | string;
|
||||
};
|
||||
|
||||
type StatisticsData = {
|
||||
count: StatisticsItem[];
|
||||
size: StatisticsItem[];
|
||||
};
|
||||
|
||||
type DatasetStatusMeta = {
|
||||
label: string;
|
||||
value: string;
|
||||
color: string;
|
||||
icon: ReactNode;
|
||||
};
|
||||
|
||||
@@ -58,6 +58,15 @@ export interface Dataset {
|
||||
distribution?: Record<string, Record<string, number>>;
|
||||
}
|
||||
|
||||
export interface DatasetImportConfig {
|
||||
source?: DataSource | string;
|
||||
target?: DataSource | string;
|
||||
dataSource?: string;
|
||||
splitByLine?: boolean;
|
||||
hasArchive?: boolean;
|
||||
[key: string]: string | number | boolean | null | undefined;
|
||||
}
|
||||
|
||||
export interface TagItem {
|
||||
id: string;
|
||||
name: string;
|
||||
@@ -84,7 +93,7 @@ export interface DatasetTask {
|
||||
status: "importing" | "waiting" | "completed" | "failed";
|
||||
progress: number;
|
||||
createdAt: string;
|
||||
importConfig: any;
|
||||
importConfig: DatasetImportConfig;
|
||||
scheduleConfig: ScheduleConfig;
|
||||
nextExecution?: string;
|
||||
lastExecution?: string;
|
||||
|
||||
Reference in New Issue
Block a user