q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'dataset_parent' into lsf

Browse Source
This commit is contained in:
Jerry Yan
2026-01-21 00:17:20 +08:00
parent d391f9fd2d d58d026256
commit 221899e194
26 changed files with 574 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
frontend/src/pages/DataManagement/Create/CreateDataset.tsx
View File

@@ -17,6 +17,7 @@ export default function DatasetCreate() {
description: "",
datasetType: DatasetType.TEXT,
tags: [],
parentDatasetId: "",
});
const handleSubmit = async () => {

12
frontend/src/pages/DataManagement/Create/EditDataset.tsx
View File

@@ -3,7 +3,7 @@ import {
queryDatasetByIdUsingGet,
updateDatasetByIdUsingPut,
} from "../dataset.api";
import { useEffect, useState } from "react";
import { useCallback, useEffect, useState } from "react";
import { Dataset, DatasetType } from "../dataset.model";
import { App, Button, Form, Modal } from "antd";
@@ -26,8 +26,9 @@ export default function EditDataset({
description: "",
datasetType: DatasetType.TEXT,
tags: [],
parentDatasetId: "",
});
const fetchDataset = async () => {
const fetchDataset = useCallback(async () => {
if (!open) return;
// 如果有id，说明是编辑模式
if (data && data.id) {
@@ -36,18 +37,19 @@ export default function EditDataset({
...newData,
type: newData.type,
tags: newData.tags.map((tag) => tag.name) || [],
parentDatasetId: newData.parentDatasetId || "",
};
setNewDataset(updatedDataset);
form.setFieldsValue(updatedDataset);
}
};
}, [data, form, open]);
useEffect(() => {
fetchDataset();
}, [data]);
}, [fetchDataset]);
const handleValuesChange = (_, allValues) => {
setNewDataset({ ...newDataset, ...allValues });
setNewDataset((prev) => ({ ...prev, ...allValues }));
};
const handleSubmit = async () => {

104
frontend/src/pages/DataManagement/Create/components/BasicInformation.tsx
View File

@@ -1,34 +1,33 @@
import RadioCard from "@/components/RadioCard";
import { Input, Select, Form } from "antd";
import { datasetTypes } from "../../dataset.const";
import { useEffect, useState } from "react";
import { queryDatasetTagsUsingGet } from "../../dataset.api";
import {queryTasksUsingGet} from "@/pages/DataCollection/collection.apis.ts";
import { useCallback, useEffect, useState } from "react";
import type { Dispatch, SetStateAction } from "react";
import { queryDatasetTagsUsingGet, queryDatasetsUsingGet } from "../../dataset.api";
import { queryTasksUsingGet } from "@/pages/DataCollection/collection.apis.ts";
import type { Dataset, TagItem } from "../../dataset.model";
export default function BasicInformation({
data,
setData,
hidden = [],
}: {
data: any;
setData: any;
data: DatasetFormData;
setData: Dispatch<SetStateAction<DatasetFormData>>;
hidden?: string[];
}) {
const [tagOptions, setTagOptions] = useState<
{
label: JSX.Element;
title: string;
options: { label: JSX.Element; value: string }[];
}[]
const [tagOptions, setTagOptions] = useState<DatasetTagOption[]>([]);
const [collectionOptions, setCollectionOptions] = useState<SelectOption[]>([]);
const [parentDatasetOptions, setParentDatasetOptions] = useState<
{ label: string; value: string }[]
>([]);
const [collectionOptions, setCollectionOptions] = useState([]);
// 获取标签
const fetchTags = async () => {
const fetchTags = useCallback(async () => {
if (hidden.includes("tags")) return;
try {
const { data } = await queryDatasetTagsUsingGet();
const customTags = data.map((tag) => ({
const tags = Array.isArray(data) ? (data as TagItem[]) : [];
const customTags = tags.map((tag) => ({
label: tag.name,
value: tag.name,
}));
@@ -36,13 +35,16 @@ export default function BasicInformation({
} catch (error) {
console.error("Error fetching tags: ", error);
}
};
}, [hidden]);
// 获取归集任务
const fetchCollectionTasks = async () => {
const fetchCollectionTasks = useCallback(async () => {
try {
const res = await queryTasksUsingGet({ page: 0, size: 100 });
const options = res.data.content.map((task: any) => ({
const tasks = Array.isArray(res?.data?.content)
? (res.data.content as CollectionTask[])
: [];
const options = tasks.map((task) => ({
label: task.name,
value: task.id,
}));
@@ -50,12 +52,40 @@ export default function BasicInformation({
} catch (error) {
console.error("Error fetching collection tasks:", error);
}
};
}, []);
const fetchParentDatasets = useCallback(async () => {
if (hidden.includes("parentDatasetId")) return;
try {
const { data: resData } = await queryDatasetsUsingGet({
parentDatasetId: "",
page: 1,
size: 1000,
});
const currentDatasetId = data?.id;
const rootDatasets = Array.isArray(resData?.content)
? (resData.content as DatasetSummary[])
: [];
const options = rootDatasets
.filter((dataset) => dataset.id !== currentDatasetId)
.map((dataset) => ({
label: dataset.name,
value: dataset.id,
}));
setParentDatasetOptions([
{ label: "根数据集", value: "" },
...options,
]);
} catch (error) {
console.error("Error fetching parent datasets:", error);
}
}, [data?.id, hidden]);
useEffect(() => {
fetchTags();
fetchCollectionTasks();
}, []);
fetchParentDatasets();
}, [fetchTags, fetchCollectionTasks, fetchParentDatasets]);
return (
<>
<Form.Item
@@ -70,6 +100,15 @@ export default function BasicInformation({
<Input.TextArea placeholder="描述数据集的用途和内容" rows={3} />
</Form.Item>
)}
{!hidden.includes("parentDatasetId") && (
<Form.Item name="parentDatasetId" label="父数据集">
<Select
className="w-full"
options={parentDatasetOptions}
placeholder="选择父数据集（仅支持一层）"
/>
</Form.Item>
)}
{/* 数据集类型选择 - 使用卡片形式 */}
{!hidden.includes("datasetType") && (
@@ -103,3 +142,28 @@ export default function BasicInformation({
</>
);
}
type DatasetFormData = Partial<Dataset> & {
type?: string;
parentDatasetId?: string;
};
type DatasetTagOption = {
label: string;
value: string;
};
type SelectOption = {
label: string;
value: string;
};
type CollectionTask = {
id: string;
name: string;
};
type DatasetSummary = {
id: string;
name: string;
};

64
frontend/src/pages/DataManagement/Detail/DatasetDetail.tsx
View File

@@ -37,29 +37,51 @@ const tabList = [
export default function DatasetDetail() {
const { id } = useParams(); // 获取动态路由参数
const navigate = useNavigate();
const [activeTab, setActiveTab] = useState("overview");
const { message } = App.useApp();
const [showEditDialog, setShowEditDialog] = useState(false);
const [dataset, setDataset] = useState<Dataset>({} as Dataset);
const filesOperation = useFilesOperation(dataset);
const [activeTab, setActiveTab] = useState("overview");
const { message } = App.useApp();
const [showEditDialog, setShowEditDialog] = useState(false);
const [dataset, setDataset] = useState<Dataset>({} as Dataset);
const [parentDataset, setParentDataset] = useState<Dataset | null>(null);
const filesOperation = useFilesOperation(dataset);
const [showUploadDialog, setShowUploadDialog] = useState(false);
const navigateItems = useMemo(
() => [
{
title: <Link to="/data/management">数据管理</Link>,
},
{
title: dataset.name || "数据集详情",
},
],
[dataset]
);
const fetchDataset = async () => {
const { data } = await queryDatasetByIdUsingGet(id as unknown as number);
setDataset(mapDataset(data));
};
const navigateItems = useMemo(() => {
const items = [
{
title: <Link to="/data/management">数据管理</Link>,
},
];
if (parentDataset) {
items.push({
title: (
<Link to={`/data/management/detail/${parentDataset.id}`}>
{parentDataset.name}
</Link>
),
});
}
items.push({
title: dataset.name || "数据集详情",
});
return items;
}, [dataset, parentDataset]);
const fetchDataset = async () => {
if (!id) {
return;
}
const { data } = await queryDatasetByIdUsingGet(id);
const mapped = mapDataset(data);
setDataset(mapped);
if (data?.parentDatasetId) {
const { data: parentData } = await queryDatasetByIdUsingGet(
data.parentDatasetId
);
setParentDataset(mapDataset(parentData));
} else {
setParentDataset(null);
}
};
useEffect(() => {
fetchDataset();

131
frontend/src/pages/DataManagement/Detail/useFilesOperation.ts
View File

@@ -3,7 +3,7 @@ import type {
DatasetFile,
} from "@/pages/DataManagement/dataset.model";
import { App } from "antd";
import { useState } from "react";
import { useState } from "react";
import {
deleteDatasetFileUsingDelete,
downloadFileByIdUsingGet,
@@ -21,7 +21,7 @@ export function useFilesOperation(dataset: Dataset) {
// 文件相关状态
const [fileList, setFileList] = useState<DatasetFile[]>([]);
const [selectedFiles, setSelectedFiles] = useState<number[]>([]);
const [selectedFiles, setSelectedFiles] = useState<string[]>([]);
const [pagination, setPagination] = useState<{
current: number;
pageSize: number;
@@ -34,16 +34,20 @@ export function useFilesOperation(dataset: Dataset) {
const [previewContent, setPreviewContent] = useState("");
const [previewFileName, setPreviewFileName] = useState("");
const fetchFiles = async (prefix?: string, current?, pageSize?) => {
// 如果明确传了 prefix（包括空字符串），使用传入的值；否则使用当前 pagination.prefix
const targetPrefix = prefix !== undefined ? prefix : (pagination.prefix || '');
const params: any = {
page: current !== undefined ? current : pagination.current,
size: pageSize !== undefined ? pageSize : pagination.pageSize,
isWithDirectory: true,
prefix: targetPrefix,
};
const fetchFiles = async (
prefix?: string,
current?: number,
pageSize?: number
) => {
// 如果明确传了 prefix（包括空字符串），使用传入的值；否则使用当前 pagination.prefix
const targetPrefix = prefix !== undefined ? prefix : (pagination.prefix || '');
const params: DatasetFilesQueryParams = {
page: current !== undefined ? current : pagination.current,
size: pageSize !== undefined ? pageSize : pagination.pageSize,
isWithDirectory: true,
prefix: targetPrefix,
};
const { data } = await queryDatasetFilesUsingGet(id!, params);
setFileList(data.content || []);
@@ -86,28 +90,28 @@ export function useFilesOperation(dataset: Dataset) {
setSelectedFiles([]); // 清空选中状态
};
const handleShowFile = (file: any) => async () => {
// 请求文件内容并弹窗预览
try {
const res = await fetch(`/api/datasets/${dataset.id}/file/${file.id}`);
const data = await res.text();
setPreviewFileName(file.fileName);
setPreviewContent(data);
setPreviewVisible(true);
} catch (err) {
message.error({ content: "文件预览失败" });
}
};
const handleDeleteFile = async (file) => {
try {
await deleteDatasetFileUsingDelete(dataset.id, file.id);
fetchFiles(); // 刷新文件列表
message.success({ content: `文件 ${file.fileName} 已删除` });
} catch (error) {
message.error({ content: `文件 ${file.fileName} 删除失败` });
}
};
const handleShowFile = (file: DatasetFile) => async () => {
// 请求文件内容并弹窗预览
try {
const res = await fetch(`/api/datasets/${dataset.id}/file/${file.id}`);
const data = await res.text();
setPreviewFileName(file.fileName);
setPreviewContent(data);
setPreviewVisible(true);
} catch {
message.error({ content: "文件预览失败" });
}
};
const handleDeleteFile = async (file: DatasetFile) => {
try {
await deleteDatasetFileUsingDelete(dataset.id, file.id);
fetchFiles(); // 刷新文件列表
message.success({ content: `文件 ${file.fileName} 已删除` });
} catch {
message.error({ content: `文件 ${file.fileName} 删除失败` });
}
};
const handleBatchExport = () => {
if (selectedFiles.length === 0) {
@@ -158,29 +162,36 @@ export function useFilesOperation(dataset: Dataset) {
// 创建成功后刷新当前目录，重置到第一页
await fetchFiles(currentPrefix, 1, pagination.pageSize);
message.success({ content: `文件夹 ${directoryName} 创建成功` });
} catch (error) {
message.error({ content: `文件夹 ${directoryName} 创建失败` });
throw error;
}
},
handleDownloadDirectory: async (directoryPath: string, directoryName: string) => {
try {
await downloadDirectoryUsingGet(dataset.id, directoryPath);
message.success({ content: `文件夹 ${directoryName} 下载成功` });
} catch (error) {
message.error({ content: `文件夹 ${directoryName} 下载失败` });
}
},
handleDeleteDirectory: async (directoryPath: string, directoryName: string) => {
try {
await deleteDirectoryUsingDelete(dataset.id, directoryPath);
// 删除成功后刷新当前目录
const currentPrefix = pagination.prefix || "";
await fetchFiles(currentPrefix, 1, pagination.pageSize);
message.success({ content: `文件夹 ${directoryName} 已删除` });
} catch (error) {
message.error({ content: `文件夹 ${directoryName} 删除失败` });
}
},
};
}
} catch (caught) {
message.error({ content: `文件夹 ${directoryName} 创建失败` });
throw caught;
}
},
handleDownloadDirectory: async (directoryPath: string, directoryName: string) => {
try {
await downloadDirectoryUsingGet(dataset.id, directoryPath);
message.success({ content: `文件夹 ${directoryName} 下载成功` });
} catch {
message.error({ content: `文件夹 ${directoryName} 下载失败` });
}
},
handleDeleteDirectory: async (directoryPath: string, directoryName: string) => {
try {
await deleteDirectoryUsingDelete(dataset.id, directoryPath);
// 删除成功后刷新当前目录
const currentPrefix = pagination.prefix || "";
await fetchFiles(currentPrefix, 1, pagination.pageSize);
message.success({ content: `文件夹 ${directoryName} 已删除` });
} catch {
message.error({ content: `文件夹 ${directoryName} 删除失败` });
}
},
};
}
interface DatasetFilesQueryParams {
page: number;
size: number;
isWithDirectory: boolean;
prefix: string;
}

90
frontend/src/pages/DataManagement/Home/DataManagement.tsx
View File

@@ -8,7 +8,8 @@ import {
} from "@ant-design/icons";
import TagManager from "@/components/business/TagManagement";
import { Link, useNavigate } from "react-router";
import { useEffect, useMemo, useState } from "react";
import { useEffect, useMemo, useState } from "react";
import type { ReactNode } from "react";
import { SearchControls } from "@/components/SearchControls";
import CardView from "@/components/CardView";
import type { Dataset } from "@/pages/DataManagement/dataset.model";
@@ -35,19 +36,19 @@ export default function DatasetManagementPage() {
const [editDatasetOpen, setEditDatasetOpen] = useState(false);
const [currentDataset, setCurrentDataset] = useState<Dataset | null>(null);
const [showUploadDialog, setShowUploadDialog] = useState(false);
const [statisticsData, setStatisticsData] = useState<any>({
count: {},
size: {},
});
const [statisticsData, setStatisticsData] = useState<StatisticsData>({
count: [],
size: [],
});
async function fetchStatistics() {
const { data } = await getDatasetStatisticsUsingGet();
const statistics = {
size: [
{
title: "数据集总数",
value: data?.totalDatasets || 0,
const statistics: StatisticsData = {
size: [
{
title: "数据集总数",
value: data?.totalDatasets || 0,
},
{
title: "文件总数",
@@ -75,10 +76,10 @@ export default function DatasetManagementPage() {
title: "视频",
value: data?.count?.video || 0,
},
],
};
setStatisticsData(statistics);
}
],
};
setStatisticsData(statistics);
}
const [tags, setTags] = useState<string[]>([]);
@@ -135,9 +136,9 @@ export default function DatasetManagementPage() {
message.success("数据集下载成功");
};
const handleDeleteDataset = async (id: number) => {
if (!id) return;
await deleteDatasetByIdUsingDelete(id);
const handleDeleteDataset = async (id: string) => {
if (!id) return;
await deleteDatasetByIdUsingDelete(id);
fetchData({ pageOffset: 0 });
message.success("数据删除成功");
};
@@ -222,12 +223,12 @@ export default function DatasetManagementPage() {
title: "状态",
dataIndex: "status",
key: "status",
render: (status: any) => {
return (
<Tag icon={status?.icon} color={status?.color}>
{status?.label}
</Tag>
);
render: (status: DatasetStatusMeta) => {
return (
<Tag icon={status?.icon} color={status?.color}>
{status?.label}
</Tag>
);
},
width: 120,
},
@@ -273,10 +274,10 @@ export default function DatasetManagementPage() {
key: "actions",
width: 200,
fixed: "right",
render: (_: any, record: Dataset) => (
<div className="flex items-center gap-2">
{operations.map((op) => (
<Tooltip key={op.key} title={op.label}>
render: (_: unknown, record: Dataset) => (
<div className="flex items-center gap-2">
{operations.map((op) => (
<Tooltip key={op.key} title={op.label}>
<Button
type="text"
icon={op.icon}
@@ -352,13 +353,13 @@ export default function DatasetManagementPage() {
<div className="grid grid-cols-1 gap-4">
<Card>
<div className="grid grid-cols-3">
{statisticsData.size?.map?.((item) => (
<Statistic
title={item.title}
key={item.title}
value={`${item.value}`}
/>
))}
{statisticsData.size.map((item) => (
<Statistic
title={item.title}
key={item.title}
value={`${item.value}`}
/>
))}
</div>
</Card>
</div>
@@ -395,5 +396,22 @@ export default function DatasetManagementPage() {
updateEvent="update:datasets"
/>
</div>
);
}
);
}
type StatisticsItem = {
title: string;
value: number | string;
};
type StatisticsData = {
count: StatisticsItem[];
size: StatisticsItem[];
};
type DatasetStatusMeta = {
label: string;
value: string;
color: string;
icon: ReactNode;
};

19
frontend/src/pages/DataManagement/dataset.model.ts
View File

@@ -33,7 +33,7 @@ export enum DataSource {
}
export interface DatasetFile {
id: number;
id: string;
fileName: string;
size: string;
uploadDate: string;
@@ -41,10 +41,10 @@ export interface DatasetFile {
}
export interface Dataset {
id: number;
id: string;
name: string;
description: string;
parentId?: number;
parentDatasetId?: string;
datasetType: DatasetType;
status: DatasetStatus;
size?: string;
@@ -58,6 +58,15 @@ export interface Dataset {
distribution?: Record<string, Record<string, number>>;
}
export interface DatasetImportConfig {
source?: DataSource | string;
target?: DataSource | string;
dataSource?: string;
splitByLine?: boolean;
hasArchive?: boolean;
[key: string]: string | number | boolean | null | undefined;
}
export interface TagItem {
id: string;
name: string;
@@ -77,14 +86,14 @@ export interface ScheduleConfig {
}
export interface DatasetTask {
id: number;
id: string;
name: string;
description: string;
type: string;
status: "importing" | "waiting" | "completed" | "failed";
progress: number;
createdAt: string;
importConfig: any;
importConfig: DatasetImportConfig;
scheduleConfig: ScheduleConfig;
nextExecution?: string;
lastExecution?: string;