q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "feat: fix the problem in the Operator Market frontend pages"

Browse Source
This commit is contained in:
Kecheng Sha
2025-12-29 12:00:37 +08:00
committed by GitHub
parent 8f30f71a68
commit 0df7a872e4
213 changed files with 45537 additions and 45547 deletions
Download Patch File Download Diff File Expand all files Collapse all files

166
frontend/src/pages/DataManagement/Create/CreateDataset.tsx
View File

@@ -1,83 +1,83 @@
import { useState } from "react";
import { ArrowLeft } from "lucide-react";
import { Button, Form, App } from "antd";
import { Link, useNavigate } from "react-router";
import { createDatasetUsingPost } from "../dataset.api";
import { DatasetType } from "../dataset.model";
import BasicInformation from "./components/BasicInformation";
export default function DatasetCreate() {
const navigate = useNavigate();
const { message } = App.useApp();
const [form] = Form.useForm();
const [newDataset, setNewDataset] = useState({
name: "",
description: "",
datasetType: DatasetType.TEXT,
tags: [],
});
const handleSubmit = async () => {
const formValues = await form.validateFields();
const params = {
...formValues,
files: undefined,
};
try {
const { data } = await createDatasetUsingPost(params);
message.success(`数据集创建成功`);
navigate("/data/management/detail/" + data.id);
} catch (error) {
console.error(error);
message.error("数据集创建失败，请重试");
return;
}
};
const handleValuesChange = (_, allValues) => {
setNewDataset({ ...newDataset, ...allValues });
};
return (
<div className="h-full flex flex-col flex-1">
{/* Header */}
<div className="flex items-center justify-between mb-2">
<div className="flex items-center">
<Link to="/data/management">
<Button type="text">
<ArrowLeft className="w-4 h-4 mr-1" />
</Button>
</Link>
<h1 className="text-xl font-bold bg-clip-text">创建数据集</h1>
</div>
</div>
{/* form */}
<div className="flex-overflow-auto border-card">
<div className="flex-1 p-6 overflow-auto">
<Form
form={form}
initialValues={newDataset}
onValuesChange={handleValuesChange}
layout="vertical"
>
<BasicInformation data={newDataset} setData={setNewDataset} />
</Form>
</div>
<div className="flex gap-2 justify-end p-6 border-top">
<Button onClick={() => navigate("/data/management")}>取消</Button>
<Button
type="primary"
disabled={!newDataset.name || !newDataset.datasetType}
onClick={handleSubmit}
>
确定
</Button>
</div>
</div>
</div>
);
}
import { useState } from "react";
import { ArrowLeft } from "lucide-react";
import { Button, Form, App } from "antd";
import { Link, useNavigate } from "react-router";
import { createDatasetUsingPost } from "../dataset.api";
import { DatasetType } from "../dataset.model";
import BasicInformation from "./components/BasicInformation";
export default function DatasetCreate() {
const navigate = useNavigate();
const { message } = App.useApp();
const [form] = Form.useForm();
const [newDataset, setNewDataset] = useState({
name: "",
description: "",
datasetType: DatasetType.TEXT,
tags: [],
});
const handleSubmit = async () => {
const formValues = await form.validateFields();
const params = {
...formValues,
files: undefined,
};
try {
const { data } = await createDatasetUsingPost(params);
message.success(`数据集创建成功`);
navigate("/data/management/detail/" + data.id);
} catch (error) {
console.error(error);
message.error("数据集创建失败，请重试");
return;
}
};
const handleValuesChange = (_, allValues) => {
setNewDataset({ ...newDataset, ...allValues });
};
return (
<div className="h-full flex flex-col flex-1">
{/* Header */}
<div className="flex items-center justify-between mb-2">
<div className="flex items-center">
<Link to="/data/management">
<Button type="text">
<ArrowLeft className="w-4 h-4 mr-1" />
</Button>
</Link>
<h1 className="text-xl font-bold bg-clip-text">创建数据集</h1>
</div>
</div>
{/* form */}
<div className="flex-overflow-auto border-card">
<div className="flex-1 p-6 overflow-auto">
<Form
form={form}
initialValues={newDataset}
onValuesChange={handleValuesChange}
layout="vertical"
>
<BasicInformation data={newDataset} setData={setNewDataset} />
</Form>
</div>
<div className="flex gap-2 justify-end p-6 border-top">
<Button onClick={() => navigate("/data/management")}>取消</Button>
<Button
type="primary"
disabled={!newDataset.name || !newDataset.datasetType}
onClick={handleSubmit}
>
确定
</Button>
</div>
</div>
</div>
);
}

204
frontend/src/pages/DataManagement/Create/EditDataset.tsx
View File

@@ -1,102 +1,102 @@
import BasicInformation from "./components/BasicInformation";
import {
queryDatasetByIdUsingGet,
updateDatasetByIdUsingPut,
} from "../dataset.api";
import { useEffect, useState } from "react";
import { Dataset, DatasetType } from "../dataset.model";
import { App, Button, Form, Modal } from "antd";
export default function EditDataset({
open,
data,
onClose,
onRefresh,
}: {
open: boolean;
data: Dataset | null;
onClose: () => void;
onRefresh?: (showMessage?: boolean) => void;
}) {
const [form] = Form.useForm();
const { message } = App.useApp();
const [newDataset, setNewDataset] = useState({
name: "",
description: "",
datasetType: DatasetType.TEXT,
tags: [],
});
const fetchDataset = async () => {
if (!open) return;
// 如果有id，说明是编辑模式
if (data && data.id) {
const { data: newData } = await queryDatasetByIdUsingGet(data.id);
const updatedDataset = {
...newData,
type: newData.type,
tags: newData.tags.map((tag) => tag.name) || [],
};
setNewDataset(updatedDataset);
form.setFieldsValue(updatedDataset);
}
};
useEffect(() => {
fetchDataset();
}, [data]);
const handleValuesChange = (_, allValues) => {
setNewDataset({ ...newDataset, ...allValues });
};
const handleSubmit = async () => {
const formValues = await form.validateFields();
const params = {
...formValues,
files: undefined,
};
try {
await updateDatasetByIdUsingPut(data?.id, params);
onClose();
message.success("数据集更新成功");
onRefresh?.(false);
} catch (error) {
console.error(error);
message.error("数据集更新失败，请重试");
return;
}
};
return (
<Modal
title={`编辑数据集${data?.name}`}
onCancel={onClose}
open={open}
width={600}
maskClosable={false}
footer={
<>
<Button onClick={onClose}>取消</Button>
<Button type="primary" onClick={handleSubmit}>
确定
</Button>
</>
}
>
<Form
form={form}
initialValues={newDataset}
onValuesChange={handleValuesChange}
layout="vertical"
>
<BasicInformation
data={newDataset}
setData={setNewDataset}
hidden={["datasetType"]}
/>
</Form>
</Modal>
);
}
import BasicInformation from "./components/BasicInformation";
import {
queryDatasetByIdUsingGet,
updateDatasetByIdUsingPut,
} from "../dataset.api";
import { useEffect, useState } from "react";
import { Dataset, DatasetType } from "../dataset.model";
import { App, Button, Form, Modal } from "antd";
export default function EditDataset({
open,
data,
onClose,
onRefresh,
}: {
open: boolean;
data: Dataset | null;
onClose: () => void;
onRefresh?: (showMessage?: boolean) => void;
}) {
const [form] = Form.useForm();
const { message } = App.useApp();
const [newDataset, setNewDataset] = useState({
name: "",
description: "",
datasetType: DatasetType.TEXT,
tags: [],
});
const fetchDataset = async () => {
if (!open) return;
// 如果有id，说明是编辑模式
if (data && data.id) {
const { data: newData } = await queryDatasetByIdUsingGet(data.id);
const updatedDataset = {
...newData,
type: newData.type,
tags: newData.tags.map((tag) => tag.name) || [],
};
setNewDataset(updatedDataset);
form.setFieldsValue(updatedDataset);
}
};
useEffect(() => {
fetchDataset();
}, [data]);
const handleValuesChange = (_, allValues) => {
setNewDataset({ ...newDataset, ...allValues });
};
const handleSubmit = async () => {
const formValues = await form.validateFields();
const params = {
...formValues,
files: undefined,
};
try {
await updateDatasetByIdUsingPut(data?.id, params);
onClose();
message.success("数据集更新成功");
onRefresh?.(false);
} catch (error) {
console.error(error);
message.error("数据集更新失败，请重试");
return;
}
};
return (
<Modal
title={`编辑数据集${data?.name}`}
onCancel={onClose}
open={open}
width={600}
maskClosable={false}
footer={
<>
<Button onClick={onClose}>取消</Button>
<Button type="primary" onClick={handleSubmit}>
确定
</Button>
</>
}
>
<Form
form={form}
initialValues={newDataset}
onValuesChange={handleValuesChange}
layout="vertical"
>
<BasicInformation
data={newDataset}
setData={setNewDataset}
hidden={["datasetType"]}
/>
</Form>
</Modal>
);
}

166
frontend/src/pages/DataManagement/Create/components/BasicInformation.tsx
View File

@@ -1,83 +1,83 @@
import RadioCard from "@/components/RadioCard";
import { Input, Select, Form } from "antd";
import { datasetTypes } from "../../dataset.const";
import { useEffect, useState } from "react";
import { queryDatasetTagsUsingGet } from "../../dataset.api";
export default function BasicInformation({
data,
setData,
hidden = [],
}: {
data: any;
setData: any;
hidden?: string[];
}) {
const [tagOptions, setTagOptions] = useState<
{
label: JSX.Element;
title: string;
options: { label: JSX.Element; value: string }[];
}[]
>([]);
// 获取标签
const fetchTags = async () => {
if (hidden.includes("tags")) return;
try {
const { data } = await queryDatasetTagsUsingGet();
const customTags = data.map((tag) => ({
label: tag.name,
value: tag.name,
}));
setTagOptions(customTags);
} catch (error) {
console.error("Error fetching tags: ", error);
}
};
useEffect(() => {
fetchTags();
}, []);
return (
<>
<Form.Item
label="名称"
name="name"
rules={[{ required: true, message: "请输入数据集名称" }]}
>
<Input placeholder="输入数据集名称" />
</Form.Item>
{!hidden.includes("description") && (
<Form.Item name="description" label="描述">
<Input.TextArea placeholder="描述数据集的用途和内容" rows={3} />
</Form.Item>
)}
{/* 数据集类型选择 - 使用卡片形式 */}
{!hidden.includes("datasetType") && (
<Form.Item
label="类型"
name="datasetType"
rules={[{ required: true, message: "请选择数据集类型" }]}
>
<RadioCard
options={datasetTypes}
value={data.type}
onChange={(datasetType) => setData({ ...data, datasetType })}
/>
</Form.Item>
)}
{!hidden.includes("tags") && (
<Form.Item label="标签" name="tags">
<Select
className="w-full"
mode="tags"
options={tagOptions}
placeholder="请选择标签"
/>
</Form.Item>
)}
</>
);
}
import RadioCard from "@/components/RadioCard";
import { Input, Select, Form } from "antd";
import { datasetTypes } from "../../dataset.const";
import { useEffect, useState } from "react";
import { queryDatasetTagsUsingGet } from "../../dataset.api";
export default function BasicInformation({
data,
setData,
hidden = [],
}: {
data: any;
setData: any;
hidden?: string[];
}) {
const [tagOptions, setTagOptions] = useState<
{
label: JSX.Element;
title: string;
options: { label: JSX.Element; value: string }[];
}[]
>([]);
// 获取标签
const fetchTags = async () => {
if (hidden.includes("tags")) return;
try {
const { data } = await queryDatasetTagsUsingGet();
const customTags = data.map((tag) => ({
label: tag.name,
value: tag.name,
}));
setTagOptions(customTags);
} catch (error) {
console.error("Error fetching tags: ", error);
}
};
useEffect(() => {
fetchTags();
}, []);
return (
<>
<Form.Item
label="名称"
name="name"
rules={[{ required: true, message: "请输入数据集名称" }]}
>
<Input placeholder="输入数据集名称" />
</Form.Item>
{!hidden.includes("description") && (
<Form.Item name="description" label="描述">
<Input.TextArea placeholder="描述数据集的用途和内容" rows={3} />
</Form.Item>
)}
{/* 数据集类型选择 - 使用卡片形式 */}
{!hidden.includes("datasetType") && (
<Form.Item
label="类型"
name="datasetType"
rules={[{ required: true, message: "请选择数据集类型" }]}
>
<RadioCard
options={datasetTypes}
value={data.type}
onChange={(datasetType) => setData({ ...data, datasetType })}
/>
</Form.Item>
)}
{!hidden.includes("tags") && (
<Form.Item label="标签" name="tags">
<Select
className="w-full"
mode="tags"
options={tagOptions}
placeholder="请选择标签"
/>
</Form.Item>
)}
</>
);
}

490
frontend/src/pages/DataManagement/Detail/DatasetDetail.tsx
View File

@@ -1,245 +1,245 @@
import { useEffect, useMemo, useState } from "react";
import { Breadcrumb, App, Tabs } from "antd";
import {
ReloadOutlined,
DownloadOutlined,
UploadOutlined,
EditOutlined,
DeleteOutlined,
} from "@ant-design/icons";
import DetailHeader from "@/components/DetailHeader";
import { mapDataset, datasetTypeMap } from "../dataset.const";
import type { Dataset } from "@/pages/DataManagement/dataset.model";
import { Link, useNavigate, useParams } from "react-router";
import { useFilesOperation } from "./useFilesOperation";
import {
createDatasetTagUsingPost,
deleteDatasetByIdUsingDelete,
downloadDatasetUsingGet,
queryDatasetByIdUsingGet,
queryDatasetTagsUsingGet,
updateDatasetByIdUsingPut,
} from "../dataset.api";
import DataQuality from "./components/DataQuality";
import DataLineageFlow from "./components/DataLineageFlow";
import Overview from "./components/Overview";
import { Activity, Clock, File, FileType } from "lucide-react";
import EditDataset from "../Create/EditDataset";
import ImportConfiguration from "./components/ImportConfiguration";
const tabList = [
{
key: "overview",
label: "概览",
},
{
key: "lineage",
label: "数据血缘",
},
{
key: "quality",
label: "数据质量",
},
];
export default function DatasetDetail() {
const { id } = useParams(); // 获取动态路由参数
const navigate = useNavigate();
const [activeTab, setActiveTab] = useState("overview");
const { message } = App.useApp();
const [showEditDialog, setShowEditDialog] = useState(false);
const [dataset, setDataset] = useState<Dataset>({} as Dataset);
const filesOperation = useFilesOperation(dataset);
const [showUploadDialog, setShowUploadDialog] = useState(false);
const navigateItems = useMemo(
() => [
{
title: <Link to="/data/management">数据管理</Link>,
},
{
title: dataset.name || "数据集详情",
},
],
[dataset]
);
const fetchDataset = async () => {
const { data } = await queryDatasetByIdUsingGet(id as unknown as number);
setDataset(mapDataset(data));
};
useEffect(() => {
fetchDataset();
filesOperation.fetchFiles();
}, []);
const handleRefresh = async (showMessage = true) => {
fetchDataset();
filesOperation.fetchFiles();
if (showMessage) message.success({ content: "数据刷新成功" });
};
const handleDownload = async () => {
await downloadDatasetUsingGet(dataset.id);
message.success("文件下载成功");
};
const handleDeleteDataset = async () => {
await deleteDatasetByIdUsingDelete(dataset.id);
navigate("/data/management");
message.success("数据集删除成功");
};
useEffect(() => {
const refreshData = () => {
handleRefresh(false);
};
window.addEventListener("update:dataset", refreshData);
return () => {
window.removeEventListener("update:dataset", refreshData);
};
}, []);
// 基本信息描述项
const statistics = [
{
icon: <File className="text-blue-400 w-4 h-4" />,
key: "file",
value: dataset?.fileCount || 0,
},
{
icon: <Activity className="text-blue-400 w-4 h-4" />,
key: "size",
value: dataset?.size || "0 B",
},
{
icon: <FileType className="text-blue-400 w-4 h-4" />,
key: "type",
value:
datasetTypeMap[dataset?.datasetType as keyof typeof datasetTypeMap]
?.label ||
dataset?.type ||
"未知",
},
{
icon: <Clock className="text-blue-400 w-4 h-4" />,
key: "time",
value: dataset?.updatedAt,
},
];
// 数据集操作列表
const operations = [
{
key: "edit",
label: "编辑",
icon: <EditOutlined />,
onClick: () => {
setShowEditDialog(true);
},
},
{
key: "upload",
label: "导入数据",
icon: <UploadOutlined />,
onClick: () => setShowUploadDialog(true),
},
{
key: "export",
label: "导出",
icon: <DownloadOutlined />,
// isDropdown: true,
// items: [
// { key: "alpaca", label: "Alpaca 格式", icon: <FileTextOutlined /> },
// { key: "jsonl", label: "JSONL 格式", icon: <DatabaseOutlined /> },
// { key: "csv", label: "CSV 格式", icon: <FileTextOutlined /> },
// { key: "coco", label: "COCO 格式", icon: <FileImageOutlined /> },
// ],
onClick: () => handleDownload(),
},
{
key: "refresh",
label: "刷新",
icon: <ReloadOutlined />,
onClick: handleRefresh,
},
{
key: "delete",
label: "删除",
danger: true,
confirm: {
title: "确认删除该数据集？",
description: "删除后该数据集将无法恢复，请谨慎操作。",
okText: "删除",
cancelText: "取消",
okType: "danger",
},
icon: <DeleteOutlined />,
onClick: handleDeleteDataset,
},
];
return (
<div className="h-full flex flex-col gap-4">
<Breadcrumb items={navigateItems} />
{/* Header */}
<DetailHeader
data={dataset}
statistics={statistics}
operations={operations}
tagConfig={{
showAdd: true,
tags: dataset.tags || [],
onFetchTags: async () => {
const res = await queryDatasetTagsUsingGet({
page: 0,
pageSize: 1000,
});
return res.data || [];
},
onCreateAndTag: async (tagName) => {
const res = await createDatasetTagUsingPost({ name: tagName });
if (res.data) {
await updateDatasetByIdUsingPut(dataset.id, {
tags: [...dataset.tags.map((tag) => tag.name), res.data.name],
});
handleRefresh();
}
},
onAddTag: async (tag) => {
const res = await updateDatasetByIdUsingPut(dataset.id, {
tags: [...dataset.tags.map((tag) => tag.name), tag],
});
if (res.data) {
handleRefresh();
}
},
}}
/>
<div className="flex-overflow-auto p-6 pt-2 bg-white rounded-md shadow">
<Tabs activeKey={activeTab} items={tabList} onChange={setActiveTab} />
<div className="h-full overflow-auto">
{activeTab === "overview" && (
<Overview dataset={dataset} filesOperation={filesOperation} fetchDataset={fetchDataset}/>
)}
{activeTab === "lineage" && <DataLineageFlow dataset={dataset} />}
{activeTab === "quality" && <DataQuality />}
</div>
</div>
<ImportConfiguration
data={dataset}
open={showUploadDialog}
onClose={() => setShowUploadDialog(false)}
updateEvent="update:dataset"
/>
<EditDataset
data={dataset}
open={showEditDialog}
onClose={() => setShowEditDialog(false)}
onRefresh={handleRefresh}
/>
</div>
);
}
import { useEffect, useMemo, useState } from "react";
import { Breadcrumb, App, Tabs } from "antd";
import {
ReloadOutlined,
DownloadOutlined,
UploadOutlined,
EditOutlined,
DeleteOutlined,
} from "@ant-design/icons";
import DetailHeader from "@/components/DetailHeader";
import { mapDataset, datasetTypeMap } from "../dataset.const";
import type { Dataset } from "@/pages/DataManagement/dataset.model";
import { Link, useNavigate, useParams } from "react-router";
import { useFilesOperation } from "./useFilesOperation";
import {
createDatasetTagUsingPost,
deleteDatasetByIdUsingDelete,
downloadDatasetUsingGet,
queryDatasetByIdUsingGet,
queryDatasetTagsUsingGet,
updateDatasetByIdUsingPut,
} from "../dataset.api";
import DataQuality from "./components/DataQuality";
import DataLineageFlow from "./components/DataLineageFlow";
import Overview from "./components/Overview";
import { Activity, Clock, File, FileType } from "lucide-react";
import EditDataset from "../Create/EditDataset";
import ImportConfiguration from "./components/ImportConfiguration";
const tabList = [
{
key: "overview",
label: "概览",
},
{
key: "lineage",
label: "数据血缘",
},
{
key: "quality",
label: "数据质量",
},
];
export default function DatasetDetail() {
const { id } = useParams(); // 获取动态路由参数
const navigate = useNavigate();
const [activeTab, setActiveTab] = useState("overview");
const { message } = App.useApp();
const [showEditDialog, setShowEditDialog] = useState(false);
const [dataset, setDataset] = useState<Dataset>({} as Dataset);
const filesOperation = useFilesOperation(dataset);
const [showUploadDialog, setShowUploadDialog] = useState(false);
const navigateItems = useMemo(
() => [
{
title: <Link to="/data/management">数据管理</Link>,
},
{
title: dataset.name || "数据集详情",
},
],
[dataset]
);
const fetchDataset = async () => {
const { data } = await queryDatasetByIdUsingGet(id as unknown as number);
setDataset(mapDataset(data));
};
useEffect(() => {
fetchDataset();
filesOperation.fetchFiles();
}, []);
const handleRefresh = async (showMessage = true) => {
fetchDataset();
filesOperation.fetchFiles();
if (showMessage) message.success({ content: "数据刷新成功" });
};
const handleDownload = async () => {
await downloadDatasetUsingGet(dataset.id);
message.success("文件下载成功");
};
const handleDeleteDataset = async () => {
await deleteDatasetByIdUsingDelete(dataset.id);
navigate("/data/management");
message.success("数据集删除成功");
};
useEffect(() => {
const refreshData = () => {
handleRefresh(false);
};
window.addEventListener("update:dataset", refreshData);
return () => {
window.removeEventListener("update:dataset", refreshData);
};
}, []);
// 基本信息描述项
const statistics = [
{
icon: <File className="text-blue-400 w-4 h-4" />,
key: "file",
value: dataset?.fileCount || 0,
},
{
icon: <Activity className="text-blue-400 w-4 h-4" />,
key: "size",
value: dataset?.size || "0 B",
},
{
icon: <FileType className="text-blue-400 w-4 h-4" />,
key: "type",
value:
datasetTypeMap[dataset?.datasetType as keyof typeof datasetTypeMap]
?.label ||
dataset?.type ||
"未知",
},
{
icon: <Clock className="text-blue-400 w-4 h-4" />,
key: "time",
value: dataset?.updatedAt,
},
];
// 数据集操作列表
const operations = [
{
key: "edit",
label: "编辑",
icon: <EditOutlined />,
onClick: () => {
setShowEditDialog(true);
},
},
{
key: "upload",
label: "导入数据",
icon: <UploadOutlined />,
onClick: () => setShowUploadDialog(true),
},
{
key: "export",
label: "导出",
icon: <DownloadOutlined />,
// isDropdown: true,
// items: [
// { key: "alpaca", label: "Alpaca 格式", icon: <FileTextOutlined /> },
// { key: "jsonl", label: "JSONL 格式", icon: <DatabaseOutlined /> },
// { key: "csv", label: "CSV 格式", icon: <FileTextOutlined /> },
// { key: "coco", label: "COCO 格式", icon: <FileImageOutlined /> },
// ],
onClick: () => handleDownload(),
},
{
key: "refresh",
label: "刷新",
icon: <ReloadOutlined />,
onClick: handleRefresh,
},
{
key: "delete",
label: "删除",
danger: true,
confirm: {
title: "确认删除该数据集？",
description: "删除后该数据集将无法恢复，请谨慎操作。",
okText: "删除",
cancelText: "取消",
okType: "danger",
},
icon: <DeleteOutlined />,
onClick: handleDeleteDataset,
},
];
return (
<div className="h-full flex flex-col gap-4">
<Breadcrumb items={navigateItems} />
{/* Header */}
<DetailHeader
data={dataset}
statistics={statistics}
operations={operations}
tagConfig={{
showAdd: true,
tags: dataset.tags || [],
onFetchTags: async () => {
const res = await queryDatasetTagsUsingGet({
page: 0,
pageSize: 1000,
});
return res.data || [];
},
onCreateAndTag: async (tagName) => {
const res = await createDatasetTagUsingPost({ name: tagName });
if (res.data) {
await updateDatasetByIdUsingPut(dataset.id, {
tags: [...dataset.tags.map((tag) => tag.name), res.data.name],
});
handleRefresh();
}
},
onAddTag: async (tag) => {
const res = await updateDatasetByIdUsingPut(dataset.id, {
tags: [...dataset.tags.map((tag) => tag.name), tag],
});
if (res.data) {
handleRefresh();
}
},
}}
/>
<div className="flex-overflow-auto p-6 pt-2 bg-white rounded-md shadow">
<Tabs activeKey={activeTab} items={tabList} onChange={setActiveTab} />
<div className="h-full overflow-auto">
{activeTab === "overview" && (
<Overview dataset={dataset} filesOperation={filesOperation} fetchDataset={fetchDataset}/>
)}
{activeTab === "lineage" && <DataLineageFlow dataset={dataset} />}
{activeTab === "quality" && <DataQuality />}
</div>
</div>
<ImportConfiguration
data={dataset}
open={showUploadDialog}
onClose={() => setShowUploadDialog(false)}
updateEvent="update:dataset"
/>
<EditDataset
data={dataset}
open={showEditDialog}
onClose={() => setShowEditDialog(false)}
onRefresh={handleRefresh}
/>
</div>
);
}

104
frontend/src/pages/DataManagement/Detail/components/DataLineageFlow.tsx
View File

@@ -1,52 +1,52 @@
import DevelopmentInProgress from "@/components/DevelopmentInProgress";
import { Dataset } from "../../dataset.model";
export default function DataLineageFlow(dataset: Dataset) {
return <DevelopmentInProgress showHome={false} />
const lineage = dataset.lineage;
if (!lineage) return null;
const steps = [
{ name: "数据源", value: lineage.source, icon: Database },
...lineage.processing.map((step, index) => ({
name: `处理${index + 1}`,
value: step,
icon: GitBranch,
})),
];
if (lineage.training) {
steps.push({
name: "模型训练",
value: `${lineage.training.model} (准确率: ${lineage.training.accuracy}%)`,
icon: Target,
});
}
return (
<div className="space-y-4">
<div className="relative">
{steps.map((step, index) => (
<div key={index} className="flex items-start gap-4 pb-8 last:pb-0">
<div className="flex flex-col items-center">
<div className="w-12 h-12 bg-gradient-to-br from-blue-500 to-indigo-600 rounded-full flex items-center justify-center shadow-lg">
<step.icon className="w-6 h-6 text-white" />
</div>
{index < steps.length - 1 && (
<div className="w-0.5 h-12 bg-gradient-to-b from-blue-200 to-indigo-200 mt-2"></div>
)}
</div>
<div className="flex-1 pt-3">
<div className="bg-white rounded-lg border border-gray-200 p-4 shadow-sm hover:shadow-md transition-shadow">
<h5 className="font-semibold text-gray-900 mb-1">
{step.name}
</h5>
<p className="text-sm text-gray-600">{step.value}</p>
</div>
</div>
</div>
))}
</div>
</div>
);
}
import DevelopmentInProgress from "@/components/DevelopmentInProgress";
import { Dataset } from "../../dataset.model";
export default function DataLineageFlow(dataset: Dataset) {
return <DevelopmentInProgress showHome={false} />
const lineage = dataset.lineage;
if (!lineage) return null;
const steps = [
{ name: "数据源", value: lineage.source, icon: Database },
...lineage.processing.map((step, index) => ({
name: `处理${index + 1}`,
value: step,
icon: GitBranch,
})),
];
if (lineage.training) {
steps.push({
name: "模型训练",
value: `${lineage.training.model} (准确率: ${lineage.training.accuracy}%)`,
icon: Target,
});
}
return (
<div className="space-y-4">
<div className="relative">
{steps.map((step, index) => (
<div key={index} className="flex items-start gap-4 pb-8 last:pb-0">
<div className="flex flex-col items-center">
<div className="w-12 h-12 bg-gradient-to-br from-blue-500 to-indigo-600 rounded-full flex items-center justify-center shadow-lg">
<step.icon className="w-6 h-6 text-white" />
</div>
{index < steps.length - 1 && (
<div className="w-0.5 h-12 bg-gradient-to-b from-blue-200 to-indigo-200 mt-2"></div>
)}
</div>
<div className="flex-1 pt-3">
<div className="bg-white rounded-lg border border-gray-200 p-4 shadow-sm hover:shadow-md transition-shadow">
<h5 className="font-semibold text-gray-900 mb-1">
{step.name}
</h5>
<p className="text-sm text-gray-600">{step.value}</p>
</div>
</div>
</div>
))}
</div>
</div>
);
}

154
frontend/src/pages/DataManagement/Detail/components/DataQuality.tsx
View File

@@ -1,77 +1,77 @@
import DevelopmentInProgress from "@/components/DevelopmentInProgress";
import { Card } from "antd";
import { AlertTriangle } from "lucide-react";
export default function DataQuality() {
return <DevelopmentInProgress showHome={false} />
return (
<div className=" mt-0">
<div className="grid md:grid-cols-2 gap-6">
<Card title="质量分布">
{[
{ metric: "图像清晰度", value: 96.2, color: "bg-green-500" },
{ metric: "色彩一致性", value: 94.8, color: "bg-blue-500" },
{ metric: "标注完整性", value: 98.1, color: "bg-purple-500" },
].map((item, index) => (
<div key={index} className="space-y-2">
<div className="flex justify-between text-sm">
<span>{item.metric}</span>
<span className="font-semibold">{item.value}%</span>
</div>
<div className="w-full bg-gray-200 rounded-full h-3">
<div
className={`${item.color} h-3 rounded-full transition-all duration-500`}
style={{ width: `${item.value}%` }}
></div>
</div>
</div>
))}
</Card>
<Card title="数据完整性">
{[
{ metric: "文件完整性", value: 99.7, color: "bg-green-500" },
{ metric: "元数据完整性", value: 97.3, color: "bg-blue-500" },
{ metric: "标签一致性", value: 95.6, color: "bg-purple-500" },
].map((item, index) => (
<div key={index} className="space-y-2">
<div className="flex justify-between text-sm">
<span>{item.metric}</span>
<span className="font-semibold">{item.value}%</span>
</div>
<div className="w-full bg-gray-200 rounded-full h-3">
<div
className={`${item.color} h-3 rounded-full transition-all duration-500`}
style={{ width: `${item.value}%` }}
></div>
</div>
</div>
))}
</Card>
</div>
<Card className="bg-gradient-to-r from-yellow-50 to-orange-50 border-yellow-200">
<div className="flex items-start gap-4">
<AlertTriangle className="w-6 h-6 text-yellow-600 mt-1 flex-shrink-0" />
<div>
<h4 className="font-semibold text-yellow-800 mb-2">质量改进建议</h4>
<ul className="text-sm text-yellow-700 space-y-2">
<li className="flex items-start gap-2">
<span className="w-1.5 h-1.5 bg-yellow-600 rounded-full mt-2 flex-shrink-0"></span>
建议对42张图像进行重新标注以提高准确性
</li>
<li className="flex items-start gap-2">
<span className="w-1.5 h-1.5 bg-yellow-600 rounded-full mt-2 flex-shrink-0"></span>
检查并补充缺失的病理分级信息
</li>
<li className="flex items-start gap-2">
<span className="w-1.5 h-1.5 bg-yellow-600 rounded-full mt-2 flex-shrink-0"></span>
考虑增加更多低分化样本以平衡数据分布
</li>
</ul>
</div>
</div>
</Card>
</div>
);
}
import DevelopmentInProgress from "@/components/DevelopmentInProgress";
import { Card } from "antd";
import { AlertTriangle } from "lucide-react";
export default function DataQuality() {
return <DevelopmentInProgress showHome={false} />
return (
<div className=" mt-0">
<div className="grid md:grid-cols-2 gap-6">
<Card title="质量分布">
{[
{ metric: "图像清晰度", value: 96.2, color: "bg-green-500" },
{ metric: "色彩一致性", value: 94.8, color: "bg-blue-500" },
{ metric: "标注完整性", value: 98.1, color: "bg-purple-500" },
].map((item, index) => (
<div key={index} className="space-y-2">
<div className="flex justify-between text-sm">
<span>{item.metric}</span>
<span className="font-semibold">{item.value}%</span>
</div>
<div className="w-full bg-gray-200 rounded-full h-3">
<div
className={`${item.color} h-3 rounded-full transition-all duration-500`}
style={{ width: `${item.value}%` }}
></div>
</div>
</div>
))}
</Card>
<Card title="数据完整性">
{[
{ metric: "文件完整性", value: 99.7, color: "bg-green-500" },
{ metric: "元数据完整性", value: 97.3, color: "bg-blue-500" },
{ metric: "标签一致性", value: 95.6, color: "bg-purple-500" },
].map((item, index) => (
<div key={index} className="space-y-2">
<div className="flex justify-between text-sm">
<span>{item.metric}</span>
<span className="font-semibold">{item.value}%</span>
</div>
<div className="w-full bg-gray-200 rounded-full h-3">
<div
className={`${item.color} h-3 rounded-full transition-all duration-500`}
style={{ width: `${item.value}%` }}
></div>
</div>
</div>
))}
</Card>
</div>
<Card className="bg-gradient-to-r from-yellow-50 to-orange-50 border-yellow-200">
<div className="flex items-start gap-4">
<AlertTriangle className="w-6 h-6 text-yellow-600 mt-1 flex-shrink-0" />
<div>
<h4 className="font-semibold text-yellow-800 mb-2">质量改进建议</h4>
<ul className="text-sm text-yellow-700 space-y-2">
<li className="flex items-start gap-2">
<span className="w-1.5 h-1.5 bg-yellow-600 rounded-full mt-2 flex-shrink-0"></span>
建议对42张图像进行重新标注以提高准确性
</li>
<li className="flex items-start gap-2">
<span className="w-1.5 h-1.5 bg-yellow-600 rounded-full mt-2 flex-shrink-0"></span>
检查并补充缺失的病理分级信息
</li>
<li className="flex items-start gap-2">
<span className="w-1.5 h-1.5 bg-yellow-600 rounded-full mt-2 flex-shrink-0"></span>
考虑增加更多低分化样本以平衡数据分布
</li>
</ul>
</div>
</div>
</Card>
</div>
);
}

554
frontend/src/pages/DataManagement/Detail/components/ImportConfiguration.tsx
View File

@@ -1,277 +1,277 @@
import { Select, Input, Form, Radio, Modal, Button, UploadFile, Switch } from "antd";
import { InboxOutlined } from "@ant-design/icons";
import { dataSourceOptions } from "../../dataset.const";
import { Dataset, DataSource } from "../../dataset.model";
import { useEffect, useMemo, useState } from "react";
import { queryTasksUsingGet } from "@/pages/DataCollection/collection.apis";
import { updateDatasetByIdUsingPut } from "../../dataset.api";
import { sliceFile } from "@/utils/file.util";
import Dragger from "antd/es/upload/Dragger";
export default function ImportConfiguration({
data,
open,
onClose,
updateEvent = "update:dataset",
}: {
data: Dataset | null;
open: boolean;
onClose: () => void;
updateEvent?: string;
}) {
const [form] = Form.useForm();
const [collectionOptions, setCollectionOptions] = useState([]);
const [importConfig, setImportConfig] = useState<any>({
source: DataSource.UPLOAD,
});
const [fileList, setFileList] = useState<UploadFile[]>([]);
const fileSliceList = useMemo(() => {
const sliceList = fileList.map((file) => {
const slices = sliceFile(file);
return { originFile: file, slices, name: file.name, size: file.size };
});
return sliceList;
}, [fileList]);
// 本地上传文件相关逻辑
const resetFiles = () => {
setFileList([]);
};
const handleUpload = async (dataset: Dataset) => {
const formData = new FormData();
fileList.forEach((file) => {
formData.append("file", file);
});
window.dispatchEvent(
new CustomEvent("upload:dataset", {
detail: {
dataset,
files: fileSliceList,
updateEvent,
hasArchive: importConfig.hasArchive,
},
})
);
resetFiles();
};
const handleBeforeUpload = (_, files: UploadFile[]) => {
setFileList([...fileList, ...files]);
return false;
};
const handleRemoveFile = (file: UploadFile) => {
setFileList((prev) => prev.filter((f) => f.uid !== file.uid));
};
const fetchCollectionTasks = async () => {
if (importConfig.source !== DataSource.COLLECTION) return;
try {
const res = await queryTasksUsingGet({ page: 0, size: 100 });
const options = res.data.content.map((task: any) => ({
label: task.name,
value: task.id,
}));
setCollectionOptions(options);
} catch (error) {
console.error("Error fetching collection tasks:", error);
}
};
const resetState = () => {
form.resetFields();
setFileList([]);
form.setFieldsValue({ files: null });
setImportConfig({ source: importConfig.source ? importConfig.source : DataSource.UPLOAD });
};
const handleImportData = async () => {
if (!data) return;
if (importConfig.source === DataSource.UPLOAD) {
await handleUpload(data);
} else if (importConfig.source === DataSource.COLLECTION) {
await updateDatasetByIdUsingPut(data.id, {
...importConfig,
});
}
onClose();
};
useEffect(() => {
if (open) {
resetState();
fetchCollectionTasks();
}
}, [open, importConfig.source]);
return (
<Modal
title="导入数据"
open={open}
width={600}
onCancel={() => {
onClose();
resetState();
}}
maskClosable={false}
footer={
<>
<Button onClick={onClose}>取消</Button>
<Button
type="primary"
disabled={!fileList?.length && !importConfig.dataSource}
onClick={handleImportData}
>
确定
</Button>
</>
}
>
<Form
form={form}
layout="vertical"
initialValues={importConfig || {}}
onValuesChange={(_, allValues) => setImportConfig(allValues)}
>
<Form.Item
label="数据源"
name="source"
rules={[{ required: true, message: "请选择数据源" }]}
>
<Radio.Group
buttonStyle="solid"
options={dataSourceOptions}
optionType="button"
/>
</Form.Item>
{importConfig?.source === DataSource.COLLECTION && (
<Form.Item name="dataSource" label="归集任务" required>
<Select placeholder="请选择归集任务" options={collectionOptions} />
</Form.Item>
)}
{/* obs import */}
{importConfig?.source === DataSource.OBS && (
<div className="grid grid-cols-2 gap-3 p-4 bg-blue-50 rounded-lg">
<Form.Item
name="endpoint"
rules={[{ required: true }]}
label="Endpoint"
>
<Input
className="h-8 text-xs"
placeholder="obs.cn-north-4.myhuaweicloud.com"
/>
</Form.Item>
<Form.Item
name="bucket"
rules={[{ required: true }]}
label="Bucket"
>
<Input className="h-8 text-xs" placeholder="my-bucket" />
</Form.Item>
<Form.Item
name="accessKey"
rules={[{ required: true }]}
label="Access Key"
>
<Input className="h-8 text-xs" placeholder="Access Key" />
</Form.Item>
<Form.Item
name="secretKey"
rules={[{ required: true }]}
label="Secret Key"
>
<Input
type="password"
className="h-8 text-xs"
placeholder="Secret Key"
/>
</Form.Item>
</div>
)}
{/* Local Upload Component */}
{importConfig?.source === DataSource.UPLOAD && (
<>
<Form.Item
label="自动解压上传的压缩包"
name="hasArchive"
valuePropName="checked"
initialValue={true}
>
<Switch />
</Form.Item>
<Form.Item
label="上传文件"
name="files"
rules={[
{
required: true,
message: "请上传文件",
},
]}
>
<Dragger
className="w-full"
onRemove={handleRemoveFile}
beforeUpload={handleBeforeUpload}
multiple
>
<p className="ant-upload-drag-icon">
<InboxOutlined />
</p>
<p className="ant-upload-text">本地文件上传</p>
<p className="ant-upload-hint">拖拽文件到此处或点击选择文件</p>
</Dragger>
</Form.Item>
</>
)}
{/* Target Configuration */}
{importConfig?.target && importConfig?.target !== DataSource.UPLOAD && (
<div className="space-y-3 p-4 bg-blue-50 rounded-lg">
{importConfig?.target === DataSource.DATABASE && (
<div className="grid grid-cols-2 gap-3">
<Form.Item
name="databaseType"
rules={[{ required: true }]}
label="数据库类型"
>
<Select
className="w-full"
options={[
{ label: "MySQL", value: "mysql" },
{ label: "PostgreSQL", value: "postgresql" },
{ label: "MongoDB", value: "mongodb" },
]}
></Select>
</Form.Item>
<Form.Item
name="tableName"
rules={[{ required: true }]}
label="表名"
>
<Input className="h-8 text-xs" placeholder="dataset_table" />
</Form.Item>
<Form.Item
name="connectionString"
rules={[{ required: true }]}
label="连接字符串"
>
<Input
className="h-8 text-xs col-span-2"
placeholder="数据库连接字符串"
/>
</Form.Item>
</div>
)}
</div>
)}
</Form>
</Modal>
);
}
import { Select, Input, Form, Radio, Modal, Button, UploadFile, Switch } from "antd";
import { InboxOutlined } from "@ant-design/icons";
import { dataSourceOptions } from "../../dataset.const";
import { Dataset, DataSource } from "../../dataset.model";
import { useEffect, useMemo, useState } from "react";
import { queryTasksUsingGet } from "@/pages/DataCollection/collection.apis";
import { updateDatasetByIdUsingPut } from "../../dataset.api";
import { sliceFile } from "@/utils/file.util";
import Dragger from "antd/es/upload/Dragger";
export default function ImportConfiguration({
data,
open,
onClose,
updateEvent = "update:dataset",
}: {
data: Dataset | null;
open: boolean;
onClose: () => void;
updateEvent?: string;
}) {
const [form] = Form.useForm();
const [collectionOptions, setCollectionOptions] = useState([]);
const [importConfig, setImportConfig] = useState<any>({
source: DataSource.UPLOAD,
});
const [fileList, setFileList] = useState<UploadFile[]>([]);
const fileSliceList = useMemo(() => {
const sliceList = fileList.map((file) => {
const slices = sliceFile(file);
return { originFile: file, slices, name: file.name, size: file.size };
});
return sliceList;
}, [fileList]);
// 本地上传文件相关逻辑
const resetFiles = () => {
setFileList([]);
};
const handleUpload = async (dataset: Dataset) => {
const formData = new FormData();
fileList.forEach((file) => {
formData.append("file", file);
});
window.dispatchEvent(
new CustomEvent("upload:dataset", {
detail: {
dataset,
files: fileSliceList,
updateEvent,
hasArchive: importConfig.hasArchive,
},
})
);
resetFiles();
};
const handleBeforeUpload = (_, files: UploadFile[]) => {
setFileList([...fileList, ...files]);
return false;
};
const handleRemoveFile = (file: UploadFile) => {
setFileList((prev) => prev.filter((f) => f.uid !== file.uid));
};
const fetchCollectionTasks = async () => {
if (importConfig.source !== DataSource.COLLECTION) return;
try {
const res = await queryTasksUsingGet({ page: 0, size: 100 });
const options = res.data.content.map((task: any) => ({
label: task.name,
value: task.id,
}));
setCollectionOptions(options);
} catch (error) {
console.error("Error fetching collection tasks:", error);
}
};
const resetState = () => {
form.resetFields();
setFileList([]);
form.setFieldsValue({ files: null });
setImportConfig({ source: importConfig.source ? importConfig.source : DataSource.UPLOAD });
};
const handleImportData = async () => {
if (!data) return;
if (importConfig.source === DataSource.UPLOAD) {
await handleUpload(data);
} else if (importConfig.source === DataSource.COLLECTION) {
await updateDatasetByIdUsingPut(data.id, {
...importConfig,
});
}
onClose();
};
useEffect(() => {
if (open) {
resetState();
fetchCollectionTasks();
}
}, [open, importConfig.source]);
return (
<Modal
title="导入数据"
open={open}
width={600}
onCancel={() => {
onClose();
resetState();
}}
maskClosable={false}
footer={
<>
<Button onClick={onClose}>取消</Button>
<Button
type="primary"
disabled={!fileList?.length && !importConfig.dataSource}
onClick={handleImportData}
>
确定
</Button>
</>
}
>
<Form
form={form}
layout="vertical"
initialValues={importConfig || {}}
onValuesChange={(_, allValues) => setImportConfig(allValues)}
>
<Form.Item
label="数据源"
name="source"
rules={[{ required: true, message: "请选择数据源" }]}
>
<Radio.Group
buttonStyle="solid"
options={dataSourceOptions}
optionType="button"
/>
</Form.Item>
{importConfig?.source === DataSource.COLLECTION && (
<Form.Item name="dataSource" label="归集任务" required>
<Select placeholder="请选择归集任务" options={collectionOptions} />
</Form.Item>
)}
{/* obs import */}
{importConfig?.source === DataSource.OBS && (
<div className="grid grid-cols-2 gap-3 p-4 bg-blue-50 rounded-lg">
<Form.Item
name="endpoint"
rules={[{ required: true }]}
label="Endpoint"
>
<Input
className="h-8 text-xs"
placeholder="obs.cn-north-4.myhuaweicloud.com"
/>
</Form.Item>
<Form.Item
name="bucket"
rules={[{ required: true }]}
label="Bucket"
>
<Input className="h-8 text-xs" placeholder="my-bucket" />
</Form.Item>
<Form.Item
name="accessKey"
rules={[{ required: true }]}
label="Access Key"
>
<Input className="h-8 text-xs" placeholder="Access Key" />
</Form.Item>
<Form.Item
name="secretKey"
rules={[{ required: true }]}
label="Secret Key"
>
<Input
type="password"
className="h-8 text-xs"
placeholder="Secret Key"
/>
</Form.Item>
</div>
)}
{/* Local Upload Component */}
{importConfig?.source === DataSource.UPLOAD && (
<>
<Form.Item
label="自动解压上传的压缩包"
name="hasArchive"
valuePropName="checked"
initialValue={true}
>
<Switch />
</Form.Item>
<Form.Item
label="上传文件"
name="files"
rules={[
{
required: true,
message: "请上传文件",
},
]}
>
<Dragger
className="w-full"
onRemove={handleRemoveFile}
beforeUpload={handleBeforeUpload}
multiple
>
<p className="ant-upload-drag-icon">
<InboxOutlined />
</p>
<p className="ant-upload-text">本地文件上传</p>
<p className="ant-upload-hint">拖拽文件到此处或点击选择文件</p>
</Dragger>
</Form.Item>
</>
)}
{/* Target Configuration */}
{importConfig?.target && importConfig?.target !== DataSource.UPLOAD && (
<div className="space-y-3 p-4 bg-blue-50 rounded-lg">
{importConfig?.target === DataSource.DATABASE && (
<div className="grid grid-cols-2 gap-3">
<Form.Item
name="databaseType"
rules={[{ required: true }]}
label="数据库类型"
>
<Select
className="w-full"
options={[
{ label: "MySQL", value: "mysql" },
{ label: "PostgreSQL", value: "postgresql" },
{ label: "MongoDB", value: "mongodb" },
]}
></Select>
</Form.Item>
<Form.Item
name="tableName"
rules={[{ required: true }]}
label="表名"
>
<Input className="h-8 text-xs" placeholder="dataset_table" />
</Form.Item>
<Form.Item
name="connectionString"
rules={[{ required: true }]}
label="连接字符串"
>
<Input
className="h-8 text-xs col-span-2"
placeholder="数据库连接字符串"
/>
</Form.Item>
</div>
)}
</div>
)}
</Form>
</Modal>
);
}

632
frontend/src/pages/DataManagement/Detail/components/Overview.tsx
View File

@@ -1,316 +1,316 @@
import { Button, Descriptions, DescriptionsProps, Modal, Table } from "antd";
import { formatBytes, formatDateTime } from "@/utils/unit";
import { Download, Trash2, Folder, File } from "lucide-react";
import { datasetTypeMap } from "../../dataset.const";
export default function Overview({ dataset, filesOperation, fetchDataset }) {
const {
fileList,
pagination,
selectedFiles,
setSelectedFiles,
previewVisible,
previewFileName,
previewContent,
setPreviewVisible,
handleDeleteFile,
handleDownloadFile,
handleBatchDeleteFiles,
handleBatchExport,
} = filesOperation;
// 文件列表多选配置
const rowSelection = {
onChange: (selectedRowKeys: React.Key[], selectedRows: any[]) => {
setSelectedFiles(selectedRowKeys as number[]);
console.log(
`selectedRowKeys: ${selectedRowKeys}`,
"selectedRows: ",
selectedRows
);
},
};
// 基本信息
const items: DescriptionsProps["items"] = [
{
key: "id",
label: "ID",
children: dataset.id,
},
{
key: "name",
label: "名称",
children: dataset.name,
},
{
key: "fileCount",
label: "文件数",
children: dataset.fileCount || 0,
},
{
key: "size",
label: "数据大小",
children: dataset.size || "0 B",
},
{
key: "datasetType",
label: "类型",
children: datasetTypeMap[dataset?.datasetType]?.label || "未知",
},
{
key: "status",
label: "状态",
children: dataset?.status?.label || "未知",
},
{
key: "createdBy",
label: "创建者",
children: dataset.createdBy || "未知",
},
{
key: "targetLocation",
label: "存储路径",
children: dataset.targetLocation || "未知",
},
{
key: "pvcName",
label: "存储名称",
children: dataset.pvcName || "未知",
},
{
key: "createdAt",
label: "创建时间",
children: dataset.createdAt,
},
{
key: "updatedAt",
label: "更新时间",
children: dataset.updatedAt,
},
{
key: "description",
label: "描述",
children: dataset.description || "无",
},
];
// 文件列表列定义
const columns = [
{
title: "文件名",
dataIndex: "fileName",
key: "fileName",
fixed: "left",
render: (text: string, record: any) => {
const isDirectory = record.id.startsWith('directory-');
const iconSize = 16;
const content = (
<div className="flex items-center">
{isDirectory ? (
<Folder className="mr-2 text-blue-500" size={iconSize} />
) : (
<File className="mr-2 text-black" size={iconSize} />
)}
<span className="truncate text-black">{text}</span>
</div>
);
if (isDirectory) {
return (
<Button
type="link"
onClick={(e) => {
const currentPath = filesOperation.pagination.prefix || '';
const newPath = `${currentPath}${record.fileName}`;
filesOperation.fetchFiles(newPath);
}}
>
{content}
</Button>
);
}
return (
<Button
type="link"
onClick={(e) => {}}
>
{content}
</Button>
);
},
},
{
title: "大小",
dataIndex: "fileSize",
key: "fileSize",
width: 150,
render: (text: number, record: any) => {
const isDirectory = record.id.startsWith('directory-');
if (isDirectory) {
return "-";
}
return formatBytes(text)
},
},
{
title: "上传时间",
dataIndex: "uploadTime",
key: "uploadTime",
width: 200,
render: (text) => formatDateTime(text),
},
{
title: "操作",
key: "action",
width: 180,
fixed: "right",
render: (_, record) => {
const isDirectory = record.id.startsWith('directory-');
if (isDirectory) {
return <div className="flex"/>;
}
return (
<div className="flex">
<Button
size="small"
type="link"
onClick={() => handleDownloadFile(record)}
>
下载
</Button>
<Button
size="small"
type="link"
onClick={async () => {
await handleDeleteFile(record);
fetchDataset()
}
}
>
删除
</Button>
</div>
)},
},
];
return (
<>
<div className=" flex flex-col gap-4">
{/* 基本信息 */}
<Descriptions
title="基本信息"
layout="vertical"
size="small"
items={items}
column={5}
/>
{/* 文件列表 */}
<h2 className="text-base font-semibold mt-8">文件列表</h2>
{selectedFiles.length > 0 && (
<div className="flex items-center gap-2 p-3 bg-blue-50 rounded-lg border border-blue-200">
<span className="text-sm text-blue-700 font-medium">
已选择 {selectedFiles.length} 个文件
</span>
<Button
onClick={handleBatchExport}
className="ml-auto bg-transparent"
>
<Download className="w-4 h-4 mr-2" />
批量导出
</Button>
<Button
onClick={handleBatchDeleteFiles}
className="text-red-600 hover:text-red-700 hover:bg-red-50 bg-transparent"
>
<Trash2 className="w-4 h-4 mr-2" />
批量删除
</Button>
</div>
)}
<div className="overflow-x-auto">
<div className="mb-2">
{(filesOperation.pagination.prefix || '') !== '' && (
<Button
type="link"
onClick={() => {
// 获取上一级目录
const currentPath = filesOperation.pagination.prefix || '';
const pathParts = currentPath.split('/').filter(Boolean);
pathParts.pop(); // 移除最后一个目录
const parentPath = pathParts.length > 0 ? `${pathParts.join('/')}/` : '';
filesOperation.fetchFiles(parentPath);
}}
className="p-0"
>
<span className="flex items-center text-blue-500">
<svg
className="w-4 h-4 mr-1"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
xmlns="http://www.w3.org/2000/svg"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M10 19l-7-7m0 0l7-7m-7 7h18"
/>
</svg>
返回上一级
</span>
</Button>
)}
{filesOperation.pagination.prefix && (
<span className="ml-2 text-gray-600">当前路径: {filesOperation.pagination.prefix}</span>
)}
</div>
<Table
size="middle"
rowKey="id"
columns={columns}
dataSource={fileList}
// rowSelection={rowSelection}
scroll={{ x: "max-content", y: 600 }}
pagination={{
...pagination,
showTotal: (total) => `共 ${total} 条`,
onChange: (page, pageSize) => {
filesOperation.setPagination(prev => ({
...prev,
current: page,
pageSize: pageSize
}));
filesOperation.fetchFiles(pagination.prefix, page, pageSize);
}
}}
/>
</div>
</div>
{/* 文件预览弹窗 */}
<Modal
title={`文件预览：${previewFileName}`}
open={previewVisible}
onCancel={() => setPreviewVisible(false)}
footer={null}
width={700}
>
<pre
style={{
whiteSpace: "pre-wrap",
wordBreak: "break-all",
fontSize: 14,
color: "#222",
}}
>
{previewContent}
</pre>
</Modal>
</>
);
}
import { Button, Descriptions, DescriptionsProps, Modal, Table } from "antd";
import { formatBytes, formatDateTime } from "@/utils/unit";
import { Download, Trash2, Folder, File } from "lucide-react";
import { datasetTypeMap } from "../../dataset.const";
export default function Overview({ dataset, filesOperation, fetchDataset }) {
const {
fileList,
pagination,
selectedFiles,
setSelectedFiles,
previewVisible,
previewFileName,
previewContent,
setPreviewVisible,
handleDeleteFile,
handleDownloadFile,
handleBatchDeleteFiles,
handleBatchExport,
} = filesOperation;
// 文件列表多选配置
const rowSelection = {
onChange: (selectedRowKeys: React.Key[], selectedRows: any[]) => {
setSelectedFiles(selectedRowKeys as number[]);
console.log(
`selectedRowKeys: ${selectedRowKeys}`,
"selectedRows: ",
selectedRows
);
},
};
// 基本信息
const items: DescriptionsProps["items"] = [
{
key: "id",
label: "ID",
children: dataset.id,
},
{
key: "name",
label: "名称",
children: dataset.name,
},
{
key: "fileCount",
label: "文件数",
children: dataset.fileCount || 0,
},
{
key: "size",
label: "数据大小",
children: dataset.size || "0 B",
},
{
key: "datasetType",
label: "类型",
children: datasetTypeMap[dataset?.datasetType]?.label || "未知",
},
{
key: "status",
label: "状态",
children: dataset?.status?.label || "未知",
},
{
key: "createdBy",
label: "创建者",
children: dataset.createdBy || "未知",
},
{
key: "targetLocation",
label: "存储路径",
children: dataset.targetLocation || "未知",
},
{
key: "pvcName",
label: "存储名称",
children: dataset.pvcName || "未知",
},
{
key: "createdAt",
label: "创建时间",
children: dataset.createdAt,
},
{
key: "updatedAt",
label: "更新时间",
children: dataset.updatedAt,
},
{
key: "description",
label: "描述",
children: dataset.description || "无",
},
];
// 文件列表列定义
const columns = [
{
title: "文件名",
dataIndex: "fileName",
key: "fileName",
fixed: "left",
render: (text: string, record: any) => {
const isDirectory = record.id.startsWith('directory-');
const iconSize = 16;
const content = (
<div className="flex items-center">
{isDirectory ? (
<Folder className="mr-2 text-blue-500" size={iconSize} />
) : (
<File className="mr-2 text-black" size={iconSize} />
)}
<span className="truncate text-black">{text}</span>
</div>
);
if (isDirectory) {
return (
<Button
type="link"
onClick={(e) => {
const currentPath = filesOperation.pagination.prefix || '';
const newPath = `${currentPath}${record.fileName}`;
filesOperation.fetchFiles(newPath);
}}
>
{content}
</Button>
);
}
return (
<Button
type="link"
onClick={(e) => {}}
>
{content}
</Button>
);
},
},
{
title: "大小",
dataIndex: "fileSize",
key: "fileSize",
width: 150,
render: (text: number, record: any) => {
const isDirectory = record.id.startsWith('directory-');
if (isDirectory) {
return "-";
}
return formatBytes(text)
},
},
{
title: "上传时间",
dataIndex: "uploadTime",
key: "uploadTime",
width: 200,
render: (text) => formatDateTime(text),
},
{
title: "操作",
key: "action",
width: 180,
fixed: "right",
render: (_, record) => {
const isDirectory = record.id.startsWith('directory-');
if (isDirectory) {
return <div className="flex"/>;
}
return (
<div className="flex">
<Button
size="small"
type="link"
onClick={() => handleDownloadFile(record)}
>
下载
</Button>
<Button
size="small"
type="link"
onClick={async () => {
await handleDeleteFile(record);
fetchDataset()
}
}
>
删除
</Button>
</div>
)},
},
];
return (
<>
<div className=" flex flex-col gap-4">
{/* 基本信息 */}
<Descriptions
title="基本信息"
layout="vertical"
size="small"
items={items}
column={5}
/>
{/* 文件列表 */}
<h2 className="text-base font-semibold mt-8">文件列表</h2>
{selectedFiles.length > 0 && (
<div className="flex items-center gap-2 p-3 bg-blue-50 rounded-lg border border-blue-200">
<span className="text-sm text-blue-700 font-medium">
已选择 {selectedFiles.length} 个文件
</span>
<Button
onClick={handleBatchExport}
className="ml-auto bg-transparent"
>
<Download className="w-4 h-4 mr-2" />
批量导出
</Button>
<Button
onClick={handleBatchDeleteFiles}
className="text-red-600 hover:text-red-700 hover:bg-red-50 bg-transparent"
>
<Trash2 className="w-4 h-4 mr-2" />
批量删除
</Button>
</div>
)}
<div className="overflow-x-auto">
<div className="mb-2">
{(filesOperation.pagination.prefix || '') !== '' && (
<Button
type="link"
onClick={() => {
// 获取上一级目录
const currentPath = filesOperation.pagination.prefix || '';
const pathParts = currentPath.split('/').filter(Boolean);
pathParts.pop(); // 移除最后一个目录
const parentPath = pathParts.length > 0 ? `${pathParts.join('/')}/` : '';
filesOperation.fetchFiles(parentPath);
}}
className="p-0"
>
<span className="flex items-center text-blue-500">
<svg
className="w-4 h-4 mr-1"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
xmlns="http://www.w3.org/2000/svg"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M10 19l-7-7m0 0l7-7m-7 7h18"
/>
</svg>
返回上一级
</span>
</Button>
)}
{filesOperation.pagination.prefix && (
<span className="ml-2 text-gray-600">当前路径: {filesOperation.pagination.prefix}</span>
)}
</div>
<Table
size="middle"
rowKey="id"
columns={columns}
dataSource={fileList}
// rowSelection={rowSelection}
scroll={{ x: "max-content", y: 600 }}
pagination={{
...pagination,
showTotal: (total) => `共 ${total} 条`,
onChange: (page, pageSize) => {
filesOperation.setPagination(prev => ({
...prev,
current: page,
pageSize: pageSize
}));
filesOperation.fetchFiles(pagination.prefix, page, pageSize);
}
}}
/>
</div>
</div>
{/* 文件预览弹窗 */}
<Modal
title={`文件预览：${previewFileName}`}
open={previewVisible}
onCancel={() => setPreviewVisible(false)}
footer={null}
width={700}
>
<pre
style={{
whiteSpace: "pre-wrap",
wordBreak: "break-all",
fontSize: 14,
color: "#222",
}}
>
{previewContent}
</pre>
</Modal>
</>
);
}

298
frontend/src/pages/DataManagement/Detail/useFilesOperation.ts
View File

@@ -1,149 +1,149 @@
import type {
Dataset,
DatasetFile,
} from "@/pages/DataManagement/dataset.model";
import { App } from "antd";
import { useState } from "react";
import {
deleteDatasetFileUsingDelete,
downloadFileByIdUsingGet,
exportDatasetUsingPost,
queryDatasetFilesUsingGet,
} from "../dataset.api";
import { useParams } from "react-router";
export function useFilesOperation(dataset: Dataset) {
const { message } = App.useApp();
const { id } = useParams(); // 获取动态路由参数
// 文件相关状态
const [fileList, setFileList] = useState<DatasetFile[]>([]);
const [selectedFiles, setSelectedFiles] = useState<number[]>([]);
const [pagination, setPagination] = useState<{
current: number;
pageSize: number;
total: number;
prefix?: string;
}>({ current: 1, pageSize: 10, total: 0, prefix: '' });
// 文件预览相关状态
const [previewVisible, setPreviewVisible] = useState(false);
const [previewContent, setPreviewContent] = useState("");
const [previewFileName, setPreviewFileName] = useState("");
const fetchFiles = async (prefix: string = '', current, pageSize) => {
const params: any = {
page: current ? current : pagination.current,
size: pageSize ? pageSize : pagination.pageSize,
isWithDirectory: true,
};
if (prefix !== undefined) {
params.prefix = prefix;
} else if (pagination.prefix) {
params.prefix = pagination.prefix;
}
const { data } = await queryDatasetFilesUsingGet(id!, params);
setFileList(data.content || []);
// Update pagination with current prefix
setPagination(prev => ({
...prev,
prefix: prefix !== undefined ? prefix : prev.prefix,
total: data.totalElements || 0,
}));
};
const handleBatchDeleteFiles = () => {
if (selectedFiles.length === 0) {
message.warning({ content: "请先选择要删除的文件" });
return;
}
// 执行批量删除逻辑
selectedFiles.forEach(async (fileId) => {
await fetch(`/api/datasets/${dataset.id}/files/${fileId}`, {
method: "DELETE",
});
});
fetchFiles(); // 刷新文件列表
setSelectedFiles([]); // 清空选中状态
message.success({
content: `已删除 ${selectedFiles.length} 个文件`,
});
};
const handleDownloadFile = async (file: DatasetFile) => {
// 实际导出逻辑
await downloadFileByIdUsingGet(dataset.id, file.id, file.fileName);
// 假设导出成功
message.success({
content: `已导出 1 个文件`,
});
setSelectedFiles([]); // 清空选中状态
};
const handleShowFile = (file: any) => async () => {
// 请求文件内容并弹窗预览
try {
const res = await fetch(`/api/datasets/${dataset.id}/file/${file.id}`);
const data = await res.text();
setPreviewFileName(file.fileName);
setPreviewContent(data);
setPreviewVisible(true);
} catch (err) {
message.error({ content: "文件预览失败" });
}
};
const handleDeleteFile = async (file) => {
try {
await deleteDatasetFileUsingDelete(dataset.id, file.id);
fetchFiles(); // 刷新文件列表
message.success({ content: `文件 ${file.fileName} 已删除` });
} catch (error) {
message.error({ content: `文件 ${file.fileName} 删除失败` });
}
};
const handleBatchExport = () => {
if (selectedFiles.length === 0) {
message.warning({ content: "请先选择要导出的文件" });
return;
}
// 执行批量导出逻辑
exportDatasetUsingPost(dataset.id, { fileIds: selectedFiles })
.then(() => {
message.success({
content: `已导出 ${selectedFiles.length} 个文件`,
});
setSelectedFiles([]); // 清空选中状态
})
.catch(() => {
message.error({
content: "导出失败，请稍后再试",
});
});
};
return {
fileList,
selectedFiles,
setSelectedFiles,
pagination,
setPagination,
previewVisible,
setPreviewVisible,
previewContent,
previewFileName,
setPreviewContent,
setPreviewFileName,
fetchFiles,
setFileList,
handleBatchDeleteFiles,
handleDownloadFile,
handleShowFile,
handleDeleteFile,
handleBatchExport,
};
}
import type {
Dataset,
DatasetFile,
} from "@/pages/DataManagement/dataset.model";
import { App } from "antd";
import { useState } from "react";
import {
deleteDatasetFileUsingDelete,
downloadFileByIdUsingGet,
exportDatasetUsingPost,
queryDatasetFilesUsingGet,
} from "../dataset.api";
import { useParams } from "react-router";
export function useFilesOperation(dataset: Dataset) {
const { message } = App.useApp();
const { id } = useParams(); // 获取动态路由参数
// 文件相关状态
const [fileList, setFileList] = useState<DatasetFile[]>([]);
const [selectedFiles, setSelectedFiles] = useState<number[]>([]);
const [pagination, setPagination] = useState<{
current: number;
pageSize: number;
total: number;
prefix?: string;
}>({ current: 1, pageSize: 10, total: 0, prefix: '' });
// 文件预览相关状态
const [previewVisible, setPreviewVisible] = useState(false);
const [previewContent, setPreviewContent] = useState("");
const [previewFileName, setPreviewFileName] = useState("");
const fetchFiles = async (prefix: string = '', current, pageSize) => {
const params: any = {
page: current ? current : pagination.current,
size: pageSize ? pageSize : pagination.pageSize,
isWithDirectory: true,
};
if (prefix !== undefined) {
params.prefix = prefix;
} else if (pagination.prefix) {
params.prefix = pagination.prefix;
}
const { data } = await queryDatasetFilesUsingGet(id!, params);
setFileList(data.content || []);
// Update pagination with current prefix
setPagination(prev => ({
...prev,
prefix: prefix !== undefined ? prefix : prev.prefix,
total: data.totalElements || 0,
}));
};
const handleBatchDeleteFiles = () => {
if (selectedFiles.length === 0) {
message.warning({ content: "请先选择要删除的文件" });
return;
}
// 执行批量删除逻辑
selectedFiles.forEach(async (fileId) => {
await fetch(`/api/datasets/${dataset.id}/files/${fileId}`, {
method: "DELETE",
});
});
fetchFiles(); // 刷新文件列表
setSelectedFiles([]); // 清空选中状态
message.success({
content: `已删除 ${selectedFiles.length} 个文件`,
});
};
const handleDownloadFile = async (file: DatasetFile) => {
// 实际导出逻辑
await downloadFileByIdUsingGet(dataset.id, file.id, file.fileName);
// 假设导出成功
message.success({
content: `已导出 1 个文件`,
});
setSelectedFiles([]); // 清空选中状态
};
const handleShowFile = (file: any) => async () => {
// 请求文件内容并弹窗预览
try {
const res = await fetch(`/api/datasets/${dataset.id}/file/${file.id}`);
const data = await res.text();
setPreviewFileName(file.fileName);
setPreviewContent(data);
setPreviewVisible(true);
} catch (err) {
message.error({ content: "文件预览失败" });
}
};
const handleDeleteFile = async (file) => {
try {
await deleteDatasetFileUsingDelete(dataset.id, file.id);
fetchFiles(); // 刷新文件列表
message.success({ content: `文件 ${file.fileName} 已删除` });
} catch (error) {
message.error({ content: `文件 ${file.fileName} 删除失败` });
}
};
const handleBatchExport = () => {
if (selectedFiles.length === 0) {
message.warning({ content: "请先选择要导出的文件" });
return;
}
// 执行批量导出逻辑
exportDatasetUsingPost(dataset.id, { fileIds: selectedFiles })
.then(() => {
message.success({
content: `已导出 ${selectedFiles.length} 个文件`,
});
setSelectedFiles([]); // 清空选中状态
})
.catch(() => {
message.error({
content: "导出失败，请稍后再试",
});
});
};
return {
fileList,
selectedFiles,
setSelectedFiles,
pagination,
setPagination,
previewVisible,
setPreviewVisible,
previewContent,
previewFileName,
setPreviewContent,
setPreviewFileName,
fetchFiles,
setFileList,
handleBatchDeleteFiles,
handleDownloadFile,
handleShowFile,
handleDeleteFile,
handleBatchExport,
};
}

796
frontend/src/pages/DataManagement/Home/DataManagement.tsx
View File

@@ -1,398 +1,398 @@
import { Card, Button, Statistic, Table, Tooltip, Tag, App } from "antd";
import {
DownloadOutlined,
EditOutlined,
DeleteOutlined,
PlusOutlined,
UploadOutlined,
} from "@ant-design/icons";
import TagManager from "@/components/business/TagManagement";
import { Link, useNavigate } from "react-router";
import { useEffect, useMemo, useState } from "react";
import { SearchControls } from "@/components/SearchControls";
import CardView from "@/components/CardView";
import type { Dataset } from "@/pages/DataManagement/dataset.model";
import { datasetStatusMap, datasetTypeMap, mapDataset } from "../dataset.const";
import useFetchData from "@/hooks/useFetchData";
import {
downloadDatasetUsingGet,
getDatasetStatisticsUsingGet,
queryDatasetsUsingGet,
deleteDatasetByIdUsingDelete,
createDatasetTagUsingPost,
queryDatasetTagsUsingGet,
deleteDatasetTagUsingDelete,
updateDatasetTagUsingPut,
} from "../dataset.api";
import { formatBytes } from "@/utils/unit";
import EditDataset from "../Create/EditDataset";
import ImportConfiguration from "../Detail/components/ImportConfiguration";
export default function DatasetManagementPage() {
const navigate = useNavigate();
const { message } = App.useApp();
const [viewMode, setViewMode] = useState<"card" | "list">("card");
const [editDatasetOpen, setEditDatasetOpen] = useState(false);
const [currentDataset, setCurrentDataset] = useState<Dataset | null>(null);
const [showUploadDialog, setShowUploadDialog] = useState(false);
const [statisticsData, setStatisticsData] = useState<any>({
count: {},
size: {},
});
async function fetchStatistics() {
const { data } = await getDatasetStatisticsUsingGet();
const statistics = {
size: [
{
title: "数据集总数",
value: data?.totalDatasets || 0,
},
{
title: "文件总数",
value: data?.totalFiles || 0,
},
{
title: "总大小",
value: formatBytes(data?.totalSize) || '0 B',
},
],
count: [
{
title: "文本",
value: data?.count?.text || 0,
},
{
title: "图像",
value: data?.count?.image || 0,
},
{
title: "音频",
value: data?.count?.audio || 0,
},
{
title: "视频",
value: data?.count?.video || 0,
},
],
};
setStatisticsData(statistics);
}
const [tags, setTags] = useState<string[]>([]);
useEffect(() => {
const fetchTags = async () => {
const { data } = await queryDatasetTagsUsingGet();
setTags(data.map((tag) => tag.name));
};
fetchTags();
}, []);
const filterOptions = useMemo(
() => [
{
key: "type",
label: "类型",
options: [...Object.values(datasetTypeMap)],
},
{
key: "status",
label: "状态",
options: [...Object.values(datasetStatusMap)],
},
{
key: "tags",
label: "标签",
mode: "multiple",
options: tags.map((tag) => ({ label: tag, value: tag })),
},
],
[tags]
);
const {
loading,
tableData,
searchParams,
pagination,
fetchData,
setSearchParams,
handleFiltersChange,
handleKeywordChange,
} = useFetchData<Dataset>(
queryDatasetsUsingGet,
mapDataset,
30000, // 30秒轮询间隔
true, // 自动刷新
[fetchStatistics], // 额外的轮询函数
0
);
const handleDownloadDataset = async (dataset: Dataset) => {
await downloadDatasetUsingGet(dataset.id, dataset.name);
message.success("数据集下载成功");
};
const handleDeleteDataset = async (id: number) => {
if (!id) return;
await deleteDatasetByIdUsingDelete(id);
fetchData({ pageOffset: 0 });
message.success("数据删除成功");
};
const handleImportData = (dataset: Dataset) => {
setCurrentDataset(dataset);
setShowUploadDialog(true);
};
const handleRefresh = async (showMessage = true) => {
await fetchData({ pageOffset: 0 });
if (showMessage) {
message.success("数据已刷新");
}
};
const operations = [
{
key: "edit",
label: "编辑",
icon: <EditOutlined />,
onClick: (item: Dataset) => {
setCurrentDataset(item);
setEditDatasetOpen(true);
},
},
{
key: "import",
label: "导入",
icon: <UploadOutlined />,
onClick: (item: Dataset) => {
handleImportData(item);
},
},
{
key: "download",
label: "下载",
icon: <DownloadOutlined />,
onClick: (item: Dataset) => {
if (!item.id) return;
handleDownloadDataset(item);
},
},
{
key: "delete",
label: "删除",
danger: true,
confirm: {
title: "确认删除该数据集？",
description: "删除后该数据集将无法恢复，请谨慎操作。",
okText: "删除",
cancelText: "取消",
okType: "danger",
},
icon: <DeleteOutlined />,
onClick: (item: Dataset) => handleDeleteDataset(item.id),
},
];
const columns = [
{
title: "名称",
dataIndex: "name",
key: "name",
fixed: "left",
render: (name, record) => (
<Button
type="link"
onClick={() => navigate(`/data/management/detail/${record.id}`)}
>
{name}
</Button>
),
},
{
title: "类型",
dataIndex: "type",
key: "type",
width: 100,
},
{
title: "状态",
dataIndex: "status",
key: "status",
render: (status: any) => {
return (
<Tag icon={status?.icon} color={status?.color}>
{status?.label}
</Tag>
);
},
width: 120,
},
{
title: "大小",
dataIndex: "size",
key: "size",
width: 120,
},
{
title: "文件数",
dataIndex: "fileCount",
key: "fileCount",
width: 100,
},
// {
// title: "创建者",
// dataIndex: "createdBy",
// key: "createdBy",
// width: 120,
// },
{
title: "存储路径",
dataIndex: "targetLocation",
key: "targetLocation",
width: 200,
ellipsis: true,
},
{
title: "创建时间",
dataIndex: "createdAt",
key: "createdAt",
width: 180,
},
{
title: "更新时间",
dataIndex: "updatedAt",
key: "updatedAt",
width: 180,
},
{
title: "操作",
key: "actions",
width: 200,
fixed: "right",
render: (_: any, record: Dataset) => (
<div className="flex items-center gap-2">
{operations.map((op) => (
<Tooltip key={op.key} title={op.label}>
<Button
type="text"
icon={op.icon}
onClick={() => op.onClick(record)}
/>
</Tooltip>
))}
</div>
),
},
];
const renderCardView = () => (
<CardView
loading={loading}
data={tableData}
pageSize={9}
operations={operations}
pagination={pagination}
onView={(dataset) => {
navigate("/data/management/detail/" + dataset.id);
}}
/>
);
const renderListView = () => (
<Card>
<Table
columns={columns}
dataSource={tableData}
pagination={pagination}
rowKey="id"
scroll={{ x: "max-content", y: "calc(100vh - 30rem)" }}
/>
</Card>
);
useEffect(() => {
const refresh = () => {
handleRefresh(true);
};
window.addEventListener("update:datasets", refresh);
return () => {
window.removeEventListener("update:datasets", refresh);
};
}, []);
return (
<div className="gap-4 h-full flex flex-col">
{/* Header */}
<div className="flex items-center justify-between">
<h1 className="text-xl font-bold">数据管理</h1>
<div className="flex gap-2 items-center">
{/* tasks */}
<TagManager
onCreate={createDatasetTagUsingPost}
onDelete={(ids: string) => deleteDatasetTagUsingDelete({ ids })}
onUpdate={updateDatasetTagUsingPut}
onFetch={queryDatasetTagsUsingGet}
/>
<Link to="/data/management/create">
<Button
type="primary"
icon={<PlusOutlined className="w-4 h-4 mr-2" />}
>
创建数据集
</Button>
</Link>
</div>
</div>
{/* Statistics */}
<div className="grid grid-cols-1 gap-4">
<Card>
<div className="grid grid-cols-3">
{statisticsData.size?.map?.((item) => (
<Statistic
title={item.title}
key={item.title}
value={`${item.value}`}
/>
))}
</div>
</Card>
</div>
<SearchControls
searchTerm={searchParams.keyword}
onSearchChange={handleKeywordChange}
searchPlaceholder="搜索数据集名称、描述"
filters={filterOptions}
onFiltersChange={handleFiltersChange}
onClearFilters={() => setSearchParams({ ...searchParams, filter: {} })}
viewMode={viewMode}
onViewModeChange={setViewMode}
showViewToggle
onReload={handleRefresh}
/>
{viewMode === "card" ? renderCardView() : renderListView()}
<EditDataset
open={editDatasetOpen}
data={currentDataset}
onClose={() => {
setCurrentDataset(null);
setEditDatasetOpen(false);
}}
onRefresh={handleRefresh}
/>
<ImportConfiguration
data={currentDataset}
open={showUploadDialog}
onClose={() => {
setCurrentDataset(null);
setShowUploadDialog(false);
}}
updateEvent="update:datasets"
/>
</div>
);
}
import { Card, Button, Statistic, Table, Tooltip, Tag, App } from "antd";
import {
DownloadOutlined,
EditOutlined,
DeleteOutlined,
PlusOutlined,
UploadOutlined,
} from "@ant-design/icons";
import TagManager from "@/components/business/TagManagement";
import { Link, useNavigate } from "react-router";
import { useEffect, useMemo, useState } from "react";
import { SearchControls } from "@/components/SearchControls";
import CardView from "@/components/CardView";
import type { Dataset } from "@/pages/DataManagement/dataset.model";
import { datasetStatusMap, datasetTypeMap, mapDataset } from "../dataset.const";
import useFetchData from "@/hooks/useFetchData";
import {
downloadDatasetUsingGet,
getDatasetStatisticsUsingGet,
queryDatasetsUsingGet,
deleteDatasetByIdUsingDelete,
createDatasetTagUsingPost,
queryDatasetTagsUsingGet,
deleteDatasetTagUsingDelete,
updateDatasetTagUsingPut,
} from "../dataset.api";
import { formatBytes } from "@/utils/unit";
import EditDataset from "../Create/EditDataset";
import ImportConfiguration from "../Detail/components/ImportConfiguration";
export default function DatasetManagementPage() {
const navigate = useNavigate();
const { message } = App.useApp();
const [viewMode, setViewMode] = useState<"card" | "list">("card");
const [editDatasetOpen, setEditDatasetOpen] = useState(false);
const [currentDataset, setCurrentDataset] = useState<Dataset | null>(null);
const [showUploadDialog, setShowUploadDialog] = useState(false);
const [statisticsData, setStatisticsData] = useState<any>({
count: {},
size: {},
});
async function fetchStatistics() {
const { data } = await getDatasetStatisticsUsingGet();
const statistics = {
size: [
{
title: "数据集总数",
value: data?.totalDatasets || 0,
},
{
title: "文件总数",
value: data?.totalFiles || 0,
},
{
title: "总大小",
value: formatBytes(data?.totalSize) || '0 B',
},
],
count: [
{
title: "文本",
value: data?.count?.text || 0,
},
{
title: "图像",
value: data?.count?.image || 0,
},
{
title: "音频",
value: data?.count?.audio || 0,
},
{
title: "视频",
value: data?.count?.video || 0,
},
],
};
setStatisticsData(statistics);
}
const [tags, setTags] = useState<string[]>([]);
useEffect(() => {
const fetchTags = async () => {
const { data } = await queryDatasetTagsUsingGet();
setTags(data.map((tag) => tag.name));
};
fetchTags();
}, []);
const filterOptions = useMemo(
() => [
{
key: "type",
label: "类型",
options: [...Object.values(datasetTypeMap)],
},
{
key: "status",
label: "状态",
options: [...Object.values(datasetStatusMap)],
},
{
key: "tags",
label: "标签",
mode: "multiple",
options: tags.map((tag) => ({ label: tag, value: tag })),
},
],
[tags]
);
const {
loading,
tableData,
searchParams,
pagination,
fetchData,
setSearchParams,
handleFiltersChange,
handleKeywordChange,
} = useFetchData<Dataset>(
queryDatasetsUsingGet,
mapDataset,
30000, // 30秒轮询间隔
true, // 自动刷新
[fetchStatistics], // 额外的轮询函数
0
);
const handleDownloadDataset = async (dataset: Dataset) => {
await downloadDatasetUsingGet(dataset.id, dataset.name);
message.success("数据集下载成功");
};
const handleDeleteDataset = async (id: number) => {
if (!id) return;
await deleteDatasetByIdUsingDelete(id);
fetchData({ pageOffset: 0 });
message.success("数据删除成功");
};
const handleImportData = (dataset: Dataset) => {
setCurrentDataset(dataset);
setShowUploadDialog(true);
};
const handleRefresh = async (showMessage = true) => {
await fetchData({ pageOffset: 0 });
if (showMessage) {
message.success("数据已刷新");
}
};
const operations = [
{
key: "edit",
label: "编辑",
icon: <EditOutlined />,
onClick: (item: Dataset) => {
setCurrentDataset(item);
setEditDatasetOpen(true);
},
},
{
key: "import",
label: "导入",
icon: <UploadOutlined />,
onClick: (item: Dataset) => {
handleImportData(item);
},
},
{
key: "download",
label: "下载",
icon: <DownloadOutlined />,
onClick: (item: Dataset) => {
if (!item.id) return;
handleDownloadDataset(item);
},
},
{
key: "delete",
label: "删除",
danger: true,
confirm: {
title: "确认删除该数据集？",
description: "删除后该数据集将无法恢复，请谨慎操作。",
okText: "删除",
cancelText: "取消",
okType: "danger",
},
icon: <DeleteOutlined />,
onClick: (item: Dataset) => handleDeleteDataset(item.id),
},
];
const columns = [
{
title: "名称",
dataIndex: "name",
key: "name",
fixed: "left",
render: (name, record) => (
<Button
type="link"
onClick={() => navigate(`/data/management/detail/${record.id}`)}
>
{name}
</Button>
),
},
{
title: "类型",
dataIndex: "type",
key: "type",
width: 100,
},
{
title: "状态",
dataIndex: "status",
key: "status",
render: (status: any) => {
return (
<Tag icon={status?.icon} color={status?.color}>
{status?.label}
</Tag>
);
},
width: 120,
},
{
title: "大小",
dataIndex: "size",
key: "size",
width: 120,
},
{
title: "文件数",
dataIndex: "fileCount",
key: "fileCount",
width: 100,
},
// {
// title: "创建者",
// dataIndex: "createdBy",
// key: "createdBy",
// width: 120,
// },
{
title: "存储路径",
dataIndex: "targetLocation",
key: "targetLocation",
width: 200,
ellipsis: true,
},
{
title: "创建时间",
dataIndex: "createdAt",
key: "createdAt",
width: 180,
},
{
title: "更新时间",
dataIndex: "updatedAt",
key: "updatedAt",
width: 180,
},
{
title: "操作",
key: "actions",
width: 200,
fixed: "right",
render: (_: any, record: Dataset) => (
<div className="flex items-center gap-2">
{operations.map((op) => (
<Tooltip key={op.key} title={op.label}>
<Button
type="text"
icon={op.icon}
onClick={() => op.onClick(record)}
/>
</Tooltip>
))}
</div>
),
},
];
const renderCardView = () => (
<CardView
loading={loading}
data={tableData}
pageSize={9}
operations={operations}
pagination={pagination}
onView={(dataset) => {
navigate("/data/management/detail/" + dataset.id);
}}
/>
);
const renderListView = () => (
<Card>
<Table
columns={columns}
dataSource={tableData}
pagination={pagination}
rowKey="id"
scroll={{ x: "max-content", y: "calc(100vh - 30rem)" }}
/>
</Card>
);
useEffect(() => {
const refresh = () => {
handleRefresh(true);
};
window.addEventListener("update:datasets", refresh);
return () => {
window.removeEventListener("update:datasets", refresh);
};
}, []);
return (
<div className="gap-4 h-full flex flex-col">
{/* Header */}
<div className="flex items-center justify-between">
<h1 className="text-xl font-bold">数据管理</h1>
<div className="flex gap-2 items-center">
{/* tasks */}
<TagManager
onCreate={createDatasetTagUsingPost}
onDelete={(ids: string) => deleteDatasetTagUsingDelete({ ids })}
onUpdate={updateDatasetTagUsingPut}
onFetch={queryDatasetTagsUsingGet}
/>
<Link to="/data/management/create">
<Button
type="primary"
icon={<PlusOutlined className="w-4 h-4 mr-2" />}
>
创建数据集
</Button>
</Link>
</div>
</div>
{/* Statistics */}
<div className="grid grid-cols-1 gap-4">
<Card>
<div className="grid grid-cols-3">
{statisticsData.size?.map?.((item) => (
<Statistic
title={item.title}
key={item.title}
value={`${item.value}`}
/>
))}
</div>
</Card>
</div>
<SearchControls
searchTerm={searchParams.keyword}
onSearchChange={handleKeywordChange}
searchPlaceholder="搜索数据集名称、描述"
filters={filterOptions}
onFiltersChange={handleFiltersChange}
onClearFilters={() => setSearchParams({ ...searchParams, filter: {} })}
viewMode={viewMode}
onViewModeChange={setViewMode}
showViewToggle
onReload={handleRefresh}
/>
{viewMode === "card" ? renderCardView() : renderListView()}
<EditDataset
open={editDatasetOpen}
data={currentDataset}
onClose={() => {
setCurrentDataset(null);
setEditDatasetOpen(false);
}}
onRefresh={handleRefresh}
/>
<ImportConfiguration
data={currentDataset}
open={showUploadDialog}
onClose={() => {
setCurrentDataset(null);
setShowUploadDialog(false);
}}
updateEvent="update:datasets"
/>
</div>
);
}

382
frontend/src/pages/DataManagement/dataset.api.ts
View File

@@ -1,191 +1,191 @@
import { get, post, put, del, download } from "@/utils/request";
// 数据集统计接口
export function getDatasetStatisticsUsingGet() {
return get("/api/data-management/datasets/statistics");
}
export function queryDatasetStatisticsByIdUsingGet(id: string | number) {
return get(`/api/data-management/datasets/${id}/statistics`);
}
// 查询数据集列表
export function queryDatasetsUsingGet(params?: any) {
return get("/api/data-management/datasets", params);
}
// 创建数据集
export function createDatasetUsingPost(data: any) {
return post("/api/data-management/datasets", data);
}
// 根据ID获取数据集详情
export function queryDatasetByIdUsingGet(id: string | number) {
return get(`/api/data-management/datasets/${id}`);
}
// 更新数据集
export function updateDatasetByIdUsingPut(id: string | number, data: any) {
return put(`/api/data-management/datasets/${id}`, data);
}
// 删除数据集
export function deleteDatasetByIdUsingDelete(id: string | number) {
return del(`/api/data-management/datasets/${id}`);
}
// 下载数据集
export function downloadDatasetUsingGet(id: string | number) {
return download(`/api/data-management/datasets/${id}/files/download`);
}
// 验证数据集
export function validateDatasetUsingPost(id: string | number, data?: any) {
return post(`/api/data-management/datasets/${id}/validate`, data);
}
// 获取数据集文件列表
export function queryDatasetFilesUsingGet(id: string | number, params?: any) {
return get(`/api/data-management/datasets/${id}/files`, params);
}
// 上传数据集文件
export function uploadDatasetFileUsingPost(id: string | number, data: any) {
return post(`/api/data-management/datasets/${id}/files`, data);
}
export function downloadFileByIdUsingGet(
id: string | number,
fileId: string | number,
fileName: string
) {
return download(
`/api/data-management/datasets/${id}/files/${fileId}/download`,
null,
fileName
);
}
// 删除数据集文件
export function deleteDatasetFileUsingDelete(
datasetId: string | number,
fileId: string | number
) {
return del(`/api/data-management/datasets/${datasetId}/files/${fileId}`);
}
// 文件预览
export function previewDatasetUsingGet(id: string | number, params?: any) {
return get(`/api/data-management/datasets/${id}/preview`, params);
}
// 获取数据集标签
export function queryDatasetTagsUsingGet(params?: any) {
return get("/api/data-management/tags", params);
}
// 创建数据集标签
export function createDatasetTagUsingPost(data: any) {
return post("/api/data-management/tags", data);
}
// 更新数据集标签
export function updateDatasetTagUsingPut(data: any) {
return put(`/api/data-management/tags`, data);
}
// 删除数据集标签
export function deleteDatasetTagUsingDelete(data: any) {
return del(`/api/data-management/tags`, data);
}
// 数据集质量检查
export function checkDatasetQualityUsingPost(id: string | number, data?: any) {
return post(`/api/data-management/datasets/${id}/quality-check`, data);
}
// 获取数据集质量报告
export function getDatasetQualityReportUsingGet(id: string | number) {
return get(`/api/data-management/datasets/${id}/quality-report`);
}
// 数据集分析
export function analyzeDatasetUsingPost(id: string | number, data?: any) {
return post(`/api/data-management/datasets/${id}/analyze`, data);
}
// 获取数据集分析结果
export function getDatasetAnalysisUsingGet(id: string | number) {
return get(`/api/data-management/datasets/${id}/analysis`);
}
// 导出数据集
export function exportDatasetUsingPost(id: string | number, data: any) {
return post(`/api/data-management/datasets/${id}/export`, data);
}
// 复制数据集
export function copyDatasetUsingPost(id: string | number, data: any) {
return post(`/api/data-management/datasets/${id}/copy`, data);
}
// 获取数据集版本列表
export function queryDatasetVersionsUsingGet(
id: string | number,
params?: any
) {
return get(`/api/data-management/datasets/${id}/versions`, params);
}
// 创建数据集版本
export function createDatasetVersionUsingPost(id: string | number, data: any) {
return post(`/api/data-management/datasets/${id}/versions`, data);
}
// 切换数据集版本
export function switchDatasetVersionUsingPut(
id: string | number,
versionId: string | number
) {
return put(
`/api/data-management/datasets/${id}/versions/${versionId}/switch`
);
}
// 删除数据集版本
export function deleteDatasetVersionUsingDelete(
id: string | number,
versionId: string | number
) {
return del(`/api/data-management/datasets/${id}/versions/${versionId}`);
}
/**
* 文件上传相关接口
*/
export function preUploadUsingPost(id: string | number, data: any) {
return post(
`/api/data-management/datasets/${id}/files/upload/pre-upload`,
data
);
}
export function cancelUploadUsingPut(id) {
return put(
`/api/data-management/datasets/upload/cancel-upload/${id}`,
{},
{ showLoading: false }
);
}
export function uploadFileChunkUsingPost(id: string | number, params, config) {
return post(
`/api/data-management/datasets/${id}/files/upload/chunk`,
params,
{
showLoading: false,
...config,
}
);
}
import { get, post, put, del, download } from "@/utils/request";
// 数据集统计接口
export function getDatasetStatisticsUsingGet() {
return get("/api/data-management/datasets/statistics");
}
export function queryDatasetStatisticsByIdUsingGet(id: string | number) {
return get(`/api/data-management/datasets/${id}/statistics`);
}
// 查询数据集列表
export function queryDatasetsUsingGet(params?: any) {
return get("/api/data-management/datasets", params);
}
// 创建数据集
export function createDatasetUsingPost(data: any) {
return post("/api/data-management/datasets", data);
}
// 根据ID获取数据集详情
export function queryDatasetByIdUsingGet(id: string | number) {
return get(`/api/data-management/datasets/${id}`);
}
// 更新数据集
export function updateDatasetByIdUsingPut(id: string | number, data: any) {
return put(`/api/data-management/datasets/${id}`, data);
}
// 删除数据集
export function deleteDatasetByIdUsingDelete(id: string | number) {
return del(`/api/data-management/datasets/${id}`);
}
// 下载数据集
export function downloadDatasetUsingGet(id: string | number) {
return download(`/api/data-management/datasets/${id}/files/download`);
}
// 验证数据集
export function validateDatasetUsingPost(id: string | number, data?: any) {
return post(`/api/data-management/datasets/${id}/validate`, data);
}
// 获取数据集文件列表
export function queryDatasetFilesUsingGet(id: string | number, params?: any) {
return get(`/api/data-management/datasets/${id}/files`, params);
}
// 上传数据集文件
export function uploadDatasetFileUsingPost(id: string | number, data: any) {
return post(`/api/data-management/datasets/${id}/files`, data);
}
export function downloadFileByIdUsingGet(
id: string | number,
fileId: string | number,
fileName: string
) {
return download(
`/api/data-management/datasets/${id}/files/${fileId}/download`,
null,
fileName
);
}
// 删除数据集文件
export function deleteDatasetFileUsingDelete(
datasetId: string | number,
fileId: string | number
) {
return del(`/api/data-management/datasets/${datasetId}/files/${fileId}`);
}
// 文件预览
export function previewDatasetUsingGet(id: string | number, params?: any) {
return get(`/api/data-management/datasets/${id}/preview`, params);
}
// 获取数据集标签
export function queryDatasetTagsUsingGet(params?: any) {
return get("/api/data-management/tags", params);
}
// 创建数据集标签
export function createDatasetTagUsingPost(data: any) {
return post("/api/data-management/tags", data);
}
// 更新数据集标签
export function updateDatasetTagUsingPut(data: any) {
return put(`/api/data-management/tags`, data);
}
// 删除数据集标签
export function deleteDatasetTagUsingDelete(data: any) {
return del(`/api/data-management/tags`, data);
}
// 数据集质量检查
export function checkDatasetQualityUsingPost(id: string | number, data?: any) {
return post(`/api/data-management/datasets/${id}/quality-check`, data);
}
// 获取数据集质量报告
export function getDatasetQualityReportUsingGet(id: string | number) {
return get(`/api/data-management/datasets/${id}/quality-report`);
}
// 数据集分析
export function analyzeDatasetUsingPost(id: string | number, data?: any) {
return post(`/api/data-management/datasets/${id}/analyze`, data);
}
// 获取数据集分析结果
export function getDatasetAnalysisUsingGet(id: string | number) {
return get(`/api/data-management/datasets/${id}/analysis`);
}
// 导出数据集
export function exportDatasetUsingPost(id: string | number, data: any) {
return post(`/api/data-management/datasets/${id}/export`, data);
}
// 复制数据集
export function copyDatasetUsingPost(id: string | number, data: any) {
return post(`/api/data-management/datasets/${id}/copy`, data);
}
// 获取数据集版本列表
export function queryDatasetVersionsUsingGet(
id: string | number,
params?: any
) {
return get(`/api/data-management/datasets/${id}/versions`, params);
}
// 创建数据集版本
export function createDatasetVersionUsingPost(id: string | number, data: any) {
return post(`/api/data-management/datasets/${id}/versions`, data);
}
// 切换数据集版本
export function switchDatasetVersionUsingPut(
id: string | number,
versionId: string | number
) {
return put(
`/api/data-management/datasets/${id}/versions/${versionId}/switch`
);
}
// 删除数据集版本
export function deleteDatasetVersionUsingDelete(
id: string | number,
versionId: string | number
) {
return del(`/api/data-management/datasets/${id}/versions/${versionId}`);
}
/**
* 文件上传相关接口
*/
export function preUploadUsingPost(id: string | number, data: any) {
return post(
`/api/data-management/datasets/${id}/files/upload/pre-upload`,
data
);
}
export function cancelUploadUsingPut(id) {
return put(
`/api/data-management/datasets/upload/cancel-upload/${id}`,
{},
{ showLoading: false }
);
}
export function uploadFileChunkUsingPost(id: string | number, params, config) {
return post(
`/api/data-management/datasets/${id}/files/upload/chunk`,
params,
{
showLoading: false,
...config,
}
);
}

446
frontend/src/pages/DataManagement/dataset.const.tsx
View File

@@ -1,223 +1,223 @@
import {
DatasetType,
DatasetStatus,
type Dataset,
DatasetSubType,
DataSource,
} from "@/pages/DataManagement/dataset.model";
import { formatBytes, formatDateTime } from "@/utils/unit";
import {
CheckCircleOutlined,
ClockCircleOutlined,
CloseCircleOutlined,
FileOutlined,
} from "@ant-design/icons";
import { AnyObject } from "antd/es/_util/type";
import {
FileImage,
FileText,
Video,
FileCode,
MessageCircleMore,
ImagePlus,
FileMusic,
Music,
Videotape,
Database,
Image,
ScanText,
} from "lucide-react";
export const datasetTypeMap: Record<
string,
{
value: DatasetType;
label: string;
order: number;
description: string;
icon?: any;
iconColor?: string;
children: DatasetSubType[];
}
> = {
[DatasetType.TEXT]: {
value: DatasetType.TEXT,
label: "文本",
order: 1,
icon: ScanText,
iconColor: "blue",
children: [
DatasetSubType.TEXT_DOCUMENT,
DatasetSubType.TEXT_WEB,
DatasetSubType.TEXT_DIALOG,
],
description: "用于处理和分析文本数据的数据集",
},
[DatasetType.IMAGE]: {
value: DatasetType.IMAGE,
label: "图像",
order: 2,
icon: Image,
iconColor: "green",
children: [DatasetSubType.IMAGE_IMAGE, DatasetSubType.IMAGE_CAPTION],
description: "用于处理和分析图像数据的数据集",
},
[DatasetType.AUDIO]: {
value: DatasetType.AUDIO,
label: "音频",
order: 3,
icon: Music,
iconColor: "orange",
children: [DatasetSubType.AUDIO_AUDIO, DatasetSubType.AUDIO_JSONL],
description: "用于处理和分析音频数据的数据集",
},
[DatasetType.VIDEO]: {
value: DatasetType.VIDEO,
label: "视频",
order: 3,
icon: Video,
iconColor: "purple",
children: [DatasetSubType.VIDEO_VIDEO, DatasetSubType.VIDEO_JSONL],
description: "用于处理和分析视频数据的数据集",
},
};
export const datasetSubTypeMap: Record<
string,
{
value: DatasetSubType;
label: string;
order?: number;
description?: string;
icon?: any;
color?: string;
}
> = {
[DatasetSubType.TEXT_DOCUMENT]: {
value: DatasetSubType.TEXT_DOCUMENT,
label: "文档",
color: "blue",
icon: FileText,
description: "用于存储和处理各种文档格式的文本数据集",
},
[DatasetSubType.TEXT_WEB]: {
value: DatasetSubType.TEXT_WEB,
label: "网页",
color: "cyan",
icon: FileCode,
description: "用于存储和处理网页数据集",
},
[DatasetSubType.TEXT_DIALOG]: {
value: DatasetSubType.TEXT_DIALOG,
label: "对话",
color: "teal",
icon: MessageCircleMore,
description: "用于存储和处理对话数据的数据集",
},
[DatasetSubType.IMAGE_IMAGE]: {
value: DatasetSubType.IMAGE_IMAGE,
label: "图像",
color: "green",
icon: FileImage,
description: "用于大规模图像预训练模型的数据集",
},
[DatasetSubType.IMAGE_CAPTION]: {
value: DatasetSubType.IMAGE_CAPTION,
label: "图像+caption",
color: "lightgreen",
icon: ImagePlus,
description: "用于图像标题生成的数据集",
},
[DatasetSubType.AUDIO_AUDIO]: {
value: DatasetSubType.AUDIO_AUDIO,
label: "音频",
color: "purple",
icon: Music,
description: "用于大规模音频预训练模型的数据集",
},
[DatasetSubType.AUDIO_JSONL]: {
value: DatasetSubType.AUDIO_JSONL,
label: "音频+JSONL",
color: "purple",
icon: FileMusic,
description: "用于大规模音频预训练模型的数据集",
},
[DatasetSubType.VIDEO_VIDEO]: {
value: DatasetSubType.VIDEO_VIDEO,
label: "视频",
color: "orange",
icon: Video,
description: "用于大规模视频预训练模型的数据集",
},
[DatasetSubType.VIDEO_JSONL]: {
value: DatasetSubType.VIDEO_JSONL,
label: "视频+JSONL",
color: "orange",
icon: Videotape,
description: "用于大规模视频预训练模型的数据集",
},
};
export const datasetStatusMap = {
[DatasetStatus.ACTIVE]: {
label: "活跃",
value: DatasetStatus.ACTIVE,
color: "#409f17ff",
icon: <CheckCircleOutlined />,
},
[DatasetStatus.PROCESSING]: {
label: "处理中",
value: DatasetStatus.PROCESSING,
color: "#2673e5",
icon: <ClockCircleOutlined />,
},
[DatasetStatus.INACTIVE]: {
label: "未激活",
value: DatasetStatus.INACTIVE,
color: "#4f4444ff",
icon: <CloseCircleOutlined />,
},
[DatasetStatus.DRAFT]: {
label: "草稿",
value: DatasetStatus.DRAFT,
color: "#a1a1a1ff",
icon: <FileOutlined />,
},
};
export const dataSourceMap: Record<string, { label: string; value: string }> = {
[DataSource.UPLOAD]: { label: "本地上传", value: DataSource.UPLOAD },
[DataSource.COLLECTION]: { label: "归集任务导入 ", value: DataSource.COLLECTION },
// [DataSource.DATABASE]: { label: "数据库导入", value: DataSource.DATABASE },
// [DataSource.NAS]: { label: "NAS导入", value: DataSource.NAS },
// [DataSource.OBS]: { label: "OBS导入", value: DataSource.OBS },
};
export const dataSourceOptions = Object.values(dataSourceMap);
export function mapDataset(dataset: AnyObject): Dataset {
const { icon: IconComponent, iconColor } =
datasetTypeMap[dataset?.datasetType] || {};
return {
...dataset,
key: dataset.id,
type: datasetTypeMap[dataset.datasetType]?.label || "未知",
size: formatBytes(dataset.totalSize || 0),
createdAt: formatDateTime(dataset.createdAt) || "--",
updatedAt: formatDateTime(dataset?.updatedAt) || "--",
icon: IconComponent ? <IconComponent className="w-full h-full" /> : <Database />,
status: datasetStatusMap[dataset.status],
statistics: [
{ label: "文件数", value: dataset.fileCount || 0 },
{ label: "大小", value: formatBytes(dataset.totalSize || 0) },
],
lastModified: dataset.updatedAt,
};
}
export const datasetTypes = Object.values(datasetTypeMap).map((type) => ({
...type,
options: type.children?.map(
(subType) => datasetSubTypeMap[subType as keyof typeof datasetSubTypeMap]
),
}));
import {
DatasetType,
DatasetStatus,
type Dataset,
DatasetSubType,
DataSource,
} from "@/pages/DataManagement/dataset.model";
import { formatBytes, formatDateTime } from "@/utils/unit";
import {
CheckCircleOutlined,
ClockCircleOutlined,
CloseCircleOutlined,
FileOutlined,
} from "@ant-design/icons";
import { AnyObject } from "antd/es/_util/type";
import {
FileImage,
FileText,
Video,
FileCode,
MessageCircleMore,
ImagePlus,
FileMusic,
Music,
Videotape,
Database,
Image,
ScanText,
} from "lucide-react";
export const datasetTypeMap: Record<
string,
{
value: DatasetType;
label: string;
order: number;
description: string;
icon?: any;
iconColor?: string;
children: DatasetSubType[];
}
> = {
[DatasetType.TEXT]: {
value: DatasetType.TEXT,
label: "文本",
order: 1,
icon: ScanText,
iconColor: "blue",
children: [
DatasetSubType.TEXT_DOCUMENT,
DatasetSubType.TEXT_WEB,
DatasetSubType.TEXT_DIALOG,
],
description: "用于处理和分析文本数据的数据集",
},
[DatasetType.IMAGE]: {
value: DatasetType.IMAGE,
label: "图像",
order: 2,
icon: Image,
iconColor: "green",
children: [DatasetSubType.IMAGE_IMAGE, DatasetSubType.IMAGE_CAPTION],
description: "用于处理和分析图像数据的数据集",
},
[DatasetType.AUDIO]: {
value: DatasetType.AUDIO,
label: "音频",
order: 3,
icon: Music,
iconColor: "orange",
children: [DatasetSubType.AUDIO_AUDIO, DatasetSubType.AUDIO_JSONL],
description: "用于处理和分析音频数据的数据集",
},
[DatasetType.VIDEO]: {
value: DatasetType.VIDEO,
label: "视频",
order: 3,
icon: Video,
iconColor: "purple",
children: [DatasetSubType.VIDEO_VIDEO, DatasetSubType.VIDEO_JSONL],
description: "用于处理和分析视频数据的数据集",
},
};
export const datasetSubTypeMap: Record<
string,
{
value: DatasetSubType;
label: string;
order?: number;
description?: string;
icon?: any;
color?: string;
}
> = {
[DatasetSubType.TEXT_DOCUMENT]: {
value: DatasetSubType.TEXT_DOCUMENT,
label: "文档",
color: "blue",
icon: FileText,
description: "用于存储和处理各种文档格式的文本数据集",
},
[DatasetSubType.TEXT_WEB]: {
value: DatasetSubType.TEXT_WEB,
label: "网页",
color: "cyan",
icon: FileCode,
description: "用于存储和处理网页数据集",
},
[DatasetSubType.TEXT_DIALOG]: {
value: DatasetSubType.TEXT_DIALOG,
label: "对话",
color: "teal",
icon: MessageCircleMore,
description: "用于存储和处理对话数据的数据集",
},
[DatasetSubType.IMAGE_IMAGE]: {
value: DatasetSubType.IMAGE_IMAGE,
label: "图像",
color: "green",
icon: FileImage,
description: "用于大规模图像预训练模型的数据集",
},
[DatasetSubType.IMAGE_CAPTION]: {
value: DatasetSubType.IMAGE_CAPTION,
label: "图像+caption",
color: "lightgreen",
icon: ImagePlus,
description: "用于图像标题生成的数据集",
},
[DatasetSubType.AUDIO_AUDIO]: {
value: DatasetSubType.AUDIO_AUDIO,
label: "音频",
color: "purple",
icon: Music,
description: "用于大规模音频预训练模型的数据集",
},
[DatasetSubType.AUDIO_JSONL]: {
value: DatasetSubType.AUDIO_JSONL,
label: "音频+JSONL",
color: "purple",
icon: FileMusic,
description: "用于大规模音频预训练模型的数据集",
},
[DatasetSubType.VIDEO_VIDEO]: {
value: DatasetSubType.VIDEO_VIDEO,
label: "视频",
color: "orange",
icon: Video,
description: "用于大规模视频预训练模型的数据集",
},
[DatasetSubType.VIDEO_JSONL]: {
value: DatasetSubType.VIDEO_JSONL,
label: "视频+JSONL",
color: "orange",
icon: Videotape,
description: "用于大规模视频预训练模型的数据集",
},
};
export const datasetStatusMap = {
[DatasetStatus.ACTIVE]: {
label: "活跃",
value: DatasetStatus.ACTIVE,
color: "#409f17ff",
icon: <CheckCircleOutlined />,
},
[DatasetStatus.PROCESSING]: {
label: "处理中",
value: DatasetStatus.PROCESSING,
color: "#2673e5",
icon: <ClockCircleOutlined />,
},
[DatasetStatus.INACTIVE]: {
label: "未激活",
value: DatasetStatus.INACTIVE,
color: "#4f4444ff",
icon: <CloseCircleOutlined />,
},
[DatasetStatus.DRAFT]: {
label: "草稿",
value: DatasetStatus.DRAFT,
color: "#a1a1a1ff",
icon: <FileOutlined />,
},
};
export const dataSourceMap: Record<string, { label: string; value: string }> = {
[DataSource.UPLOAD]: { label: "本地上传", value: DataSource.UPLOAD },
[DataSource.COLLECTION]: { label: "归集任务导入 ", value: DataSource.COLLECTION },
// [DataSource.DATABASE]: { label: "数据库导入", value: DataSource.DATABASE },
// [DataSource.NAS]: { label: "NAS导入", value: DataSource.NAS },
// [DataSource.OBS]: { label: "OBS导入", value: DataSource.OBS },
};
export const dataSourceOptions = Object.values(dataSourceMap);
export function mapDataset(dataset: AnyObject): Dataset {
const { icon: IconComponent, iconColor } =
datasetTypeMap[dataset?.datasetType] || {};
return {
...dataset,
key: dataset.id,
type: datasetTypeMap[dataset.datasetType]?.label || "未知",
size: formatBytes(dataset.totalSize || 0),
createdAt: formatDateTime(dataset.createdAt) || "--",
updatedAt: formatDateTime(dataset?.updatedAt) || "--",
icon: IconComponent ? <IconComponent className="w-full h-full" /> : <Database />,
status: datasetStatusMap[dataset.status],
statistics: [
{ label: "文件数", value: dataset.fileCount || 0 },
{ label: "大小", value: formatBytes(dataset.totalSize || 0) },
],
lastModified: dataset.updatedAt,
};
}
export const datasetTypes = Object.values(datasetTypeMap).map((type) => ({
...type,
options: type.children?.map(
(subType) => datasetSubTypeMap[subType as keyof typeof datasetSubTypeMap]
),
}));

210
frontend/src/pages/DataManagement/dataset.model.ts
View File

@@ -1,105 +1,105 @@
export enum DatasetType {
TEXT = "TEXT",
IMAGE = "IMAGE",
AUDIO = "AUDIO",
VIDEO = "VIDEO",
}
export enum DatasetSubType {
TEXT_DOCUMENT = "TEXT_DOCUMENT",
TEXT_WEB = "TEXT_WEB",
TEXT_DIALOG = "TEXT_DIALOG",
IMAGE_IMAGE = "IMAGE_IMAGE",
IMAGE_CAPTION = "IMAGE_CAPTION",
AUDIO_AUDIO = "AUDIO_AUDIO",
AUDIO_JSONL = "AUDIO_JSONL",
VIDEO_VIDEO = "VIDEO_VIDEO",
VIDEO_JSONL = "VIDEO_JSONL",
}
export enum DatasetStatus {
ACTIVE = "ACTIVE",
INACTIVE = "INACTIVE",
PROCESSING = "PROCESSING",
DRAFT = "DRAFT",
}
export enum DataSource {
UPLOAD = "UPLOAD",
COLLECTION = "COLLECTION",
DATABASE = "DATABASE",
NAS = "NAS",
OBS = "OBS",
}
export interface DatasetFile {
id: number;
fileName: string;
size: string;
uploadDate: string;
path: string;
}
export interface Dataset {
id: number;
name: string;
description: string;
parentId?: number;
datasetType: DatasetType;
status: DatasetStatus;
size?: string;
itemCount?: number;
fileCount?: number;
createdBy: string;
createdAt: string;
updatedAt: string;
tags: string[];
targetLocation?: string;
distribution?: Record<string, Record<string, number>>;
}
export interface TagItem {
id: string;
name: string;
color: string;
description: string;
}
export interface ScheduleConfig {
type: "immediate" | "scheduled";
scheduleType?: "daily" | "weekly" | "monthly" | "custom";
time?: string;
dayOfWeek?: string;
dayOfMonth?: string;
cronExpression?: string;
maxExecutions?: number;
executionCount?: number;
}
export interface DatasetTask {
id: number;
name: string;
description: string;
type: string;
status: "importing" | "waiting" | "completed" | "failed";
progress: number;
createdAt: string;
importConfig: any;
scheduleConfig: ScheduleConfig;
nextExecution?: string;
lastExecution?: string;
executionHistory?: { time: string; status: string }[];
}
export interface TaskItem {
key: string;
title: string;
percent: number;
reqId: number;
isCancel?: boolean;
controller: AbortController;
cancelFn?: () => void;
updateEvent?: string;
size?: number;
hasArchive?: boolean;
}
export enum DatasetType {
TEXT = "TEXT",
IMAGE = "IMAGE",
AUDIO = "AUDIO",
VIDEO = "VIDEO",
}
export enum DatasetSubType {
TEXT_DOCUMENT = "TEXT_DOCUMENT",
TEXT_WEB = "TEXT_WEB",
TEXT_DIALOG = "TEXT_DIALOG",
IMAGE_IMAGE = "IMAGE_IMAGE",
IMAGE_CAPTION = "IMAGE_CAPTION",
AUDIO_AUDIO = "AUDIO_AUDIO",
AUDIO_JSONL = "AUDIO_JSONL",
VIDEO_VIDEO = "VIDEO_VIDEO",
VIDEO_JSONL = "VIDEO_JSONL",
}
export enum DatasetStatus {
ACTIVE = "ACTIVE",
INACTIVE = "INACTIVE",
PROCESSING = "PROCESSING",
DRAFT = "DRAFT",
}
export enum DataSource {
UPLOAD = "UPLOAD",
COLLECTION = "COLLECTION",
DATABASE = "DATABASE",
NAS = "NAS",
OBS = "OBS",
}
export interface DatasetFile {
id: number;
fileName: string;
size: string;
uploadDate: string;
path: string;
}
export interface Dataset {
id: number;
name: string;
description: string;
parentId?: number;
datasetType: DatasetType;
status: DatasetStatus;
size?: string;
itemCount?: number;
fileCount?: number;
createdBy: string;
createdAt: string;
updatedAt: string;
tags: string[];
targetLocation?: string;
distribution?: Record<string, Record<string, number>>;
}
export interface TagItem {
id: string;
name: string;
color: string;
description: string;
}
export interface ScheduleConfig {
type: "immediate" | "scheduled";
scheduleType?: "daily" | "weekly" | "monthly" | "custom";
time?: string;
dayOfWeek?: string;
dayOfMonth?: string;
cronExpression?: string;
maxExecutions?: number;
executionCount?: number;
}
export interface DatasetTask {
id: number;
name: string;
description: string;
type: string;
status: "importing" | "waiting" | "completed" | "failed";
progress: number;
createdAt: string;
importConfig: any;
scheduleConfig: ScheduleConfig;
nextExecution?: string;
lastExecution?: string;
executionHistory?: { time: string; status: string }[];
}
export interface TaskItem {
key: string;
title: string;
percent: number;
reqId: number;
isCancel?: boolean;
controller: AbortController;
cancelFn?: () => void;
updateEvent?: string;
size?: number;
hasArchive?: boolean;
}