You've already forked DataMate
init datamate
This commit is contained in:
Dallas98
23
frontend/src/pages/Layout/MainLayout.tsx
Normal file
23
frontend/src/pages/Layout/MainLayout.tsx
Normal file
@@ -0,0 +1,23 @@
|
||||
import React, { memo } from "react";
|
||||
import { Outlet } from "react-router";
|
||||
import Sidebar from "./Sidebar";
|
||||
|
||||
const MainLayout = () => {
|
||||
return (
|
||||
<div className="w-full h-screen flex flex-col bg-gray-50 min-w-6xl">
|
||||
<div className="w-full h-full flex">
|
||||
{/* Sidebar */}
|
||||
<Sidebar />
|
||||
{/* Main Content */}
|
||||
<div className="flex-1 flex flex-col overflow-auto p-6">
|
||||
{/* Content Area */}
|
||||
<div className="flex-1 flex flex-col overflow-auto">
|
||||
<Outlet />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default memo(MainLayout);
|
||||
183
frontend/src/pages/Layout/Sidebar.tsx
Normal file
183
frontend/src/pages/Layout/Sidebar.tsx
Normal file
@@ -0,0 +1,183 @@
|
||||
import React, { memo, useEffect, useState } from "react";
|
||||
import { Button, Menu, Popover } from "antd";
|
||||
import {
|
||||
CloseOutlined,
|
||||
MenuOutlined,
|
||||
SettingOutlined,
|
||||
} from "@ant-design/icons";
|
||||
import { ClipboardList, Sparkles, X } from "lucide-react";
|
||||
import { menuItems } from "@/pages/Layout/menu";
|
||||
import { NavLink, useLocation, useNavigate } from "react-router";
|
||||
import TaskUpload from "./TaskUpload";
|
||||
|
||||
const AsiderAndHeaderLayout = () => {
|
||||
const { pathname } = useLocation();
|
||||
const navigate = useNavigate();
|
||||
const [activeItem, setActiveItem] = useState<string>("management");
|
||||
const [sidebarOpen, setSidebarOpen] = useState(true);
|
||||
const [taskCenterVisible, setTaskCenterVisible] = useState(false);
|
||||
|
||||
// Initialize active item based on current pathname
|
||||
const initActiveItem = () => {
|
||||
for (let index = 0; index < menuItems.length; index++) {
|
||||
const element = menuItems[index];
|
||||
if (element.children) {
|
||||
element.children.forEach((subItem) => {
|
||||
if (pathname.includes(subItem.id)) {
|
||||
setActiveItem(subItem.id);
|
||||
return;
|
||||
}
|
||||
});
|
||||
} else if (pathname.includes(element.id)) {
|
||||
setActiveItem(element.id);
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
initActiveItem();
|
||||
}, [pathname]);
|
||||
|
||||
useEffect(() => {
|
||||
const handleShowTaskPopover = (event: CustomEvent) => {
|
||||
const { show } = event.detail;
|
||||
setTaskCenterVisible(show);
|
||||
};
|
||||
|
||||
window.addEventListener(
|
||||
"show:task-popover",
|
||||
handleShowTaskPopover as EventListener
|
||||
);
|
||||
|
||||
return () => {
|
||||
window.removeEventListener(
|
||||
"show:task-popover",
|
||||
handleShowTaskPopover as EventListener
|
||||
);
|
||||
};
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div
|
||||
className={`${
|
||||
sidebarOpen ? "w-64" : "w-20"
|
||||
} bg-white border-r border-gray-200 transition-all duration-300 flex flex-col relative`}
|
||||
>
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between p-4 border-b border-gray-200">
|
||||
{sidebarOpen && (
|
||||
<NavLink to="/" className="flex items-center gap-2 cursor-pointer">
|
||||
<div className="w-8 h-8 bg-gradient-to-br from-blue-500 to-indigo-600 rounded-lg flex items-center justify-center">
|
||||
<Sparkles className="w-5 h-5 text-white" />
|
||||
</div>
|
||||
<span className="text-lg font-bold text-gray-900">ModelEngine</span>
|
||||
</NavLink>
|
||||
)}
|
||||
<span
|
||||
className="cursor-pointer hover:text-blue-500"
|
||||
onClick={() => setSidebarOpen(!sidebarOpen)}
|
||||
>
|
||||
{sidebarOpen ? <CloseOutlined /> : <MenuOutlined className="ml-4" />}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{/* Navigation */}
|
||||
<div className="flex-1">
|
||||
<Menu
|
||||
mode="inline"
|
||||
inlineCollapsed={!sidebarOpen}
|
||||
items={menuItems.map((item) => ({
|
||||
key: item.id,
|
||||
label: item.title,
|
||||
icon: item.icon ? <item.icon className="w-4 h-4" /> : null,
|
||||
children: item.children
|
||||
? item.children.map((subItem) => ({
|
||||
key: subItem.id,
|
||||
label: subItem.title,
|
||||
icon: subItem.icon ? (
|
||||
<subItem.icon className="w-4 h-4" />
|
||||
) : null,
|
||||
}))
|
||||
: undefined,
|
||||
}))}
|
||||
selectedKeys={[activeItem]}
|
||||
defaultOpenKeys={["synthesis"]}
|
||||
onClick={({ key }) => {
|
||||
setActiveItem(key);
|
||||
console.log(`/data/${key}`);
|
||||
navigate(`/data/${key}`);
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Footer */}
|
||||
<div className="p-4 border-t border-gray-200">
|
||||
{sidebarOpen ? (
|
||||
<div className="space-y-2">
|
||||
<Popover
|
||||
forceRender
|
||||
title={
|
||||
<div className="flex items-center justify-between gap-2 border-b border-gray-200 pb-2 mb-2">
|
||||
<h4 className="font-bold">任务中心</h4>
|
||||
<X
|
||||
onClick={() => setTaskCenterVisible(false)}
|
||||
className="cursor-pointer w-4 h-4 text-gray-500 hover:text-gray-900"
|
||||
/>
|
||||
</div>
|
||||
}
|
||||
open={taskCenterVisible}
|
||||
content={<TaskUpload />}
|
||||
trigger="click"
|
||||
destroyOnHidden={false}
|
||||
>
|
||||
<Button block onClick={() => setTaskCenterVisible(true)}>
|
||||
任务中心
|
||||
</Button>
|
||||
</Popover>
|
||||
<Button block onClick={() => navigate("/data/settings")}>
|
||||
设置
|
||||
</Button>
|
||||
</div>
|
||||
) : (
|
||||
<div className="space-y-2">
|
||||
<div className="relative">
|
||||
<Popover
|
||||
forceRender
|
||||
title="任务中心"
|
||||
open={taskCenterVisible}
|
||||
content={<TaskUpload />}
|
||||
trigger="click"
|
||||
destroyOnHidden={false}
|
||||
>
|
||||
<Button
|
||||
block
|
||||
onClick={() => setTaskCenterVisible(true)}
|
||||
icon={<ClipboardList className="w-4 h-4" />}
|
||||
></Button>
|
||||
</Popover>
|
||||
</div>
|
||||
<Button block onClick={() => navigate("/data/settings")}>
|
||||
<SettingOutlined />
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* 添加遮罩层，点击外部区域时关闭 */}
|
||||
{taskCenterVisible && (
|
||||
<div
|
||||
className="fixed inset-0 z-40"
|
||||
onClick={() => {
|
||||
console.log("clicked outside");
|
||||
|
||||
setTaskCenterVisible(false);
|
||||
toggleShowTaskPopover(false);
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default memo(AsiderAndHeaderLayout);
|
||||
217
frontend/src/pages/Layout/TaskUpload.tsx
Normal file
217
frontend/src/pages/Layout/TaskUpload.tsx
Normal file
@@ -0,0 +1,217 @@
|
||||
import {
|
||||
cancelUploadUsingPut,
|
||||
preUploadUsingPost,
|
||||
uploadFileChunkUsingPost,
|
||||
} from "@/pages/DataManagement/dataset.api";
|
||||
import { TaskItem } from "@/pages/DataManagement/dataset.model";
|
||||
import { calculateSHA256, checkIsFilesExist } from "@/utils/file.util";
|
||||
import { App, Button, Empty, Progress } from "antd";
|
||||
import { DeleteOutlined } from "@ant-design/icons";
|
||||
import { useState, useRef, useEffect } from "react";
|
||||
|
||||
export default function TaskUpload() {
|
||||
const { message } = App.useApp();
|
||||
const [taskList, setTaskList] = useState<TaskItem[]>([]);
|
||||
const taskListRef = useRef<TaskItem[]>([]); // 用于固定任务顺序
|
||||
|
||||
const createTask = (detail: any = {}) => {
|
||||
const { dataset } = detail;
|
||||
const title = `上传数据集: ${dataset.name} `;
|
||||
const controller = new AbortController();
|
||||
const task: TaskItem = {
|
||||
key: dataset.id,
|
||||
title,
|
||||
percent: 0,
|
||||
reqId: -1,
|
||||
controller,
|
||||
};
|
||||
taskListRef.current = [task, ...taskListRef.current];
|
||||
|
||||
setTaskList(taskListRef.current);
|
||||
return task;
|
||||
};
|
||||
|
||||
const updateTaskList = (task: TaskItem) => {
|
||||
taskListRef.current = taskListRef.current.map((item) =>
|
||||
item.key === task.key ? task : item
|
||||
);
|
||||
setTaskList(taskListRef.current);
|
||||
};
|
||||
|
||||
const removeTask = (task: TaskItem) => {
|
||||
const { key } = task;
|
||||
taskListRef.current = taskListRef.current.filter(
|
||||
(item) => item.key !== key
|
||||
);
|
||||
setTaskList(taskListRef.current);
|
||||
if (task.isCancel && task.cancelFn) {
|
||||
task.cancelFn();
|
||||
}
|
||||
window.dispatchEvent(new Event("update:dataset"));
|
||||
window.dispatchEvent(
|
||||
new CustomEvent("show:task-popover", { detail: { show: false } })
|
||||
);
|
||||
};
|
||||
|
||||
async function buildFormData({ file, reqId, i, j }) {
|
||||
const formData = new FormData();
|
||||
const { slices, name, size } = file;
|
||||
const checkSum = await calculateSHA256(slices[j]);
|
||||
formData.append("file", slices[j]);
|
||||
formData.append("reqId", reqId.toString());
|
||||
formData.append("fileNo", (i + 1).toString());
|
||||
formData.append("chunkNo", (j + 1).toString());
|
||||
formData.append("fileName", name);
|
||||
formData.append("fileSize", size.toString());
|
||||
formData.append("totalChunkNum", slices.length.toString());
|
||||
formData.append("checkSumHex", checkSum);
|
||||
return formData;
|
||||
}
|
||||
|
||||
async function uploadSlice(task: TaskItem, fileInfo) {
|
||||
if (!task) {
|
||||
return;
|
||||
}
|
||||
const { reqId, key, signal } = task;
|
||||
const { loaded, i, j, files, totalSize } = fileInfo;
|
||||
const formData = await buildFormData({
|
||||
file: files[i],
|
||||
i,
|
||||
j,
|
||||
reqId,
|
||||
});
|
||||
|
||||
let newTask = { ...task };
|
||||
await uploadFileChunkUsingPost(key, formData, {
|
||||
onUploadProgress: (e) => {
|
||||
const loadedSize = loaded + e.loaded;
|
||||
const curPercent = Math.round(loadedSize / totalSize) * 100;
|
||||
newTask = {
|
||||
...newTask,
|
||||
...taskListRef.current.find((item) => item.key === key),
|
||||
percent: curPercent >= 100 ? 99.99 : curPercent,
|
||||
};
|
||||
updateTaskList(newTask);
|
||||
},
|
||||
signal,
|
||||
});
|
||||
}
|
||||
|
||||
async function uploadFile({ task, files, totalSize }) {
|
||||
const { data: reqId } = await preUploadUsingPost(task.key, {
|
||||
totalFileNum: files.length,
|
||||
totalSize,
|
||||
datasetId: task.key,
|
||||
});
|
||||
|
||||
const newTask: TaskItem = {
|
||||
...task,
|
||||
reqId,
|
||||
isCancel: false,
|
||||
cancelFn: () => {
|
||||
task.controller.abort();
|
||||
cancelUploadUsingPut(reqId);
|
||||
window.dispatchEvent(new Event("update:dataset"));
|
||||
},
|
||||
};
|
||||
updateTaskList(newTask);
|
||||
window.dispatchEvent(
|
||||
new CustomEvent("show:task-popover", { detail: { show: true } })
|
||||
);
|
||||
// 更新数据状态
|
||||
window.dispatchEvent(new Event("update:dataset-status"));
|
||||
|
||||
let loaded = 0;
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
const { slices } = files[i];
|
||||
for (let j = 0; j < slices.length; j++) {
|
||||
await uploadSlice(newTask, {
|
||||
loaded,
|
||||
i,
|
||||
j,
|
||||
files,
|
||||
totalSize,
|
||||
});
|
||||
loaded += slices[j].size;
|
||||
}
|
||||
}
|
||||
removeTask(newTask);
|
||||
}
|
||||
|
||||
const handleUpload = async ({ task, files }) => {
|
||||
const isErrorFile = await checkIsFilesExist(files);
|
||||
if (isErrorFile) {
|
||||
message.error("文件被修改或删除，请重新选择文件上传");
|
||||
removeTask({
|
||||
...task,
|
||||
isCancel: false,
|
||||
...taskListRef.current.find((item) => item.key === task.key),
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const totalSize = files.reduce((acc, file) => acc + file.size, 0);
|
||||
await uploadFile({ task, files, totalSize });
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
message.error("文件上传失败，请稍后重试");
|
||||
removeTask({
|
||||
...task,
|
||||
isCancel: true,
|
||||
...taskListRef.current.find((item) => item.key === task.key),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
const uploadHandler = (e: any) => {
|
||||
const { files } = e.detail;
|
||||
const task = createTask(e.detail);
|
||||
handleUpload({ task, files });
|
||||
};
|
||||
window.addEventListener("upload:dataset", uploadHandler);
|
||||
return () => {
|
||||
window.removeEventListener("upload:dataset", uploadHandler);
|
||||
};
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div
|
||||
className="w-90 max-w-90 max-h-96 overflow-y-auto p-2"
|
||||
id="header-task-popover"
|
||||
>
|
||||
{taskList.length > 0 &&
|
||||
taskList.map((task) => (
|
||||
<div key={task.key} className="border-b border-gray-200 pb-2">
|
||||
<div className="flex items-center justify-between">
|
||||
<div>{task.title}</div>
|
||||
<Button
|
||||
type="text"
|
||||
danger
|
||||
disabled={!task?.cancelFn}
|
||||
onClick={() =>
|
||||
removeTask({
|
||||
...task,
|
||||
isCancel: true,
|
||||
...taskListRef.current.find(
|
||||
(item) => item.key === task.key
|
||||
),
|
||||
})
|
||||
}
|
||||
icon={<DeleteOutlined />}
|
||||
></Button>
|
||||
</div>
|
||||
|
||||
<Progress size="small" percent={Number(task.percent.toFixed(2))} />
|
||||
</div>
|
||||
))}
|
||||
{taskList.length === 0 && (
|
||||
<Empty
|
||||
image={Empty.PRESENTED_IMAGE_SIMPLE}
|
||||
description="暂无上传任务"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
115
frontend/src/pages/Layout/menu.tsx
Normal file
115
frontend/src/pages/Layout/menu.tsx
Normal file
@@ -0,0 +1,115 @@
|
||||
import {
|
||||
FolderOpen,
|
||||
Tag,
|
||||
Target,
|
||||
BookOpen,
|
||||
Shuffle,
|
||||
BarChart3,
|
||||
MessageSquare,
|
||||
GitBranch,
|
||||
Zap,
|
||||
Shield,
|
||||
Database,
|
||||
Store,
|
||||
Merge,
|
||||
} from "lucide-react";
|
||||
|
||||
export const menuItems = [
|
||||
{
|
||||
id: "collection",
|
||||
title: "数据归集",
|
||||
icon: Database,
|
||||
description: "创建、导入和管理数据集",
|
||||
color: "bg-orange-500",
|
||||
},
|
||||
{
|
||||
id: "management",
|
||||
title: "数据管理",
|
||||
icon: FolderOpen,
|
||||
description: "创建、导入和管理数据集",
|
||||
color: "bg-blue-500",
|
||||
},
|
||||
{
|
||||
id: "cleansing",
|
||||
title: "数据清洗",
|
||||
icon: GitBranch,
|
||||
description: "数据清洗和预处理",
|
||||
color: "bg-purple-500",
|
||||
},
|
||||
{
|
||||
id: "annotation",
|
||||
title: "数据标注",
|
||||
icon: Tag,
|
||||
description: "对数据进行标注和标记",
|
||||
color: "bg-green-500",
|
||||
},
|
||||
{
|
||||
id: "synthesis",
|
||||
title: "数据合成",
|
||||
icon: Shuffle,
|
||||
description: "智能数据合成和配比",
|
||||
color: "bg-pink-500",
|
||||
children: [
|
||||
{
|
||||
id: "synthesis/task",
|
||||
title: "合成任务",
|
||||
icon: Merge,
|
||||
},
|
||||
{
|
||||
id: "synthesis/ratio-task",
|
||||
title: "配比任务",
|
||||
icon: BarChart3,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "evaluation",
|
||||
title: "数据评估",
|
||||
icon: Target,
|
||||
badge: 4,
|
||||
description: "质量分析、性能评估和偏见检测",
|
||||
color: "bg-indigo-500",
|
||||
},
|
||||
{
|
||||
id: "knowledge-generation",
|
||||
title: "知识生成",
|
||||
icon: BookOpen,
|
||||
description: "面向RAG的知识库构建",
|
||||
color: "bg-teal-500",
|
||||
},
|
||||
{
|
||||
id: "operator-market",
|
||||
title: "算子市场",
|
||||
icon: Store,
|
||||
description: "算子上传与管理",
|
||||
color: "bg-yellow-500",
|
||||
},
|
||||
];
|
||||
|
||||
export const features = [
|
||||
{
|
||||
icon: GitBranch,
|
||||
title: "智能编排",
|
||||
description: "可视化数据清洗流程编排，拖拽式设计复杂的数据清洗管道",
|
||||
},
|
||||
{
|
||||
icon: MessageSquare,
|
||||
title: "对话助手",
|
||||
description: "通过自然语言对话完成复杂的数据集操作和业务流程",
|
||||
},
|
||||
{
|
||||
icon: Target,
|
||||
title: "全面评估",
|
||||
description: "多维度数据质量评估，包含统计分析、性能测试和偏见检测",
|
||||
},
|
||||
{
|
||||
icon: Zap,
|
||||
title: "高效处理",
|
||||
description: "完整的数据清洗流水线，从原始数据到可用数据集",
|
||||
},
|
||||
{
|
||||
icon: Shield,
|
||||
title: "知识管理",
|
||||
description: "构建面向RAG的知识库，支持智能问答和检索",
|
||||
},
|
||||
];
|
||||
Reference in New Issue
Block a user