-
- {item?.icon && (
-
- {item?.icon}
-
- )}
-
-
-
onView?.(item)}
+ onView?.(item)}
+ style={{ cursor: onView ? "pointer" : "default" }}
+ >
+ {/* Header */}
+
+
+ {item?.icon && (
+
- {item?.name}
-
- {item?.status && (
-
-
- {item?.status?.icon}
- {item?.status?.label}
-
-
- )}
+ {item?.icon}
+
+ )}
+
+
+
+ {item?.name}
+
+ {item?.status && (
+
+
+ {item?.status?.icon}
+ {item?.status?.label}
+
+
+ )}
+
+ {onFavorite && (
+
onFavorite?.(item)}
+ />
+ )}
- {onFavorite && (
-
onFavorite?.(item)}
- />
- )}
-
- {/* Tags */}
-
+
+ {/* Tags */}
+
- {/* Description */}
-
-
- {item?.description}
-
-
+ {/* Description */}
+
+
+ {item?.description}
+
+
- {/* Statistics */}
-
- {item?.statistics?.map((stat, idx) => (
-
-
- {stat?.label}:
+ {/* Statistics */}
+
+ {item?.statistics?.map((stat, idx) => (
+
+
+ {stat?.label}:
+
+
+ {stat?.value}
+
-
- {stat?.value}
-
-
- ))}
+ ))}
+
@@ -257,24 +308,15 @@ function CardView
(props: CardViewProps) {
-
-
-
+ />
)}
diff --git a/frontend/src/components/DetailHeader.tsx b/frontend/src/components/DetailHeader.tsx
index b3c910e..2b77f45 100644
--- a/frontend/src/components/DetailHeader.tsx
+++ b/frontend/src/components/DetailHeader.tsx
@@ -1,8 +1,9 @@
import React from "react";
import { Database } from "lucide-react";
-import { Card, Dropdown, Button, Tag, Tooltip } from "antd";
+import { Card, Button, Tag, Tooltip, Popconfirm } from "antd";
import type { ItemType } from "antd/es/menu/interface";
import AddTagPopover from "./AddTagPopover";
+import ActionDropdown from "./ActionDropdown";
interface StatisticItem {
icon: React.ReactNode;
@@ -100,30 +101,38 @@ function DetailHeader
({
{operations.map((op) => {
if (op.isDropdown) {
return (
-
-
-
-
-
+
+ );
+ }
+ if (op.confirm) {
+ return (
+
+ {
+ op?.onClick();
+ }}
+ okText={op.confirm.okText || "确定"}
+ cancelText={op.confirm.cancelText || "取消"}
+ okType={op.danger ? "danger" : "primary"}
+ overlayStyle={{ zIndex: 9999 }}
+ >
+
+
+
);
}
return (
);
diff --git a/frontend/src/components/TagManagement.tsx b/frontend/src/components/TagManagement.tsx
index 26f2572..42913b3 100644
--- a/frontend/src/components/TagManagement.tsx
+++ b/frontend/src/components/TagManagement.tsx
@@ -128,6 +128,7 @@ const TagManager: React.FC = ({
name: tag,
});
fetchTags();
+ setNewTag("");
message.success("标签添加成功");
} catch (error) {
message.error("添加标签失败");
diff --git a/frontend/src/hooks/useFetchData.ts b/frontend/src/hooks/useFetchData.ts
index 441eeba..6042ad4 100644
--- a/frontend/src/hooks/useFetchData.ts
+++ b/frontend/src/hooks/useFetchData.ts
@@ -1,14 +1,36 @@
// 首页数据获取
-import { useState } from "react";
+// 支持轮询功能，使用示例：
+// const { fetchData, startPolling, stopPolling, isPolling } = useFetchData(
+// fetchFunction,
+// mapFunction,
+// 5000, // 5秒轮询一次，默认30秒
+// true, // 是否自动开始轮询，默认 true
+// [fetchStatistics, fetchOtherData] // 额外的轮询函数数组
+// );
+//
+// startPolling(); // 开始轮询
+// stopPolling(); // 停止轮询
+// 手动调用 fetchData() 时，如果正在轮询，会重新开始轮询计时
+// 轮询时会同时执行主要的 fetchFunction 和所有额外的轮询函数
+import { useState, useRef, useEffect, useCallback } from "react";
import { useDebouncedEffect } from "./useDebouncedEffect";
import Loading from "@/utils/loading";
import { App } from "antd";
+import { AnyObject } from "antd/es/_util/type";
export default function useFetchData(
fetchFunc: (params?: any) => Promise,
- mapDataFunc: (data: any) => T = (data) => data as T
+ mapDataFunc: (data: AnyObject) => T = (data) => data as T,
+ pollingInterval: number = 30000, // 默认30秒轮询一次
+ autoRefresh: boolean = true,
+ additionalPollingFuncs: (() => Promise)[] = [] // 额外的轮询函数
) {
const { message } = App.useApp();
+
+ // 轮询相关状态
+ const [isPolling, setIsPolling] = useState(false);
+ const pollingTimerRef = useRef(null);
+
// 表格数据
const [tableData, setTableData] = useState([]);
// 设置加载状态
@@ -55,39 +77,117 @@ export default function useFetchData(
return arr[0];
}
- async function fetchData(extraParams = {}) {
- const { keyword, filter, current, pageSize } = searchParams;
- Loading.show();
- setLoading(true);
- try {
- const { data } = await fetchFunc({
- ...filter,
- ...extraParams,
- keyword,
- type: getFirstOfArray(filter?.type) || undefined,
- status: getFirstOfArray(filter?.status) || undefined,
- tags: filter?.tags?.length ? filter.tags.join(",") : undefined,
- page: current - 1,
- size: pageSize,
- });
- setPagination((prev) => ({
- ...prev,
- total: data?.totalElements || 0,
- }));
- let result = [];
- if (mapDataFunc) {
- result = data?.content.map(mapDataFunc) ?? [];
- }
- setTableData(result);
- } catch (error) {
- console.error(error)
- message.error("数据获取失败，请稍后重试");
- } finally {
- Loading.hide();
- setLoading(false);
+ // 清除轮询定时器
+ const clearPollingTimer = useCallback(() => {
+ if (pollingTimerRef.current) {
+ clearTimeout(pollingTimerRef.current);
+ pollingTimerRef.current = null;
}
- }
+ }, []);
+ const fetchData = useCallback(
+ async (extraParams = {}, skipPollingRestart = false) => {
+ const { keyword, filter, current, pageSize } = searchParams;
+
+ if (!skipPollingRestart) {
+ Loading.show();
+ setLoading(true);
+ }
+
+ // 如果正在轮询且不是轮询触发的调用，先停止当前轮询
+ const wasPolling = isPolling && !skipPollingRestart;
+ if (wasPolling) {
+ clearPollingTimer();
+ }
+
+ try {
+ // 同时执行主要数据获取和额外的轮询函数
+ const promises = [
+ fetchFunc({
+ ...filter,
+ ...extraParams,
+ keyword,
+ type: getFirstOfArray(filter?.type) || undefined,
+ status: getFirstOfArray(filter?.status) || undefined,
+ tags: filter?.tags?.length ? filter.tags.join(",") : undefined,
+ page: current - 1,
+ size: pageSize,
+ }),
+ ...additionalPollingFuncs.map((func) => func()),
+ ];
+
+ const results = await Promise.all(promises);
+ const { data } = results[0]; // 主要数据结果
+
+ setPagination((prev) => ({
+ ...prev,
+ total: data?.totalElements || 0,
+ }));
+ let result = [];
+ if (mapDataFunc) {
+ result = data?.content.map(mapDataFunc) ?? [];
+ }
+ setTableData(result);
+
+ // 如果之前正在轮询且不是轮询触发的调用，重新开始轮询
+ if (wasPolling) {
+ const poll = () => {
+ pollingTimerRef.current = setTimeout(() => {
+ fetchData({}, true).then(() => {
+ if (pollingTimerRef.current) {
+ poll();
+ }
+ });
+ }, pollingInterval);
+ };
+ poll();
+ }
+ } catch (error) {
+ console.error(error);
+ message.error("数据获取失败，请稍后重试");
+ } finally {
+ Loading.hide();
+ setLoading(false);
+ }
+ },
+ [
+ searchParams,
+ fetchFunc,
+ mapDataFunc,
+ isPolling,
+ clearPollingTimer,
+ pollingInterval,
+ message,
+ additionalPollingFuncs,
+ ]
+ );
+
+ // 开始轮询
+ const startPolling = useCallback(() => {
+ clearPollingTimer();
+ setIsPolling(true);
+
+ const poll = () => {
+ pollingTimerRef.current = setTimeout(() => {
+ fetchData({}, true).then(() => {
+ if (pollingTimerRef.current) {
+ poll();
+ }
+ });
+ }, pollingInterval);
+ };
+
+ poll();
+ }, [pollingInterval, clearPollingTimer, fetchData]);
+
+ // 停止轮询
+ const stopPolling = useCallback(() => {
+ clearPollingTimer();
+ setIsPolling(false);
+ }, [clearPollingTimer]);
+
+ // 搜索参数变化时，自动刷新数据
+ // keyword 变化时，防抖500ms后刷新
useDebouncedEffect(
() => {
fetchData();
@@ -96,6 +196,16 @@ export default function useFetchData(
searchParams?.keyword ? 500 : 0
);
+ // 组件卸载时清理轮询
+ useEffect(() => {
+ if (autoRefresh) {
+ startPolling();
+ }
+ return () => {
+ clearPollingTimer();
+ };
+ }, [clearPollingTimer]);
+
return {
loading,
tableData,
@@ -109,5 +219,8 @@ export default function useFetchData(
setPagination,
handleFiltersChange,
fetchData,
+ isPolling,
+ startPolling,
+ stopPolling,
};
}
diff --git a/frontend/src/mock/cleansing.tsx b/frontend/src/mock/cleansing.tsx
deleted file mode 100644
index 9d90184..0000000
--- a/frontend/src/mock/cleansing.tsx
+++ /dev/null
@@ -1,56 +0,0 @@
-import {
- DatabaseOutlined,
- BarChartOutlined,
- FileTextOutlined,
- ThunderboltOutlined,
- PictureOutlined,
- CalculatorOutlined,
- SwapOutlined,
-} from "@ant-design/icons";
-import { FileImage, FileText, Music, Repeat, Video } from "lucide-react";
-
-// 模板类型选项
-export const templateTypes = [
- {
- value: "text",
- label: "文本",
- icon: FileText,
- description: "处理文本数据的清洗模板",
- },
- {
- value: "image",
- label: "图片",
- icon: FileImage,
- description: "处理图像数据的清洗模板",
- },
- {
- value: "video",
- label: "视频",
- icon: Video,
- description: "处理视频数据的清洗模板",
- },
- {
- value: "audio",
- label: "音频",
- icon: Music,
- description: "处理音频数据的清洗模板",
- },
- {
- value: "image-to-text",
- label: "图片转文本",
- icon: Repeat,
- description: "图像识别转文本的处理模板",
- },
-];
-
-// 算子分类
-export const OPERATOR_CATEGORIES = {
- data: { name: "数据清洗", icon: , color: "#1677ff" },
- ml: { name: "机器学习", icon: , color: "#722ed1" },
- vision: { name: "计算机视觉", icon: , color: "#52c41a" },
- nlp: { name: "自然语言处理", icon: , color: "#faad14" },
- analysis: { name: "数据分析", icon: , color: "#f5222d" },
- transform: { name: "数据转换", icon: , color: "#13c2c2" },
- io: { name: "输入输出", icon: , color: "#595959" },
- math: { name: "数学计算", icon: , color: "#fadb14" },
-};
diff --git a/frontend/src/mock/mock-seed/data-cleansing.cjs b/frontend/src/mock/mock-seed/data-cleansing.cjs
index 6ecf051..c2b9a23 100644
--- a/frontend/src/mock/mock-seed/data-cleansing.cjs
+++ b/frontend/src/mock/mock-seed/data-cleansing.cjs
@@ -93,9 +93,13 @@ function cleaningTaskItem() {
srcDatasetName: Mock.Random.ctitle(5, 15),
destDatasetId: Mock.Random.guid().replace(/[^a-zA-Z0-9]/g, ""),
destDatasetName: Mock.Random.ctitle(5, 15),
- progress: Mock.Random.float(0, 100, 2, 2),
+ progress: {
+ finishedFileNum: Mock.Random.integer(0, 100),
+ process: Mock.Random.integer(0, 100),
+ totalFileNum: 100,
+ },
startedAt: Mock.Random.datetime("yyyy-MM-dd HH:mm:ss"),
- endedAt: Mock.Random.datetime("yyyy-MM-dd HH:mm:ss"),
+ finishedAt: Mock.Random.datetime("yyyy-MM-dd HH:mm:ss"),
createdAt: Mock.Random.datetime("yyyy-MM-dd HH:mm:ss"),
updatedAt: Mock.Random.datetime("yyyy-MM-dd HH:mm:ss"),
instance: operatorList,
@@ -244,7 +248,7 @@ module.exports = function (router) {
const task = cleaningTaskList.find((j) => j.id === taskId);
if (task) {
- task.status = "running";
+ task.status = "RUNNING";
task.startTime = new Date().toISOString();
res.send({
@@ -252,7 +256,7 @@ module.exports = function (router) {
msg: "Cleaning task execution started",
data: {
executionId: Mock.Random.guid().replace(/[^a-zA-Z0-9]/g, ""),
- status: "running",
+ status: "RUNNING",
message: "Task execution started successfully",
},
});
@@ -271,7 +275,7 @@ module.exports = function (router) {
const task = cleaningTaskList.find((j) => j.id === taskId);
if (task) {
- task.status = "pending";
+ task.status = "PENDING";
task.endTime = new Date().toISOString();
res.send({
diff --git a/frontend/src/pages/DataCleansing/Create/DragDrop.css b/frontend/src/pages/DataCleansing/Create/DragDrop.css
deleted file mode 100644
index 2762059..0000000
--- a/frontend/src/pages/DataCleansing/Create/DragDrop.css
+++ /dev/null
@@ -1,411 +0,0 @@
-/* PreciseDragDrop.css */
-.precise-drag-drop {
- max-width: 1200px;
- margin: 0 auto;
- padding: 20px;
- font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
- color: #333;
-}
-
-.header {
- text-align: center;
- margin-bottom: 40px;
-}
-
-.header h1 {
- color: #2c3e50;
- margin-bottom: 10px;
- font-weight: 600;
- font-size: 2rem;
-}
-
-.header p {
- color: #7f8c8d;
- font-size: 1.1rem;
-}
-
-.containers {
- display: grid;
- grid-template-columns: 1fr 1fr;
- gap: 30px;
- margin-bottom: 40px;
-}
-
-.container {
- background: white;
- border-radius: 12px;
- box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
- overflow: hidden;
- transition: all 0.3s ease;
- border: 2px solid transparent;
-}
-
-.container.drag-over {
- border-color: #3498db;
- background-color: #f8fafc;
- transform: scale(1.02);
-}
-
-.container-header {
- display: flex;
- justify-content: space-between;
- align-items: center;
- padding: 20px;
- background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
- color: white;
-}
-
-.container-header h2 {
- margin: 0;
- font-size: 1.4rem;
- display: flex;
- align-items: center;
- gap: 8px;
-}
-
-.count {
- background: rgba(255, 255, 255, 0.2);
- padding: 6px 12px;
- border-radius: 20px;
- font-size: 0.9rem;
- font-weight: 600;
-}
-
-.header-actions {
- display: flex;
- align-items: center;
- gap: 10px;
-}
-
-.clear-btn {
- background: rgba(255, 255, 255, 0.2);
- border: 1px solid rgba(255, 255, 255, 0.3);
- color: white;
- padding: 6px 12px;
- border-radius: 6px;
- cursor: pointer;
- font-size: 0.8rem;
- transition: all 0.3s ease;
-}
-
-.clear-btn:hover {
- background: rgba(255, 255, 255, 0.3);
- transform: translateY(-1px);
-}
-
-.items-list {
- padding: 20px;
- min-height: 500px;
- max-height: 600px;
- overflow-y: auto;
- position: relative;
-}
-
-.item {
- display: flex;
- justify-content: space-between;
- align-items: center;
- padding: 16px;
- margin-bottom: 8px;
- background: white;
- border-radius: 8px;
- border-left: 4px solid var(--item-color);
- box-shadow: 0 2px 6px rgba(0, 0, 0, 0.08);
- transition: all 0.3s ease;
- cursor: grab;
- position: relative;
-}
-
-.item:hover {
- transform: translateY(-2px);
- box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15);
-}
-
-.item.dragging {
- opacity: 0.6;
- cursor: grabbing;
- transform: rotate(3deg) scale(1.05);
- box-shadow: 0 8px 20px rgba(0, 0, 0, 0.2);
-}
-
-.item.drag-over.insert-above {
- border-top: 2px dashed var(--item-color);
- margin-top: 4px;
-}
-
-.item.drag-over.insert-below {
- border-bottom: 2px dashed var(--item-color);
- margin-bottom: 4px;
-}
-
-.item-content {
- display: flex;
- align-items: center;
- gap: 12px;
- flex: 1;
-}
-
-.item-index {
- display: flex;
- align-items: center;
- justify-content: center;
- width: 28px;
- height: 28px;
- background-color: var(--item-color);
- color: white;
- border-radius: 50%;
- font-size: 0.9rem;
- font-weight: bold;
- flex-shrink: 0;
-}
-
-.item-icon {
- font-size: 1.3rem;
- flex-shrink: 0;
-}
-
-.item-info {
- display: flex;
- flex-direction: column;
- gap: 4px;
- flex: 1;
-}
-
-.item-title {
- font-weight: 500;
- font-size: 1rem;
-}
-
-.priority-tag {
- padding: 2px 8px;
- border-radius: 12px;
- font-size: 0.7rem;
- font-weight: 600;
- text-transform: uppercase;
- width: fit-content;
-}
-
-.priority-high {
- background: #ffebee;
- color: #c62828;
-}
-
-.priority-medium {
- background: #fff3e0;
- color: #ef6c00;
-}
-
-.priority-low {
- background: #e8f5e8;
- color: #2e7d32;
-}
-
-.item-type {
- background: #f1f3f4;
- padding: 4px 8px;
- border-radius: 4px;
- font-size: 0.8rem;
- color: #666;
- text-transform: capitalize;
- flex-shrink: 0;
-}
-
-.item-actions {
- display: flex;
- align-items: center;
-}
-
-.drag-handle {
- color: #bdc3c7;
- font-size: 16px;
- cursor: grab;
- padding: 8px;
- user-select: none;
- border-radius: 4px;
- transition: all 0.3s ease;
-}
-
-.drag-handle:hover {
- color: #7f8c8d;
- background: #f5f5f5;
-}
-
-.empty-state {
- text-align: center;
- padding: 80px 20px;
- color: #95a5a6;
-}
-
-.empty-state p {
- margin: 0 0 8px 0;
- font-size: 1.1rem;
- font-weight: 500;
-}
-
-.empty-state span {
- font-size: 0.9rem;
-}
-
-/* 插入位置指示器 */
-.insert-indicator {
- display: flex;
- align-items: center;
- gap: 8px;
- padding: 8px 0;
- margin: 4px 0;
- opacity: 0.8;
- animation: pulse 1.5s infinite;
-}
-
-.insert-indicator.above {
- margin-bottom: 0;
-}
-
-.insert-indicator.below {
- margin-top: 0;
-}
-
-.indicator-line {
- flex: 1;
- height: 2px;
- background: linear-gradient(90deg, transparent, var(--item-color, #3498db), transparent);
-}
-
-.indicator-arrow {
- color: var(--item-color, #3498db);
- font-weight: bold;
- font-size: 0.9rem;
- padding: 0 8px;
-}
-
-@keyframes pulse {
- 0%, 100% { opacity: 0.6; }
- 50% { opacity: 1; }
-}
-
-.instructions {
- background: #f8f9fa;
- padding: 25px;
- border-radius: 12px;
- border-left: 4px solid #3498db;
-}
-
-.instructions h3 {
- margin-top: 0;
- color: #2c3e50;
- text-align: center;
- margin-bottom: 20px;
- font-size: 1.3rem;
-}
-
-.instruction-grid {
- display: grid;
- grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
- gap: 20px;
-}
-
-.instruction {
- display: flex;
- align-items: flex-start;
- gap: 15px;
- padding: 20px;
- background: white;
- border-radius: 8px;
- box-shadow: 0 2px 6px rgba(0, 0, 0, 0.05);
- transition: all 0.3s ease;
-}
-
-.instruction:hover {
- transform: translateY(-2px);
- box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
-}
-
-.instruction .icon {
- font-size: 1.8rem;
- flex-shrink: 0;
-}
-
-.instruction strong {
- display: block;
- margin-bottom: 5px;
- color: #2c3e50;
- font-size: 1rem;
-}
-
-.instruction p {
- margin: 0;
- color: #7f8c8d;
- font-size: 0.9rem;
- line-height: 1.4;
-}
-
-/* 动画效果 */
-@keyframes slideIn {
- from {
- opacity: 0;
- transform: translateY(10px);
- }
- to {
- opacity: 1;
- transform: translateY(0);
- }
-}
-
-.item {
- animation: slideIn 0.3s ease;
-}
-
-/* 滚动条样式 */
-.items-list::-webkit-scrollbar {
- width: 6px;
-}
-
-.items-list::-webkit-scrollbar-track {
- background: #f1f1f1;
- border-radius: 3px;
-}
-
-.items-list::-webkit-scrollbar-thumb {
- background: #c1c1c1;
- border-radius: 3px;
-}
-
-.items-list::-webkit-scrollbar-thumb:hover {
- background: #a8a8a8;
-}
-
-/* 响应式设计 */
-@media (max-width: 768px) {
- .containers {
- grid-template-columns: 1fr;
- gap: 20px;
- }
-
- .precise-drag-drop {
- padding: 15px;
- }
-
- .instruction-grid {
- grid-template-columns: 1fr;
- }
-
- .container-header {
- flex-direction: column;
- gap: 10px;
- align-items: flex-start;
- }
-
- .header-actions {
- align-self: flex-end;
- }
-
- .item-content {
- flex-direction: column;
- align-items: flex-start;
- gap: 8px;
- }
-
- .item-info {
- width: 100%;
- }
-}
\ No newline at end of file
diff --git a/frontend/src/pages/DataCleansing/Create/DragExample.tsx b/frontend/src/pages/DataCleansing/Create/DragExample.tsx
deleted file mode 100644
index bd4a2bd..0000000
--- a/frontend/src/pages/DataCleansing/Create/DragExample.tsx
+++ /dev/null
@@ -1,430 +0,0 @@
-import React, { useState } from "react";
-import "./DragDrop.css";
-
-const PreciseDragDrop = () => {
- // 初始数据
- const [leftItems, setLeftItems] = useState([
- {
- id: 1,
- title: "需求分析",
- type: "analysis",
- color: "#4CAF50",
- priority: "high",
- },
- {
- id: 2,
- title: "UI设计",
- type: "design",
- color: "#2196F3",
- priority: "medium",
- },
- {
- id: 3,
- title: "前端开发",
- type: "development",
- color: "#FF9800",
- priority: "high",
- },
- {
- id: 4,
- title: "后端开发",
- type: "development",
- color: "#9C27B0",
- priority: "high",
- },
- {
- id: 5,
- title: "功能测试",
- type: "testing",
- color: "#3F51B5",
- priority: "medium",
- },
- {
- id: 6,
- title: "部署上线",
- type: "deployment",
- color: "#009688",
- priority: "low",
- },
- ]);
-
- const [rightItems, setRightItems] = useState([
- {
- id: 7,
- title: "项目启动",
- type: "planning",
- color: "#E91E63",
- priority: "high",
- },
- ]);
-
- const [draggingItem, setDraggingItem] = useState(null);
- const [insertPosition, setInsertPosition] = useState(null); // 'above' 或 'below'
-
- // 处理拖拽开始
- const handleDragStart = (e, item, source) => {
- setDraggingItem({ ...item, source });
- e.dataTransfer.effectAllowed = "move";
-
- setTimeout(() => {
- e.target.classList.add("dragging");
- }, 0);
- };
-
- // 处理拖拽结束
- const handleDragEnd = (e) => {
- setDraggingItem(null);
- setInsertPosition(null);
- e.target.classList.remove("dragging");
- };
-
- // 处理容器拖拽经过
- const handleContainerDragOver = (e) => {
- e.preventDefault();
- };
-
- // 处理容器拖拽离开
- const handleContainerDragLeave = (e) => {
- if (!e.currentTarget.contains(e.relatedTarget)) {
- setInsertPosition(null);
- }
- };
-
- // 处理项目拖拽经过（用于精确插入）
- const handleItemDragOver = (e, itemId) => {
- e.preventDefault();
- e.stopPropagation();
-
- const rect = e.currentTarget.getBoundingClientRect();
- const mouseY = e.clientY;
- const elementMiddle = rect.top + rect.height;
-
- // 判断鼠标在元素的上半部分还是下半部分
- const newPosition = mouseY < elementMiddle ? "above" : "below";
-
- setInsertPosition(newPosition);
- };
-
- // 处理项目拖拽离开
- const handleItemDragLeave = (e) => {
- if (!e.currentTarget.contains(e.relatedTarget)) {
- setInsertPosition(null);
- }
- };
-
- // 处理放置到右侧容器空白区域
- const handleDropToRightContainer = (e) => {
- e.preventDefault();
-
- if (!draggingItem) return;
-
- // 如果是从左侧拖拽过来的
- if (draggingItem.source === "left") {
- // 检查是否已存在
- const exists = rightItems.some((item) => item.id === draggingItem.id);
- if (!exists) {
- setRightItems((prev) => [
- ...prev,
- {
- ...draggingItem,
- source: "right",
- },
- ]);
-
- setLeftItems((prev) =>
- prev.filter((item) => item.id !== draggingItem.id)
- );
- }
- }
-
- resetDragState();
- };
-
- // 处理放置到右侧容器的特定位置
- const handleDropToRightItem = (e, targetItemId) => {
- e.preventDefault();
- e.stopPropagation();
-
- if (!draggingItem) return;
-
- // 从左侧拖拽到右侧的精确插入
- if (draggingItem.source === "left") {
- const targetIndex = rightItems.findIndex(
- (item) => item.id === targetItemId
- );
-
- if (targetIndex !== -1) {
- const insertIndex =
- insertPosition === "above" ? targetIndex : targetIndex + 1;
-
- // 检查是否已存在
- const exists = rightItems.some((item) => item.id === draggingItem.id);
- if (!exists) {
- const newRightItems = [...rightItems];
- newRightItems.splice(insertIndex, 0, {
- ...draggingItem,
- source: "right",
- });
-
- setRightItems(newRightItems);
- setLeftItems((prev) =>
- prev.filter((item) => item.id !== draggingItem.id)
- );
- }
- }
- }
- // 右侧容器内的重新排序
- else if (draggingItem.source === "right") {
- const draggedIndex = rightItems.findIndex(
- (item) => item.id === draggingItem.id
- );
- const targetIndex = rightItems.findIndex(
- (item) => item.id === targetItemId
- );
-
- if (
- draggedIndex !== -1 &&
- targetIndex !== -1 &&
- draggedIndex !== targetIndex
- ) {
- const newItems = [...rightItems];
- const [draggedItem] = newItems.splice(draggedIndex, 1);
-
- // 计算正确的插入位置
- let insertIndex =
- insertPosition === "above" ? targetIndex : targetIndex + 1;
- if (draggedIndex < insertIndex) {
- insertIndex--; // 调整插入位置，因为已经移除了原元素
- }
-
- newItems.splice(insertIndex, 0, draggedItem);
- setRightItems(newItems);
- }
- }
-
- resetDragState();
- };
-
- // 处理拖拽回左侧容器
- const handleDropToLeft = (e) => {
- e.preventDefault();
-
- if (!draggingItem || draggingItem.source !== "right") return;
-
- setRightItems((prev) => prev.filter((item) => item.id !== draggingItem.id));
- setLeftItems((prev) => [
- ...prev,
- {
- ...draggingItem,
- source: "left",
- },
- ]);
-
- resetDragState();
- };
-
- // 重置拖拽状态
- const resetDragState = () => {
- setDraggingItem(null);
- setInsertPosition(null);
- };
-
- // 清空右侧容器
- const clearRightContainer = () => {
- setLeftItems((prev) => [
- ...prev,
- ...rightItems.map((item) => ({
- ...item,
- source: "left",
- })),
- ]);
- setRightItems([]);
- };
-
- // 获取类型图标
- const getTypeIcon = (type) => {
- switch (type) {
- case "analysis":
- return "📊";
- case "design":
- return "🎨";
- case "development":
- return "💻";
- case "testing":
- return "🧪";
- case "deployment":
- return "🚀";
- case "planning":
- return "📋";
- default:
- return "📌";
- }
- };
-
- // 获取优先级标签
- const getPriorityLabel = (priority) => {
- switch (priority) {
- case "high":
- return { label: "高优先级", class: "priority-high" };
- case "medium":
- return { label: "中优先级", class: "priority-medium" };
- case "low":
- return { label: "低优先级", class: "priority-low" };
- default:
- return { label: "普通", class: "priority-medium" };
- }
- };
-
- return (
-
-
-
精确位置拖拽排序
-
拖拽时悬停在项目上方或下方可选择精确插入位置
-
-
-
- {/* 左侧容器 - 待办事项 */}
-
handleContainerDragOver(e, "left")}
- onDragLeave={handleContainerDragLeave}
- onDrop={handleDropToLeft}
- >
-
-
📋 待办事项
- {leftItems.length} 项
-
-
- {leftItems.map((item) => (
-
handleDragStart(e, item, "left")}
- onDragEnd={handleDragEnd}
- style={{ "--item-color": item.color }}
- >
-
-
{getTypeIcon(item.type)}
-
- {item.title}
-
- {getPriorityLabel(item.priority).label}
-
-
-
-
{item.type}
-
- ))}
- {leftItems.length === 0 && (
-
-
🎉 所有任务已完成！
-
从右侧拖拽项目回来重新安排
-
- )}
-
-
-
- {/* 右侧容器 - 进行中的任务 */}
-
handleContainerDragOver(e, "right")}
- onDragLeave={handleContainerDragLeave}
- onDrop={handleDropToRightContainer}
- >
-
-
🚀 进行中的任务
-
- {rightItems.length} 项
- {rightItems.length > 0 && (
-
- )}
-
-
-
- {rightItems.length === 0 ? (
-
-
📥 暂无进行中的任务
-
从左侧拖拽项目过来开始工作
-
- ) : (
- rightItems.map((item, index) => (
-
handleDragStart(e, item, "right")}
- onDragEnd={handleDragEnd}
- onDragOver={(e) => handleItemDragOver(e, item.id)}
- onDragLeave={handleItemDragLeave}
- onDrop={(e) => handleDropToRightItem(e, item.id)}
- style={{ "--item-color": item.color }}
- >
-
-
{index + 1}
-
{getTypeIcon(item.type)}
-
- {item.title}
-
- {getPriorityLabel(item.priority).label}
-
-
-
-
- ⋮⋮
-
-
- ))
- )}
-
-
-
-
-
-
🎯 操作指南
-
-
-
🎯
-
-
精确插入
-
拖拽时悬停在项目上方或下方选择插入位置
-
-
-
-
🔄
-
-
重新排序
-
在右侧容器内拖拽调整任务顺序
-
-
-
-
-
🧹
-
-
批量操作
-
使用"清空所有"按钮快速重置
-
-
-
-
-
-
diff --git a/frontend/src/pages/DataManagement/Create/EditDataset.tsx b/frontend/src/pages/DataManagement/Create/EditDataset.tsx
index 0847372..f43ba7f 100644
--- a/frontend/src/pages/DataManagement/Create/EditDataset.tsx
+++ b/frontend/src/pages/DataManagement/Create/EditDataset.tsx
@@ -5,7 +5,7 @@ import {
} from "../dataset.api";
import { useEffect, useState } from "react";
import { Dataset, DatasetType } from "../dataset.model";
-import { App, Button, Drawer, Form, Modal } from "antd";
+import { App, Button, Form, Modal } from "antd";
export default function EditDataset({
open,
@@ -16,7 +16,7 @@ export default function EditDataset({
open: boolean;
data: Dataset | null;
onClose: () => void;
- onRefresh?: () => void;
+ onRefresh?: (showMessage?: boolean) => void;
}) {
const [form] = Form.useForm();
const { message } = App.useApp();
@@ -60,7 +60,7 @@ export default function EditDataset({
await updateDatasetByIdUsingPut(data?.id, params);
onClose();
message.success("数据集更新成功");
- onRefresh?.();
+ onRefresh?.(false);
} catch (error) {
console.error(error);
message.error("数据集更新失败，请重试");
diff --git a/frontend/src/pages/DataManagement/Detail/DatasetDetail.tsx b/frontend/src/pages/DataManagement/Detail/DatasetDetail.tsx
index 6a6442e..23546d2 100644
--- a/frontend/src/pages/DataManagement/Detail/DatasetDetail.tsx
+++ b/frontend/src/pages/DataManagement/Detail/DatasetDetail.tsx
@@ -5,15 +5,17 @@ import {
DownloadOutlined,
UploadOutlined,
EditOutlined,
+ DeleteOutlined,
} from "@ant-design/icons";
import DetailHeader from "@/components/DetailHeader";
import { mapDataset, datasetTypeMap } from "../dataset.const";
import type { Dataset } from "@/pages/DataManagement/dataset.model";
-import { Link, useParams } from "react-router";
-import { useFilesOperation } from "../hooks";
+import { Link, useNavigate, useParams } from "react-router";
+import { useFilesOperation } from "./useFilesOperation";
import {
createDatasetTagUsingPost,
- downloadFile,
+ deleteDatasetByIdUsingDelete,
+ downloadDatasetUsingGet,
queryDatasetByIdUsingGet,
queryDatasetTagsUsingGet,
updateDatasetByIdUsingPut,
@@ -42,6 +44,7 @@ const tabList = [
export default function DatasetDetail() {
const { id } = useParams(); // 获取动态路由参数
+ const navigate = useNavigate();
const [activeTab, setActiveTab] = useState("overview");
const { message } = App.useApp();
const [showEditDialog, setShowEditDialog] = useState(false);
@@ -77,11 +80,17 @@ export default function DatasetDetail() {
if (showMessage) message.success({ content: "数据刷新成功" });
};
- const handleExportFormat = async ({ type }) => {
- await downloadFile(dataset.id, type, `${dataset.name}-${type}.zip`);
+ const handleDownload = async () => {
+ await downloadDatasetUsingGet(dataset.id);
message.success("文件下载成功");
};
+ const handleDeleteDataset = async () => {
+ await deleteDatasetByIdUsingDelete(dataset.id);
+ navigate("/data/management");
+ message.success("数据集删除成功");
+ };
+
useEffect(() => {
const refreshDataset = () => {
fetchDataset();
@@ -153,7 +162,7 @@ export default function DatasetDetail() {
// { key: "csv", label: "CSV 格式", icon:
},
// { key: "coco", label: "COCO 格式", icon:
},
// ],
- onMenuClick: handleExportFormat,
+ onClick: () => handleDownload(),
},
{
key: "refresh",
@@ -161,6 +170,20 @@ export default function DatasetDetail() {
icon:
,
onClick: handleRefresh,
},
+ {
+ key: "delete",
+ label: "删除",
+ danger: true,
+ confirm: {
+ title: "确认删除该数据集？",
+ description: "删除后该数据集将无法恢复，请谨慎操作。",
+ okText: "删除",
+ cancelText: "取消",
+ okType: "danger",
+ },
+ icon:
,
+ onClick: handleDeleteDataset,
+ },
];
return (
diff --git a/frontend/src/pages/DataManagement/Detail/components/ImportConfiguration.tsx b/frontend/src/pages/DataManagement/Detail/components/ImportConfiguration.tsx
index a35deed..ebb710f 100644
--- a/frontend/src/pages/DataManagement/Detail/components/ImportConfiguration.tsx
+++ b/frontend/src/pages/DataManagement/Detail/components/ImportConfiguration.tsx
@@ -1,10 +1,21 @@
-import { Select, Input, Form, Radio, Modal, Button } from "antd";
+import {
+ Select,
+ Input,
+ Form,
+ Radio,
+ Modal,
+ Button,
+ App,
+ UploadFile,
+} from "antd";
+import { InboxOutlined } from "@ant-design/icons";
import { dataSourceOptions } from "../../dataset.const";
import { Dataset, DataSource } from "../../dataset.model";
-import { useEffect, useState } from "react";
+import { useEffect, useMemo, useState } from "react";
import { queryTasksUsingGet } from "@/pages/DataCollection/collection.apis";
-import { useImportFile } from "../../hooks";
import { updateDatasetByIdUsingPut } from "../../dataset.api";
+import { sliceFile } from "@/utils/file.util";
+import Dragger from "antd/es/upload/Dragger";
export default function ImportConfiguration({
data,
@@ -15,16 +26,52 @@ export default function ImportConfiguration({
data?: Dataset;
open: boolean;
onClose: () => void;
- onRefresh?: () => void;
+ onRefresh?: (showMessage?: boolean) => void;
}) {
+ const { message } = App.useApp();
const [form] = Form.useForm();
const [collectionOptions, setCollectionOptions] = useState([]);
const [importConfig, setImportConfig] = useState
({
source: DataSource.UPLOAD,
});
- const { importFileRender, handleUpload } = useImportFile();
- // 获取归集任务列表
+ const [fileList, setFileList] = useState([]);
+ const fileSliceList = useMemo(() => {
+ const sliceList = fileList.map((file) => {
+ const slices = sliceFile(file);
+ return { originFile: file, slices, name: file.name, size: file.size };
+ });
+ return sliceList;
+ }, [fileList]);
+
+ // 本地上传文件相关逻辑
+
+ const resetFiles = () => {
+ setFileList([]);
+ };
+
+ const handleUpload = async (dataset: Dataset) => {
+ const formData = new FormData();
+ fileList.forEach((file) => {
+ formData.append("file", file);
+ });
+ window.dispatchEvent(
+ new CustomEvent("upload:dataset", {
+ detail: { dataset, files: fileSliceList },
+ })
+ );
+ resetFiles();
+ };
+
+ const handleBeforeUpload = (_, files: UploadFile[]) => {
+ setFileList([...fileList, ...files]);
+ return false;
+ };
+
+ const handleRemoveFile = (file: UploadFile) => {
+ setFileList((prev) => prev.filter((f) => f.uid !== file.uid));
+ };
+
const fetchCollectionTasks = async () => {
try {
const res = await queryTasksUsingGet({ page: 0, size: 100 });
@@ -40,6 +87,8 @@ export default function ImportConfiguration({
const resetState = () => {
form.resetFields();
+ setFileList([]);
+ form.setFieldsValue({ files: null });
setImportConfig({ source: DataSource.UPLOAD });
};
@@ -51,13 +100,16 @@ export default function ImportConfiguration({
...importConfig,
});
}
- resetState();
- onRefresh?.();
+ message.success("数据已更新");
+ onRefresh?.(false);
onClose();
};
useEffect(() => {
- if (open) fetchCollectionTasks();
+ if (open) {
+ resetState();
+ fetchCollectionTasks();
+ }
}, [open]);
return (
@@ -65,12 +117,19 @@ export default function ImportConfiguration({
title="导入数据"
open={open}
width={600}
- onCancel={onClose}
+ onCancel={() => {
+ onClose();
+ resetState();
+ }}
maskClosable={false}
footer={
<>
-
)}
+
{/* obs import */}
{importConfig?.source === DataSource.OBS && (
@@ -185,7 +245,18 @@ export default function ImportConfiguration({
},
]}
>
- {importFileRender()}
+
+
+
+
+ 本地文件上传
+ 拖拽文件到此处或点击选择文件
+
)}
diff --git a/frontend/src/pages/DataManagement/Detail/components/Overview.tsx b/frontend/src/pages/DataManagement/Detail/components/Overview.tsx
index ed3754b..f5466e7 100644
--- a/frontend/src/pages/DataManagement/Detail/components/Overview.tsx
+++ b/frontend/src/pages/DataManagement/Detail/components/Overview.tsx
@@ -82,11 +82,6 @@ export default function Overview({ dataset, filesOperation }) {
label: "更新时间",
children: dataset.updatedAt,
},
- {
- key: "dataSource",
- label: "数据源",
- children: dataset.dataSource || "未知",
- },
{
key: "description",
label: "描述",
diff --git a/frontend/src/pages/DataManagement/hooks/useFilesOperation.ts b/frontend/src/pages/DataManagement/Detail/useFilesOperation.ts
similarity index 97%
rename from frontend/src/pages/DataManagement/hooks/useFilesOperation.ts
rename to frontend/src/pages/DataManagement/Detail/useFilesOperation.ts
index 17ca811..1c58b19 100644
--- a/frontend/src/pages/DataManagement/hooks/useFilesOperation.ts
+++ b/frontend/src/pages/DataManagement/Detail/useFilesOperation.ts
@@ -6,7 +6,7 @@ import { App } from "antd";
import { useState } from "react";
import {
deleteDatasetFileUsingDelete,
- downloadFile,
+ downloadFileByIdUsingGet,
exportDatasetUsingPost,
queryDatasetFilesUsingGet,
} from "../dataset.api";
@@ -51,7 +51,7 @@ export function useFilesOperation(dataset: Dataset) {
const handleDownloadFile = async (file: DatasetFile) => {
console.log("批量下载文件:", selectedFiles);
// 实际导出逻辑
- await downloadFile(dataset.id, file.id, file.fileName);
+ await downloadFileByIdUsingGet(dataset.id, file.id, file.fileName);
// 假设导出成功
message.success({
content: `已导出 1 个文件`,
diff --git a/frontend/src/pages/DataManagement/Home/DataManagement.tsx b/frontend/src/pages/DataManagement/Home/DataManagement.tsx
index a5a66fa..ec81a64 100644
--- a/frontend/src/pages/DataManagement/Home/DataManagement.tsx
+++ b/frontend/src/pages/DataManagement/Home/DataManagement.tsx
@@ -4,6 +4,7 @@ import {
EditOutlined,
DeleteOutlined,
PlusOutlined,
+ UploadOutlined,
} from "@ant-design/icons";
import TagManager from "@/components/TagManagement";
import { Link, useNavigate } from "react-router";
@@ -25,6 +26,7 @@ import {
} from "../dataset.api";
import { formatBytes } from "@/utils/unit";
import EditDataset from "../Create/EditDataset";
+import ImportConfiguration from "../Detail/components/ImportConfiguration";
export default function DatasetManagementPage() {
const navigate = useNavigate();
@@ -32,7 +34,7 @@ export default function DatasetManagementPage() {
const [viewMode, setViewMode] = useState<"card" | "list">("card");
const [editDatasetOpen, setEditDatasetOpen] = useState(false);
const [currentDataset, setCurrentDataset] = useState
(null);
-
+ const [showUploadDialog, setShowUploadDialog] = useState(false);
const [statisticsData, setStatisticsData] = useState({
count: {},
size: {},
@@ -117,7 +119,13 @@ export default function DatasetManagementPage() {
fetchData,
setSearchParams,
handleFiltersChange,
- } = useFetchData(queryDatasetsUsingGet, mapDataset);
+ } = useFetchData(
+ queryDatasetsUsingGet,
+ mapDataset,
+ 30000, // 30秒轮询间隔
+ true, // 自动刷新
+ [fetchStatistics] // 额外的轮询函数
+ );
const handleDownloadDataset = async (dataset: Dataset) => {
await downloadDatasetUsingGet(dataset.id, dataset.name);
@@ -131,9 +139,17 @@ export default function DatasetManagementPage() {
message.success("数据删除成功");
};
- useEffect(() => {
- fetchStatistics();
- }, []);
+ const handleImportData = (dataset: Dataset) => {
+ setCurrentDataset(dataset);
+ setShowUploadDialog(true);
+ };
+
+ const handleRefresh = async (showMessage = true) => {
+ await fetchData();
+ if (showMessage) {
+ message.success("数据已刷新");
+ }
+ };
const operations = [
{
@@ -141,11 +157,18 @@ export default function DatasetManagementPage() {
label: "编辑",
icon: ,
onClick: (item: Dataset) => {
- console.log(item);
setCurrentDataset(item);
setEditDatasetOpen(true);
},
},
+ {
+ key: "import",
+ label: "导入",
+ icon: ,
+ onClick: (item: Dataset) => {
+ handleImportData(item);
+ },
+ },
{
key: "download",
label: "下载",
@@ -158,6 +181,14 @@ export default function DatasetManagementPage() {
{
key: "delete",
label: "删除",
+ danger: true,
+ confirm: {
+ title: "确认删除该数据集？",
+ description: "删除后该数据集将无法恢复，请谨慎操作。",
+ okText: "删除",
+ cancelText: "取消",
+ okType: "danger",
+ },
icon: ,
onClick: (item: Dataset) => handleDeleteDataset(item.id),
},
@@ -291,7 +322,7 @@ export default function DatasetManagementPage() {
{/* Header */}
数据管理
-
+
{/* tasks */}
{viewMode === "card" ? renderCardView() : renderListView()}
setEditDatasetOpen(false)}
- onRefresh={fetchData}
+ onClose={() => {
+ setCurrentDataset(null);
+ setEditDatasetOpen(false);
+ }}
+ onRefresh={handleRefresh}
+ />
+ {
+ setCurrentDataset(null);
+ setShowUploadDialog(false);
+ }}
+ onRefresh={handleRefresh}
/>
);
diff --git a/frontend/src/pages/DataManagement/dataset.api.ts b/frontend/src/pages/DataManagement/dataset.api.ts
index c800b82..e2cb656 100644
--- a/frontend/src/pages/DataManagement/dataset.api.ts
+++ b/frontend/src/pages/DataManagement/dataset.api.ts
@@ -1,4 +1,4 @@
-import { get, post, put, del, download, upload } from "@/utils/request";
+import { get, post, put, del, download } from "@/utils/request";
// 数据集统计接口
export function getDatasetStatisticsUsingGet() {
@@ -35,15 +35,8 @@ export function deleteDatasetByIdUsingDelete(id: string | number) {
}
// 下载数据集
-export function downloadDatasetUsingGet(
- id: string | number,
- filename?: string
-) {
- return download(
- `/api/data-management/datasets/${id}/download`,
- null,
- filename
- );
+export function downloadDatasetUsingGet(id: string | number) {
+ return download(`/api/data-management/datasets/${id}/files/download`);
}
// 验证数据集
@@ -61,15 +54,15 @@ export function uploadDatasetFileUsingPost(id: string | number, data: any) {
return post(`/api/data-management/datasets/${id}/files`, data);
}
-export function downloadFile(
+export function downloadFileByIdUsingGet(
id: string | number,
fileId: string | number,
- filename?: string
+ fileName: string
) {
return download(
- `/api/data-management/datasets/${id}/files/download`,
+ `/api/data-management/datasets/${id}/files/${fileId}/download`,
null,
- filename
+ fileName
);
}
diff --git a/frontend/src/pages/DataManagement/dataset.const.tsx b/frontend/src/pages/DataManagement/dataset.const.tsx
index ecee1cc..8950506 100644
--- a/frontend/src/pages/DataManagement/dataset.const.tsx
+++ b/frontend/src/pages/DataManagement/dataset.const.tsx
@@ -12,6 +12,7 @@ import {
CloseCircleOutlined,
FileOutlined,
} from "@ant-design/icons";
+import { AnyObject } from "antd/es/_util/type";
import {
FileImage,
FileText,
@@ -186,7 +187,7 @@ export const datasetStatusMap = {
export const dataSourceMap: Record
= {
[DataSource.UPLOAD]: { label: "本地上传", value: DataSource.UPLOAD },
- [DataSource.COLLECTION]: { label: "本地归集 ", value: DataSource.COLLECTION },
+ // [DataSource.COLLECTION]: { label: "本地归集 ", value: DataSource.COLLECTION },
// [DataSource.DATABASE]: { label: "数据库导入", value: DataSource.DATABASE },
// [DataSource.NAS]: { label: "NAS导入", value: DataSource.NAS },
// [DataSource.OBS]: { label: "OBS导入", value: DataSource.OBS },
@@ -194,7 +195,7 @@ export const dataSourceMap: Record = {
export const dataSourceOptions = Object.values(dataSourceMap);
-export function mapDataset(dataset: Dataset) {
+export function mapDataset(dataset: AnyObject): Dataset {
const { icon: IconComponent, iconColor } =
datasetTypeMap[dataset?.datasetType] || {};
return {
diff --git a/frontend/src/pages/DataManagement/hooks/index.ts b/frontend/src/pages/DataManagement/hooks/index.ts
deleted file mode 100644
index dda61af..0000000
--- a/frontend/src/pages/DataManagement/hooks/index.ts
+++ /dev/null
@@ -1,2 +0,0 @@
-export { useFilesOperation } from "./useFilesOperation";
-export { useImportFile } from "./useImportFile";
\ No newline at end of file
diff --git a/frontend/src/pages/DataManagement/hooks/useImportFile.tsx b/frontend/src/pages/DataManagement/hooks/useImportFile.tsx
deleted file mode 100644
index 8b8b7b9..0000000
--- a/frontend/src/pages/DataManagement/hooks/useImportFile.tsx
+++ /dev/null
@@ -1,61 +0,0 @@
-import { Upload, type UploadFile } from "antd";
-import { InboxOutlined } from "@ant-design/icons";
-import { useMemo, useState } from "react";
-import type { Dataset } from "@/pages/DataManagement/dataset.model";
-import { sliceFile } from "@/utils/file.util";
-
-const { Dragger } = Upload;
-
-export const useImportFile = () => {
- const [fileList, setFileList] = useState([]);
- const fileSliceList = useMemo(() => {
- const sliceList = fileList.map((file) => {
- const slices = sliceFile(file);
- return { originFile: file, slices, name: file.name, size: file.size };
- });
- return sliceList;
- }, [fileList]);
-
- const resetFiles = () => {
- setFileList([]);
- };
-
- const handleUpload = async (dataset: Dataset) => {
- const formData = new FormData();
- fileList.forEach((file) => {
- formData.append("file", file);
- });
- window.dispatchEvent(
- new CustomEvent("upload:dataset", {
- detail: { dataset, files: fileSliceList },
- })
- );
- resetFiles();
- };
-
- const handleBeforeUpload = (_, files: UploadFile[]) => {
- setFileList([...fileList, ...files]);
- return false;
- };
-
- const handleRemoveFile = (file: UploadFile) => {
- setFileList((prev) => prev.filter((f) => f.uid !== file.uid));
- };
-
- const importFileRender = () => (
-
-
-
-
- 本地文件上传
- 拖拽文件到此处或点击选择文件
-
- );
-
- return { fileList, resetFiles, handleUpload, importFileRender };
-};
diff --git a/frontend/src/pages/Layout/Sidebar.tsx b/frontend/src/pages/Layout/Sidebar.tsx
index a63a6ed..f881d80 100644
--- a/frontend/src/pages/Layout/Sidebar.tsx
+++ b/frontend/src/pages/Layout/Sidebar.tsx
@@ -71,7 +71,7 @@ const AsiderAndHeaderLayout = () => {
- ModelEngine
+ DataMate
)}
{
+ if (seconds < 0) return "--";
if (seconds < 60) {
return `${seconds} 秒`;
} else if (seconds < 3600) {
From f86d4fae250f85d1c2cd8238ab1810af333c2d9b Mon Sep 17 00:00:00 2001
From: Startalker <103120663+Startalker@users.noreply.github.com>
Date: Thu, 23 Oct 2025 16:49:03 +0800
Subject: [PATCH 2/2] feature: add unstructured formatter operator for doc/docx
(#17)
* feature: add UnstructuredFormatter
* feature: add UnstructuredFormatter in db
* feature: add unstructured[docx]==0.18.15
* feature: support doc
---------
Co-authored-by: Startalker <438747480@qq.com>
---
runtime/ops/formatter/__init__.py | 1 +
.../unstructured_formatter/__init__.py | 6 ++++
.../unstructured_formatter/metadata.yml | 16 +++++++++
.../unstructured_formatter/process.py | 35 +++++++++++++++++++
runtime/ops/requirements.txt | 1 +
scripts/db/data-operator-init.sql | 5 +--
scripts/images/runtime/Dockerfile | 2 +-
7 files changed, 63 insertions(+), 3 deletions(-)
create mode 100644 runtime/ops/formatter/unstructured_formatter/__init__.py
create mode 100644 runtime/ops/formatter/unstructured_formatter/metadata.yml
create mode 100644 runtime/ops/formatter/unstructured_formatter/process.py
diff --git a/runtime/ops/formatter/__init__.py b/runtime/ops/formatter/__init__.py
index c08289b..bc02387 100644
--- a/runtime/ops/formatter/__init__.py
+++ b/runtime/ops/formatter/__init__.py
@@ -20,6 +20,7 @@ def _import_operators():
from . import img_formatter
from . import file_exporter
from . import slide_formatter
+ from . import unstructured_formatter
_import_operators()
diff --git a/runtime/ops/formatter/unstructured_formatter/__init__.py b/runtime/ops/formatter/unstructured_formatter/__init__.py
new file mode 100644
index 0000000..ab5ad41
--- /dev/null
+++ b/runtime/ops/formatter/unstructured_formatter/__init__.py
@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from datamate.core.base_op import OPERATORS
+
+OPERATORS.register_module(module_name='UnstructuredFormatter',
+ module_path="ops.formatter.unstructured_formatter.process")
diff --git a/runtime/ops/formatter/unstructured_formatter/metadata.yml b/runtime/ops/formatter/unstructured_formatter/metadata.yml
new file mode 100644
index 0000000..fc2956c
--- /dev/null
+++ b/runtime/ops/formatter/unstructured_formatter/metadata.yml
@@ -0,0 +1,16 @@
+name: '非结构化文本抽取'
+name_en: 'Unstructured Text Extraction'
+description: '抽取非结构化文件的文本，目前支持word文档'
+description_en: 'Extracts text from Unstructured files, currently supporting Word documents.'
+language: 'python'
+vendor: 'huawei'
+raw_id: 'UnstructuredFormatter'
+version: '1.0.0'
+types:
+ - 'collect'
+modal: 'text'
+effect:
+ before: ''
+ after: ''
+inputs: 'text'
+outputs: 'text'
diff --git a/runtime/ops/formatter/unstructured_formatter/process.py b/runtime/ops/formatter/unstructured_formatter/process.py
new file mode 100644
index 0000000..3107f8b
--- /dev/null
+++ b/runtime/ops/formatter/unstructured_formatter/process.py
@@ -0,0 +1,35 @@
+
+#!/user/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+Description: 非结构化文本抽取
+Create: 2025/10/22 15:15
+"""
+import time
+from typing import Dict, Any
+
+from loguru import logger
+from unstructured.partition.auto import partition
+
+from datamate.core.base_op import Mapper
+
+
+class UnstructuredFormatter(Mapper):
+ """把输入的非结构化文本抽取为txt"""
+
+ def __init__(self, *args, **kwargs):
+ super(UnstructuredFormatter, self).__init__(*args, **kwargs)
+
+ def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:
+ start = time.time()
+ filepath = sample.get(self.filepath_key)
+ filename = sample.get(self.filename_key)
+ try:
+ elements = partition(filename=filepath)
+ sample[self.text_key] = "\n\n".join([str(el) for el in elements])
+ logger.info(f"fileName: {filename}, method: UnstructuredFormatter costs {(time.time() - start):6f} s")
+ except UnicodeDecodeError as err:
+ logger.exception(f"fileName: {filename}, method: UnstructuredFormatter causes decode error: {err}")
+ raise
+ return sample
diff --git a/runtime/ops/requirements.txt b/runtime/ops/requirements.txt
index 92dcfc6..8c842b0 100644
--- a/runtime/ops/requirements.txt
+++ b/runtime/ops/requirements.txt
@@ -19,3 +19,4 @@ xmltodict==1.0.2
zhconv==1.4.3
sqlalchemy==2.0.40
pymysql==1.1.1
+unstructured[docx]==0.18.15
\ No newline at end of file
diff --git a/scripts/db/data-operator-init.sql b/scripts/db/data-operator-init.sql
index 9ee826a..4a1b8a6 100644
--- a/scripts/db/data-operator-init.sql
+++ b/scripts/db/data-operator-init.sql
@@ -68,6 +68,7 @@ VALUES (1, '模态', 'predefined', 0),
INSERT IGNORE INTO t_operator
(id, name, description, version, inputs, outputs, runtime, settings, file_name, is_star)
VALUES ('TextFormatter', 'TXT文本抽取', '抽取TXT中的文本。', '1.0.0', 'text', 'text', null, null, '', false),
+ ('UnstructuredFormatter', '非结构化文本抽取', '抽取非结构化文件的文本，目前支持word文档。', '1.0.0', 'text', 'text', null, null, '', false),
('FileExporter', '落盘算子', '将文件保存到本地目录。', '1.0.0', 'all', 'all', null, null, '', false),
('FileWithHighRepeatPhraseRateFilter', '文档词重复率检查', '去除重复词过多的文档。', '1.0.0', 'text', 'text', null, '{"repeatPhraseRatio": {"name": "文档词重复率", "description": "某个词的统计数/文档总词数 > 设定值，该文档被去除。", "type": "slider", "defaultVal": 0.5, "min": 0, "max": 1, "step": 0.1}, "hitStopwords": {"name": "去除停用词", "description": "统计重复词时，选择是否要去除停用词。", "type": "switch", "defaultVal": false, "required": true, "checkedLabel": "去除", "unCheckedLabel": "不去除"}}', '', 'false'),
('FileWithHighRepeatWordRateFilter', '文档字重复率检查', '去除重复字过多的文档。', '1.0.0', 'text', 'text', null, '{"repeatWordRatio": {"name": "文档字重复率", "description": "某个字的统计数/文档总字数 > 设定值，该文档被去除。", "type": "slider", "defaultVal": 0.5, "min": 0, "max": 1, "step": 0.1}}', '', 'false'),
@@ -121,7 +122,7 @@ AND o.id IN ('TextFormatter', 'FileWithShortOrLongLengthFilter', 'FileWithHighRe
'AnonymizedIpAddress', 'AnonymizedPhoneNumber', 'AnonymizedUrlCleaner', 'HtmlTagCleaner', 'XMLTagCleaner',
'ContentCleaner', 'EmailNumberCleaner', 'EmojiCleaner', 'ExtraSpaceCleaner', 'FullWidthCharacterCleaner',
'GrableCharactersCleaner', 'InvisibleCharactersCleaner', 'LegendCleaner', 'PoliticalWordCleaner',
- 'SexualAndViolentWordCleaner', 'TraditionalChineseCleaner', 'UnicodeSpaceCleaner');
+ 'SexualAndViolentWordCleaner', 'TraditionalChineseCleaner', 'UnicodeSpaceCleaner', 'UnstructuredFormatter');
INSERT IGNORE INTO t_operator_category_relation(category_id, operator_id)
SELECT c.id, o.id
@@ -137,4 +138,4 @@ SELECT c.id, o.id
FROM t_operator_category c
CROSS JOIN t_operator o
WHERE c.id IN (7, 8, 11)
- AND o.id IN ('FileExporter');
+ AND o.id IN ('FileExporter', 'UnstructuredFormatter');
diff --git a/scripts/images/runtime/Dockerfile b/scripts/images/runtime/Dockerfile
index 147fe5f..6d3f8ef 100644
--- a/scripts/images/runtime/Dockerfile
+++ b/scripts/images/runtime/Dockerfile
@@ -7,7 +7,7 @@ ENV PYTHONPATH=/opt/runtime/datamate/
RUN sed -i 's/deb.debian.org/mirrors.huaweicloud.com/g' /etc/apt/sources.list.d/debian.sources \
&& apt update \
- && apt install -y libgl1 libglib2.0-0 vim poppler-utils tesseract-ocr tesseract-ocr-chi-sim libmagic1t64 \
+ && apt install -y libgl1 libglib2.0-0 vim poppler-utils tesseract-ocr tesseract-ocr-chi-sim libmagic1t64 libreoffice\
&& apt clean \
&& rm -rf /var/lib/apt/lists/*