q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: modify data collection to python implementation (#214)

Browse Source 
* feature: LabelStudio jumps without login

* refactor: modify data collection to python implementation

* refactor: modify data collection to python implementation

* refactor: modify data collection to python implementation

* refactor: modify data collection to python implementation

* refactor: modify data collection to python implementation

* refactor: modify data collection to python implementation

* fix: remove terrabase dependency

* feature: add the collection task executions page and the collection template page

* fix: fix the collection task creation

* fix: fix the collection task creation
This commit is contained in:
hefanli
2025-12-30 18:48:43 +08:00
committed by GitHub
parent 80d4dfd285
commit 63f4e3e447
71 changed files with 1861 additions and 2557 deletions
Download Patch File Download Diff File Expand all files Collapse all files

602
frontend/src/pages/DataCollection/Create/CreateTask.tsx
View File

@@ -1,79 +1,60 @@
import { useState } from "react";
import { Input, Button, Radio, Form, App, Select } from "antd";
import { useEffect, useState } from "react";
import { Input, Button, Radio, Form, App, Select, InputNumber } from "antd";
import { Link, useNavigate } from "react-router";
import { ArrowLeft } from "lucide-react";
import { createTaskUsingPost } from "../collection.apis";
import { createTaskUsingPost, queryDataXTemplatesUsingGet } from "../collection.apis";
import SimpleCronScheduler from "@/pages/DataCollection/Create/SimpleCronScheduler";
import RadioCard from "@/components/RadioCard";
import { datasetTypes } from "@/pages/DataManagement/dataset.const";
import { SyncModeMap } from "../collection.const";
import { SyncMode } from "../collection.model";
import { DatasetSubType } from "@/pages/DataManagement/dataset.model";
const { TextArea } = Input;
const defaultTemplates = [
{
id: "NAS",
name: "NAS到本地",
description: "从NAS文件系统导入数据到本地文件系统",
config: {
reader: "nfsreader",
writer: "localwriter",
},
},
{
id: "OBS",
name: "OBS到本地",
description: "从OBS文件系统导入数据到本地文件系统",
config: {
reader: "obsreader",
writer: "localwriter",
},
},
{
id: "MYSQL",
name: "Mysql到本地",
description: "从Mysql中导入数据到本地文件系统",
config: {
reader: "mysqlreader",
writer: "localwriter",
},
},
];
const syncModeOptions = Object.values(SyncModeMap);
enum TemplateType {
NAS = "NAS",
OBS = "OBS",
MYSQL = "MYSQL",
}
type CollectionTemplate = {
id: string;
name: string;
description?: string;
sourceType?: string;
sourceName?: string;
targetType?: string;
targetName?: string;
templateContent?: {
parameter?: any;
reader?: any;
writer?: any;
};
builtIn?: boolean;
};
type TemplateFieldDef = {
name?: string;
type?: string;
description?: string;
required?: boolean;
options?: Array<{ label: string; value: string | number } | string | number>;
defaultValue?: any;
};
export default function CollectionTaskCreate() {
const navigate = useNavigate();
const [form] = Form.useForm();
const { message } = App.useApp();
const [templateType, setTemplateType] = useState<"default" | "custom">(
"default"
);
// 默认模板类型设为 NAS
const [selectedTemplate, setSelectedTemplate] = useState<TemplateType>(
TemplateType.NAS
);
const [customConfig, setCustomConfig] = useState("");
const [templates, setTemplates] = useState<CollectionTemplate[]>([]);
const [templatesLoading, setTemplatesLoading] = useState(false);
const [selectedTemplateId, setSelectedTemplateId] = useState<string | undefined>(undefined);
// 将 newTask 设为 any，并初始化 config.templateType 为 NAS
const [newTask, setNewTask] = useState<any>({
name: "",
description: "",
syncMode: SyncMode.ONCE,
cronExpression: "",
maxRetries: 10,
dataset: null,
config: { templateType: TemplateType.NAS },
createDataset: false,
scheduleExpression: "",
timeoutSeconds: 3600,
templateId: "",
config: {
parameter: {},
},
});
const [scheduleExpression, setScheduleExpression] = useState({
type: "once",
@@ -81,33 +62,37 @@ export default function CollectionTaskCreate() {
cronExpression: "0 0 0 * * ?",
});
const [isCreateDataset, setIsCreateDataset] = useState(false);
useEffect(() => {
const run = async () => {
setTemplatesLoading(true);
try {
const resp: any = await queryDataXTemplatesUsingGet({ page: 1, size: 1000 });
const list: CollectionTemplate[] = resp?.data?.content || [];
setTemplates(list);
} catch (e) {
message.error("加载归集模板失败");
} finally {
setTemplatesLoading(false);
}
};
run()
}, []);
const handleSubmit = async () => {
try {
await form.validateFields();
if (templateType === "default" && !selectedTemplate) {
window.alert("请选择默认模板");
return;
}
if (templateType === "custom" && !customConfig.trim()) {
window.alert("请填写自定义配置");
return;
}
// 构建最终 payload，不依赖异步 setState
const values = form.getFieldsValue(true);
const payload = {
...newTask,
taskType:
templateType === "default" ? selectedTemplate : "CUSTOM",
config: {
...((newTask && newTask.config) || {}),
...(templateType === "custom" ? { dataxJson: customConfig } : {}),
},
name: values.name,
description: values.description,
syncMode: values.syncMode,
scheduleExpression: values.scheduleExpression,
timeoutSeconds: values.timeoutSeconds,
templateId: values.templateId,
config: values.config,
};
console.log("创建任务 payload:", payload);
await createTaskUsingPost(payload);
message.success("任务创建成功");
navigate("/data/collection");
@@ -116,6 +101,102 @@ export default function CollectionTaskCreate() {
}
};
const selectedTemplate = templates.find((t) => t.id === selectedTemplateId);
const renderTemplateFields = (
section: "parameter" | "reader" | "writer",
defs: Record<string, TemplateFieldDef> | undefined
) => {
if (!defs || typeof defs !== "object") return null;
const items = Object.entries(defs).map(([key, def]) => {
const label = def?.name || key;
const description = def?.description;
const fieldType = (def?.type || "input").toLowerCase();
const required = def?.required !== false;
const rules = required
? [{ required: true, message: `请输入${label}` }]
: undefined;
if (fieldType === "password") {
return (
<Form.Item
key={`${section}.${key}`}
name={["config", section, key]}
label={label}
tooltip={description}
rules={rules}
>
<Input.Password placeholder={description || `请输入${label}`} />
</Form.Item>
);
}
if (fieldType === "textarea") {
return (
<Form.Item
key={`${section}.${key}`}
name={["config", section, key]}
label={label}
tooltip={description}
rules={rules}
className="md:col-span-2"
>
<TextArea rows={4} placeholder={description || `请输入${label}`} />
</Form.Item>
);
}
if (fieldType === "select") {
const options = (def?.options || []).map((opt: any) => {
if (typeof opt === "string" || typeof opt === "number") {
return { label: String(opt), value: opt };
}
return { label: opt?.label ?? String(opt?.value), value: opt?.value };
});
return (
<Form.Item
key={`${section}.${key}`}
name={["config", section, key]}
label={label}
tooltip={description}
rules={rules}
>
<Select placeholder={description || `请选择${label}`} options={options} />
</Form.Item>
);
}
return (
<Form.Item
key={`${section}.${key}`}
name={["config", section, key]}
label={label}
tooltip={description}
rules={rules}
>
<Input placeholder={description || `请输入${label}`} />
</Form.Item>
);
});
return (
<div className="grid grid-cols-1 md:grid-cols-2 gap-x-4 gap-y-2">
{items}
</div>
);
};
const getPropertyCountSafe = (obj: any) => {
// 类型检查
if (obj === null || obj === undefined) {
return 0;
}
// 处理普通对象
return Object.keys(obj).length;
}
return (
<div className="h-full flex flex-col">
<div className="flex items-center justify-between mb-2">
@@ -130,10 +211,11 @@ export default function CollectionTaskCreate() {
</div>
<div className="flex-overflow-auto border-card">
<div className="flex-1 overflow-auto p-6">
<div className="flex-1 overflow-auto p-4">
<Form
form={form}
layout="vertical"
className="[&_.ant-form-item]:mb-3 [&_.ant-form-item-label]:pb-1"
initialValues={newTask}
onValuesChange={(_, allValues) => {
setNewTask({ ...newTask, ...allValues });
@@ -142,19 +224,36 @@ export default function CollectionTaskCreate() {
{/* 基本信息 */}
<h2 className="font-medium text-gray-900 text-lg mb-2">基本信息</h2>
<Form.Item
label="名称"
name="name"
rules={[{ required: true, message: "请输入任务名称" }]}
>
<Input placeholder="请输入任务名称" />
</Form.Item>
<Form.Item label="描述" name="description">
<TextArea placeholder="请输入任务描述" rows={3} />
</Form.Item>
<div className="grid grid-cols-1 md:grid-cols-2 gap-x-4 gap-y-2">
<Form.Item
label="名称"
name="name"
rules={[{ required: true, message: "请输入任务名称" }]}
>
<Input placeholder="请输入任务名称" />
</Form.Item>
<Form.Item
label="超时时间（秒）"
name="timeoutSeconds"
rules={[{ required: true, message: "请输入超时时间" }]}
initialValue={3600}
>
<InputNumber
className="w-full"
min={1}
precision={0}
placeholder="默认 3600"
/>
</Form.Item>
<Form.Item className="md:col-span-2" label="描述" name="description">
<TextArea placeholder="请输入任务描述" rows={2} />
</Form.Item>
</div>
{/* 同步配置 */}
<h2 className="font-medium text-gray-900 pt-6 mb-2 text-lg">
<h2 className="font-medium text-gray-900 pt-2 mb-1 text-lg">
同步配置
</h2>
<Form.Item name="syncMode" label="同步方式">
@@ -180,7 +279,7 @@ export default function CollectionTaskCreate() {
rules={[{ required: true, message: "请输入Cron表达式" }]}
>
<SimpleCronScheduler
className="px-2 rounded"
className="px-2 py-1 rounded"
value={scheduleExpression}
onChange={(value) => {
setScheduleExpression(value);
@@ -194,271 +293,90 @@ export default function CollectionTaskCreate() {
)}
{/* 模板配置 */}
<h2 className="font-medium text-gray-900 pt-6 mb-2 text-lg">
<h2 className="font-medium text-gray-900 pt-4 mb-2 text-lg">
模板配置
</h2>
{/* <Form.Item label="模板类型">
<Radio.Group
value={templateType}
onChange={(e) => setTemplateType(e.target.value)}
>
<Radio value="default">使用默认模板</Radio>
<Radio value="custom">自定义DataX JSON配置</Radio>
</Radio.Group>
</Form.Item> */}
{templateType === "default" && (
<>
{
<Form.Item label="选择模板">
<div className="grid grid-cols-1 sm:grid-cols-2 md:grid-cols-3 gap-4">
{defaultTemplates.map((template) => (
<div
key={template.id}
className={`border p-4 rounded-md hover:shadow-lg transition-shadow ${
selectedTemplate === template.id
? "border-blue-500"
: "border-gray-300"
}`}
onClick={() => {
setSelectedTemplate(template.id as TemplateType);
// 使用函数式更新，合并之前的 config
setNewTask((prev: any) => ({
...prev,
config: {
templateType: template.id,
},
}));
// 同步表单显示
form.setFieldsValue({
config: { templateType: template.id },
});
}}
>
<div className="font-medium">{template.name}</div>
<div className="text-gray-500">
{template.description}
</div>
<div className="text-gray-400">
{template.config.reader} → {template.config.writer}
</div>
</div>
))}
<Form.Item
label="选择模板"
name="templateId"
rules={[{ required: true, message: "请选择归集模板" }]}
>
<Select
placeholder="请选择归集模板"
loading={templatesLoading}
onChange={(templateId) => {
setSelectedTemplateId(templateId);
form.setFieldsValue({
templateId,
config: {},
});
setNewTask((prev: any) => ({
...prev,
templateId,
config: {},
}));
}}
optionRender={(option) => {
const tpl = templates.find((t) => t.id === option.value);
return (
<div>
<div className="font-medium">{tpl?.name || option.label}</div>
<div className="text-xs text-gray-500 line-clamp-2">
{tpl?.description || ""}
</div>
</div>
</Form.Item>
}
{/* nas import */}
{selectedTemplate === TemplateType.NAS && (
<div className="grid grid-cols-2 gap-3 px-2 bg-blue-50 rounded">
<Form.Item
name={["config", "ip"]}
rules={[{ required: true, message: "请输入NAS地址" }]}
label="NAS地址"
>
<Input placeholder="192.168.1.100" />
</Form.Item>
<Form.Item
name={["config", "path"]}
rules={[{ required: true, message: "请输入共享路径" }]}
label="共享路径"
>
<Input placeholder="/share/importConfig" />
</Form.Item>
<Form.Item
name={["config", "files"]}
label="文件列表"
className="col-span-2"
>
<Select placeholder="请选择文件列表" mode="tags" />
</Form.Item>
</div>
)}
);
}}
options={templates.map((template) => ({
label: template.name,
value: template.id,
}))}
/>
</Form.Item>
{/* obs import */}
{selectedTemplate === TemplateType.OBS && (
<div className="grid grid-cols-2 gap-3 p-4 bg-blue-50 rounded-lg">
<Form.Item
name={["config", "endpoint"]}
rules={[{ required: true }]}
label="Endpoint"
>
<Input
className="h-8 text-xs"
placeholder="obs.cn-north-4.myhuaweicloud.com"
/>
</Form.Item>
<Form.Item
name={["config", "bucket"]}
rules={[{ required: true }]}
label="Bucket"
>
<Input className="h-8 text-xs" placeholder="my-bucket" />
</Form.Item>
<Form.Item
name={["config", "accessKey"]}
rules={[{ required: true }]}
label="Access Key"
>
<Input className="h-8 text-xs" placeholder="Access Key" />
</Form.Item>
<Form.Item
name={["config", "secretKey"]}
rules={[{ required: true }]}
label="Secret Key"
>
<Input
type="password"
className="h-8 text-xs"
placeholder="Secret Key"
/>
</Form.Item>
<Form.Item
name={["config", "prefix"]}
rules={[{ required: true }]}
label="Prefix"
>
<Input className="h-8 text-xs" placeholder="Prefix" />
</Form.Item>
</div>
)}
{/* mysql import */}
{selectedTemplate === TemplateType.MYSQL && (
<div className="grid grid-cols-2 gap-3 px-2 bg-blue-50 rounded">
<Form.Item
name={["config", "jdbcUrl"]}
rules={[{ required: true, message: "请输入数据库链接" }]}
label="数据库链接"
className="col-span-2"
>
<Input placeholder="jdbc:mysql://localhost:3306/mysql?useUnicode=true&characterEncoding=utf8" />
</Form.Item>
<Form.Item
name={["config", "username"]}
rules={[{ required: true, message: "请输入用户名" }]}
label="用户名"
>
<Input placeholder="mysql" />
</Form.Item>
<Form.Item
name={["config", "password"]}
rules={[{ required: true, message: "请输入密码" }]}
label="密码"
>
<Input type="password" className="h-8 text-xs" placeholder="Secret Key" />
</Form.Item>
<Form.Item
name={["config", "querySql"]}
rules={[{ required: true, message: "请输入查询语句" }]}
label="查询语句"
>
<Input placeholder="select * from your_table" />
</Form.Item>
<Form.Item
name={["config", "headers"]}
label="列名"
className="col-span-2"
>
<Select placeholder="请输入列名" mode="tags" />
</Form.Item>
</div>
)}
</>
)}
{templateType === "custom" && (
<Form.Item label="DataX JSON配置">
<TextArea
placeholder="请输入DataX JSON配置..."
value={customConfig}
onChange={(e) => setCustomConfig(e.target.value)}
rows={12}
className="w-full"
/>
</Form.Item>
)}
{/* 数据集配置 */}
{templateType === "default" && (
{selectedTemplate ? (
<>
<h2 className="font-medium text-gray-900 my-4 text-lg">
数据集配置
</h2>
<Form.Item
label="是否创建数据集"
name="createDataset"
required
rules={[{ required: true, message: "请选择是否创建数据集" }]}
tooltip={"支持后续在【数据管理】中手动创建数据集并关联至此任务。"}
>
<Radio.Group
value={isCreateDataset}
onChange={(e) => {
const value = e.target.value;
let datasetInit = null;
if (value === true) {
datasetInit = {};
}
form.setFieldsValue({
dataset: datasetInit,
});
setNewTask((prev: any) => ({
...prev,
dataset: datasetInit,
}));
setIsCreateDataset(e.target.value);
}}
>
<Radio value={true}>是</Radio>
<Radio value={false}>否</Radio>
</Radio.Group>
</Form.Item>
{isCreateDataset && (
{getPropertyCountSafe(selectedTemplate.templateContent?.parameter) > 0 ? (
<>
<Form.Item
label="数据集名称"
name={["dataset", "name"]}
required
>
<Input
placeholder="输入数据集名称"
onChange={(e) => {
setNewTask((prev: any) => ({
...prev,
dataset: {
...(prev.dataset || {}),
name: e.target.value,
},
}));
}}
/>
</Form.Item>
<Form.Item
label="数据集类型"
name={["dataset", "datasetType"]}
rules={[{ required: true, message: "请选择数据集类型" }]}
>
<RadioCard
options={datasetTypes}
value={newTask.dataset?.datasetType}
onChange={(type) => {
form.setFieldValue(["dataset", "datasetType"], type);
setNewTask((prev: any) => ({
...prev,
dataset: {
...(prev.dataset || {}),
datasetType: type as DatasetSubType,
},
}));
}}
/>
</Form.Item>
<h3 className="font-medium text-gray-900 pt-2 mb-2">
模板参数
</h3>
{renderTemplateFields(
"parameter",
selectedTemplate.templateContent?.parameter as Record<string, TemplateFieldDef>
)}
</>
)}
): null}
{getPropertyCountSafe(selectedTemplate.templateContent?.reader) > 0 ? (
<>
<h3 className="font-medium text-gray-900 pt-2 mb-2">
源端参数
</h3>
{renderTemplateFields(
"reader",
selectedTemplate.templateContent?.reader as Record<string, TemplateFieldDef>
)}
</>
) : null}
{getPropertyCountSafe(selectedTemplate.templateContent?.writer) > 0 ? (
<>
<h3 className="font-medium text-gray-900 pt-2 mb-2">
目标端参数
</h3>
{renderTemplateFields(
"writer",
selectedTemplate.templateContent?.writer as Record<string, TemplateFieldDef>
)}
</>
) : null}
</>
)}
) : null}
</Form>
</div>
<div className="flex gap-2 justify-end border-top p-6">
<div className="flex gap-2 justify-end border-top p-4">
<Button onClick={() => navigate("/data/collection")}>取消</Button>
<Button type="primary" onClick={handleSubmit}>
创建任务

31
frontend/src/pages/DataCollection/Home/DataCollectionPage.tsx
View File

@@ -1,13 +1,27 @@
import { useState } from "react";
import { useEffect, useState } from "react";
import { Button, Tabs } from "antd";
import { PlusOutlined } from "@ant-design/icons";
import TaskManagement from "./TaskManagement";
import ExecutionLog from "./ExecutionLog";
import { useNavigate } from "react-router";
import Execution from "./Execution.tsx";
import TemplateManagement from "./TemplateManagement";
import { useLocation, useNavigate } from "react-router";
export default function DataCollection() {
const navigate = useNavigate();
const location = useLocation();
const [activeTab, setActiveTab] = useState("task-management");
const [taskId, setTaskId] = useState<string | undefined>(undefined);
useEffect(() => {
const params = new URLSearchParams(location.search);
const tab = params.get("tab") || undefined;
const nextTaskId = params.get("taskId") || undefined;
if (tab === "task-execution" || tab === "task-management" || tab === "task-template") {
setActiveTab(tab);
}
setTaskId(nextTaskId);
}, [location.search]);
return (
<div className="gap-4 h-full flex flex-col">
@@ -29,13 +43,20 @@ export default function DataCollection() {
activeKey={activeTab}
items={[
{ label: "任务管理", key: "task-management" },
// { label: "执行日志", key: "execution-log" },
{ label: "执行记录", key: "task-execution" },
{ label: "模板管理", key: "task-template" },
]}
onChange={(tab) => {
setActiveTab(tab);
setTaskId(undefined);
const params = new URLSearchParams();
params.set("tab", tab);
navigate({ pathname: location.pathname, search: params.toString() }, { replace: true });
}}
/>
{activeTab === "task-management" ? <TaskManagement /> : <ExecutionLog />}
{activeTab === "task-management" ? <TaskManagement /> : null}
{activeTab === "task-execution" ? <Execution taskId={taskId} /> : null}
{activeTab === "task-template" ? <TemplateManagement /> : null}
</div>
);
}

291
frontend/src/pages/DataCollection/Home/Execution.tsx Normal file
View File

@@ -0,0 +1,291 @@
import {Card, Badge, Button, Modal, Table, Tag} from "antd";
import type { ColumnsType } from "antd/es/table";
import { SearchControls } from "@/components/SearchControls";
import { queryExecutionLogUsingPost } from "../collection.apis";
import useFetchData from "@/hooks/useFetchData";
import { useEffect, useState } from "react";
import {TaskExecution} from "@/pages/DataCollection/collection.model.ts";
import {mapTaskExecution} from "@/pages/DataCollection/collection.const.ts";
import { queryExecutionLogFileByIdUsingGet } from "../collection.apis";
import { FileTextOutlined } from "@ant-design/icons";
const filterOptions = [
{
key: "status",
label: "状态筛选",
options: [
{ value: "all", label: "全部状态" },
{ value: "RUNNING", label: "运行中" },
{ value: "SUCCESS", label: "成功" },
{ value: "FAILED", label: "失败" },
{ value: "STOPPED", label: "停止" },
],
},
];
export default function Execution({ taskId }: { taskId?: string }) {
const [dateRange, setDateRange] = useState<[any, any] | null>(null);
const [logOpen, setLogOpen] = useState(false);
const [logLoading, setLogLoading] = useState(false);
const [logTitle, setLogTitle] = useState<string>("");
const [logContent, setLogContent] = useState<string>("");
const [logFilename, setLogFilename] = useState<string>("");
const [logBlobUrl, setLogBlobUrl] = useState<string>("");
const formatDuration = (seconds?: number) => {
if (seconds === undefined || seconds === null) return "-";
const total = Math.max(0, Math.floor(seconds));
if (total < 60) return `${total}s`;
const min = Math.floor(total / 60);
const sec = total % 60;
return `${min}min${sec}s`;
};
const handleReset = () => {
setSearchParams({
keyword: "",
filter: {
type: [],
status: [],
tags: [],
},
current: 1,
pageSize: 10,
});
setDateRange(null);
};
const {
loading,
tableData,
pagination,
searchParams,
setSearchParams,
handleFiltersChange,
handleKeywordChange,
} = useFetchData<TaskExecution>(
(params) => {
const { keyword, start_time, end_time, ...rest } = params || {};
return queryExecutionLogUsingPost({
...rest,
task_id: taskId || undefined,
task_name: keyword || undefined,
start_time,
end_time,
});
},
mapTaskExecution,
30000,
false,
[],
0
);
useEffect(() => {
setSearchParams((prev) => ({
...prev,
current: 1,
}));
}, [taskId, setSearchParams]);
const handleViewLog = async (record: TaskExecution) => {
setLogOpen(true);
setLogLoading(true);
setLogTitle(`${record.taskName} / ${record.id}`);
setLogContent("");
setLogFilename("");
if (logBlobUrl) {
URL.revokeObjectURL(logBlobUrl);
setLogBlobUrl("");
}
try {
const { blob, filename } = await queryExecutionLogFileByIdUsingGet(record.id);
setLogFilename(filename);
const url = URL.createObjectURL(blob);
setLogBlobUrl(url);
const text = await blob.text();
setLogContent(text);
} catch (e: any) {
setLogContent(e?.data?.detail || e?.message || "Failed to load log");
} finally {
setLogLoading(false);
}
};
const columns: ColumnsType<any> = [
{
title: "任务名称",
dataIndex: "taskName",
key: "taskName",
fixed: "left",
render: (text: string) => (
<span style={{ fontWeight: 500 }}>{text}</span>
),
},
{
title: "状态",
dataIndex: "status",
key: "status",
render: (status: any) => ((
<Tag color={status.color}>{status.label}</Tag>
)
),
},
{
title: "开始时间",
dataIndex: "startedAt",
key: "startedAt",
},
{
title: "结束时间",
dataIndex: "completedAt",
key: "completedAt",
},
{
title: "执行时长",
dataIndex: "durationSeconds",
key: "durationSeconds",
render: (v?: number) => formatDuration(v),
},
{
title: "错误信息",
dataIndex: "errorMessage",
key: "errorMessage",
render: (msg?: string) =>
msg ? (
<span style={{ color: "#f5222d" }} title={msg}>
{msg}
</span>
) : (
<span style={{ color: "#bbb" }}>-</span>
),
},
{
title: "操作",
key: "action",
fixed: "right",
width: 120,
render: (_: any, record: TaskExecution) => (
<Button
type="link"
icon={<FileTextOutlined />}
onClick={() => handleViewLog(record)}
>
查看日志
</Button>
),
},
];
return (
<div className="flex flex-col gap-4">
{/* Filter Controls */}
<div className="flex items-center justify-between gap-4">
<SearchControls
searchTerm={searchParams.keyword}
onSearchChange={handleKeywordChange}
filters={filterOptions}
onFiltersChange={handleFiltersChange}
showViewToggle={false}
onClearFilters={() =>
setSearchParams((prev) => ({
...prev,
filter: { ...prev.filter, status: [] },
current: 1,
}))
}
showDatePicker
dateRange={dateRange as any}
onDateChange={(date) => {
setDateRange(date as any);
const start = (date?.[0] as any)?.toISOString?.() || undefined;
const end = (date?.[1] as any)?.toISOString?.() || undefined;
setSearchParams((prev) => ({
...prev,
current: 1,
start_time: start,
end_time: end,
}));
}}
onReload={handleReset}
searchPlaceholder="搜索任务名称..."
className="flex-1"
/>
</div>
<Card>
<Table
loading={loading}
columns={columns}
dataSource={tableData}
rowKey="id"
pagination={pagination}
scroll={{ x: "max-content" }}
/>
</Card>
<Modal
title={logTitle || "执行日志"}
open={logOpen}
onCancel={() => {
setLogOpen(false);
if (logBlobUrl) {
URL.revokeObjectURL(logBlobUrl);
setLogBlobUrl("");
}
}}
footer={
<div style={{ display: "flex", justifyContent: "space-between", width: "100%" }}>
<div style={{ color: "#6b7280", fontSize: 12 }}>{logFilename || ""}</div>
<div style={{ display: "flex", gap: 8 }}>
{logBlobUrl ? (
<Button
onClick={() => {
const a = document.createElement("a");
a.href = logBlobUrl;
a.download = logFilename || "execution.log";
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
}}
>
下载日志
</Button>
) : null}
<Button
type="primary"
onClick={() => {
setLogOpen(false);
if (logBlobUrl) {
URL.revokeObjectURL(logBlobUrl);
setLogBlobUrl("");
}
}}
>
关闭
</Button>
</div>
</div>
}
width={900}
>
<div
style={{
background: "#0b1020",
color: "#e5e7eb",
borderRadius: 8,
padding: 12,
maxHeight: "60vh",
overflow: "auto",
fontFamily: "ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, 'Liberation Mono', 'Courier New', monospace",
fontSize: 12,
lineHeight: 1.5,
whiteSpace: "pre-wrap",
wordBreak: "break-word",
}}
>
{logLoading ? "Loading..." : (logContent || "(empty)")}
</div>
</Modal>
</div>
);
}

149
frontend/src/pages/DataCollection/Home/ExecutionLog.tsx
View File

@@ -1,149 +0,0 @@
import { Card, Badge, Table } from "antd";
import type { ColumnsType } from "antd/es/table";
import { SearchControls } from "@/components/SearchControls";
import type { CollectionLog } from "@/pages/DataCollection/collection.model";
import { queryExecutionLogUsingPost } from "../collection.apis";
import { LogStatusMap, LogTriggerTypeMap } from "../collection.const";
import useFetchData from "@/hooks/useFetchData";
const filterOptions = [
{
key: "status",
label: "状态筛选",
options: Object.values(LogStatusMap),
},
{
key: "triggerType",
label: "触发类型",
options: Object.values(LogTriggerTypeMap),
},
];
export default function ExecutionLog() {
const handleReset = () => {
setSearchParams({
keyword: "",
filters: {},
current: 1,
pageSize: 10,
dateRange: null,
});
};
const {
loading,
tableData,
pagination,
searchParams,
setSearchParams,
handleFiltersChange,
handleKeywordChange,
} = useFetchData(queryExecutionLogUsingPost);
const columns: ColumnsType<CollectionLog> = [
{
title: "任务名称",
dataIndex: "taskName",
key: "taskName",
fixed: "left",
render: (text: string) => <span style={{ fontWeight: 500 }}>{text}</span>,
},
{
title: "状态",
dataIndex: "status",
key: "status",
render: (status: string) => (
<Badge
text={LogStatusMap[status]?.label}
color={LogStatusMap[status]?.color}
/>
),
},
{
title: "触发类型",
dataIndex: "triggerType",
key: "triggerType",
render: (type: string) => LogTriggerTypeMap[type].label,
},
{
title: "开始时间",
dataIndex: "startTime",
key: "startTime",
},
{
title: "结束时间",
dataIndex: "endTime",
key: "endTime",
},
{
title: "执行时长",
dataIndex: "duration",
key: "duration",
},
{
title: "重试次数",
dataIndex: "retryCount",
key: "retryCount",
},
{
title: "进程ID",
dataIndex: "processId",
key: "processId",
render: (text: string) => (
<span style={{ fontFamily: "monospace" }}>{text}</span>
),
},
{
title: "错误信息",
dataIndex: "errorMessage",
key: "errorMessage",
render: (msg?: string) =>
msg ? (
<span style={{ color: "#f5222d" }} title={msg}>
{msg}
</span>
) : (
<span style={{ color: "#bbb" }}>-</span>
),
},
];
return (
<div className="flex flex-col gap-4">
{/* Filter Controls */}
<div className="flex items-center justify-between gap-4">
<SearchControls
searchTerm={searchParams.keyword}
onSearchChange={handleKeywordChange}
filters={filterOptions}
onFiltersChange={handleFiltersChange}
showViewToggle={false}
onClearFilters={() =>
setSearchParams((prev) => ({
...prev,
filters: {},
}))
}
showDatePicker
dateRange={searchParams.dateRange || [null, null]}
onDateChange={(date) =>
setSearchParams((prev) => ({ ...prev, dateRange: date }))
}
onReload={handleReset}
searchPlaceholder="搜索任务名称、进程ID或错误信息..."
className="flex-1"
/>
</div>
<Card>
<Table
loading={loading}
columns={columns}
dataSource={tableData}
rowKey="id"
pagination={pagination}
scroll={{ x: "max-content" }}
/>
</Card>
</div>
);
}

132
frontend/src/pages/DataCollection/Home/TaskManagement.tsx
View File

@@ -1,34 +1,16 @@
import {
Card,
Button,
Badge,
Table,
Dropdown,
App,
Tooltip,
Popconfirm,
} from "antd";
import {
DeleteOutlined,
EditOutlined,
EllipsisOutlined,
PauseCircleOutlined,
PauseOutlined,
PlayCircleOutlined,
StopOutlined,
} from "@ant-design/icons";
import { SearchControls } from "@/components/SearchControls";
import {App, Button, Card, Popconfirm, Table, Tag, Tooltip,} from "antd";
import {DeleteOutlined, PauseCircleOutlined, PlayCircleOutlined, ProfileOutlined,} from "@ant-design/icons";
import {SearchControls} from "@/components/SearchControls";
import {
deleteTaskByIdUsingDelete,
executeTaskByIdUsingPost,
queryTasksUsingGet,
stopTaskByIdUsingPost,
} from "../collection.apis";
import { TaskStatus, type CollectionTask } from "../collection.model";
import { StatusMap, SyncModeMap } from "../collection.const";
import {type CollectionTask, TaskStatus} from "../collection.model";
import {mapCollectionTask, StatusMap} from "../collection.const";
import useFetchData from "@/hooks/useFetchData";
import { useNavigate } from "react-router";
import { mapCollectionTask } from "../collection.const";
import {useNavigate} from "react-router";
export default function TaskManagement() {
const { message } = App.useApp();
@@ -51,8 +33,20 @@ export default function TaskManagement() {
searchParams,
setSearchParams,
fetchData,
handleFiltersChange,
} = useFetchData(queryTasksUsingGet, mapCollectionTask);
} = useFetchData(
(params) => {
const { keyword, ...rest } = params || {};
return queryTasksUsingGet({
...rest,
name: keyword || undefined,
});
},
mapCollectionTask,
30000,
false,
[],
0
);
const handleStartTask = async (taskId: string) => {
await executeTaskByIdUsingPost(taskId);
@@ -86,21 +80,21 @@ export default function TaskManagement() {
icon: <PauseCircleOutlined />,
onClick: () => handleStopTask(record.id),
};
const items = [
// isStopped ? startButton : stopButton,
// {
// key: "edit",
// label: "编辑",
// icon: <EditOutlined />,
// onClick: () => {
// showEditTaskModal(record);
// },
// },
return [
{
key: "executions",
label: "执行记录",
icon: <ProfileOutlined/>,
onClick: () =>
navigate(
`/data/collection?tab=task-execution&taskId=${encodeURIComponent(record.id)}`
),
},
{
key: "delete",
label: "删除",
danger: true,
icon: <DeleteOutlined />,
icon: <DeleteOutlined/>,
confirm: {
title: "确定要删除该任务吗？此操作不可撤销。",
okText: "删除",
@@ -110,7 +104,6 @@ export default function TaskManagement() {
onClick: () => handleDeleteTask(record.id),
},
];
return items;
};
const columns = [
@@ -128,17 +121,49 @@ export default function TaskManagement() {
key: "status",
width: 150,
ellipsis: true,
render: (status: string) => (
<Badge text={status.label} color={status.color} />
render: (status: any) => (
<Tag color={status.color}>{status.label}</Tag>
),
},
{
title: "所用模板",
dataIndex: "templateName",
key: "templateName",
width: 180,
ellipsis: true,
render: (v?: string) => v || "-",
},
{
title: "同步方式",
dataIndex: "syncMode",
key: "syncMode",
width: 150,
ellipsis: true,
render: (text: string) => <span>{SyncModeMap[text]?.label}</span>,
render: (text: any) => (
<Tag color={text.color}>{text.label}</Tag>
),
},
{
title: "Cron调度表达式",
dataIndex: "scheduleExpression",
key: "scheduleExpression",
width: 200,
ellipsis: true,
},
{
title: "超时时间",
dataIndex: "timeoutSeconds",
key: "timeoutSeconds",
width: 140,
ellipsis: true,
render: (v?: number) => (v === undefined || v === null ? "-" : `${v}s`),
},
{
title: "描述",
dataIndex: "description",
key: "description",
ellipsis: true,
width: 200,
},
{
title: "创建时间",
@@ -154,20 +179,6 @@ export default function TaskManagement() {
width: 150,
ellipsis: true,
},
{
title: "最近执行ID",
dataIndex: "lastExecutionId",
key: "lastExecutionId",
width: 150,
ellipsis: true,
},
{
title: "描述",
dataIndex: "description",
key: "description",
ellipsis: true,
width: 200,
},
{
title: "操作",
key: "action",
@@ -180,7 +191,7 @@ export default function TaskManagement() {
type="text"
icon={op.icon}
danger={op?.danger}
onClick={() => op.onClick(record)}
onClick={() => op.onClick()}
/>
</Tooltip>
);
@@ -192,7 +203,7 @@ export default function TaskManagement() {
okText={op.confirm.okText}
cancelText={op.confirm.cancelText}
okType={op.danger ? "danger" : "primary"}
onConfirm={() => op.onClick(record)}
onConfirm={() => op.onClick()}
>
<Tooltip key={op.key} title={op.label}>
<Button type="text" icon={op.icon} danger={op?.danger} />
@@ -218,14 +229,15 @@ export default function TaskManagement() {
current: 1,
}))
}
searchPlaceholder="搜索任务名称或描述..."
searchPlaceholder="搜索任务名称..."
filters={filters}
onFiltersChange={handleFiltersChange}
onFiltersChange={() => {}}
showViewToggle={false}
onClearFilters={() =>
setSearchParams((prev) => ({
...prev,
filters: {},
filter: { ...prev.filter, status: [] },
current: 1,
}))
}
onReload={fetchData}

173
frontend/src/pages/DataCollection/Home/TemplateManagement.tsx Normal file
View File

@@ -0,0 +1,173 @@
import { App, Card, Table, Tag } from "antd";
import type { ColumnsType } from "antd/es/table";
import { SearchControls } from "@/components/SearchControls";
import useFetchData from "@/hooks/useFetchData";
import { queryDataXTemplatesUsingGet } from "../collection.apis";
import { formatDateTime } from "@/utils/unit";
type CollectionTemplate = {
id: string;
name: string;
description?: string;
sourceType: string;
sourceName: string;
targetType: string;
targetName: string;
builtIn?: boolean;
createdAt?: string;
updatedAt?: string;
};
export default function TemplateManagement() {
const { message } = App.useApp();
const filters = [
{
key: "builtIn",
label: "模板类型",
options: [
{ value: "all", label: "全部" },
{ value: "true", label: "内置" },
{ value: "false", label: "自定义" },
],
},
];
const {
loading,
tableData,
pagination,
searchParams,
setSearchParams,
fetchData,
handleFiltersChange,
} = useFetchData<CollectionTemplate>(
(params) => {
const { keyword, builtIn, ...rest } = params || {};
const builtInValue = Array.isArray(builtIn)
? builtIn?.[0]
: builtIn;
return queryDataXTemplatesUsingGet({
...rest,
name: keyword || undefined,
built_in:
builtInValue && builtInValue !== "all"
? builtInValue === "true"
: undefined,
});
},
(tpl) => ({
...tpl,
createdAt: tpl.createdAt ? formatDateTime(tpl.createdAt) : "-",
updatedAt: tpl.updatedAt ? formatDateTime(tpl.updatedAt) : "-",
}),
30000,
false,
[],
0
);
const columns: ColumnsType<CollectionTemplate> = [
{
title: "模板名称",
dataIndex: "name",
key: "name",
fixed: "left",
width: 200,
ellipsis: true,
},
{
title: "模板类型",
dataIndex: "builtIn",
key: "builtIn",
width: 120,
render: (v?: boolean) => (
<Tag color={v ? "blue" : "default"}>{v ? "内置" : "自定义"}</Tag>
),
},
{
title: "源端",
key: "source",
width: 220,
ellipsis: true,
render: (_: any, record: CollectionTemplate) => (
<span>{`${record.sourceType} / ${record.sourceName}`}</span>
),
},
{
title: "目标端",
key: "target",
width: 220,
ellipsis: true,
render: (_: any, record: CollectionTemplate) => (
<span>{`${record.targetType} / ${record.targetName}`}</span>
),
},
{
title: "描述",
dataIndex: "description",
key: "description",
width: 260,
ellipsis: true,
render: (v?: string) => v || "-",
},
{
title: "创建时间",
dataIndex: "createdAt",
key: "createdAt",
width: 160,
},
{
title: "更新时间",
dataIndex: "updatedAt",
key: "updatedAt",
width: 160,
},
];
return (
<div className="space-y-4">
<SearchControls
searchTerm={searchParams.keyword}
onSearchChange={(newSearchTerm) =>
setSearchParams((prev) => ({
...prev,
keyword: newSearchTerm,
current: 1,
}))
}
searchPlaceholder="搜索模板名称..."
filters={filters}
onFiltersChange={handleFiltersChange}
showViewToggle={false}
onClearFilters={() =>
setSearchParams((prev) => ({
...prev,
filter: { ...prev.filter, builtIn: [] },
current: 1,
}))
}
onReload={() => {
fetchData().catch(() => message.error("刷新失败"));
}}
/>
<Card>
<Table
columns={columns}
dataSource={tableData}
loading={loading}
rowKey="id"
pagination={{
...pagination,
current: searchParams.current,
pageSize: searchParams.pageSize,
total: pagination.total,
}}
scroll={{ x: "max-content", y: "calc(100vh - 25rem)" }}
/>
</Card>
</div>
);
}

38
frontend/src/pages/DataCollection/collection.apis.ts
View File

@@ -28,7 +28,7 @@ export function queryDataXTemplatesUsingGet(params?: any) {
return get("/api/data-collection/templates", params);
}
export function deleteTaskByIdUsingDelete(id: string | number) {
return del(`/api/data-collection/tasks/${id}`);
return del("/api/data-collection/tasks", { ids: [id] });
}
export function executeTaskByIdUsingPost(
@@ -47,13 +47,47 @@ export function stopTaskByIdUsingPost(
// 执行日志相关接口
export function queryExecutionLogUsingPost(params?: any) {
return post("/api/data-collection/executions", params);
return get("/api/data-collection/executions", params);
}
export function queryExecutionLogByIdUsingGet(id: string | number) {
return get(`/api/data-collection/executions/${id}`);
}
export function queryExecutionLogContentByIdUsingGet(id: string | number) {
return get(`/api/data-collection/executions/${id}/log`);
}
export async function queryExecutionLogFileByIdUsingGet(id: string | number) {
const token = localStorage.getItem("token") || sessionStorage.getItem("token");
const resp = await fetch(`/api/data-collection/executions/${id}/log`, {
method: "GET",
headers: {
...(token ? { Authorization: `Bearer ${token}` } : {}),
},
credentials: "include",
});
if (!resp.ok) {
let detail = "";
try {
detail = await resp.text();
} catch {
detail = resp.statusText;
}
const err: any = new Error(detail || `HTTP error ${resp.status}`);
err.status = resp.status;
err.data = detail;
throw err;
}
const contentDisposition = resp.headers.get("content-disposition") || "";
const filenameMatch = contentDisposition.match(/filename\*?=(?:UTF-8''|\")?([^;\"\n]+)/i);
const filename = filenameMatch?.[1] ? decodeURIComponent(filenameMatch[1].replace(/\"/g, "").trim()) : `execution_${id}.log`;
const blob = await resp.blob();
return { blob, filename };
}
// 监控统计相关接口
export function queryCollectionStatisticsUsingGet(params?: any) {
return get("/api/data-collection/monitor/statistics", params);

34
frontend/src/pages/DataCollection/collection.const.ts
View File

@@ -1,9 +1,11 @@
import {
CollectionTask,
LogStatus,
SyncMode,
SyncMode, TaskExecution,
TaskStatus,
TriggerType,
} from "./collection.model";
import {formatDateTime} from "@/utils/unit.ts";
export const StatusMap: Record<
TaskStatus,
@@ -24,23 +26,27 @@ export const StatusMap: Record<
color: "red",
value: TaskStatus.FAILED,
},
[TaskStatus.SUCCESS]: {
[TaskStatus.COMPLETED]: {
label: "成功",
color: "green",
value: TaskStatus.SUCCESS,
value: TaskStatus.COMPLETED,
},
[TaskStatus.DRAFT]: {
label: "草稿",
color: "orange",
value: TaskStatus.DRAFT,
},
[TaskStatus.READY]: { label: "就绪", color: "cyan", value: TaskStatus.READY },
[TaskStatus.PENDING]: {
label: "就绪",
color: "cyan",
value: TaskStatus.PENDING
},
};
export const SyncModeMap: Record<SyncMode, { label: string; value: SyncMode }> =
export const SyncModeMap: Record<SyncMode, { label: string; value: SyncMode, color: string }> =
{
[SyncMode.ONCE]: { label: "立即同步", value: SyncMode.ONCE },
[SyncMode.SCHEDULED]: { label: "定时同步", value: SyncMode.SCHEDULED },
[SyncMode.ONCE]: { label: "立即同步", value: SyncMode.ONCE, color: "orange" },
[SyncMode.SCHEDULED]: { label: "定时同步", value: SyncMode.SCHEDULED, color: "blue" },
};
export const LogStatusMap: Record<
@@ -73,9 +79,21 @@ export const LogTriggerTypeMap: Record<
[TriggerType.API]: { label: "API", value: TriggerType.API },
};
export function mapCollectionTask(task: CollectionTask): CollectionTask {
export function mapCollectionTask(task: CollectionTask): any {
return {
...task,
status: StatusMap[task.status],
syncMode: SyncModeMap[task.syncMode],
createdAt: formatDateTime(task.createdAt),
updatedAt: formatDateTime(task.updatedAt)
};
}
export function mapTaskExecution(execution: TaskExecution): any {
return {
...execution,
status: StatusMap[execution.status],
startedAt: formatDateTime(execution.startedAt),
completedAt: formatDateTime(execution.completedAt)
};
}

18
frontend/src/pages/DataCollection/collection.model.ts
View File

@@ -1,8 +1,8 @@
export enum TaskStatus {
DRAFT = "DRAFT",
READY = "READY",
PENDING = "PENDING",
RUNNING = "RUNNING",
SUCCESS = "SUCCESS",
COMPLETED = "COMPLETED",
FAILED = "FAILED",
STOPPED = "STOPPED",
}
@@ -19,12 +19,26 @@ export interface CollectionTask {
config: object; // 具体配置结构根据实际需求定义
status: TaskStatus;
syncMode: SyncMode;
templateName?: string;
scheduleExpression?: string; // 仅当 syncMode 为 SCHEDULED 时存在
timeoutSeconds?: number;
lastExecutionId: string;
createdAt: string; // ISO date string
updatedAt: string; // ISO date string
}
export interface TaskExecution {
id: string;
taskId: string;
taskName: string;
status: string;
logPath: string;
startedAt: string;
completedAt: string;
durationSeconds: number;
errorMessage: string;
}
export enum LogStatus {
RUNNING = "RUNNING",
SUCCESS = "SUCCESS",