You've already forked DataMate
feature:增加数据配比功能 (#52)
* refactor: 修改调整数据归集实现,删除无用代码,优化代码结构 * feature: 每天凌晨00:00扫描所有数据集,检查数据集是否超过了预设的保留天数,超出保留天数的数据集调用删除接口进行删除 * fix: 修改删除数据集文件的逻辑,上传到数据集中的文件会同时删除数据库中的记录和文件系统中的文件,归集过来的文件仅删除数据库中的记录 * fix: 增加参数校验和接口定义,删除不使用的接口 * fix: 数据集统计数据默认为0 * feature: 数据集状态增加流转,创建时为草稿状态,上传文件或者归集文件后修改为活动状态 * refactor: 修改分页查询归集任务的代码 * fix: 更新后重新执行;归集任务执行增加事务控制 * feature: 创建归集任务时能够同步创建数据集,更新归集任务时能更新到指定数据集 * fix: 创建归集任务不需要创建数据集时不应该报错 * fix: 修复删除文件时数据集的统计数据不变动 * feature: 查询数据集详情时能够获取到文件标签分布 * fix: tags为空时不进行分析 * fix: 状态修改为ACTIVE * fix: 修改解析tag的方法 * feature: 实现创建、分页查询、删除配比任务 * feature: 实现创建、分页查询、删除配比任务的前端交互 * fix: 修复进度计算异常导致的页面报错
This commit is contained in:
@@ -0,0 +1,34 @@
|
||||
import React from "react";
|
||||
import { Form, Input } from "antd";
|
||||
|
||||
const { TextArea } = Input;
|
||||
|
||||
interface BasicInformationProps {
|
||||
totalTargetCount: number;
|
||||
}
|
||||
|
||||
const BasicInformation: React.FC<BasicInformationProps> = ({ totalTargetCount }) => {
|
||||
return (
|
||||
<div className="grid grid-cols-2 gap-4 mb-4">
|
||||
<Form.Item
|
||||
label="任务名称"
|
||||
name="name"
|
||||
rules={[{ required: true, message: "请输入配比任务名称" }]}
|
||||
>
|
||||
<Input placeholder="输入配比任务名称" />
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label="目标总数量"
|
||||
name="totalTargetCount"
|
||||
rules={[{ required: true, message: "请输入目标总数量" }]}
|
||||
>
|
||||
<Input type="number" placeholder="目标总数量" min={1} />
|
||||
</Form.Item>
|
||||
<Form.Item label="任务描述" name="description" className="col-span-2">
|
||||
<TextArea placeholder="描述配比任务的目的和要求(可选)" rows={2} />
|
||||
</Form.Item>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default BasicInformation;
|
||||
132
frontend/src/pages/RatioTask/Create/components/RatioConfig.tsx
Normal file
132
frontend/src/pages/RatioTask/Create/components/RatioConfig.tsx
Normal file
@@ -0,0 +1,132 @@
|
||||
import React from "react";
|
||||
import { Badge, Card, Input, Progress } from "antd";
|
||||
import { BarChart3 } from "lucide-react";
|
||||
import type { Dataset } from "@/pages/DataManagement/dataset.model.ts";
|
||||
|
||||
interface RatioConfigItem {
|
||||
id: string;
|
||||
name: string;
|
||||
type: "dataset" | "label";
|
||||
quantity: number;
|
||||
percentage: number;
|
||||
source: string;
|
||||
}
|
||||
|
||||
interface RatioConfigProps {
|
||||
ratioType: "dataset" | "label";
|
||||
selectedDatasets: string[];
|
||||
datasets: Dataset[];
|
||||
ratioConfigs: RatioConfigItem[];
|
||||
totalTargetCount: number;
|
||||
distributions: Record<string, Record<string, number>>;
|
||||
onUpdateDatasetQuantity: (datasetId: string, quantity: number) => void;
|
||||
onUpdateLabelQuantity: (datasetId: string, label: string, quantity: number) => void;
|
||||
}
|
||||
|
||||
const RatioConfig: React.FC<RatioConfigProps> = ({
|
||||
ratioType,
|
||||
selectedDatasets,
|
||||
datasets,
|
||||
ratioConfigs,
|
||||
totalTargetCount,
|
||||
distributions,
|
||||
onUpdateDatasetQuantity,
|
||||
onUpdateLabelQuantity,
|
||||
}) => {
|
||||
const totalConfigured = ratioConfigs.reduce((sum, c) => sum + (c.quantity || 0), 0);
|
||||
|
||||
return (
|
||||
<div className="mb-4">
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-sm font-medium">配比设置</span>
|
||||
<span className="text-xs text-gray-500">
|
||||
已配置: {totalConfigured} / {totalTargetCount}
|
||||
</span>
|
||||
</div>
|
||||
{selectedDatasets.length === 0 ? (
|
||||
<div className="text-center py-8 text-gray-500">
|
||||
<BarChart3 className="w-12 h-12 mx-auto mb-2 text-gray-300" />
|
||||
<p className="text-sm">请先选择数据集</p>
|
||||
</div>
|
||||
) : (
|
||||
<div style={{ maxHeight: 500, overflowY: "auto" }}>
|
||||
{selectedDatasets.map((datasetId) => {
|
||||
const dataset = datasets.find((d) => String(d.id) === datasetId);
|
||||
const config = ratioConfigs.find((c) => c.source === datasetId);
|
||||
const currentQuantity = config?.quantity || 0;
|
||||
if (!dataset) return null;
|
||||
return (
|
||||
<Card key={datasetId} size="small" className="mb-2">
|
||||
<div className="flex items-center justify-between mb-3">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="font-medium text-sm">{dataset.name}</span>
|
||||
<Badge color="gray">{dataset.fileCount}条</Badge>
|
||||
</div>
|
||||
<div className="text-xs text-gray-500">{config?.percentage || 0}%</div>
|
||||
</div>
|
||||
{ratioType === "dataset" ? (
|
||||
<div>
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
<span className="text-xs">数量:</span>
|
||||
<Input
|
||||
type="number"
|
||||
value={currentQuantity}
|
||||
onChange={(e) => onUpdateDatasetQuantity(datasetId, Number(e.target.value))}
|
||||
style={{ width: 80 }}
|
||||
min={0}
|
||||
max={Math.min(dataset.fileCount || 0, totalTargetCount)}
|
||||
/>
|
||||
<span className="text-xs text-gray-500">条</span>
|
||||
</div>
|
||||
<Progress
|
||||
percent={Math.round((currentQuantity / totalTargetCount) * 100)}
|
||||
size="small"
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
<div>
|
||||
{!distributions[String(dataset.id)] ? (
|
||||
<div className="text-xs text-gray-400">加载标签分布...</div>
|
||||
) : Object.entries(distributions[String(dataset.id)]).length === 0 ? (
|
||||
<div className="text-xs text-gray-400">该数据集暂无标签</div>
|
||||
) : (
|
||||
<div className="flex flex-col gap-2">
|
||||
{Object.entries(distributions[String(dataset.id)]).map(([label, count]) => {
|
||||
const sourceKey = `${datasetId}_${label}`;
|
||||
const labelConfig = ratioConfigs.find((c) => c.source === sourceKey);
|
||||
const labelQuantity = labelConfig?.quantity || 0;
|
||||
return (
|
||||
<div key={label} className="flex items-center justify-between gap-2">
|
||||
<div className="flex items-center gap-2">
|
||||
<Badge color="gray">{label}</Badge>
|
||||
<span className="text-xs text-gray-500">{count}条</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-xs">数量:</span>
|
||||
<Input
|
||||
type="number"
|
||||
value={labelQuantity}
|
||||
onChange={(e) => onUpdateLabelQuantity(datasetId, label, Number(e.target.value))}
|
||||
style={{ width: 80 }}
|
||||
min={0}
|
||||
max={Math.min(Number(count) || 0, totalTargetCount)}
|
||||
/>
|
||||
<span className="text-xs text-gray-500">条</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</Card>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default RatioConfig;
|
||||
250
frontend/src/pages/RatioTask/Create/components/SelectDataset.tsx
Normal file
250
frontend/src/pages/RatioTask/Create/components/SelectDataset.tsx
Normal file
@@ -0,0 +1,250 @@
|
||||
import React, { useEffect, useState } from "react";
|
||||
import { Badge, Button, Card, Checkbox, Input, Pagination, Select } from "antd";
|
||||
import { Database, Search as SearchIcon } from "lucide-react";
|
||||
import type { Dataset } from "@/pages/DataManagement/dataset.model.ts";
|
||||
import { queryDatasetsUsingGet, queryDatasetByIdUsingGet, queryDatasetStatisticsByIdUsingGet } from "@/pages/DataManagement/dataset.api.ts";
|
||||
|
||||
interface SelectDatasetProps {
|
||||
selectedDatasets: string[];
|
||||
ratioType: "dataset" | "label";
|
||||
onRatioTypeChange: (val: "dataset" | "label") => void;
|
||||
onSelectedDatasetsChange: (next: string[]) => void;
|
||||
onDistributionsChange?: (next: Record<string, Record<string, number>>) => void;
|
||||
onDatasetsChange?: (list: Dataset[]) => void;
|
||||
}
|
||||
|
||||
const SelectDataset: React.FC<SelectDatasetProps> = ({
|
||||
selectedDatasets,
|
||||
ratioType,
|
||||
onRatioTypeChange,
|
||||
onSelectedDatasetsChange,
|
||||
onDistributionsChange,
|
||||
onDatasetsChange,
|
||||
}) => {
|
||||
const [datasets, setDatasets] = useState<Dataset[]>([]);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [searchQuery, setSearchQuery] = useState("");
|
||||
const [pagination, setPagination] = useState({ page: 1, size: 10, total: 0 });
|
||||
const [distributions, setDistributions] = useState<Record<string, Record<string, number>>>({});
|
||||
|
||||
// Fetch dataset list
|
||||
useEffect(() => {
|
||||
const fetchDatasets = async () => {
|
||||
try {
|
||||
setLoading(true);
|
||||
const { data } = await queryDatasetsUsingGet({
|
||||
page: pagination.page,
|
||||
size: pagination.size,
|
||||
keyword: searchQuery?.trim() || undefined,
|
||||
});
|
||||
const list = data?.content || data?.data || [];
|
||||
setDatasets(list);
|
||||
onDatasetsChange?.(list);
|
||||
setPagination((prev) => ({ ...prev, total: data?.totalElements ?? data?.total ?? 0 }));
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
fetchDatasets();
|
||||
}, [pagination.page, pagination.size, searchQuery]);
|
||||
|
||||
// Fetch label distributions when in label mode
|
||||
useEffect(() => {
|
||||
const fetchDistributions = async () => {
|
||||
if (ratioType !== "label" || !datasets?.length) return;
|
||||
const idsToFetch = datasets.map((d) => String(d.id)).filter((id) => !distributions[id]);
|
||||
if (!idsToFetch.length) return;
|
||||
try {
|
||||
const results = await Promise.all(
|
||||
idsToFetch.map(async (id) => {
|
||||
try {
|
||||
const statRes = await queryDatasetStatisticsByIdUsingGet(id);
|
||||
return { id, stats: statRes?.data };
|
||||
} catch {
|
||||
return { id, stats: null };
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
const next: Record<string, Record<string, number>> = { ...distributions };
|
||||
for (const { id, stats } of results) {
|
||||
let dist: Record<string, number> | undefined = undefined;
|
||||
if (stats) {
|
||||
const candidates: any[] = [
|
||||
(stats as any).labelDistribution,
|
||||
(stats as any).tagDistribution,
|
||||
(stats as any).label_stats,
|
||||
(stats as any).labels,
|
||||
(stats as any).distribution,
|
||||
];
|
||||
let picked = candidates.find((c) => c && (typeof c === "object" || Array.isArray(c)));
|
||||
if (Array.isArray(picked)) {
|
||||
const obj: Record<string, number> = {};
|
||||
picked.forEach((it: any) => {
|
||||
const key = it?.label ?? it?.name ?? it?.tag ?? it?.key;
|
||||
const val = it?.count ?? it?.value ?? it?.num ?? it?.total;
|
||||
if (key != null && typeof val === "number") obj[String(key)] = val;
|
||||
});
|
||||
dist = obj;
|
||||
} else if (picked && typeof picked === "object") {
|
||||
dist = picked as Record<string, number>;
|
||||
}
|
||||
}
|
||||
if (!dist) {
|
||||
try {
|
||||
const detRes = await queryDatasetByIdUsingGet(id);
|
||||
const det = detRes?.data;
|
||||
if (det) {
|
||||
let picked =
|
||||
(det as any).distribution ||
|
||||
(det as any).labelDistribution ||
|
||||
(det as any).tagDistribution ||
|
||||
(det as any).label_stats ||
|
||||
(det as any).labels ||
|
||||
undefined;
|
||||
if (Array.isArray(picked)) {
|
||||
const obj: Record<string, number> = {};
|
||||
picked.forEach((it: any) => {
|
||||
const key = it?.label ?? it?.name ?? it?.tag ?? it?.key;
|
||||
const val = it?.count ?? it?.value ?? it?.num ?? it?.total;
|
||||
if (key != null && typeof val === "number") obj[String(key)] = val;
|
||||
});
|
||||
dist = obj;
|
||||
} else if (picked && typeof picked === "object") {
|
||||
dist = picked as Record<string, number>;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
dist = undefined;
|
||||
}
|
||||
}
|
||||
next[String(id)] = dist || {};
|
||||
}
|
||||
setDistributions(next);
|
||||
onDistributionsChange?.(next);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
};
|
||||
fetchDistributions();
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [ratioType, datasets]);
|
||||
|
||||
const onToggleDataset = (datasetId: string, checked: boolean) => {
|
||||
if (checked) {
|
||||
const next = Array.from(new Set([...selectedDatasets, datasetId]));
|
||||
onSelectedDatasetsChange(next);
|
||||
} else {
|
||||
onSelectedDatasetsChange(selectedDatasets.filter((id) => id !== datasetId));
|
||||
}
|
||||
};
|
||||
|
||||
const onClearSelection = () => {
|
||||
onSelectedDatasetsChange([]);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="col-span-5">
|
||||
<h2 className="font-medium text-gray-900 text-lg mb-2 flex items-center gap-2">
|
||||
<Database className="w-5 h-5" />
|
||||
数据集选择
|
||||
</h2>
|
||||
<Card>
|
||||
<div className="flex items-center gap-4 mb-4">
|
||||
<span className="text-sm">配比方式:</span>
|
||||
<Select
|
||||
style={{ width: 120 }}
|
||||
value={ratioType}
|
||||
onChange={(v) => onRatioTypeChange(v)}
|
||||
options={[
|
||||
{ label: "按数据集", value: "dataset" },
|
||||
{ label: "按标签", value: "label" },
|
||||
]}
|
||||
/>
|
||||
</div>
|
||||
<Input
|
||||
prefix={<SearchIcon className="text-gray-400" />}
|
||||
placeholder="搜索数据集"
|
||||
value={searchQuery}
|
||||
onChange={(e) => {
|
||||
setSearchQuery(e.target.value);
|
||||
setPagination((p) => ({ ...p, page: 1 }));
|
||||
}}
|
||||
/>
|
||||
<div style={{ maxHeight: 500, overflowY: "auto" }}>
|
||||
{loading && (
|
||||
<div className="text-center text-gray-500 py-8">正在加载数据集...</div>
|
||||
)}
|
||||
{!loading &&
|
||||
datasets.map((dataset) => {
|
||||
const idStr = String(dataset.id);
|
||||
const checked = selectedDatasets.includes(idStr);
|
||||
return (
|
||||
<Card
|
||||
key={dataset.id}
|
||||
size="small"
|
||||
className={`mb-2 cursor-pointer ${checked ? "border-blue-500" : "hover:border-blue-200"}`}
|
||||
onClick={() => onToggleDataset(idStr, !checked)}
|
||||
>
|
||||
<div className="flex items-start gap-3">
|
||||
<Checkbox
|
||||
checked={checked}
|
||||
onChange={(e) => onToggleDataset(idStr, e.target.checked)}
|
||||
/>
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="font-medium text-sm truncate">{dataset.name}</span>
|
||||
<Badge color="blue">{dataset.datasetType}</Badge>
|
||||
</div>
|
||||
<div className="text-xs text-gray-500 mt-1">{dataset.description}</div>
|
||||
<div className="flex items-center gap-4 mt-2 text-xs text-gray-500">
|
||||
<span>{dataset.fileCount}条</span>
|
||||
<span>{dataset.size}</span>
|
||||
</div>
|
||||
{ratioType === "label" && (
|
||||
<div className="mt-2">
|
||||
{distributions[idStr] ? (
|
||||
Object.entries(distributions[idStr]).length > 0 ? (
|
||||
<div className="flex flex-wrap gap-2 text-xs">
|
||||
{Object.entries(distributions[idStr])
|
||||
.slice(0, 8)
|
||||
.map(([tag, count]) => (
|
||||
<Badge key={tag} color="gray">{`${tag}: ${count}`}</Badge>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div className="text-xs text-gray-400">未检测到标签分布</div>
|
||||
)
|
||||
) : (
|
||||
<div className="text-xs text-gray-400">加载标签分布...</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</Card>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
<div className="flex justify-between mt-3 items-center">
|
||||
<span className="text-sm text-gray-600">已选择 {selectedDatasets.length} 个数据集</span>
|
||||
<div className="flex items-center gap-3">
|
||||
<Button size="small" onClick={onClearSelection}>
|
||||
清空选择
|
||||
</Button>
|
||||
<Pagination
|
||||
size="small"
|
||||
current={pagination.page}
|
||||
pageSize={pagination.size}
|
||||
total={pagination.total}
|
||||
showSizeChanger
|
||||
onChange={(p, ps) => setPagination((prev) => ({ ...prev, page: p, size: ps }))}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default SelectDataset;
|
||||
Reference in New Issue
Block a user