feature:增加数据配比功能 (#52)

* refactor: 修改调整数据归集实现,删除无用代码,优化代码结构

* feature: 每天凌晨00:00扫描所有数据集,检查数据集是否超过了预设的保留天数,超出保留天数的数据集调用删除接口进行删除

* fix: 修改删除数据集文件的逻辑,上传到数据集中的文件会同时删除数据库中的记录和文件系统中的文件,归集过来的文件仅删除数据库中的记录

* fix: 增加参数校验和接口定义,删除不使用的接口

* fix: 数据集统计数据默认为0

* feature: 数据集状态增加流转,创建时为草稿状态,上传文件或者归集文件后修改为活动状态

* refactor: 修改分页查询归集任务的代码

* fix: 更新后重新执行;归集任务执行增加事务控制

* feature: 创建归集任务时能够同步创建数据集,更新归集任务时能更新到指定数据集

* fix: 创建归集任务不需要创建数据集时不应该报错

* fix: 修复删除文件时数据集的统计数据不变动

* feature: 查询数据集详情时能够获取到文件标签分布

* fix: tags为空时不进行分析

* fix: 状态修改为ACTIVE

* fix: 修改解析tag的方法

* feature: 实现创建、分页查询、删除配比任务

* feature: 实现创建、分页查询、删除配比任务的前端交互

* fix: 修复进度计算异常导致的页面报错
This commit is contained in:
hefanli
2025-11-03 10:17:39 +08:00
committed by GitHub
parent 07edf16044
commit 08bd4eca5c
32 changed files with 1894 additions and 1028 deletions

View File

@@ -0,0 +1,34 @@
import React from "react";
import { Form, Input } from "antd";
const { TextArea } = Input;
interface BasicInformationProps {
totalTargetCount: number;
}
const BasicInformation: React.FC<BasicInformationProps> = ({ totalTargetCount }) => {
return (
<div className="grid grid-cols-2 gap-4 mb-4">
<Form.Item
label="任务名称"
name="name"
rules={[{ required: true, message: "请输入配比任务名称" }]}
>
<Input placeholder="输入配比任务名称" />
</Form.Item>
<Form.Item
label="目标总数量"
name="totalTargetCount"
rules={[{ required: true, message: "请输入目标总数量" }]}
>
<Input type="number" placeholder="目标总数量" min={1} />
</Form.Item>
<Form.Item label="任务描述" name="description" className="col-span-2">
<TextArea placeholder="描述配比任务的目的和要求(可选)" rows={2} />
</Form.Item>
</div>
);
};
export default BasicInformation;

View File

@@ -0,0 +1,132 @@
import React from "react";
import { Badge, Card, Input, Progress } from "antd";
import { BarChart3 } from "lucide-react";
import type { Dataset } from "@/pages/DataManagement/dataset.model.ts";
interface RatioConfigItem {
id: string;
name: string;
type: "dataset" | "label";
quantity: number;
percentage: number;
source: string;
}
interface RatioConfigProps {
ratioType: "dataset" | "label";
selectedDatasets: string[];
datasets: Dataset[];
ratioConfigs: RatioConfigItem[];
totalTargetCount: number;
distributions: Record<string, Record<string, number>>;
onUpdateDatasetQuantity: (datasetId: string, quantity: number) => void;
onUpdateLabelQuantity: (datasetId: string, label: string, quantity: number) => void;
}
const RatioConfig: React.FC<RatioConfigProps> = ({
ratioType,
selectedDatasets,
datasets,
ratioConfigs,
totalTargetCount,
distributions,
onUpdateDatasetQuantity,
onUpdateLabelQuantity,
}) => {
const totalConfigured = ratioConfigs.reduce((sum, c) => sum + (c.quantity || 0), 0);
return (
<div className="mb-4">
<div className="flex items-center justify-between">
<span className="text-sm font-medium"></span>
<span className="text-xs text-gray-500">
: {totalConfigured} / {totalTargetCount}
</span>
</div>
{selectedDatasets.length === 0 ? (
<div className="text-center py-8 text-gray-500">
<BarChart3 className="w-12 h-12 mx-auto mb-2 text-gray-300" />
<p className="text-sm"></p>
</div>
) : (
<div style={{ maxHeight: 500, overflowY: "auto" }}>
{selectedDatasets.map((datasetId) => {
const dataset = datasets.find((d) => String(d.id) === datasetId);
const config = ratioConfigs.find((c) => c.source === datasetId);
const currentQuantity = config?.quantity || 0;
if (!dataset) return null;
return (
<Card key={datasetId} size="small" className="mb-2">
<div className="flex items-center justify-between mb-3">
<div className="flex items-center gap-2">
<span className="font-medium text-sm">{dataset.name}</span>
<Badge color="gray">{dataset.fileCount}</Badge>
</div>
<div className="text-xs text-gray-500">{config?.percentage || 0}%</div>
</div>
{ratioType === "dataset" ? (
<div>
<div className="flex items-center gap-2 mb-2">
<span className="text-xs">:</span>
<Input
type="number"
value={currentQuantity}
onChange={(e) => onUpdateDatasetQuantity(datasetId, Number(e.target.value))}
style={{ width: 80 }}
min={0}
max={Math.min(dataset.fileCount || 0, totalTargetCount)}
/>
<span className="text-xs text-gray-500"></span>
</div>
<Progress
percent={Math.round((currentQuantity / totalTargetCount) * 100)}
size="small"
/>
</div>
) : (
<div>
{!distributions[String(dataset.id)] ? (
<div className="text-xs text-gray-400">...</div>
) : Object.entries(distributions[String(dataset.id)]).length === 0 ? (
<div className="text-xs text-gray-400"></div>
) : (
<div className="flex flex-col gap-2">
{Object.entries(distributions[String(dataset.id)]).map(([label, count]) => {
const sourceKey = `${datasetId}_${label}`;
const labelConfig = ratioConfigs.find((c) => c.source === sourceKey);
const labelQuantity = labelConfig?.quantity || 0;
return (
<div key={label} className="flex items-center justify-between gap-2">
<div className="flex items-center gap-2">
<Badge color="gray">{label}</Badge>
<span className="text-xs text-gray-500">{count}</span>
</div>
<div className="flex items-center gap-2">
<span className="text-xs">:</span>
<Input
type="number"
value={labelQuantity}
onChange={(e) => onUpdateLabelQuantity(datasetId, label, Number(e.target.value))}
style={{ width: 80 }}
min={0}
max={Math.min(Number(count) || 0, totalTargetCount)}
/>
<span className="text-xs text-gray-500"></span>
</div>
</div>
);
})}
</div>
)}
</div>
)}
</Card>
);
})}
</div>
)}
</div>
);
};
export default RatioConfig;

View File

@@ -0,0 +1,250 @@
import React, { useEffect, useState } from "react";
import { Badge, Button, Card, Checkbox, Input, Pagination, Select } from "antd";
import { Database, Search as SearchIcon } from "lucide-react";
import type { Dataset } from "@/pages/DataManagement/dataset.model.ts";
import { queryDatasetsUsingGet, queryDatasetByIdUsingGet, queryDatasetStatisticsByIdUsingGet } from "@/pages/DataManagement/dataset.api.ts";
interface SelectDatasetProps {
selectedDatasets: string[];
ratioType: "dataset" | "label";
onRatioTypeChange: (val: "dataset" | "label") => void;
onSelectedDatasetsChange: (next: string[]) => void;
onDistributionsChange?: (next: Record<string, Record<string, number>>) => void;
onDatasetsChange?: (list: Dataset[]) => void;
}
const SelectDataset: React.FC<SelectDatasetProps> = ({
selectedDatasets,
ratioType,
onRatioTypeChange,
onSelectedDatasetsChange,
onDistributionsChange,
onDatasetsChange,
}) => {
const [datasets, setDatasets] = useState<Dataset[]>([]);
const [loading, setLoading] = useState(false);
const [searchQuery, setSearchQuery] = useState("");
const [pagination, setPagination] = useState({ page: 1, size: 10, total: 0 });
const [distributions, setDistributions] = useState<Record<string, Record<string, number>>>({});
// Fetch dataset list
useEffect(() => {
const fetchDatasets = async () => {
try {
setLoading(true);
const { data } = await queryDatasetsUsingGet({
page: pagination.page,
size: pagination.size,
keyword: searchQuery?.trim() || undefined,
});
const list = data?.content || data?.data || [];
setDatasets(list);
onDatasetsChange?.(list);
setPagination((prev) => ({ ...prev, total: data?.totalElements ?? data?.total ?? 0 }));
} finally {
setLoading(false);
}
};
fetchDatasets();
}, [pagination.page, pagination.size, searchQuery]);
// Fetch label distributions when in label mode
useEffect(() => {
const fetchDistributions = async () => {
if (ratioType !== "label" || !datasets?.length) return;
const idsToFetch = datasets.map((d) => String(d.id)).filter((id) => !distributions[id]);
if (!idsToFetch.length) return;
try {
const results = await Promise.all(
idsToFetch.map(async (id) => {
try {
const statRes = await queryDatasetStatisticsByIdUsingGet(id);
return { id, stats: statRes?.data };
} catch {
return { id, stats: null };
}
})
);
const next: Record<string, Record<string, number>> = { ...distributions };
for (const { id, stats } of results) {
let dist: Record<string, number> | undefined = undefined;
if (stats) {
const candidates: any[] = [
(stats as any).labelDistribution,
(stats as any).tagDistribution,
(stats as any).label_stats,
(stats as any).labels,
(stats as any).distribution,
];
let picked = candidates.find((c) => c && (typeof c === "object" || Array.isArray(c)));
if (Array.isArray(picked)) {
const obj: Record<string, number> = {};
picked.forEach((it: any) => {
const key = it?.label ?? it?.name ?? it?.tag ?? it?.key;
const val = it?.count ?? it?.value ?? it?.num ?? it?.total;
if (key != null && typeof val === "number") obj[String(key)] = val;
});
dist = obj;
} else if (picked && typeof picked === "object") {
dist = picked as Record<string, number>;
}
}
if (!dist) {
try {
const detRes = await queryDatasetByIdUsingGet(id);
const det = detRes?.data;
if (det) {
let picked =
(det as any).distribution ||
(det as any).labelDistribution ||
(det as any).tagDistribution ||
(det as any).label_stats ||
(det as any).labels ||
undefined;
if (Array.isArray(picked)) {
const obj: Record<string, number> = {};
picked.forEach((it: any) => {
const key = it?.label ?? it?.name ?? it?.tag ?? it?.key;
const val = it?.count ?? it?.value ?? it?.num ?? it?.total;
if (key != null && typeof val === "number") obj[String(key)] = val;
});
dist = obj;
} else if (picked && typeof picked === "object") {
dist = picked as Record<string, number>;
}
}
} catch {
dist = undefined;
}
}
next[String(id)] = dist || {};
}
setDistributions(next);
onDistributionsChange?.(next);
} catch {
// ignore
}
};
fetchDistributions();
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [ratioType, datasets]);
const onToggleDataset = (datasetId: string, checked: boolean) => {
if (checked) {
const next = Array.from(new Set([...selectedDatasets, datasetId]));
onSelectedDatasetsChange(next);
} else {
onSelectedDatasetsChange(selectedDatasets.filter((id) => id !== datasetId));
}
};
const onClearSelection = () => {
onSelectedDatasetsChange([]);
};
return (
<div className="col-span-5">
<h2 className="font-medium text-gray-900 text-lg mb-2 flex items-center gap-2">
<Database className="w-5 h-5" />
</h2>
<Card>
<div className="flex items-center gap-4 mb-4">
<span className="text-sm">:</span>
<Select
style={{ width: 120 }}
value={ratioType}
onChange={(v) => onRatioTypeChange(v)}
options={[
{ label: "按数据集", value: "dataset" },
{ label: "按标签", value: "label" },
]}
/>
</div>
<Input
prefix={<SearchIcon className="text-gray-400" />}
placeholder="搜索数据集"
value={searchQuery}
onChange={(e) => {
setSearchQuery(e.target.value);
setPagination((p) => ({ ...p, page: 1 }));
}}
/>
<div style={{ maxHeight: 500, overflowY: "auto" }}>
{loading && (
<div className="text-center text-gray-500 py-8">...</div>
)}
{!loading &&
datasets.map((dataset) => {
const idStr = String(dataset.id);
const checked = selectedDatasets.includes(idStr);
return (
<Card
key={dataset.id}
size="small"
className={`mb-2 cursor-pointer ${checked ? "border-blue-500" : "hover:border-blue-200"}`}
onClick={() => onToggleDataset(idStr, !checked)}
>
<div className="flex items-start gap-3">
<Checkbox
checked={checked}
onChange={(e) => onToggleDataset(idStr, e.target.checked)}
/>
<div className="flex-1 min-w-0">
<div className="flex items-center gap-2">
<span className="font-medium text-sm truncate">{dataset.name}</span>
<Badge color="blue">{dataset.datasetType}</Badge>
</div>
<div className="text-xs text-gray-500 mt-1">{dataset.description}</div>
<div className="flex items-center gap-4 mt-2 text-xs text-gray-500">
<span>{dataset.fileCount}</span>
<span>{dataset.size}</span>
</div>
{ratioType === "label" && (
<div className="mt-2">
{distributions[idStr] ? (
Object.entries(distributions[idStr]).length > 0 ? (
<div className="flex flex-wrap gap-2 text-xs">
{Object.entries(distributions[idStr])
.slice(0, 8)
.map(([tag, count]) => (
<Badge key={tag} color="gray">{`${tag}: ${count}`}</Badge>
))}
</div>
) : (
<div className="text-xs text-gray-400"></div>
)
) : (
<div className="text-xs text-gray-400">...</div>
)}
</div>
)}
</div>
</div>
</Card>
);
})}
</div>
<div className="flex justify-between mt-3 items-center">
<span className="text-sm text-gray-600"> {selectedDatasets.length} </span>
<div className="flex items-center gap-3">
<Button size="small" onClick={onClearSelection}>
</Button>
<Pagination
size="small"
current={pagination.page}
pageSize={pagination.size}
total={pagination.total}
showSizeChanger
onChange={(p, ps) => setPagination((prev) => ({ ...prev, page: p, size: ps }))}
/>
</div>
</div>
</Card>
</div>
);
};
export default SelectDataset;