q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(data-management): 添加数据集父子层级结构功能

Browse Source 
- 在OpenAPI规范中新增parentDatasetId字段用于层级过滤
- 实现数据集父子关系的创建、更新和删除逻辑
- 添加数据集移动时的路径重命名和文件路径前缀更新
- 增加子数据集数量验证防止误删父数据集
- 更新前端界面支持选择父数据集和导航显示
- 优化Python后端自动标注任务的路径处理逻辑
- 修改数据库表结构添加外键约束确保数据一致性
This commit is contained in:
Jerry Yan
2026-01-20 13:34:50 +08:00
parent bde00c3c38
commit 79371ba078
26 changed files with 394 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
backend/openapi/specs/data-management.yaml
View File

@@ -67,6 +67,11 @@ paths:
type: string
enum: [DRAFT, ACTIVE, PROCESSING, ARCHIVED, PUBLISHED, DEPRECATED]
description: 数据集状态过滤
- name: parentDatasetId
in: query
schema:
type: string
description: 父数据集ID过滤（传空字符串表示根数据集）
responses:
'200':
description: 成功
@@ -544,6 +549,9 @@ components:
id:
type: string
description: 数据集ID
parentDatasetId:
type: string
description: 父数据集ID
name:
type: string
description: 数据集名称
@@ -608,6 +616,9 @@ components:
type:
type: string
description: 数据集类型
parentDatasetId:
type: string
description: 父数据集ID
tags:
type: array
items:
@@ -631,6 +642,9 @@ components:
type: string
description: 数据集描述
maxLength: 500
parentDatasetId:
type: string
description: 父数据集ID
tags:
type: array
items:

85
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java
View File

@@ -6,6 +6,9 @@ import com.datamate.common.domain.utils.ChunksSaver;
import com.datamate.common.setting.application.SysParamApplicationService;
import com.datamate.datamanagement.interfaces.dto.*;
import com.datamate.common.infrastructure.exception.BusinessAssert;
import com.datamate.common.infrastructure.exception.BusinessException;
import com.datamate.common.infrastructure.exception.CommonErrorCode;
import com.datamate.common.infrastructure.exception.SystemErrorCode;
import com.datamate.common.interfaces.PagedResponse;
import com.datamate.datamanagement.domain.model.dataset.Dataset;
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
@@ -27,6 +30,7 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.StringUtils;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -62,7 +66,9 @@ public class DatasetApplicationService {
BusinessAssert.isTrue(datasetRepository.findByName(createDatasetRequest.getName()) == null, DataManagementErrorCode.DATASET_ALREADY_EXISTS);
// 创建数据集对象
Dataset dataset = DatasetConverter.INSTANCE.convertToDataset(createDatasetRequest);
dataset.initCreateParam(datasetBasePath);
Dataset parentDataset = resolveParentDataset(createDatasetRequest.getParentDatasetId(), dataset.getId());
dataset.setParentDatasetId(parentDataset == null ? null : parentDataset.getId());
dataset.initCreateParam(datasetBasePath, parentDataset == null ? null : parentDataset.getPath());
// 处理标签
Set<Tag> processedTags = Optional.ofNullable(createDatasetRequest.getTags())
.filter(CollectionUtils::isNotEmpty)
@@ -98,6 +104,9 @@ public class DatasetApplicationService {
if (Objects.nonNull(updateDatasetRequest.getStatus())) {
dataset.setStatus(updateDatasetRequest.getStatus());
}
if (updateDatasetRequest.getParentDatasetId() != null) {
handleParentChange(dataset, updateDatasetRequest.getParentDatasetId());
}
if (StringUtils.hasText(updateDatasetRequest.getDataSource())) {
// 数据源id不为空，使用异步线程进行文件扫盘落库
processDataSourceAsync(dataset.getId(), updateDatasetRequest.getDataSource());
@@ -112,10 +121,11 @@ public class DatasetApplicationService {
@Transactional
public void deleteDataset(String datasetId) {
Dataset dataset = datasetRepository.getById(datasetId);
BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND);
long childCount = datasetRepository.countByParentId(datasetId);
BusinessAssert.isTrue(childCount == 0, DataManagementErrorCode.DATASET_HAS_CHILDREN);
datasetRepository.removeById(datasetId);
if (dataset != null) {
ChunksSaver.deleteFolder(dataset.getPath());
}
ChunksSaver.deleteFolder(dataset.getPath());
}
/**
@@ -164,6 +174,73 @@ public class DatasetApplicationService {
return tags;
}
private Dataset resolveParentDataset(String parentDatasetId, String currentDatasetId) {
String normalized = normalizeParentId(parentDatasetId);
if (normalized == null) {
return null;
}
BusinessAssert.isTrue(!normalized.equals(currentDatasetId), CommonErrorCode.PARAM_ERROR);
Dataset parent = datasetRepository.getById(normalized);
BusinessAssert.notNull(parent, DataManagementErrorCode.DATASET_NOT_FOUND);
BusinessAssert.isTrue(parent.getParentDatasetId() == null, CommonErrorCode.PARAM_ERROR);
return parent;
}
private void handleParentChange(Dataset dataset, String parentDatasetId) {
String normalized = normalizeParentId(parentDatasetId);
if (Objects.equals(dataset.getParentDatasetId(), normalized)) {
return;
}
long childCount = datasetRepository.countByParentId(dataset.getId());
if (childCount > 0 && normalized != null) {
throw BusinessException.of(DataManagementErrorCode.DATASET_HAS_CHILDREN);
}
Dataset parent = normalized == null ? null : resolveParentDataset(normalized, dataset.getId());
String newPath = buildDatasetPath(parent == null ? datasetBasePath : parent.getPath(), dataset.getId());
moveDatasetPath(dataset, newPath);
dataset.setParentDatasetId(parent == null ? null : parent.getId());
}
private String normalizeParentId(String parentDatasetId) {
if (!StringUtils.hasText(parentDatasetId)) {
return null;
}
return parentDatasetId.trim();
}
private String buildDatasetPath(String basePath, String datasetId) {
String normalized = basePath;
while (normalized.endsWith(File.separator)) {
normalized = normalized.substring(0, normalized.length() - 1);
}
return normalized + File.separator + datasetId;
}
private void moveDatasetPath(Dataset dataset, String newPath) {
String oldPath = dataset.getPath();
if (Objects.equals(oldPath, newPath)) {
return;
}
Path sourcePath = Paths.get(oldPath);
Path targetPath = Paths.get(newPath);
try {
if (Files.exists(sourcePath)) {
if (Files.exists(targetPath)) {
throw BusinessException.of(CommonErrorCode.PARAM_ERROR);
}
Files.createDirectories(targetPath.getParent());
Files.move(sourcePath, targetPath);
} else {
Files.createDirectories(targetPath);
}
} catch (IOException e) {
log.error("move dataset path error, from {} to {}", oldPath, newPath, e);
throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR);
}
datasetFileRepository.updateFilePathPrefix(dataset.getId(), oldPath, newPath);
dataset.setPath(newPath);
}
/**
* 获取数据集统计信息
*/

8
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java
View File

@@ -324,7 +324,8 @@ public class DatasetFileApplicationService {
*/
@Transactional
public String preUpload(UploadFilesPreRequest chunkUploadRequest, String datasetId) {
if (Objects.isNull(datasetRepository.getById(datasetId))) {
Dataset dataset = datasetRepository.getById(datasetId);
if (Objects.isNull(dataset)) {
throw BusinessException.of(DataManagementErrorCode.DATASET_NOT_FOUND);
}
@@ -335,7 +336,10 @@ public class DatasetFileApplicationService {
prefix = prefix.substring(1);
}
String uploadPath = datasetBasePath + File.separator + datasetId;
String uploadPath = dataset.getPath();
if (uploadPath == null || uploadPath.isBlank()) {
uploadPath = datasetBasePath + File.separator + datasetId;
}
if (!prefix.isEmpty()) {
uploadPath = uploadPath + File.separator + prefix.replace("/", File.separator);
}

17
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/domain/model/dataset/Dataset.java
View File

@@ -28,6 +28,10 @@ public class Dataset extends BaseEntity<String> {
* 数据集描述
*/
private String description;
/**
* 父数据集ID
*/
private String parentDatasetId;
/**
* 数据集类型
*/
@@ -110,14 +114,23 @@ public class Dataset extends BaseEntity<String> {
this.updatedAt = LocalDateTime.now();
}
public void initCreateParam(String datasetBasePath) {
public void initCreateParam(String datasetBasePath, String parentPath) {
this.id = UUID.randomUUID().toString();
this.path = datasetBasePath + File.separator + this.id;
String basePath = normalizeBasePath(parentPath != null && !parentPath.isBlank() ? parentPath : datasetBasePath);
this.path = basePath + File.separator + this.id;
if (this.status == null) {
this.status = DatasetStatusType.DRAFT;
}
}
private String normalizeBasePath(String basePath) {
String normalized = basePath;
while (normalized.endsWith(File.separator)) {
normalized = normalized.substring(0, normalized.length() - 1);
}
return normalized;
}
public void updateBasicInfo(String name, String description, String category) {
if (name != null && !name.isEmpty()) this.name = name;
if (description != null) this.description = description;

6
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/exception/DataManagementErrorCode.java
View File

@@ -40,7 +40,11 @@ public enum DataManagementErrorCode implements ErrorCode {
/**
* 目录不存在
*/
DIRECTORY_NOT_FOUND("data_management.0007", "目录不存在");
DIRECTORY_NOT_FOUND("data_management.0007", "目录不存在"),
/**
* 存在子数据集
*/
DATASET_HAS_CHILDREN("data_management.0008", "存在子数据集，禁止删除或移动");
private final String code;
private final String message;

3
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/mapper/DatasetFileMapper.java
View File

@@ -26,4 +26,7 @@ public interface DatasetFileMapper extends BaseMapper<DatasetFile> {
int update(DatasetFile file);
int deleteById(@Param("id") String id);
int updateFilePathPrefix(@Param("datasetId") String datasetId,
@Param("oldPrefix") String oldPrefix,
@Param("newPrefix") String newPrefix);
}

2
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/repository/DatasetFileRepository.java
View File

@@ -25,4 +25,6 @@ public interface DatasetFileRepository extends IRepository<DatasetFile> {
IPage<DatasetFile> findByCriteria(String datasetId, String fileType, String status, String name,
IPage<DatasetFile> page);
int updateFilePathPrefix(String datasetId, String oldPrefix, String newPrefix);
}

2
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/repository/DatasetRepository.java
View File

@@ -26,4 +26,6 @@ public interface DatasetRepository extends IRepository<Dataset> {
AllDatasetStatisticsResponse getAllDatasetStatistics();
IPage<Dataset> findByCriteria(IPage<Dataset> page, DatasetPagingQuery query);
long countByParentId(String parentDatasetId);
}

5
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/repository/impl/DatasetFileRepositoryImpl.java
View File

@@ -56,4 +56,9 @@ public class DatasetFileRepositoryImpl extends CrudRepository<DatasetFileMapper,
.eq(StringUtils.hasText(status), DatasetFile::getStatus, status)
.like(StringUtils.hasText(name), DatasetFile::getFileName, name));
}
@Override
public int updateFilePathPrefix(String datasetId, String oldPrefix, String newPrefix) {
return datasetFileMapper.updateFilePathPrefix(datasetId, oldPrefix, newPrefix);
}
}

14
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/repository/impl/DatasetRepositoryImpl.java
View File

@@ -54,6 +54,14 @@ public class DatasetRepositoryImpl extends CrudRepository<DatasetMapper, Dataset
.eq(query.getType() != null, Dataset::getDatasetType, query.getType())
.eq(query.getStatus() != null, Dataset::getStatus, query.getStatus());
if (query.getParentDatasetId() != null) {
if (StringUtils.isBlank(query.getParentDatasetId())) {
wrapper.isNull(Dataset::getParentDatasetId);
} else {
wrapper.eq(Dataset::getParentDatasetId, query.getParentDatasetId());
}
}
if (StringUtils.isNotBlank(query.getKeyword())) {
wrapper.and(w ->
w.like(Dataset::getName, query.getKeyword()).or()
@@ -74,4 +82,10 @@ public class DatasetRepositoryImpl extends CrudRepository<DatasetMapper, Dataset
wrapper.orderByDesc(Dataset::getCreatedAt);
return datasetMapper.selectPage(page, wrapper);
}
@Override
public long countByParentId(String parentDatasetId) {
return datasetMapper.selectCount(new LambdaQueryWrapper<Dataset>()
.eq(Dataset::getParentDatasetId, parentDatasetId));
}
}

2
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/dto/CreateDatasetRequest.java
View File

@@ -29,6 +29,8 @@ public class CreateDatasetRequest {
/** 数据集类型 */
@NotNull(message = "数据集类型不能为空")
private DatasetType datasetType;
/** 父数据集ID */
private String parentDatasetId;
/** 标签列表 */
private List<String> tags;
/** 数据源 */

5
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/dto/DatasetPagingQuery.java
View File

@@ -39,4 +39,9 @@ public class DatasetPagingQuery extends PagingQuery {
* 状态过滤
*/
private DatasetStatusType status;
/**
* 父数据集ID过滤
*/
private String parentDatasetId;
}

2
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/dto/DatasetResponse.java
View File

@@ -15,6 +15,8 @@ import java.util.Map;
public class DatasetResponse {
/** 数据集ID */
private String id;
/** 父数据集ID */
private String parentDatasetId;
/** 数据集名称 */
private String name;
/** 数据集描述 */

2
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/dto/UpdateDatasetRequest.java
View File

@@ -23,6 +23,8 @@ public class UpdateDatasetRequest {
private String description;
/** 归集任务id */
private String dataSource;
/** 父数据集ID */
private String parentDatasetId;
/** 标签列表 */
private List<String> tags;
/** 数据集状态 */

7
backend/services/data-management-service/src/main/resources/mappers/DatasetFileMapper.xml
View File

@@ -95,4 +95,11 @@
<delete id="deleteById" parameterType="string">
DELETE FROM t_dm_dataset_files WHERE id = #{id}
</delete>
<update id="updateFilePathPrefix">
UPDATE t_dm_dataset_files
SET file_path = CONCAT(#{newPrefix}, SUBSTRING(file_path, LENGTH(#{oldPrefix}) + 1))
WHERE dataset_id = #{datasetId}
AND file_path LIKE CONCAT(#{oldPrefix}, '%')
</update>
</mapper>

3
backend/services/data-management-service/src/main/resources/mappers/DatasetMapper.xml
View File

@@ -4,12 +4,13 @@
<mapper namespace="com.datamate.datamanagement.infrastructure.persistence.mapper.DatasetMapper">
<sql id="Base_Column_List">
id, name, description, dataset_type, category, path, format, schema_info, size_bytes, file_count, record_count,
id, parent_dataset_id, name, description, dataset_type, category, path, format, schema_info, size_bytes, file_count, record_count,
retention_days, tags, metadata, status, is_public, is_featured, version, created_at, updated_at, created_by, updated_by
</sql>
<sql id="Alias_D_Column_List">
d.id AS id,
d.parent_dataset_id AS parent_dataset_id,
d.name AS name,
d.description AS description,
d.dataset_type AS dataset_type,

1
frontend/src/pages/DataManagement/Create/CreateDataset.tsx
View File

@@ -17,6 +17,7 @@ export default function DatasetCreate() {
description: "",
datasetType: DatasetType.TEXT,
tags: [],
parentDatasetId: "",
});
const handleSubmit = async () => {

2
frontend/src/pages/DataManagement/Create/EditDataset.tsx
View File

@@ -26,6 +26,7 @@ export default function EditDataset({
description: "",
datasetType: DatasetType.TEXT,
tags: [],
parentDatasetId: "",
});
const fetchDataset = async () => {
if (!open) return;
@@ -36,6 +37,7 @@ export default function EditDataset({
...newData,
type: newData.type,
tags: newData.tags.map((tag) => tag.name) || [],
parentDatasetId: newData.parentDatasetId || "",
};
setNewDataset(updatedDataset);
form.setFieldsValue(updatedDataset);

42
frontend/src/pages/DataManagement/Create/components/BasicInformation.tsx
View File

@@ -2,7 +2,7 @@ import RadioCard from "@/components/RadioCard";
import { Input, Select, Form } from "antd";
import { datasetTypes } from "../../dataset.const";
import { useEffect, useState } from "react";
import { queryDatasetTagsUsingGet } from "../../dataset.api";
import { queryDatasetTagsUsingGet, queryDatasetsUsingGet } from "../../dataset.api";
import {queryTasksUsingGet} from "@/pages/DataCollection/collection.apis.ts";
export default function BasicInformation({
@@ -22,6 +22,9 @@ export default function BasicInformation({
}[]
>([]);
const [collectionOptions, setCollectionOptions] = useState([]);
const [parentDatasetOptions, setParentDatasetOptions] = useState<
{ label: string; value: string }[]
>([]);
// 获取标签
const fetchTags = async () => {
@@ -52,10 +55,36 @@ export default function BasicInformation({
}
};
const fetchParentDatasets = async () => {
if (hidden.includes("parentDatasetId")) return;
try {
const { data: resData } = await queryDatasetsUsingGet({
parentDatasetId: "",
page: 1,
size: 1000,
});
const currentDatasetId = data?.id;
const rootDatasets = resData?.content || [];
const options = rootDatasets
.filter((dataset) => dataset.id !== currentDatasetId)
.map((dataset) => ({
label: dataset.name,
value: dataset.id,
}));
setParentDatasetOptions([
{ label: "根数据集", value: "" },
...options,
]);
} catch (error) {
console.error("Error fetching parent datasets:", error);
}
};
useEffect(() => {
fetchTags();
fetchCollectionTasks();
}, []);
fetchParentDatasets();
}, [data?.id, hidden.join(",")]);
return (
<>
<Form.Item
@@ -70,6 +99,15 @@ export default function BasicInformation({
<Input.TextArea placeholder="描述数据集的用途和内容" rows={3} />
</Form.Item>
)}
{!hidden.includes("parentDatasetId") && (
<Form.Item name="parentDatasetId" label="父数据集">
<Select
className="w-full"
options={parentDatasetOptions}
placeholder="选择父数据集（仅支持一层）"
/>
</Form.Item>
)}
{/* 数据集类型选择 - 使用卡片形式 */}
{!hidden.includes("datasetType") && (

42
frontend/src/pages/DataManagement/Detail/DatasetDetail.tsx
View File

@@ -42,23 +42,45 @@ export default function DatasetDetail() {
const [showEditDialog, setShowEditDialog] = useState(false);
const [dataset, setDataset] = useState<Dataset>({} as Dataset);
const [parentDataset, setParentDataset] = useState<Dataset | null>(null);
const filesOperation = useFilesOperation(dataset);
const [showUploadDialog, setShowUploadDialog] = useState(false);
const navigateItems = useMemo(
() => [
const navigateItems = useMemo(() => {
const items = [
{
title: <Link to="/data/management">数据管理</Link>,
},
{
title: dataset.name || "数据集详情",
},
],
[dataset]
);
];
if (parentDataset) {
items.push({
title: (
<Link to={`/data/management/detail/${parentDataset.id}`}>
{parentDataset.name}
</Link>
),
});
}
items.push({
title: dataset.name || "数据集详情",
});
return items;
}, [dataset, parentDataset]);
const fetchDataset = async () => {
const { data } = await queryDatasetByIdUsingGet(id as unknown as number);
setDataset(mapDataset(data));
if (!id) {
return;
}
const { data } = await queryDatasetByIdUsingGet(id);
const mapped = mapDataset(data);
setDataset(mapped);
if (data?.parentDatasetId) {
const { data: parentData } = await queryDatasetByIdUsingGet(
data.parentDatasetId
);
setParentDataset(mapDataset(parentData));
} else {
setParentDataset(null);
}
};
useEffect(() => {

2
frontend/src/pages/DataManagement/Detail/useFilesOperation.ts
View File

@@ -21,7 +21,7 @@ export function useFilesOperation(dataset: Dataset) {
// 文件相关状态
const [fileList, setFileList] = useState<DatasetFile[]>([]);
const [selectedFiles, setSelectedFiles] = useState<number[]>([]);
const [selectedFiles, setSelectedFiles] = useState<string[]>([]);
const [pagination, setPagination] = useState<{
current: number;
pageSize: number;

2
frontend/src/pages/DataManagement/Home/DataManagement.tsx
View File

@@ -135,7 +135,7 @@ export default function DatasetManagementPage() {
message.success("数据集下载成功");
};
const handleDeleteDataset = async (id: number) => {
const handleDeleteDataset = async (id: string) => {
if (!id) return;
await deleteDatasetByIdUsingDelete(id);
fetchData({ pageOffset: 0 });

8
frontend/src/pages/DataManagement/dataset.model.ts
View File

@@ -33,7 +33,7 @@ export enum DataSource {
}
export interface DatasetFile {
id: number;
id: string;
fileName: string;
size: string;
uploadDate: string;
@@ -41,10 +41,10 @@ export interface DatasetFile {
}
export interface Dataset {
id: number;
id: string;
name: string;
description: string;
parentId?: number;
parentDatasetId?: string;
datasetType: DatasetType;
status: DatasetStatus;
size?: string;
@@ -77,7 +77,7 @@ export interface ScheduleConfig {
}
export interface DatasetTask {
id: number;
id: string;
name: string;
description: string;
type: string;

1
runtime/datamate-python/app/db/models/dataset_management.py
View File

@@ -14,6 +14,7 @@ class Dataset(Base):
__tablename__ = "t_dm_datasets"
id = Column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()), comment="UUID")
parent_dataset_id = Column(String(36), nullable=True, comment="父数据集ID（UUID）")
name = Column(String(255), nullable=False, comment="数据集名称")
description = Column(Text, nullable=True, comment="数据集描述")
dataset_type = Column(String(50), nullable=False, comment="数据集类型：IMAGE/TEXT/QA/MULTIMODAL/OTHER")

43
runtime/python-executor/datamate/auto_annotation_worker.py
View File

@@ -213,6 +213,42 @@ def _load_dataset_files(dataset_id: str) -> List[Tuple[str, str, str]]:
return [(str(r[0]), str(r[1]), str(r[2])) for r in rows]
def _load_dataset_meta(dataset_id: str) -> Optional[Dict[str, Any]]:
"""加载数据集基础信息（含父ID与路径）。"""
sql = text(
"""
SELECT id, name, parent_dataset_id, path
FROM t_dm_datasets
WHERE id = :dataset_id
"""
)
with SQLManager.create_connect() as conn:
row = conn.execute(sql, {"dataset_id": dataset_id}).fetchone()
if not row:
return None
return dict(row._mapping) # type: ignore[attr-defined]
def _resolve_output_parent(source_dataset_id: str) -> Tuple[Optional[str], str]:
"""根据源数据集确定产出数据集的父级与基路径（产出挂在父级下）。"""
base_path = DEFAULT_OUTPUT_ROOT.rstrip("/") or "/dataset"
source_meta = _load_dataset_meta(source_dataset_id)
if not source_meta:
return None, base_path
parent_dataset_id = source_meta.get("parent_dataset_id")
if not parent_dataset_id:
return None, base_path
parent_meta = _load_dataset_meta(str(parent_dataset_id))
parent_path = parent_meta.get("path") if parent_meta else None
if not parent_path:
return None, base_path
return str(parent_dataset_id), str(parent_path)
def _load_files_by_ids(file_ids: List[str]) -> List[Tuple[str, str, str]]:
"""根据文件ID列表加载文件记录，支持跨多个数据集。"""
@@ -253,7 +289,7 @@ def _create_output_dataset(
"""为自动标注结果创建一个新的数据集并返回 (dataset_id, path)。"""
new_dataset_id = str(uuid.uuid4())
dataset_base_path = DEFAULT_OUTPUT_ROOT.rstrip("/") or "/dataset"
parent_dataset_id, dataset_base_path = _resolve_output_parent(source_dataset_id)
output_dir = os.path.join(dataset_base_path, new_dataset_id)
description = (
@@ -262,12 +298,13 @@ def _create_output_dataset(
sql = text(
"""
INSERT INTO t_dm_datasets (id, name, description, dataset_type, path, status)
VALUES (:id, :name, :description, :dataset_type, :path, :status)
INSERT INTO t_dm_datasets (id, parent_dataset_id, name, description, dataset_type, path, status)
VALUES (:id, :parent_dataset_id, :name, :description, :dataset_type, :path, :status)
"""
)
params = {
"id": new_dataset_id,
"parent_dataset_id": parent_dataset_id,
"name": output_dataset_name,
"description": description,
"dataset_type": "IMAGE",

5
scripts/db/data-management-init.sql
View File

@@ -15,6 +15,7 @@ USE datamate;
-- 数据集表（支持医学影像、文本、问答等多种类型）
CREATE TABLE IF NOT EXISTS t_dm_datasets (
id VARCHAR(36) PRIMARY KEY COMMENT 'UUID',
parent_dataset_id VARCHAR(36) NULL COMMENT '父数据集ID（UUID）',
name VARCHAR(255) NOT NULL COMMENT '数据集名称',
description TEXT COMMENT '数据集描述',
dataset_type VARCHAR(50) NOT NULL COMMENT '数据集类型：IMAGE/TEXT/QA/MULTIMODAL/OTHER',
@@ -36,13 +37,15 @@ CREATE TABLE IF NOT EXISTS t_dm_datasets (
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
created_by VARCHAR(255) COMMENT '创建者',
updated_by VARCHAR(255) COMMENT '更新者',
INDEX idx_dm_parent_dataset_id (parent_dataset_id),
INDEX idx_dm_dataset_type (dataset_type),
INDEX idx_dm_category (category),
INDEX idx_dm_format (format),
INDEX idx_dm_status (status),
INDEX idx_dm_public (is_public),
INDEX idx_dm_featured (is_featured),
INDEX idx_dm_created_at (created_at)
INDEX idx_dm_created_at (created_at),
FOREIGN KEY (parent_dataset_id) REFERENCES t_dm_datasets(id) ON DELETE RESTRICT
) COMMENT='数据集表（UUID 主键）';
-- 数据集文件表