You've already forked DataMate
feat(data-management): 实现数据集文件版本管理和内部路径保护
- 将数据集文件查询方法替换为只查询可见文件的版本 - 引入文件状态管理（ACTIVE/ARCHIVED）和内部目录结构 - 实现文件重复处理策略，支持版本控制模式而非覆盖 - 添加内部数据目录保护，防止访问.datamate等系统目录 - 重构文件上传流程，引入暂存目录和事务后清理机制 - 实现文件版本归档功能，保留历史版本到专用存储位置 - 优化文件路径规范化和安全验证逻辑 - 修复文件删除逻辑，确保归档文件不会被错误移除 - 更新数据集压缩下载功能以排除内部系统文件
This commit is contained in:
@@ -61,13 +61,15 @@ class DatasetFiles(Base):
|
||||
dataset_id = Column(String(36), nullable=False, comment="所属数据集ID（UUID）")
|
||||
file_name = Column(String(255), nullable=False, comment="文件名")
|
||||
file_path = Column(String(1000), nullable=False, comment="文件路径")
|
||||
logical_path = Column(String(1000), nullable=False, comment="文件逻辑路径（相对数据集根目录）")
|
||||
version = Column(BigInteger, nullable=False, default=1, comment="文件版本号（同 logical_path 递增）")
|
||||
file_type = Column(String(50), nullable=True, comment="文件格式：JPG/PNG/DCM/TXT等")
|
||||
file_size = Column(BigInteger, default=0, comment="文件大小(字节)")
|
||||
check_sum = Column(String(64), nullable=True, comment="文件校验和")
|
||||
tags = Column(JSON, nullable=True, comment="文件标签信息")
|
||||
tags_updated_at = Column(TIMESTAMP, nullable=True, comment="标签最后更新时间")
|
||||
dataset_filemetadata = Column("metadata", JSON, nullable=True, comment="文件元数据")
|
||||
status = Column(String(50), default='ACTIVE', comment="文件状态：ACTIVE/DELETED/PROCESSING")
|
||||
status = Column(String(50), default='ACTIVE', comment="文件状态：ACTIVE/ARCHIVED/DELETED/PROCESSING")
|
||||
upload_time = Column(TIMESTAMP, server_default=func.current_timestamp(), comment="上传时间")
|
||||
last_access_time = Column(TIMESTAMP, nullable=True, comment="最后访问时间")
|
||||
created_at = Column(TIMESTAMP, server_default=func.current_timestamp(), comment="创建时间")
|
||||
@@ -112,4 +114,4 @@ class Tag(Base):
|
||||
updated_at = Column(TIMESTAMP, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), comment="更新时间")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Tag(id={self.id}, name={self.name}, category={self.category})>"
|
||||
return f"<Tag(id={self.id}, name={self.name}, category={self.category})>"
|
||||
|
||||
@@ -372,15 +372,15 @@ def _register_output_dataset(
|
||||
)
|
||||
return
|
||||
|
||||
insert_file_sql = text(
|
||||
"""
|
||||
INSERT INTO t_dm_dataset_files (
|
||||
id, dataset_id, file_name, file_path, file_type, file_size, status
|
||||
) VALUES (
|
||||
:id, :dataset_id, :file_name, :file_path, :file_type, :file_size, :status
|
||||
)
|
||||
"""
|
||||
)
|
||||
insert_file_sql = text(
|
||||
"""
|
||||
INSERT INTO t_dm_dataset_files (
|
||||
id, dataset_id, file_name, file_path, logical_path, version, file_type, file_size, status
|
||||
) VALUES (
|
||||
:id, :dataset_id, :file_name, :file_path, :logical_path, :version, :file_type, :file_size, :status
|
||||
)
|
||||
"""
|
||||
)
|
||||
update_dataset_stat_sql = text(
|
||||
"""
|
||||
UPDATE t_dm_datasets
|
||||
@@ -393,37 +393,43 @@ def _register_output_dataset(
|
||||
with SQLManager.create_connect() as conn:
|
||||
added_count = 0
|
||||
|
||||
for file_name, file_path, file_size in image_files:
|
||||
ext = os.path.splitext(file_name)[1].lstrip(".").upper() or None
|
||||
conn.execute(
|
||||
insert_file_sql,
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"dataset_id": output_dataset_id,
|
||||
"file_name": file_name,
|
||||
"file_path": file_path,
|
||||
"file_type": ext,
|
||||
"file_size": int(file_size),
|
||||
"status": "ACTIVE",
|
||||
},
|
||||
)
|
||||
added_count += 1
|
||||
|
||||
for file_name, file_path, file_size in annotation_files:
|
||||
ext = os.path.splitext(file_name)[1].lstrip(".").upper() or None
|
||||
conn.execute(
|
||||
insert_file_sql,
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"dataset_id": output_dataset_id,
|
||||
"file_name": file_name,
|
||||
"file_path": file_path,
|
||||
"file_type": ext,
|
||||
"file_size": int(file_size),
|
||||
"status": "ACTIVE",
|
||||
},
|
||||
)
|
||||
added_count += 1
|
||||
for file_name, file_path, file_size in image_files:
|
||||
ext = os.path.splitext(file_name)[1].lstrip(".").upper() or None
|
||||
logical_path = os.path.relpath(file_path, output_dir).replace("\\", "/")
|
||||
conn.execute(
|
||||
insert_file_sql,
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"dataset_id": output_dataset_id,
|
||||
"file_name": file_name,
|
||||
"file_path": file_path,
|
||||
"logical_path": logical_path,
|
||||
"version": 1,
|
||||
"file_type": ext,
|
||||
"file_size": int(file_size),
|
||||
"status": "ACTIVE",
|
||||
},
|
||||
)
|
||||
added_count += 1
|
||||
|
||||
for file_name, file_path, file_size in annotation_files:
|
||||
ext = os.path.splitext(file_name)[1].lstrip(".").upper() or None
|
||||
logical_path = os.path.relpath(file_path, output_dir).replace("\\", "/")
|
||||
conn.execute(
|
||||
insert_file_sql,
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"dataset_id": output_dataset_id,
|
||||
"file_name": file_name,
|
||||
"file_path": file_path,
|
||||
"logical_path": logical_path,
|
||||
"version": 1,
|
||||
"file_type": ext,
|
||||
"file_size": int(file_size),
|
||||
"status": "ACTIVE",
|
||||
},
|
||||
)
|
||||
added_count += 1
|
||||
|
||||
if added_count > 0:
|
||||
conn.execute(
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
{
|
||||
"query_sql": "SELECT * FROM t_task_instance_info WHERE instance_id IN (:instance_id)",
|
||||
"insert_sql": "INSERT INTO t_task_instance_info (instance_id, meta_file_name, meta_file_type, meta_file_id, meta_file_size, file_id, file_size, file_type, file_name, file_path, status, operator_id, error_code, incremental, child_id, slice_num) VALUES (:instance_id, :meta_file_name, :meta_file_type, :meta_file_id, :meta_file_size, :file_id, :file_size, :file_type, :file_name, :file_path, :status, :operator_id, :error_code, :incremental, :child_id, :slice_num)",
|
||||
"insert_dataset_file_sql": "INSERT INTO t_dm_dataset_files (id, dataset_id, file_name, file_path, file_type, file_size, status, upload_time, last_access_time, created_at, updated_at) VALUES (:id, :dataset_id, :file_name, :file_path, :file_type, :file_size, :status, :upload_time, :last_access_time, :created_at, :updated_at)",
|
||||
"insert_dataset_file_sql": "INSERT INTO t_dm_dataset_files (id, dataset_id, file_name, file_path, logical_path, version, file_type, file_size, status, upload_time, last_access_time, created_at, updated_at) VALUES (:id, :dataset_id, :file_name, :file_path, :logical_path, :version, :file_type, :file_size, :status, :upload_time, :last_access_time, :created_at, :updated_at)",
|
||||
"insert_clean_result_sql": "INSERT INTO t_clean_result (instance_id, src_file_id, dest_file_id, src_name, dest_name, src_type, dest_type, src_size, dest_size, status, result) VALUES (:instance_id, :src_file_id, :dest_file_id, :src_name, :dest_name, :src_type, :dest_type, :src_size, :dest_size, :status, :result)",
|
||||
"query_dataset_sql": "SELECT file_size FROM t_dm_dataset_files WHERE dataset_id = :dataset_id",
|
||||
"query_dataset_sql": "SELECT file_size FROM t_dm_dataset_files WHERE dataset_id = :dataset_id AND (status IS NULL OR status <> 'ARCHIVED')",
|
||||
"update_dataset_sql": "UPDATE t_dm_datasets SET size_bytes = :total_size, file_count = :file_count WHERE id = :dataset_id;",
|
||||
"update_task_sql": "UPDATE t_clean_task SET status = :status, after_size = :total_size, finished_at = :finished_time WHERE id = :task_id",
|
||||
"create_tables_sql": "CREATE TABLE IF NOT EXISTS t_task_instance_info (instance_id VARCHAR(255), meta_file_name TEXT, meta_file_type VARCHAR(100), meta_file_id BIGINT, meta_file_size VARCHAR(100), file_id BIGINT, file_size VARCHAR(100), file_type VARCHAR(100), file_name TEXT, file_path TEXT, status INT, operator_id VARCHAR(255), error_code VARCHAR(100), incremental VARCHAR(50), child_id BIGINT, slice_num INT DEFAULT 0);",
|
||||
@@ -14,4 +14,4 @@
|
||||
"delete_similar_img_tables_sql": "DELETE FROM operator_similar_img_features WHERE flow_id = :flow_id",
|
||||
"create_similar_text_tables_sql": "CREATE TABLE IF NOT EXISTS operators_similar_text_features (id INT AUTO_INCREMENT PRIMARY KEY, task_uuid VARCHAR(255),file_feature TEXT,file_name TEXT,timestamp DATETIME);",
|
||||
"delete_similar_text_tables_sql": "DELETE FROM operators_similar_text_features WHERE flow_id = :flow_id"
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user