q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'gitea/lsf' into lsf

Browse Source
This commit is contained in:
Jerry Yan
2026-02-06 18:29:58 +08:00
parent 38e58ba864 38add27d84
commit 0f1dd9ec8d
5 changed files with 636 additions and 529 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java
View File

@@ -36,6 +36,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.Resource;
@@ -581,25 +582,100 @@ public class DatasetFileApplicationService {
@Transactional
public void deleteDatasetFile(String datasetId, String fileId) {
DatasetFile file = getDatasetFile(datasetId, fileId);
Dataset dataset = datasetRepository.getById(datasetId);
datasetFileRepository.removeById(fileId);
if (!isArchivedStatus(file)) {
dataset.setFiles(new ArrayList<>(Collections.singleton(file)));
dataset.removeFile(file);
datasetRepository.updateById(dataset);
if (file == null) {
log.warn("File not found: datasetId={}, fileId={}", datasetId, fileId);
return;
}
datasetFilePreviewService.deletePreviewFileQuietly(datasetId, fileId);
// 删除文件时，上传到数据集中的文件会同时删除数据库中的记录和文件系统中的文件，归集过来的文件仅删除数据库中的记录
if (file.getFilePath().startsWith(dataset.getPath())) {
String logicalPath = file.getLogicalPath();
// 如果 logicalPath 为 null、空字符串或纯空白字符，直接删除当前文件（兼容旧数据）
if (StringUtils.isBlank(logicalPath)) {
deleteDatasetFileInternal(datasetId, file);
return;
}
List<DatasetFile> allVersions = datasetFileRepository.findAllByDatasetIdAndLogicalPath(datasetId, logicalPath);
for (DatasetFile versionFile : allVersions) {
deleteDatasetFileInternal(datasetId, versionFile);
}
}
private void deleteDatasetFileInternal(String datasetId, DatasetFile file) {
Dataset dataset = datasetRepository.getById(datasetId);
if (file == null || dataset == null) {
return;
}
// 先删除数据库记录，确保数据库操作成功后再清理派生文件
try {
datasetFileRepository.removeById(file.getId());
} catch (Exception e) {
log.error("Failed to delete file record from database: fileId={}", file.getId(), e);
// 数据库删除失败时，跳过后续清理以避免数据不一致
return;
}
// 数据库删除成功后，再删除派生文件
if (isSourceDocument(file)) {
deleteDerivedTextFileQuietly(datasetId, file.getId());
}
if (!isArchivedStatus(file)) {
try {
dataset.setFiles(new ArrayList<>(Collections.singleton(file)));
dataset.removeFile(file);
datasetRepository.updateById(dataset);
} catch (Exception e) {
log.error("Failed to update dataset: datasetId={}", datasetId, e);
}
}
datasetFilePreviewService.deletePreviewFileQuietly(datasetId, file.getId());
if (file.getFilePath() != null && file.getFilePath().startsWith(dataset.getPath())) {
try {
Path filePath = Paths.get(file.getFilePath());
Files.deleteIfExists(filePath);
} catch (IOException ex) {
throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR);
log.error("Failed to delete physical file: filePath={}", file.getFilePath(), ex);
}
}
}
private void deleteDerivedTextFileQuietly(String datasetId, String sourceFileId) {
if (sourceFileId == null || sourceFileId.isBlank()) {
return;
}
try {
List<DatasetFile> derivedFiles = datasetFileRepository.findAllByDatasetId(datasetId).stream()
.filter(f -> isDerivedFileFromSource(f, sourceFileId))
.toList();
for (DatasetFile derivedFile : derivedFiles) {
deleteDatasetFileInternal(datasetId, derivedFile);
}
} catch (Exception e) {
log.error("Failed to delete derived text files for sourceFileId: {}", sourceFileId, e);
}
}
private boolean isDerivedFileFromSource(DatasetFile file, String sourceFileId) {
if (file == null || file.getMetadata() == null || file.getMetadata().isBlank()) {
return false;
}
try {
ObjectMapper mapper = new ObjectMapper();
Map<String, Object> metadataMap = mapper.readValue(file.getMetadata(), new TypeReference<Map<String, Object>>() {});
Object derivedFromFileId = metadataMap.get(DERIVED_METADATA_KEY);
return derivedFromFileId != null && sourceFileId.equals(String.valueOf(derivedFromFileId));
} catch (Exception e) {
log.debug("Failed to parse metadata for derived detection: fileId={}", file.getId(), e);
return false;
}
}
/**
* 下载文件
*/

9
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/mapper/DatasetFileMapper.java
View File

@@ -46,4 +46,13 @@ public interface DatasetFileMapper extends BaseMapper<DatasetFile> {
* @return 文件数统计列表
*/
List<DatasetFileCount> countNonDerivedByDatasetIds(@Param("datasetIds") List<String> datasetIds);
/**
* 查询指定逻辑路径的所有文件（包括所有状态）
*
* @param datasetId 数据集ID
* @param logicalPath 逻辑路径
* @return 文件列表
*/
List<DatasetFile> findAllByDatasetIdAndLogicalPath(@Param("datasetId") String datasetId, @Param("logicalPath") String logicalPath);
}

9
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/repository/DatasetFileRepository.java
View File

@@ -37,6 +37,15 @@ public interface DatasetFileRepository extends IRepository<DatasetFile> {
*/
DatasetFile findLatestByDatasetIdAndLogicalPath(String datasetId, String logicalPath);
/**
* 查询指定逻辑路径的所有文件（包括所有状态）
*
* @param datasetId 数据集ID
* @param logicalPath 逻辑路径
* @return 文件列表
*/
List<DatasetFile> findAllByDatasetIdAndLogicalPath(String datasetId, String logicalPath);
IPage<DatasetFile> findByCriteria(String datasetId, String fileType, String status, String name,
Boolean hasAnnotation, IPage<DatasetFile> page);

5
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/repository/impl/DatasetFileRepositoryImpl.java
View File

@@ -84,6 +84,11 @@ public class DatasetFileRepositoryImpl extends CrudRepository<DatasetFileMapper,
.last("LIMIT 1"));
}
@Override
public List<DatasetFile> findAllByDatasetIdAndLogicalPath(String datasetId, String logicalPath) {
return datasetFileMapper.findAllByDatasetIdAndLogicalPath(datasetId, logicalPath);
}
public IPage<DatasetFile> findByCriteria(String datasetId, String fileType, String status, String name,
Boolean hasAnnotation, IPage<DatasetFile> page) {
LambdaQueryWrapper<DatasetFile> wrapper = new LambdaQueryWrapper<DatasetFile>()

14
backend/services/data-management-service/src/main/resources/mappers/DatasetFileMapper.xml
View File

@@ -64,7 +64,7 @@
AND (status IS NULL OR status &lt;&gt; 'ARCHIVED')
</select>
<select id="findByDatasetIdAndFileName" resultType="com.datamate.datamanagement.domain.model.dataset.DatasetFile">
<select id="findByDatasetIdAndFileName" resultType="com.datamate.datamanagement.domain.model.dataset.DatasetFile">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_dataset_files
WHERE dataset_id = #{datasetId}
@@ -74,8 +74,16 @@
LIMIT 1
</select>
<select id="findAllByDatasetId" parameterType="string"
resultType="com.datamate.datamanagement.domain.model.dataset.DatasetFile">
<select id="findAllByDatasetIdAndLogicalPath" resultType="com.datamate.datamanagement.domain.model.dataset.DatasetFile">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_dataset_files
WHERE dataset_id = #{datasetId}
AND logical_path = #{logicalPath}
ORDER BY version DESC, upload_time DESC
</select>
<select id="findAllByDatasetId" parameterType="string"
resultType="com.datamate.datamanagement.domain.model.dataset.DatasetFile">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_dataset_files
WHERE dataset_id = #{datasetId}