From e300d13c21ee7e92e8510e1c453d3372d9a2bec3 Mon Sep 17 00:00:00 2001 From: Dallas98 <40557804+Dallas98@users.noreply.github.com> Date: Fri, 14 Nov 2025 23:30:28 +0800 Subject: [PATCH] feat: Enhance dataset file management with improved file copying --- .../DatasetFileApplicationService.java | 57 +++++++++++-------- backend/services/main-application/pom.xml | 2 +- ...lication.java => DataMateApplication.java} | 4 +- 3 files changed, 35 insertions(+), 28 deletions(-) rename backend/services/main-application/src/main/java/com/datamate/main/{DataMatePlatformApplication.java => DataMateApplication.java} (88%) diff --git a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java index e129a38..5f467a2 100644 --- a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java +++ b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; import org.apache.ibatis.session.RowBounds; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -45,6 +44,7 @@ import java.nio.file.Paths; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.*; +import java.util.concurrent.CompletableFuture; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -76,7 +76,7 @@ public class DatasetFileApplicationService { */ @Transactional(readOnly = true) public Page getDatasetFiles(String datasetId, String fileType, - String status, Pageable pageable) { + String status, Pageable pageable) { RowBounds bounds = new RowBounds(pageable.getPageNumber() * pageable.getPageSize(), pageable.getPageSize()); List content = datasetFileRepository.findByCriteria(datasetId, fileType, status, bounds); long total = content.size() < pageable.getPageSize() && pageable.getPageNumber() == 0 ? content.size() : content.size() + (long) pageable.getPageNumber() * pageable.getPageSize(); @@ -148,7 +148,7 @@ public class DatasetFileApplicationService { fileRename(allByDatasetId); response.setContentType("application/zip"); String zipName = String.format("dataset_%s.zip", - LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss"))); + LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss"))); response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + zipName); try (ZipOutputStream zos = new ZipOutputStream(response.getOutputStream())) { for (DatasetFile file : allByDatasetId) { @@ -203,7 +203,7 @@ public class DatasetFileApplicationService { * 预上传 * * @param chunkUploadRequest 上传请求 - * @param datasetId 数据集id + * @param datasetId 数据集id * @return 请求id */ @Transactional @@ -245,15 +245,15 @@ public class DatasetFileApplicationService { File savedFile = fileUploadResult.getSavedFile(); LocalDateTime currentTime = LocalDateTime.now(); DatasetFile datasetFile = DatasetFile.builder() - .id(UUID.randomUUID().toString()) - .datasetId(datasetId) - .fileSize(savedFile.length()) - .uploadTime(currentTime) - .lastAccessTime(currentTime) - .fileName(uploadFile.getFileName()) - .filePath(savedFile.getPath()) - .fileType(AnalyzerUtils.getExtension(uploadFile.getFileName())) - .build(); + .id(UUID.randomUUID().toString()) + .datasetId(datasetId) + .fileSize(savedFile.length()) + .uploadTime(currentTime) + .lastAccessTime(currentTime) + .fileName(uploadFile.getFileName()) + .filePath(savedFile.getPath()) + .fileType(AnalyzerUtils.getExtension(uploadFile.getFileName())) + .build(); datasetFileRepository.save(datasetFile); dataset.addFile(datasetFile); @@ -265,7 +265,7 @@ public class DatasetFileApplicationService { * 复制文件到数据集目录 * * @param datasetId 数据集id - * @param req 复制文件请求 + * @param req 复制文件请求 * @return 复制的文件列表 */ @Transactional @@ -280,31 +280,38 @@ public class DatasetFileApplicationService { continue; } String fileName = sourcePath.getFileName().toString(); - File targetFile = new File(dataset.getPath(), fileName); - try { - FileUtils.copyInputStreamToFile(Files.newInputStream(sourcePath), targetFile); - } catch (IOException e) { - log.error("Failed to copy file: {}", sourceFilePath, e); - continue; - } - + File sourceFile = sourcePath.toFile(); LocalDateTime currentTime = LocalDateTime.now(); DatasetFile datasetFile = DatasetFile.builder() .id(UUID.randomUUID().toString()) .datasetId(datasetId) .fileName(fileName) .fileType(AnalyzerUtils.getExtension(fileName)) - .fileSize(targetFile.length()) - .filePath(targetFile.getPath()) + .fileSize(sourceFile.length()) + .filePath(Paths.get(dataset.getPath(), fileName).toString()) .uploadTime(currentTime) .lastAccessTime(currentTime) .build(); - datasetFileRepository.save(datasetFile); dataset.addFile(datasetFile); copiedFiles.add(datasetFile); } + datasetFileRepository.saveBatch(copiedFiles, 100); dataset.active(); datasetRepository.updateById(dataset); + CompletableFuture.runAsync(() -> copyFilesToDatasetDir(req.sourcePaths(), dataset)); return copiedFiles; } + + private void copyFilesToDatasetDir(List sourcePaths, Dataset dataset) { + for (String sourcePath : sourcePaths) { + Path sourceFilePath = Paths.get(sourcePath); + Path targetFilePath = Paths.get(dataset.getPath(), sourceFilePath.getFileName().toString()); + try { + Files.createDirectories(Path.of(dataset.getPath())); + Files.copy(sourceFilePath, targetFilePath); + } catch (IOException e) { + log.error("Failed to copy file from {} to {}", sourcePath, targetFilePath, e); + } + } + } } diff --git a/backend/services/main-application/pom.xml b/backend/services/main-application/pom.xml index 23a72d4..b5b31ee 100644 --- a/backend/services/main-application/pom.xml +++ b/backend/services/main-application/pom.xml @@ -162,7 +162,7 @@ ${spring-boot.version} data-mate - com.datamate.main.DataMatePlatformApplication + com.datamate.main.DataMateApplication diff --git a/backend/services/main-application/src/main/java/com/datamate/main/DataMatePlatformApplication.java b/backend/services/main-application/src/main/java/com/datamate/main/DataMateApplication.java similarity index 88% rename from backend/services/main-application/src/main/java/com/datamate/main/DataMatePlatformApplication.java rename to backend/services/main-application/src/main/java/com/datamate/main/DataMateApplication.java index f570ff3..01a4631 100644 --- a/backend/services/main-application/src/main/java/com/datamate/main/DataMatePlatformApplication.java +++ b/backend/services/main-application/src/main/java/com/datamate/main/DataMateApplication.java @@ -23,8 +23,8 @@ import org.springframework.transaction.annotation.EnableTransactionManagement; @EnableAsync @EnableScheduling @EnableCaching -public class DataMatePlatformApplication { +public class DataMateApplication { public static void main(String[] args) { - SpringApplication.run(DataMatePlatformApplication.class, args); + SpringApplication.run(DataMateApplication.class, args); } }