feat: Enhance dataset file management with improved file copying

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java

@@ -21,7 +21,6 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import jakarta.servlet.http.HttpServletResponse;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.io.FileUtils;
 import org.apache.ibatis.session.RowBounds;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -45,6 +44,7 @@ import java.nio.file.Paths;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
+import java.util.concurrent.CompletableFuture;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
@@ -280,31 +280,38 @@ public class DatasetFileApplicationService {
                 continue;
             }
             String fileName = sourcePath.getFileName().toString();
-            File targetFile = new File(dataset.getPath(), fileName);
+            File sourceFile = sourcePath.toFile();
-            try {
-                FileUtils.copyInputStreamToFile(Files.newInputStream(sourcePath), targetFile);
-            } catch (IOException e) {
-                log.error("Failed to copy file: {}", sourceFilePath, e);
-                continue;
-            }
-
             LocalDateTime currentTime = LocalDateTime.now();
             DatasetFile datasetFile = DatasetFile.builder()
                     .id(UUID.randomUUID().toString())
                     .datasetId(datasetId)
                     .fileName(fileName)
                     .fileType(AnalyzerUtils.getExtension(fileName))
-                    .fileSize(targetFile.length())
+                    .fileSize(sourceFile.length())
-                    .filePath(targetFile.getPath())
+                    .filePath(Paths.get(dataset.getPath(), fileName).toString())
                     .uploadTime(currentTime)
                     .lastAccessTime(currentTime)
                     .build();
-            datasetFileRepository.save(datasetFile);
             dataset.addFile(datasetFile);
             copiedFiles.add(datasetFile);
         }
+        datasetFileRepository.saveBatch(copiedFiles, 100);
         dataset.active();
         datasetRepository.updateById(dataset);
+        CompletableFuture.runAsync(() -> copyFilesToDatasetDir(req.sourcePaths(), dataset));
         return copiedFiles;
     }
+
+    private void copyFilesToDatasetDir(List<String> sourcePaths, Dataset dataset) {
+        for (String sourcePath : sourcePaths) {
+            Path sourceFilePath = Paths.get(sourcePath);
+            Path targetFilePath = Paths.get(dataset.getPath(), sourceFilePath.getFileName().toString());
+            try {
+                Files.createDirectories(Path.of(dataset.getPath()));
+                Files.copy(sourceFilePath, targetFilePath);
+            } catch (IOException e) {
+                log.error("Failed to copy file from {} to {}", sourcePath, targetFilePath, e);
+            }
+        }
+    }
 }

backend/services/main-application/pom.xml

@@ -162,7 +162,7 @@
                 <version>${spring-boot.version}</version>
                 <configuration>
                     <finalName>data-mate</finalName>
-                    <mainClass>com.datamate.main.DataMatePlatformApplication</mainClass>
+                    <mainClass>com.datamate.main.DataMateApplication</mainClass>
                 </configuration>
                 <executions>
                   <execution>

backend/services/main-application/src/main/java/com/datamate/main/DataMateApplication.java

@@ -23,8 +23,8 @@ import org.springframework.transaction.annotation.EnableTransactionManagement;
 @EnableAsync
 @EnableScheduling
 @EnableCaching
-public class DataMatePlatformApplication {
+public class DataMateApplication {
     public static void main(String[] args) {
-        SpringApplication.run(DataMatePlatformApplication.class, args);
+        SpringApplication.run(DataMateApplication.class, args);
     }
 }