init datamate

This commit is contained in:
Dallas98
2025-10-21 23:00:48 +08:00
commit 1c97afed7d
692 changed files with 135442 additions and 0 deletions

View File

@@ -0,0 +1,22 @@
package com.datamate.datamanagement;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableAsync;
/**
* Data Management Service Configuration
* 数据管理服务配置类 - 多源接入、元数据、血缘治理
*/
@Configuration
@EnableFeignClients(basePackages = "com.datamate.datamanagement.infrastructure.client")
@EnableAsync
@ComponentScan(basePackages = {
"com.datamate.datamanagement",
"com.datamate.shared"
})
public class DataManagementServiceConfiguration {
// Service configuration class for JAR packaging
// 作为jar包形式提供服务的配置类
}

View File

@@ -0,0 +1,288 @@
package com.datamate.datamanagement.application;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.datamate.datamanagement.interfaces.dto.*;
import com.datamate.common.infrastructure.exception.BusinessAssert;
import com.datamate.common.interfaces.PagedResponse;
import com.datamate.datamanagement.domain.model.dataset.Dataset;
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
import com.datamate.datamanagement.domain.model.dataset.Tag;
import com.datamate.datamanagement.infrastructure.client.CollectionTaskClient;
import com.datamate.datamanagement.infrastructure.client.dto.CollectionTaskDetailResponse;
import com.datamate.datamanagement.infrastructure.client.dto.LocalCollectionConfig;
import com.datamate.datamanagement.infrastructure.exception.DataManagementErrorCode;
import com.datamate.datamanagement.infrastructure.persistence.mapper.TagMapper;
import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetFileRepository;
import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetRepository;
import com.datamate.datamanagement.interfaces.converter.DatasetConverter;
import com.datamate.datamanagement.interfaces.dto.*;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.StringUtils;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* 数据集应用服务(对齐 DB schema,使用 UUID 字符串主键)
*/
@Slf4j
@Service
@Transactional
@RequiredArgsConstructor
public class DatasetApplicationService {
private final DatasetRepository datasetRepository;
private final TagMapper tagMapper;
private final DatasetFileRepository datasetFileRepository;
private final CollectionTaskClient collectionTaskClient;
private final FileMetadataService fileMetadataService;
private final ObjectMapper objectMapper;
@Value("${dataset.base.path:/dataset}")
private String datasetBasePath;
/**
* 创建数据集
*/
@Transactional
public Dataset createDataset(CreateDatasetRequest createDatasetRequest) {
BusinessAssert.isTrue(datasetRepository.findByName(createDatasetRequest.getName()) == null, DataManagementErrorCode.DATASET_ALREADY_EXISTS);
// 创建数据集对象
Dataset dataset = DatasetConverter.INSTANCE.convertToDataset(createDatasetRequest);
dataset.initCreateParam(datasetBasePath);
// 处理标签
Set<Tag> processedTags = Optional.ofNullable(createDatasetRequest.getTags())
.filter(CollectionUtils::isNotEmpty)
.map(this::processTagNames)
.orElseGet(HashSet::new);
dataset.setTags(processedTags);
datasetRepository.save(dataset);
//todo 需要解耦这块逻辑
if (StringUtils.hasText(createDatasetRequest.getDataSource())) {
// 数据源id不为空,使用异步线程进行文件扫盘落库
processDataSourceAsync(dataset.getId(), createDatasetRequest.getDataSource());
}
return dataset;
}
public Dataset updateDataset(String datasetId, UpdateDatasetRequest updateDatasetRequest) {
Dataset dataset = datasetRepository.getById(datasetId);
BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND);
if (StringUtils.hasText(updateDatasetRequest.getName())) {
dataset.setName(updateDatasetRequest.getName());
}
if (StringUtils.hasText(updateDatasetRequest.getDescription())) {
dataset.setDescription(updateDatasetRequest.getDescription());
}
if (CollectionUtils.isNotEmpty(updateDatasetRequest.getTags())) {
dataset.setTags(processTagNames(updateDatasetRequest.getTags()));
}
if (Objects.nonNull(updateDatasetRequest.getStatus())) {
dataset.setStatus(updateDatasetRequest.getStatus());
}
if (StringUtils.hasText(updateDatasetRequest.getDataSource())) {
// 数据源id不为空,使用异步线程进行文件扫盘落库
processDataSourceAsync(dataset.getId(), updateDatasetRequest.getDataSource());
}
datasetRepository.updateById(dataset);
return dataset;
}
/**
* 删除数据集
*/
public void deleteDataset(String datasetId) {
datasetRepository.removeById(datasetId);
}
/**
* 获取数据集详情
*/
@Transactional(readOnly = true)
public Dataset getDataset(String datasetId) {
Dataset dataset = datasetRepository.getById(datasetId);
BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND);
return dataset;
}
/**
* 分页查询数据集
*/
@Transactional(readOnly = true)
public PagedResponse<DatasetResponse> getDatasets(DatasetPagingQuery query) {
IPage<Dataset> page = new Page<>(query.getPage(), query.getSize());
page = datasetRepository.findByCriteria(page, query);
return PagedResponse.of(DatasetConverter.INSTANCE.convertToResponse(page.getRecords()), page.getCurrent(), page.getTotal(), page.getPages());
}
/**
* 处理标签名称,创建或获取标签
*/
private Set<Tag> processTagNames(List<String> tagNames) {
Set<Tag> tags = new HashSet<>();
for (String tagName : tagNames) {
Tag tag = tagMapper.findByName(tagName);
if (tag == null) {
Tag newTag = new Tag(tagName, null, null, "#007bff");
newTag.setUsageCount(0L);
newTag.setId(UUID.randomUUID().toString());
tagMapper.insert(newTag);
tag = newTag;
}
tag.setUsageCount(tag.getUsageCount() == null ? 1L : tag.getUsageCount() + 1);
tagMapper.updateUsageCount(tag.getId(), tag.getUsageCount());
tags.add(tag);
}
return tags;
}
/**
* 获取数据集统计信息
*/
@Transactional(readOnly = true)
public Map<String, Object> getDatasetStatistics(String datasetId) {
Dataset dataset = datasetRepository.getById(datasetId);
if (dataset == null) {
throw new IllegalArgumentException("Dataset not found: " + datasetId);
}
Map<String, Object> statistics = new HashMap<>();
// 基础统计
Long totalFiles = datasetFileRepository.countByDatasetId(datasetId);
Long completedFiles = datasetFileRepository.countCompletedByDatasetId(datasetId);
Long totalSize = datasetFileRepository.sumSizeByDatasetId(datasetId);
statistics.put("totalFiles", totalFiles != null ? totalFiles.intValue() : 0);
statistics.put("completedFiles", completedFiles != null ? completedFiles.intValue() : 0);
statistics.put("totalSize", totalSize != null ? totalSize : 0L);
// 完成率计算
float completionRate = 0.0f;
if (totalFiles != null && totalFiles > 0) {
completionRate = (completedFiles != null ? completedFiles.floatValue() : 0.0f) / totalFiles.floatValue() * 100.0f;
}
statistics.put("completionRate", completionRate);
// 文件类型分布统计
Map<String, Integer> fileTypeDistribution = new HashMap<>();
List<DatasetFile> allFiles = datasetFileRepository.findAllByDatasetId(datasetId);
if (allFiles != null) {
for (DatasetFile file : allFiles) {
String fileType = file.getFileType() != null ? file.getFileType() : "unknown";
fileTypeDistribution.put(fileType, fileTypeDistribution.getOrDefault(fileType, 0) + 1);
}
}
statistics.put("fileTypeDistribution", fileTypeDistribution);
// 状态分布统计
Map<String, Integer> statusDistribution = new HashMap<>();
if (allFiles != null) {
for (DatasetFile file : allFiles) {
String status = file.getStatus() != null ? file.getStatus() : "unknown";
statusDistribution.put(status, statusDistribution.getOrDefault(status, 0) + 1);
}
}
statistics.put("statusDistribution", statusDistribution);
return statistics;
}
/**
* 获取所有数据集的汇总统计信息
*/
public AllDatasetStatisticsResponse getAllDatasetStatistics() {
return datasetRepository.getAllDatasetStatistics();
}
/**
* 异步处理数据源文件扫描
*
* @param datasetId 数据集ID
* @param dataSourceId 数据源ID(归集任务ID)
*/
@Async
public void processDataSourceAsync(String datasetId, String dataSourceId) {
try {
log.info("开始处理数据源文件扫描,数据集ID: {}, 数据源ID: {}", datasetId, dataSourceId);
// 1. 调用数据归集服务获取任务详情
CollectionTaskDetailResponse taskDetail = collectionTaskClient.getTaskDetail(dataSourceId).getData();
if (taskDetail == null) {
log.error("获取归集任务详情失败,任务ID: {}", dataSourceId);
return;
}
log.info("获取到归集任务详情: {}", taskDetail);
// 2. 解析任务配置
LocalCollectionConfig config = parseTaskConfig(taskDetail.getConfig());
if (config == null) {
log.error("解析任务配置失败,任务ID: {}", dataSourceId);
return;
}
// 4. 获取文件路径列表
List<String> filePaths = config.getFilePaths();
if (CollectionUtils.isEmpty(filePaths)) {
log.warn("文件路径列表为空,任务ID: {}", dataSourceId);
return;
}
log.info("开始扫描文件,共 {} 个文件路径", filePaths.size());
// 5. 扫描文件元数据
List<DatasetFile> datasetFiles = fileMetadataService.scanFiles(filePaths, datasetId);
// 查询数据集中已存在的文件
List<DatasetFile> existDatasetFileList = datasetFileRepository.findAllByDatasetId(datasetId);
Map<String, DatasetFile> existDatasetFilePathMap = existDatasetFileList.stream().collect(Collectors.toMap(DatasetFile::getFilePath, Function.identity()));
Dataset dataset = datasetRepository.getById(datasetId);
// 6. 批量插入数据集文件表
if (CollectionUtils.isNotEmpty(datasetFiles)) {
for (DatasetFile datasetFile : datasetFiles) {
if (existDatasetFilePathMap.containsKey(datasetFile.getFilePath())) {
DatasetFile existDatasetFile = existDatasetFilePathMap.get(datasetFile.getFilePath());
dataset.removeFile(existDatasetFile);
existDatasetFile.setFileSize(datasetFile.getFileSize());
dataset.addFile(existDatasetFile);
datasetFileRepository.updateById(existDatasetFile);
} else {
dataset.addFile(datasetFile);
datasetFileRepository.save(datasetFile);
}
}
log.info("文件元数据写入完成,共写入 {} 条记录", datasetFiles.size());
} else {
log.warn("未扫描到有效文件");
}
datasetRepository.updateById(dataset);
} catch (Exception e) {
log.error("处理数据源文件扫描失败,数据集ID: {}, 数据源ID: {}", datasetId, dataSourceId, e);
}
}
/**
* 解析任务配置
*/
private LocalCollectionConfig parseTaskConfig(Map<String, Object> configMap) {
try {
if (configMap == null || configMap.isEmpty()) {
return null;
}
return objectMapper.convertValue(configMap, LocalCollectionConfig.class);
} catch (Exception e) {
log.error("解析任务配置失败", e);
return null;
}
}
}

View File

@@ -0,0 +1,306 @@
package com.datamate.datamanagement.application;
import com.datamate.common.domain.model.ChunkUploadPreRequest;
import com.datamate.common.domain.model.FileUploadResult;
import com.datamate.common.domain.service.FileService;
import com.datamate.common.domain.utils.AnalyzerUtils;
import com.datamate.common.infrastructure.exception.BusinessException;
import com.datamate.common.infrastructure.exception.SystemErrorCode;
import com.datamate.datamanagement.domain.contants.DatasetConstant;
import com.datamate.datamanagement.domain.model.dataset.Dataset;
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
import com.datamate.datamanagement.domain.model.dataset.DatasetFileUploadCheckInfo;
import com.datamate.datamanagement.domain.model.dataset.StatusConstants;
import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetFileRepository;
import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetRepository;
import com.datamate.datamanagement.interfaces.converter.DatasetConverter;
import com.datamate.datamanagement.interfaces.dto.UploadFileRequest;
import com.datamate.datamanagement.interfaces.dto.UploadFilesPreRequest;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
import org.apache.ibatis.session.RowBounds;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.Resource;
import org.springframework.core.io.UrlResource;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.http.HttpHeaders;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
/**
* 数据集文件应用服务
*/
@Slf4j
@Service
@Transactional
public class DatasetFileApplicationService {
private final DatasetFileRepository datasetFileRepository;
private final DatasetRepository datasetRepository;
private final Path fileStorageLocation;
private final FileService fileService;
@Value("${dataset.base.path:/dataset}")
private String datasetBasePath;
@Autowired
public DatasetFileApplicationService(DatasetFileRepository datasetFileRepository,
DatasetRepository datasetRepository, FileService fileService,
@Value("${app.file.upload-dir:./dataset}") String uploadDir) {
this.datasetFileRepository = datasetFileRepository;
this.datasetRepository = datasetRepository;
this.fileStorageLocation = Paths.get(uploadDir).toAbsolutePath().normalize();
this.fileService = fileService;
try {
Files.createDirectories(this.fileStorageLocation);
} catch (Exception ex) {
throw new RuntimeException("Could not create the directory where the uploaded files will be stored.", ex);
}
}
/**
* 上传文件到数据集
*/
public DatasetFile uploadFile(String datasetId, MultipartFile file) {
Dataset dataset = datasetRepository.getById(datasetId);
if (dataset == null) {
throw new IllegalArgumentException("Dataset not found: " + datasetId);
}
String originalFilename = file.getOriginalFilename();
String fileName = originalFilename != null ? originalFilename : "file";
try {
// 保存文件到磁盘
Path targetLocation = this.fileStorageLocation.resolve(datasetId + File.separator + fileName);
// 确保目标目录存在
Files.createDirectories(targetLocation);
Files.copy(file.getInputStream(), targetLocation, StandardCopyOption.REPLACE_EXISTING);
// 创建文件实体(UUID 主键)
DatasetFile datasetFile = new DatasetFile();
datasetFile.setId(UUID.randomUUID().toString());
datasetFile.setDatasetId(datasetId);
datasetFile.setFileName(fileName);
datasetFile.setFilePath(targetLocation.toString());
datasetFile.setFileType(getFileExtension(originalFilename));
datasetFile.setFileSize(file.getSize());
datasetFile.setUploadTime(LocalDateTime.now());
datasetFile.setStatus(StatusConstants.DatasetFileStatuses.COMPLETED);
// 保存到数据库
datasetFileRepository.save(datasetFile);
// 更新数据集统计
dataset.addFile(datasetFile);
datasetRepository.updateById(dataset);
return datasetFileRepository.findByDatasetIdAndFileName(datasetId, fileName);
} catch (IOException ex) {
log.error("Could not store file {}", fileName, ex);
throw new RuntimeException("Could not store file " + fileName, ex);
}
}
/**
* 获取数据集文件列表
*/
@Transactional(readOnly = true)
public Page<DatasetFile> getDatasetFiles(String datasetId, String fileType,
String status, Pageable pageable) {
RowBounds bounds = new RowBounds(pageable.getPageNumber() * pageable.getPageSize(), pageable.getPageSize());
List<DatasetFile> content = datasetFileRepository.findByCriteria(datasetId, fileType, status, bounds);
long total = content.size() < pageable.getPageSize() && pageable.getPageNumber() == 0 ? content.size() : content.size() + (long) pageable.getPageNumber() * pageable.getPageSize();
return new PageImpl<>(content, pageable, total);
}
/**
* 获取文件详情
*/
@Transactional(readOnly = true)
public DatasetFile getDatasetFile(String datasetId, String fileId) {
DatasetFile file = datasetFileRepository.getById(fileId);
if (file == null) {
throw new IllegalArgumentException("File not found: " + fileId);
}
if (!file.getDatasetId().equals(datasetId)) {
throw new IllegalArgumentException("File does not belong to the specified dataset");
}
return file;
}
/**
* 删除文件
*/
public void deleteDatasetFile(String datasetId, String fileId) {
DatasetFile file = getDatasetFile(datasetId, fileId);
try {
Path filePath = Paths.get(file.getFilePath());
Files.deleteIfExists(filePath);
} catch (IOException ex) {
// ignore
}
datasetFileRepository.removeById(fileId);
Dataset dataset = datasetRepository.getById(datasetId);
// 简单刷新统计(精确处理可从DB统计)
dataset.setFileCount(Math.max(0, dataset.getFileCount() - 1));
dataset.setSizeBytes(Math.max(0, dataset.getSizeBytes() - (file.getFileSize() != null ? file.getFileSize() : 0)));
datasetRepository.updateById(dataset);
}
/**
* 下载文件
*/
@Transactional(readOnly = true)
public Resource downloadFile(String datasetId, String fileId) {
DatasetFile file = getDatasetFile(datasetId, fileId);
try {
Path filePath = Paths.get(file.getFilePath()).normalize();
Resource resource = new UrlResource(filePath.toUri());
if (resource.exists()) {
return resource;
} else {
throw new RuntimeException("File not found: " + file.getFileName());
}
} catch (MalformedURLException ex) {
throw new RuntimeException("File not found: " + file.getFileName(), ex);
}
}
/**
* 下载文件
*/
@Transactional(readOnly = true)
public void downloadDatasetFileAsZip(String datasetId, HttpServletResponse response) {
List<DatasetFile> allByDatasetId = datasetFileRepository.findAllByDatasetId(datasetId);
response.setContentType("application/zip");
String zipName = String.format("dataset_%s.zip",
LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss")));
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + zipName);
try (ZipOutputStream zos = new ZipOutputStream(response.getOutputStream())) {
for (DatasetFile file : allByDatasetId) {
addToZipFile(file, zos);
}
} catch (IOException e) {
log.error("Failed to download files in batches.", e);
throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR);
}
}
private void addToZipFile(DatasetFile file, ZipOutputStream zos) throws IOException {
if (file.getFilePath() == null || !Files.exists(Paths.get(file.getFilePath()))) {
log.warn("The file hasn't been found on filesystem, id: {}", file.getId());
return;
}
try (InputStream fis = Files.newInputStream(Paths.get(file.getFilePath()));
BufferedInputStream bis = new BufferedInputStream(fis)) {
ZipEntry zipEntry = new ZipEntry(file.getFileName());
zos.putNextEntry(zipEntry);
byte[] buffer = new byte[8192];
int length;
while ((length = bis.read(buffer)) >= 0) {
zos.write(buffer, 0, length);
}
zos.closeEntry();
}
}
private String getFileExtension(String fileName) {
if (fileName == null || fileName.isEmpty()) {
return null;
}
int lastDotIndex = fileName.lastIndexOf(".");
if (lastDotIndex == -1) {
return null;
}
return fileName.substring(lastDotIndex + 1);
}
/**
* 预上传
*
* @param chunkUploadRequest 上传请求
* @param datasetId 数据集id
* @return 请求id
*/
@Transactional
public String preUpload(UploadFilesPreRequest chunkUploadRequest, String datasetId) {
ChunkUploadPreRequest request = ChunkUploadPreRequest.builder().build();
request.setUploadPath(datasetBasePath + File.separator + datasetId);
request.setTotalFileNum(chunkUploadRequest.getTotalFileNum());
request.setServiceId(DatasetConstant.SERVICE_ID);
DatasetFileUploadCheckInfo checkInfo = new DatasetFileUploadCheckInfo();
checkInfo.setDatasetId(datasetId);
checkInfo.setHasArchive(chunkUploadRequest.isHasArchive());
try {
ObjectMapper objectMapper = new ObjectMapper();
String checkInfoJson = objectMapper.writeValueAsString(checkInfo);
request.setCheckInfo(checkInfoJson);
} catch (JsonProcessingException e) {
throw new IllegalArgumentException("Failed to serialize checkInfo to JSON", e);
}
return fileService.preUpload(request);
}
/**
* 切片上传
*
* @param uploadFileRequest 上传请求
*/
@Transactional
public void chunkUpload(String datasetId, UploadFileRequest uploadFileRequest) {
FileUploadResult uploadResult = fileService.chunkUpload(DatasetConverter.INSTANCE.toChunkUploadRequest(uploadFileRequest));
saveFileInfoToDb(uploadResult, uploadFileRequest, datasetId);
if (uploadResult.isAllFilesUploaded()) {
// 解析文件,后续依据需求看是否添加校验文件元数据和解析半结构化文件的逻辑,
}
}
private void saveFileInfoToDb(FileUploadResult fileUploadResult, UploadFileRequest uploadFile, String datasetId) {
if (Objects.isNull(fileUploadResult.getSavedFile())) {
// 文件切片上传没有完成
return;
}
Dataset dataset = datasetRepository.getById(datasetId);
File savedFile = fileUploadResult.getSavedFile();
LocalDateTime currentTime = LocalDateTime.now();
DatasetFile datasetFile = DatasetFile.builder()
.id(UUID.randomUUID().toString())
.datasetId(datasetId)
.fileSize(savedFile.length())
.uploadTime(currentTime)
.lastAccessTime(currentTime)
.fileName(uploadFile.getFileName())
.filePath(savedFile.getPath())
.fileType(AnalyzerUtils.getExtension(uploadFile.getFileName()))
.build();
datasetFileRepository.save(datasetFile);
dataset.addFile(datasetFile);
datasetRepository.updateById(dataset);
}
}

View File

@@ -0,0 +1,127 @@
package com.datamate.datamanagement.application;
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
/**
* 文件元数据扫描服务
*/
@Slf4j
@Service
public class FileMetadataService {
/**
* 扫描文件路径列表,提取文件元数据
* @param datasetId 数据集ID
* @return 数据集文件列表
*/
public List<DatasetFile> scanFiles(List<String> filePaths, String datasetId) {
List<DatasetFile> datasetFiles = new ArrayList<>();
if (filePaths == null || filePaths.isEmpty()) {
log.warn("文件路径列表为空,跳过扫描");
return datasetFiles;
}
for (String filePath : filePaths) {
try {
Path path = Paths.get(filePath);
if (!Files.exists(path)) {
log.warn("路径不存在: {}", filePath);
continue;
}
if (Files.isDirectory(path)) {
scanDirectory(datasetId, filePath, path, datasetFiles);
} else {
// 如果是文件,直接处理
DatasetFile datasetFile = extractFileMetadata(filePath, datasetId);
if (datasetFile != null) {
datasetFiles.add(datasetFile);
}
}
} catch (Exception e) {
log.error("扫描路径失败: {}, 错误: {}", filePath, e.getMessage(), e);
}
}
log.info("文件扫描完成,共扫描 {} 个文件", datasetFiles.size());
return datasetFiles;
}
private void scanDirectory(String datasetId, String filePath, Path path,
List<DatasetFile> datasetFiles) throws IOException {
// 如果是目录,扫描该目录下的所有文件(非递归)
List<Path> filesInDir = Files.list(path)
.filter(Files::isRegularFile)
.toList();
for (Path file : filesInDir) {
try {
DatasetFile datasetFile = extractFileMetadata(file.toString(), datasetId);
if (datasetFile != null) {
datasetFiles.add(datasetFile);
}
} catch (Exception e) {
log.error("处理目录中的文件失败: {}, 错误: {}", file, e.getMessage(), e);
}
}
log.info("已扫描目录 {} 下的 {} 个文件", filePath, filesInDir.size());
}
/**
* @param filePath 文件路径
* @param datasetId 数据集ID
* @return 数据集文件对象
*/
private DatasetFile extractFileMetadata(String filePath, String datasetId) throws IOException {
Path path = Paths.get(filePath);
if (!Files.exists(path)) {
log.warn("文件不存在: {}", filePath);
return null;
}
if (!Files.isRegularFile(path)) {
log.warn("路径不是文件: {}", filePath);
return null;
}
String fileName = path.getFileName().toString();
long fileSize = Files.size(path);
String fileType = getFileExtension(fileName);
return DatasetFile.builder()
.id(UUID.randomUUID().toString())
.datasetId(datasetId)
.fileName(fileName)
.filePath(filePath)
.fileSize(fileSize)
.fileType(fileType)
.uploadTime(LocalDateTime.now())
.lastAccessTime(LocalDateTime.now())
.status("UPLOADED")
.build();
}
/**
* 获取文件扩展名
*/
private String getFileExtension(String fileName) {
int lastDotIndex = fileName.lastIndexOf('.');
if (lastDotIndex > 0 && lastDotIndex < fileName.length() - 1) {
return fileName.substring(lastDotIndex + 1).toLowerCase();
}
return "unknown";
}
}

View File

@@ -0,0 +1,116 @@
package com.datamate.datamanagement.application;
import com.datamate.datamanagement.domain.model.dataset.Tag;
import com.datamate.datamanagement.infrastructure.persistence.mapper.TagMapper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.UUID;
/**
* 标签应用服务(UUID 主键)
*/
@Service
@Transactional
public class TagApplicationService {
private final TagMapper tagMapper;
@Autowired
public TagApplicationService(TagMapper tagMapper) {
this.tagMapper = tagMapper;
}
/**
* 创建标签
*/
public Tag createTag(String name, String color, String description) {
// 检查名称是否已存在
if (tagMapper.findByName(name) != null) {
throw new IllegalArgumentException("Tag with name '" + name + "' already exists");
}
Tag tag = new Tag(name, description, null, color);
tag.setUsageCount(0L);
tag.setId(UUID.randomUUID().toString());
tagMapper.insert(tag);
return tagMapper.findById(tag.getId());
}
/**
* 更新标签
*
* @param tag 待更新的标签实体,必须包含有效的 ID
* @return 更新结果
*/
@Transactional
public Tag updateTag(Tag tag) {
Tag existingTag = tagMapper.findById(tag.getId());
if (existingTag == null) {
throw new IllegalArgumentException("Tag not found: " + tag.getId());
}
existingTag.setName(tag.getName());
existingTag.setColor(tag.getColor());
existingTag.setDescription(tag.getDescription());
tagMapper.update(existingTag);
return tagMapper.findById(existingTag.getId());
}
@Transactional
public void deleteTag(List<String> tagIds) {
List<Tag> tags = tagMapper.findByIdIn(tagIds);
if (tags.stream().anyMatch(tag -> tag.getUsageCount() > 0)) {
throw new IllegalArgumentException("Cannot delete tags that are in use");
}
if (CollectionUtils.isEmpty(tags)) {
return;
}
tagMapper.deleteTagsById(tags.stream().map(Tag::getId).toList());
}
/**
* 获取所有标签
*/
@Transactional(readOnly = true)
public List<Tag> getAllTags() {
return tagMapper.findAllByOrderByUsageCountDesc();
}
/**
* 根据关键词搜索标签
*/
@Transactional(readOnly = true)
public List<Tag> searchTags(String keyword) {
if (keyword == null || keyword.trim().isEmpty()) {
return getAllTags();
}
return tagMapper.findByKeyword(keyword.trim());
}
/**
* 获取标签详情
*/
@Transactional(readOnly = true)
public Tag getTag(String tagId) {
Tag tag = tagMapper.findById(tagId);
if (tag == null) {
throw new IllegalArgumentException("Tag not found: " + tagId);
}
return tag;
}
/**
* 根据名称获取标签
*/
@Transactional(readOnly = true)
public Tag getTagByName(String name) {
Tag tag = tagMapper.findByName(name);
if (tag == null) {
throw new IllegalArgumentException("Tag not found: " + name);
}
return tag;
}
}

View File

@@ -0,0 +1,41 @@
package com.datamate.datamanagement.common.enums;
/**
* 数据集状态类型
* <p>数据集可以处于以下几种状态:
* <p>草稿(DRAFT):数据集正在创建中,尚未完成。
* <p>活动(ACTIVE):数据集处于活动状态, 可以被查询和使用,也可以被更新和删除。
* <p>处理中(PROCESSING):数据集正在处理中,可能需要一些时间,处理完成后会变成活动状态。
* <p>已归档(ARCHIVED):数据集已被归档,不可以更新文件,可以解锁变成活动状态。
* <p>已发布(PUBLISHED):数据集已被发布,可供外部使用,外部用户可以查询和使用数据集。
* <p>已弃用(DEPRECATED):数据集已被弃用,不建议再使用。
*
* @author dallas
* @since 2025-10-17
*/
public enum DatasetStatusType {
/**
* 草稿状态
*/
DRAFT,
/**
* 活动状态
*/
ACTIVE,
/**
* 处理中状态
*/
PROCESSING,
/**
* 已归档状态
*/
ARCHIVED,
/**
* 已发布状态
*/
PUBLISHED,
/**
* 已弃用状态
*/
DEPRECATED
}

View File

@@ -0,0 +1,28 @@
package com.datamate.datamanagement.common.enums;
import lombok.Getter;
/**
* 数据集类型值对象
*
* @author DataMate
* @since 2025-10-15
*/
public enum DatasetType {
TEXT("text", "文本数据集"),
IMAGE("image", "图像数据集"),
AUDIO("audio", "音频数据集"),
VIDEO("video", "视频数据集"),
OTHER("other", "其他数据集");
@Getter
private final String code;
@Getter
private final String description;
DatasetType(String code, String description) {
this.code = code;
this.description = description;
}
}

View File

@@ -0,0 +1,11 @@
package com.datamate.datamanagement.domain.contants;
/**
* 数据集常量
*/
public interface DatasetConstant {
/**
* 服务ID
*/
String SERVICE_ID = "DATA_MANAGEMENT";
}

View File

@@ -0,0 +1,146 @@
package com.datamate.datamanagement.domain.model.dataset;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableName;
import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler;
import com.datamate.common.domain.model.base.BaseEntity;
import com.datamate.datamanagement.common.enums.DatasetStatusType;
import com.datamate.datamanagement.common.enums.DatasetType;
import lombok.Getter;
import lombok.Setter;
import java.io.File;
import java.time.LocalDateTime;
import java.util.*;
/**
* 数据集实体(与数据库表 t_dm_datasets 对齐)
*/
@Getter
@Setter
@TableName(value = "t_dm_datasets", autoResultMap = true)
public class Dataset extends BaseEntity<String> {
/**
* 数据集名称
*/
private String name;
/**
* 数据集描述
*/
private String description;
/**
* 数据集类型
*/
private DatasetType datasetType;
/**
* 数据集分类
*/
private String category;
/**
* 数据集路径
*/
private String path;
/**
* 数据集格式
*/
private String format;
/**
* 数据集模式信息,JSON格式, 用于解析当前数据集的文件结构
*/
private String schemaInfo;
/**
* 数据集大小(字节)
*/
private Long sizeBytes = 0L;
/**
* 文件数量
*/
private Long fileCount = 0L;
/**
* 记录数量
*/
private Long recordCount = 0L;
/**
* 数据集保留天数
*/
private Integer retentionDays = 0;
/**
* 标签列表, JSON格式
*/
@TableField(typeHandler = JacksonTypeHandler.class)
private Collection<Tag> tags = new HashSet<>();
/**
* 额外元数据,JSON格式
*/
private String metadata;
/**
* 数据集状态
*/
private DatasetStatusType status;
/**
* 是否为公共数据集
*/
private Boolean isPublic = false;
/**
* 是否为精选数据集
*/
private Boolean isFeatured = false;
/**
* 数据集版本号
*/
private Long version = 0L;
@TableField(exist = false)
private List<DatasetFile> files = new ArrayList<>();
public Dataset() {
}
public Dataset(String name, String description, DatasetType datasetType, String category, String path,
String format, DatasetStatusType status, String createdBy) {
this.name = name;
this.description = description;
this.datasetType = datasetType;
this.category = category;
this.path = path;
this.format = format;
this.status = status;
this.createdBy = createdBy;
this.createdAt = LocalDateTime.now();
this.updatedAt = LocalDateTime.now();
}
public void initCreateParam(String datasetBasePath) {
this.id = UUID.randomUUID().toString();
this.path = datasetBasePath + File.separator + this.id;
this.status = DatasetStatusType.DRAFT;
}
public void updateBasicInfo(String name, String description, String category) {
if (name != null && !name.isEmpty()) this.name = name;
if (description != null) this.description = description;
if (category != null) this.category = category;
this.updatedAt = LocalDateTime.now();
}
public void updateStatus(DatasetStatusType status, String updatedBy) {
this.status = status;
this.updatedBy = updatedBy;
this.updatedAt = LocalDateTime.now();
}
public void addFile(DatasetFile file) {
this.files.add(file);
this.fileCount = this.fileCount + 1;
this.sizeBytes = this.sizeBytes + (file.getFileSize() != null ? file.getFileSize() : 0L);
this.updatedAt = LocalDateTime.now();
}
public void removeFile(DatasetFile file) {
if (this.files.remove(file)) {
this.fileCount = Math.max(0, this.fileCount - 1);
this.sizeBytes = Math.max(0, this.sizeBytes - (file.getFileSize() != null ? file.getFileSize() : 0L));
this.updatedAt = LocalDateTime.now();
}
}
}

View File

@@ -0,0 +1,35 @@
package com.datamate.datamanagement.domain.model.dataset;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.*;
import java.time.LocalDateTime;
import java.util.List;
/**
* 数据集文件实体(与数据库表 t_dm_dataset_files 对齐)
*/
@Getter
@Setter
@Builder
@NoArgsConstructor
@AllArgsConstructor
@TableName("t_dm_dataset_files")
public class DatasetFile {
@TableId
private String id; // UUID
private String datasetId; // UUID
private String fileName;
private String filePath;
private String fileType; // JPG/PNG/DCM/TXT
private Long fileSize; // bytes
private String checkSum;
private List<String> tags;
private String metadata;
private String status; // UPLOADED, PROCESSING, COMPLETED, ERROR
private LocalDateTime uploadTime;
private LocalDateTime lastAccessTime;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}

View File

@@ -0,0 +1,18 @@
package com.datamate.datamanagement.domain.model.dataset;
import com.datamate.common.domain.model.UploadCheckInfo;
import lombok.Getter;
import lombok.Setter;
/**
* 数据集文件上传检查信息
*/
@Getter
@Setter
public class DatasetFileUploadCheckInfo extends UploadCheckInfo {
/** 数据集id */
private String datasetId;
/** 是否为压缩包上传 */
private boolean hasArchive;
}

View File

@@ -0,0 +1,33 @@
package com.datamate.datamanagement.domain.model.dataset;
/**
* 状态常量类 - 统一管理所有状态枚举值
*/
public final class StatusConstants {
/**
* 数据集状态
*/
public static final class DatasetStatuses {
public static final String DRAFT = "DRAFT";
public static final String ACTIVE = "ACTIVE";
public static final String ARCHIVED = "ARCHIVED";
public static final String PROCESSING = "PROCESSING";
private DatasetStatuses() {}
}
/**
* 数据集文件状态
*/
public static final class DatasetFileStatuses {
public static final String UPLOADED = "UPLOADED";
public static final String PROCESSING = "PROCESSING";
public static final String COMPLETED = "COMPLETED";
public static final String ERROR = "ERROR";
private DatasetFileStatuses() {}
}
private StatusConstants() {}
}

View File

@@ -0,0 +1,33 @@
package com.datamate.datamanagement.domain.model.dataset;
import com.datamate.common.domain.model.base.BaseEntity;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
/**
* 标签实体(与数据库表 t_dm_tags 对齐)
*/
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
public class Tag extends BaseEntity<String> {
private String name;
private String description;
private String category;
private String color;
private Long usageCount = 0L;
public Tag(String name, String description, String category, String color) {
this.name = name;
this.description = description;
this.category = category;
this.color = color;
}
public void decrementUsage() {
if (this.usageCount != null && this.usageCount > 0) this.usageCount--;
}
}

View File

@@ -0,0 +1,22 @@
package com.datamate.datamanagement.infrastructure.client;
import com.datamate.common.infrastructure.common.Response;
import com.datamate.datamanagement.infrastructure.client.dto.CollectionTaskDetailResponse;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
/**
* 数据归集服务 Feign Client
*/
@FeignClient(name = "collection-service", url = "${collection.service.url:http://localhost:8080}")
public interface CollectionTaskClient {
/**
* 获取归集任务详情
* @param taskId 任务ID
* @return 任务详情
*/
@GetMapping("/api/data-collection/tasks/{id}")
Response<CollectionTaskDetailResponse> getTaskDetail(@PathVariable("id") String taskId);
}

View File

@@ -0,0 +1,23 @@
package com.datamate.datamanagement.infrastructure.client.dto;
import lombok.Data;
import java.time.LocalDateTime;
import java.util.Map;
/**
* 归集任务详情响应
*/
@Data
public class CollectionTaskDetailResponse {
private String id;
private String name;
private String description;
private Map<String, Object> config;
private String status;
private String syncMode;
private String scheduleExpression;
private String lastExecutionId;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}

View File

@@ -0,0 +1,21 @@
package com.datamate.datamanagement.infrastructure.client.dto;
import lombok.Data;
import java.util.List;
/**
* 本地归集任务配置
*/
@Data
public class LocalCollectionConfig {
/**
* 归集类型
*/
private String type;
/**
* 文件路径列表
*/
private List<String> filePaths;
}

View File

@@ -0,0 +1,37 @@
package com.datamate.datamanagement.infrastructure.config;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cache.CacheManager;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.cache.concurrent.ConcurrentMapCacheManager;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.transaction.annotation.EnableTransactionManagement;
import org.springframework.web.multipart.support.StandardServletMultipartResolver;
/**
* 数据管理服务配置
*/
@Configuration
@EnableTransactionManagement
@EnableCaching
@EnableConfigurationProperties(DataManagementProperties.class)
public class DataManagementConfig {
/**
* 缓存管理器
*/
@Bean
public CacheManager cacheManager() {
return new ConcurrentMapCacheManager("datasets", "datasetFiles", "tags");
}
/**
* 文件上传解析器
*/
@Bean
public StandardServletMultipartResolver multipartResolver() {
StandardServletMultipartResolver resolver = new StandardServletMultipartResolver();
return resolver;
}
}

View File

@@ -0,0 +1,82 @@
package com.datamate.datamanagement.infrastructure.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
/**
* 数据管理服务配置属性
*/
@Configuration
@ConfigurationProperties(prefix = "datamanagement")
public class DataManagementProperties {
private FileStorage fileStorage = new FileStorage();
private Cache cache = new Cache();
public FileStorage getFileStorage() {
return fileStorage;
}
public void setFileStorage(FileStorage fileStorage) {
this.fileStorage = fileStorage;
}
public Cache getCache() {
return cache;
}
public void setCache(Cache cache) {
this.cache = cache;
}
public static class FileStorage {
private String uploadDir = "./uploads";
private long maxFileSize = 10485760; // 10MB
private long maxRequestSize = 52428800; // 50MB
public String getUploadDir() {
return uploadDir;
}
public void setUploadDir(String uploadDir) {
this.uploadDir = uploadDir;
}
public long getMaxFileSize() {
return maxFileSize;
}
public void setMaxFileSize(long maxFileSize) {
this.maxFileSize = maxFileSize;
}
public long getMaxRequestSize() {
return maxRequestSize;
}
public void setMaxRequestSize(long maxRequestSize) {
this.maxRequestSize = maxRequestSize;
}
}
public static class Cache {
private int ttl = 3600; // 1 hour
private int maxSize = 1000;
public int getTtl() {
return ttl;
}
public void setTtl(int ttl) {
this.ttl = ttl;
}
public int getMaxSize() {
return maxSize;
}
public void setMaxSize(int maxSize) {
this.maxSize = maxSize;
}
}
}

View File

@@ -0,0 +1,39 @@
package com.datamate.datamanagement.infrastructure.exception;
import com.datamate.common.infrastructure.exception.ErrorCode;
import lombok.AllArgsConstructor;
import lombok.Getter;
/**
* 数据管理模块错误码
*
* @author dallas
* @since 2025-10-20
*/
@Getter
@AllArgsConstructor
public enum DataManagementErrorCode implements ErrorCode {
/**
* 数据集不存在
*/
DATASET_NOT_FOUND("data_management.0001", "数据集不存在"),
/**
* 数据集已存在
*/
DATASET_ALREADY_EXISTS("data_management.0002", "数据集已存在"),
/**
* 数据集状态错误
*/
DATASET_STATUS_ERROR("data_management.0003", "数据集状态错误"),
/**
* 数据集标签不存在
*/
DATASET_TAG_NOT_FOUND("data_management.0004", "数据集标签不存在"),
/**
* 数据集标签已存在
*/
DATASET_TAG_ALREADY_EXISTS("data_management.0005", "数据集标签已存在");
private final String code;
private final String message;
}

View File

@@ -0,0 +1,30 @@
package com.datamate.datamanagement.infrastructure.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.session.RowBounds;
import java.util.List;
@Mapper
public interface DatasetFileMapper extends BaseMapper<DatasetFile> {
DatasetFile findById(@Param("id") String id);
List<DatasetFile> findByDatasetId(@Param("datasetId") String datasetId, RowBounds rowBounds);
List<DatasetFile> findByDatasetIdAndStatus(@Param("datasetId") String datasetId, @Param("status") String status, RowBounds rowBounds);
List<DatasetFile> findByDatasetIdAndFileType(@Param("datasetId") String datasetId, @Param("fileType") String fileType, RowBounds rowBounds);
Long countByDatasetId(@Param("datasetId") String datasetId);
Long countCompletedByDatasetId(@Param("datasetId") String datasetId);
Long sumSizeByDatasetId(@Param("datasetId") String datasetId);
DatasetFile findByDatasetIdAndFileName(@Param("datasetId") String datasetId, @Param("fileName") String fileName);
List<DatasetFile> findAllByDatasetId(@Param("datasetId") String datasetId);
List<DatasetFile> findByCriteria(@Param("datasetId") String datasetId,
@Param("fileType") String fileType,
@Param("status") String status,
RowBounds rowBounds);
int insert(DatasetFile file);
int update(DatasetFile file);
int deleteById(@Param("id") String id);
}

View File

@@ -0,0 +1,33 @@
package com.datamate.datamanagement.infrastructure.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.datamate.datamanagement.domain.model.dataset.Dataset;
import com.datamate.datamanagement.interfaces.dto.AllDatasetStatisticsResponse;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.session.RowBounds;
import java.util.List;
@Mapper
public interface DatasetMapper extends BaseMapper<Dataset> {
Dataset findById(@Param("id") String id);
Dataset findByName(@Param("name") String name);
List<Dataset> findByStatus(@Param("status") String status);
List<Dataset> findByCreatedBy(@Param("createdBy") String createdBy, RowBounds rowBounds);
List<Dataset> findByTypeCode(@Param("typeCode") String typeCode, RowBounds rowBounds);
List<Dataset> findByTagNames(@Param("tagNames") List<String> tagNames, RowBounds rowBounds);
List<Dataset> findByKeyword(@Param("keyword") String keyword, RowBounds rowBounds);
List<Dataset> findByCriteria(@Param("typeCode") String typeCode,
@Param("status") String status,
@Param("keyword") String keyword,
@Param("tagNames") List<String> tagNames,
RowBounds rowBounds);
long countByCriteria(@Param("typeCode") String typeCode,
@Param("status") String status,
@Param("keyword") String keyword,
@Param("tagNames") List<String> tagNames);
int deleteById(@Param("id") String id);
AllDatasetStatisticsResponse getAllDatasetStatistics();
}

View File

@@ -0,0 +1,27 @@
package com.datamate.datamanagement.infrastructure.persistence.mapper;
import com.datamate.datamanagement.domain.model.dataset.Tag;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import java.util.List;
@Mapper
public interface TagMapper {
Tag findById(@Param("id") String id);
Tag findByName(@Param("name") String name);
List<Tag> findByNameIn(@Param("list") List<String> names);
List<Tag> findByIdIn(@Param("ids") List<String> ids);
List<Tag> findByKeyword(@Param("keyword") String keyword);
List<Tag> findAllByOrderByUsageCountDesc();
int insert(Tag tag);
int update(Tag tag);
int updateUsageCount(@Param("id") String id, @Param("usageCount") Long usageCount);
// Relations with dataset
int insertDatasetTag(@Param("datasetId") String datasetId, @Param("tagId") String tagId);
int deleteDatasetTagsByDatasetId(@Param("datasetId") String datasetId);
List<Tag> findByDatasetId(@Param("datasetId") String datasetId);
void deleteTagsById(@Param("ids") List<String> ids);
}

View File

@@ -0,0 +1,27 @@
package com.datamate.datamanagement.infrastructure.persistence.repository;
import com.baomidou.mybatisplus.extension.repository.IRepository;
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
import org.apache.ibatis.session.RowBounds;
import java.util.List;
/**
* 数据集文件仓储接口
*
* @author dallas
* @since 2025-10-15
*/
public interface DatasetFileRepository extends IRepository<DatasetFile> {
Long countByDatasetId(String datasetId);
Long countCompletedByDatasetId(String datasetId);
Long sumSizeByDatasetId(String datasetId);
List<DatasetFile> findAllByDatasetId(String datasetId);
DatasetFile findByDatasetIdAndFileName(String datasetId, String fileName);
List<DatasetFile> findByCriteria(String datasetId, String fileType, String status, RowBounds bounds);
}

View File

@@ -0,0 +1,29 @@
package com.datamate.datamanagement.infrastructure.persistence.repository;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.repository.IRepository;
import com.datamate.datamanagement.domain.model.dataset.Dataset;
import com.datamate.datamanagement.interfaces.dto.AllDatasetStatisticsResponse;
import com.datamate.datamanagement.interfaces.dto.DatasetPagingQuery;
import org.apache.ibatis.session.RowBounds;
import java.util.List;
/**
* 数据集仓储层
*
* @author dallas
* @since 2025-10-15
*/
public interface DatasetRepository extends IRepository<Dataset> {
Dataset findByName(String name);
List<Dataset> findByCriteria(String type, String status, String keyword, List<String> tagList, RowBounds bounds);
long countByCriteria(String type, String status, String keyword, List<String> tagList);
AllDatasetStatisticsResponse getAllDatasetStatistics();
IPage<Dataset> findByCriteria(IPage<Dataset> page, DatasetPagingQuery query);
}

View File

@@ -0,0 +1,54 @@
package com.datamate.datamanagement.infrastructure.persistence.repository.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.repository.CrudRepository;
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
import com.datamate.datamanagement.infrastructure.persistence.mapper.DatasetFileMapper;
import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetFileRepository;
import lombok.RequiredArgsConstructor;
import org.apache.ibatis.session.RowBounds;
import org.springframework.stereotype.Repository;
import java.util.List;
/**
* 数据集文件仓储实现类
*
* @author dallas
* @since 2025-10-15
*/
@Repository
@RequiredArgsConstructor
public class DatasetFileRepositoryImpl extends CrudRepository<DatasetFileMapper, DatasetFile> implements DatasetFileRepository {
private final DatasetFileMapper datasetFileMapper;
@Override
public Long countByDatasetId(String datasetId) {
return datasetFileMapper.selectCount(new LambdaQueryWrapper<DatasetFile>().eq(DatasetFile::getDatasetId, datasetId));
}
@Override
public Long countCompletedByDatasetId(String datasetId) {
return datasetFileMapper.countCompletedByDatasetId(datasetId);
}
@Override
public Long sumSizeByDatasetId(String datasetId) {
return datasetFileMapper.sumSizeByDatasetId(datasetId);
}
@Override
public List<DatasetFile> findAllByDatasetId(String datasetId) {
return datasetFileMapper.findAllByDatasetId(datasetId);
}
@Override
public DatasetFile findByDatasetIdAndFileName(String datasetId, String fileName) {
return datasetFileMapper.findByDatasetIdAndFileName(datasetId, fileName);
}
@Override
public List<DatasetFile> findByCriteria(String datasetId, String fileType, String status, RowBounds bounds) {
return datasetFileMapper.findByCriteria(datasetId, fileType, status, bounds);
}
}

View File

@@ -0,0 +1,73 @@
package com.datamate.datamanagement.infrastructure.persistence.repository.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.repository.CrudRepository;
import com.datamate.datamanagement.domain.model.dataset.Dataset;
import com.datamate.datamanagement.infrastructure.persistence.mapper.DatasetMapper;
import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetRepository;
import com.datamate.datamanagement.interfaces.dto.AllDatasetStatisticsResponse;
import com.datamate.datamanagement.interfaces.dto.DatasetPagingQuery;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.apache.ibatis.session.RowBounds;
import org.springframework.stereotype.Repository;
import java.util.List;
/**
* 数据集仓储层实现类
*
* @author dallas
* @since 2025-10-15
*/
@Repository
@RequiredArgsConstructor
public class DatasetRepositoryImpl extends CrudRepository<DatasetMapper, Dataset> implements DatasetRepository {
private final DatasetMapper datasetMapper;
@Override
public Dataset findByName(String name) {
return datasetMapper.selectOne(new LambdaQueryWrapper<Dataset>().eq(Dataset::getName, name));
}
@Override
public List<Dataset> findByCriteria(String type, String status, String keyword, List<String> tagList,
RowBounds bounds) {
return datasetMapper.findByCriteria(type, status, keyword, tagList, bounds);
}
@Override
public long countByCriteria(String type, String status, String keyword, List<String> tagList) {
return datasetMapper.countByCriteria(type, status, keyword, tagList);
}
@Override
public AllDatasetStatisticsResponse getAllDatasetStatistics() {
return datasetMapper.getAllDatasetStatistics();
}
@Override
public IPage<Dataset> findByCriteria(IPage<Dataset> page, DatasetPagingQuery query) {
LambdaQueryWrapper<Dataset> wrapper = new LambdaQueryWrapper<Dataset>()
.eq(query.getType() != null, Dataset::getDatasetType, query.getType())
.eq(query.getStatus() != null, Dataset::getStatus, query.getStatus())
.like(StringUtils.isNotBlank(query.getKeyword()), Dataset::getName, query.getKeyword())
.like(StringUtils.isNotBlank(query.getKeyword()), Dataset::getDescription, query.getKeyword());
/*
标签过滤 {@link Tag}
*/
for (String tagName : query.getTags()) {
wrapper.and(w ->
w.apply("tags IS NOT NULL " +
"AND JSON_VALID(tags) = 1 " +
"AND JSON_LENGTH(tags) > 0 " +
"AND JSON_SEARCH(tags, 'one', {0}, NULL, '$[*].name') IS NOT NULL", tagName)
);
}
wrapper.orderByDesc(Dataset::getCreatedAt);
return datasetMapper.selectPage(page, wrapper);
}
}

View File

@@ -0,0 +1,53 @@
package com.datamate.datamanagement.interfaces.converter;
import com.datamate.datamanagement.interfaces.dto.CreateDatasetRequest;
import com.datamate.datamanagement.interfaces.dto.DatasetFileResponse;
import com.datamate.datamanagement.interfaces.dto.DatasetResponse;
import com.datamate.datamanagement.interfaces.dto.UploadFileRequest;
import com.datamate.common.domain.model.ChunkUploadRequest;
import com.datamate.datamanagement.domain.model.dataset.Dataset;
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
import com.datamate.datamanagement.interfaces.dto.*;
import org.mapstruct.Mapper;
import org.mapstruct.Mapping;
import org.mapstruct.factory.Mappers;
import java.util.List;
/**
* 数据集文件转换器
*/
@Mapper
public interface DatasetConverter {
/** 单例实例 */
DatasetConverter INSTANCE = Mappers.getMapper(DatasetConverter.class);
/**
* 将数据集转换为响应
*/
@Mapping(source = "sizeBytes", target = "totalSize")
@Mapping(source = "path", target = "targetLocation")
DatasetResponse convertToResponse(Dataset dataset);
/**
* 将数据集转换为响应
*/
@Mapping(target = "tags", ignore = true)
Dataset convertToDataset(CreateDatasetRequest createDatasetRequest);
/**
* 将上传文件请求转换为分片上传请求
*/
ChunkUploadRequest toChunkUploadRequest(UploadFileRequest uploadFileRequest);
/**
* 将数据集转换为响应
*/
List<DatasetResponse> convertToResponse(List<Dataset> datasets);
/**
*
* 将数据集文件转换为响应
*/
DatasetFileResponse convertToResponse(DatasetFile datasetFile);
}

View File

@@ -0,0 +1,30 @@
package com.datamate.datamanagement.interfaces.converter;
import com.datamate.datamanagement.domain.model.dataset.Tag;
import com.datamate.datamanagement.interfaces.dto.TagResponse;
import com.datamate.datamanagement.interfaces.dto.UpdateTagRequest;
import org.mapstruct.Mapper;
import org.mapstruct.factory.Mappers;
/**
* 标签转换器
*/
@Mapper
public interface TagConverter {
/** 单例实例 */
TagConverter INSTANCE = Mappers.getMapper(TagConverter.class);
/**
* 将 UpdateTagRequest 转换为 Tag 实体
* @param request 更新标签请求DTO
* @return 标签实体
*/
Tag updateRequestToTag(UpdateTagRequest request);
/**
* 将 Tag 实体转换为 TagResponse DTO
* @param tag 标签实体
* @return 标签响应DTO
*/
TagResponse convertToResponse(Tag tag);
}

View File

@@ -0,0 +1,20 @@
package com.datamate.datamanagement.interfaces.dto;
import lombok.Getter;
import lombok.Setter;
/**
* 所有数据集统计信息响应DTO
*/
@Getter
@Setter
public class AllDatasetStatisticsResponse {
/** 总数据集数 */
private Integer totalDatasets;
/** 总文件数 */
private Long totalSize;
/** 总大小(字节) */
private Long totalFiles;
}

View File

@@ -0,0 +1,35 @@
package com.datamate.datamanagement.interfaces.dto;
import com.datamate.datamanagement.common.enums.DatasetType;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import java.util.List;
/**
* 创建数据集请求DTO
*/
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
public class CreateDatasetRequest {
/** 数据集名称 */
@NotBlank(message = "数据集名称不能为空")
private String name;
/** 数据集描述 */
private String description;
/** 数据集类型 */
@NotNull(message = "数据集类型不能为空")
private DatasetType datasetType;
/** 标签列表 */
private List<String> tags;
/** 数据源 */
private String dataSource;
/** 目标位置 */
private String targetLocation;
}

View File

@@ -0,0 +1,18 @@
package com.datamate.datamanagement.interfaces.dto;
import lombok.Getter;
import lombok.Setter;
/**
* 创建标签请求DTO
*/
@Getter
@Setter
public class CreateTagRequest {
/** 标签名称 */
private String name;
/** 标签颜色 */
private String color;
/** 标签描述 */
private String description;
}

View File

@@ -0,0 +1,36 @@
package com.datamate.datamanagement.interfaces.dto;
import lombok.Getter;
import lombok.Setter;
import java.time.LocalDateTime;
/**
* 数据集文件响应DTO
*/
@Getter
@Setter
public class DatasetFileResponse {
/** 文件ID */
private String id;
/** 文件名 */
private String fileName;
/** 原始文件名 */
private String originalName;
/** 文件类型 */
private String fileType;
/** 文件大小(字节) */
private Long fileSize;
/** 文件状态 */
private String status;
/** 文件描述 */
private String description;
/** 文件路径 */
private String filePath;
/** 上传时间 */
private LocalDateTime uploadTime;
/** 最后更新时间 */
private LocalDateTime lastAccessTime;
/** 上传者 */
private String uploadedBy;
}

View File

@@ -0,0 +1,42 @@
package com.datamate.datamanagement.interfaces.dto;
import com.datamate.common.interfaces.PagingQuery;
import com.datamate.datamanagement.common.enums.DatasetStatusType;
import com.datamate.datamanagement.common.enums.DatasetType;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import java.util.ArrayList;
import java.util.List;
/**
* 数据集分页查询请求
*
*/
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
public class DatasetPagingQuery extends PagingQuery {
/**
* 数据集类型过滤
*/
private DatasetType type;
/**
* 标签名过滤
*/
private List<String> tags = new ArrayList<>();
/**
* 关键词搜索(名称或描述)
*/
private String keyword;
/**
* 状态过滤
*/
private DatasetStatusType status;
}

View File

@@ -0,0 +1,47 @@
package com.datamate.datamanagement.interfaces.dto;
import lombok.Getter;
import lombok.Setter;
import java.time.LocalDateTime;
import java.util.List;
/**
* 数据集响应DTO
*/
@Getter
@Setter
public class DatasetResponse {
/** 数据集ID */
private String id;
/** 数据集名称 */
private String name;
/** 数据集描述 */
private String description;
/** 数据集类型 */
private String datasetType;
/** 数据集状态 */
private String status;
/** 标签列表 */
private List<TagResponse> tags;
/** 数据源 */
private String dataSource;
/** 目标位置 */
private String targetLocation;
/** 文件数量 */
private Integer fileCount;
/** 总大小(字节) */
private Long totalSize;
/** 完成率(0-100) */
private Float completionRate;
/** 创建时间 */
private LocalDateTime createdAt;
/** 更新时间 */
private LocalDateTime updatedAt;
/** 创建者 */
private String createdBy;
/**
* 更新者
*/
private String updatedBy;
}

View File

@@ -0,0 +1,26 @@
package com.datamate.datamanagement.interfaces.dto;
import lombok.Getter;
import lombok.Setter;
import java.util.Map;
/**
* 数据集统计信息响应DTO
*/
@Getter
@Setter
public class DatasetStatisticsResponse {
/** 总文件数 */
private Integer totalFiles;
/** 已完成文件数 */
private Integer completedFiles;
/** 总大小(字节) */
private Long totalSize;
/** 完成率(0-100) */
private Float completionRate;
/** 文件类型分布 */
private Map<String, Integer> fileTypeDistribution;
/** 状态分布 */
private Map<String, Integer> statusDistribution;
}

View File

@@ -0,0 +1,24 @@
package com.datamate.datamanagement.interfaces.dto;
import lombok.Getter;
import lombok.Setter;
import java.util.List;
/**
* 数据集类型响应DTO
*/
@Getter
@Setter
public class DatasetTypeResponse {
/** 类型编码 */
private String code;
/** 类型名称 */
private String name;
/** 类型描述 */
private String description;
/** 支持的文件格式 */
private List<String> supportedFormats;
/** 图标 */
private String icon;
}

View File

@@ -0,0 +1,28 @@
package com.datamate.datamanagement.interfaces.dto;
import lombok.Getter;
import lombok.Setter;
import java.util.List;
/**
* 数据集文件分页响应DTO
*/
@Getter
@Setter
public class PagedDatasetFileResponse {
/** 文件内容列表 */
private List<DatasetFileResponse> content;
/** 当前页码 */
private Integer page;
/** 每页大小 */
private Integer size;
/** 总元素数 */
private Integer totalElements;
/** 总页数 */
private Integer totalPages;
/** 是否为第一页 */
private Boolean first;
/** 是否为最后一页 */
private Boolean last;
}

View File

@@ -0,0 +1,28 @@
package com.datamate.datamanagement.interfaces.dto;
import lombok.Getter;
import lombok.Setter;
import java.util.List;
/**
* 数据集分页响应DTO
*/
@Getter
@Setter
public class PagedDatasetResponse {
/** 数据集内容列表 */
private List<DatasetResponse> content;
/** 当前页码 */
private Integer page;
/** 每页大小 */
private Integer size;
/** 总元素数 */
private Integer totalElements;
/** 总页数 */
private Integer totalPages;
/** 是否为第一页 */
private Boolean first;
/** 是否为最后一页 */
private Boolean last;
}

View File

@@ -0,0 +1,22 @@
package com.datamate.datamanagement.interfaces.dto;
import lombok.Getter;
import lombok.Setter;
/**
* 标签响应DTO
*/
@Getter
@Setter
public class TagResponse {
/** 标签ID */
private String id;
/** 标签名称 */
private String name;
/** 标签颜色 */
private String color;
/** 标签描述 */
private String description;
/** 使用次数 */
private Integer usageCount;
}

View File

@@ -0,0 +1,25 @@
package com.datamate.datamanagement.interfaces.dto;
import com.datamate.datamanagement.common.enums.DatasetStatusType;
import lombok.Getter;
import lombok.Setter;
import java.util.List;
/**
* 更新数据集请求DTO
*/
@Getter
@Setter
public class UpdateDatasetRequest {
/** 数据集名称 */
private String name;
/** 数据集描述 */
private String description;
/** 归集任务id */
private String dataSource;
/** 标签列表 */
private List<String> tags;
/** 数据集状态 */
private DatasetStatusType status;
}

View File

@@ -0,0 +1,20 @@
package com.datamate.datamanagement.interfaces.dto;
import lombok.Getter;
import lombok.Setter;
/**
* 更新标签请求DTO
*/
@Getter
@Setter
public class UpdateTagRequest {
/** 标签 ID */
private String id;
/** 标签名称 */
private String name;
/** 标签颜色 */
private String color;
/** 标签描述 */
private String description;
}

View File

@@ -0,0 +1,34 @@
package com.datamate.datamanagement.interfaces.dto;
import lombok.Getter;
import lombok.Setter;
import org.springframework.web.multipart.MultipartFile;
/**
* 上传文件请求
* 用于分块上传文件时的请求参数封装,支持大文件分片上传功能
*/
@Getter
@Setter
public class UploadFileRequest {
/** 预上传返回的id,用来确认同一个任务 */
private String reqId;
/** 文件编号,用于标识批量上传中的第几个文件 */
private int fileNo;
/** 文件名称 */
private String fileName;
/** 文件总分块数量 */
private int totalChunkNum;
/** 当前分块编号,从1开始 */
private int chunkNo;
/** 上传的文件分块内容 */
private MultipartFile file;
/** 文件分块的校验和(十六进制字符串),用于验证文件完整性 */
private String checkSumHex;
}

View File

@@ -0,0 +1,22 @@
package com.datamate.datamanagement.interfaces.dto;
import jakarta.validation.constraints.Min;
import lombok.Getter;
import lombok.Setter;
/**
* 切片上传预上传请求
*/
@Getter
@Setter
public class UploadFilesPreRequest {
/** 是否为压缩包上传 */
private boolean hasArchive;
/** 总文件数量 */
@Min(1)
private int totalFileNum;
/** 总文件大小 */
private long totalSize;
}

View File

@@ -0,0 +1,115 @@
package com.datamate.datamanagement.interfaces.rest;
import com.datamate.datamanagement.interfaces.dto.*;
import com.datamate.common.infrastructure.common.Response;
import com.datamate.common.infrastructure.exception.SystemErrorCode;
import com.datamate.common.interfaces.PagedResponse;
import com.datamate.datamanagement.application.DatasetApplicationService;
import com.datamate.datamanagement.domain.model.dataset.Dataset;
import com.datamate.datamanagement.interfaces.converter.DatasetConverter;
import com.datamate.datamanagement.interfaces.dto.*;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import java.util.Map;
/**
* 数据集 REST 控制器
*/
@Slf4j
@RestController
@RequiredArgsConstructor
@RequestMapping("/data-management/datasets")
public class DatasetController {
private final DatasetApplicationService datasetApplicationService;
/**
* 获取数据集列表
*
* @param query 分页查询参数
* @return 分页的数据集列表
*/
@GetMapping
public PagedResponse<DatasetResponse> getDatasets(DatasetPagingQuery query) {
return datasetApplicationService.getDatasets(query);
}
/**
* 创建数据集
*
* @param createDatasetRequest 创建数据集请求参数
* @return 创建的数据集响应
*/
@PostMapping
public DatasetResponse createDataset(@RequestBody @Valid CreateDatasetRequest createDatasetRequest) {
Dataset dataset = datasetApplicationService.createDataset(createDatasetRequest);
return DatasetConverter.INSTANCE.convertToResponse(dataset);
}
/**
* 根据ID获取数据集详情
*
* @param datasetId 数据集ID
* @return 数据集响应
*/
@GetMapping("/{datasetId}")
public DatasetResponse getDatasetById(@PathVariable("datasetId") String datasetId) {
Dataset dataset = datasetApplicationService.getDataset(datasetId);
return DatasetConverter.INSTANCE.convertToResponse(dataset);
}
/**
* 根据ID更新数据集
*
* @param datasetId 数据集ID
* @param updateDatasetRequest 更新数据集请求参数
* @return 更新后的数据集响应
*/
@PutMapping("/{datasetId}")
public DatasetResponse updateDataset(@PathVariable("datasetId") String datasetId,
@RequestBody UpdateDatasetRequest updateDatasetRequest) {
Dataset dataset = datasetApplicationService.updateDataset(datasetId, updateDatasetRequest);
return DatasetConverter.INSTANCE.convertToResponse(dataset);
}
/**
* 根据ID删除数据集
*
* @param datasetId 数据集ID
*/
@DeleteMapping("/{datasetId}")
public void deleteDataset(@PathVariable("datasetId") String datasetId) {
datasetApplicationService.deleteDataset(datasetId);
}
@GetMapping("/{datasetId}/statistics")
public ResponseEntity<Response<DatasetStatisticsResponse>> getDatasetStatistics(
@PathVariable("datasetId") String datasetId) {
try {
Map<String, Object> stats = datasetApplicationService.getDatasetStatistics(datasetId);
DatasetStatisticsResponse response = new DatasetStatisticsResponse();
response.setTotalFiles((Integer) stats.get("totalFiles"));
response.setCompletedFiles((Integer) stats.get("completedFiles"));
response.setTotalSize((Long) stats.get("totalSize"));
response.setCompletionRate((Float) stats.get("completionRate"));
response.setFileTypeDistribution((Map<String, Integer>) stats.get("fileTypeDistribution"));
response.setStatusDistribution((Map<String, Integer>) stats.get("statusDistribution"));
return ResponseEntity.ok(Response.ok(response));
} catch (IllegalArgumentException e) {
return ResponseEntity.status(HttpStatus.NOT_FOUND).body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
} catch (Exception e) {
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
}
}
@GetMapping("/statistics")
public ResponseEntity<Response<AllDatasetStatisticsResponse>> getAllStatistics() {
return ResponseEntity.ok(Response.ok(datasetApplicationService.getAllDatasetStatistics()));
}
}

View File

@@ -0,0 +1,163 @@
package com.datamate.datamanagement.interfaces.rest;
import com.datamate.common.infrastructure.common.IgnoreResponseWrap;
import com.datamate.common.infrastructure.common.Response;
import com.datamate.common.infrastructure.exception.SystemErrorCode;
import com.datamate.datamanagement.application.DatasetFileApplicationService;
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
import com.datamate.datamanagement.interfaces.converter.DatasetConverter;
import com.datamate.datamanagement.interfaces.dto.DatasetFileResponse;
import com.datamate.datamanagement.interfaces.dto.PagedDatasetFileResponse;
import com.datamate.datamanagement.interfaces.dto.UploadFileRequest;
import com.datamate.datamanagement.interfaces.dto.UploadFilesPreRequest;
import jakarta.servlet.http.HttpServletResponse;
import jakarta.validation.Valid;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.Resource;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import java.util.stream.Collectors;
/**
* 数据集文件 REST 控制器(UUID 模式)
*/
@Slf4j
@RestController
@RequestMapping("/data-management/datasets/{datasetId}/files")
public class DatasetFileController {
private final DatasetFileApplicationService datasetFileApplicationService;
@Autowired
public DatasetFileController(DatasetFileApplicationService datasetFileApplicationService) {
this.datasetFileApplicationService = datasetFileApplicationService;
}
@GetMapping
public ResponseEntity<Response<PagedDatasetFileResponse>> getDatasetFiles(
@PathVariable("datasetId") String datasetId,
@RequestParam(value = "page", required = false, defaultValue = "0") Integer page,
@RequestParam(value = "size", required = false, defaultValue = "20") Integer size,
@RequestParam(value = "fileType", required = false) String fileType,
@RequestParam(value = "status", required = false) String status) {
Pageable pageable = PageRequest.of(page != null ? page : 0, size != null ? size : 20);
Page<DatasetFile> filesPage = datasetFileApplicationService.getDatasetFiles(
datasetId, fileType, status, pageable);
PagedDatasetFileResponse response = new PagedDatasetFileResponse();
response.setContent(filesPage.getContent().stream()
.map(DatasetConverter.INSTANCE::convertToResponse)
.collect(Collectors.toList()));
response.setPage(filesPage.getNumber());
response.setSize(filesPage.getSize());
response.setTotalElements((int) filesPage.getTotalElements());
response.setTotalPages(filesPage.getTotalPages());
response.setFirst(filesPage.isFirst());
response.setLast(filesPage.isLast());
return ResponseEntity.ok(Response.ok(response));
}
@PostMapping(consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public ResponseEntity<Response<DatasetFileResponse>> uploadDatasetFile(
@PathVariable("datasetId") String datasetId,
@RequestPart(value = "file", required = false) MultipartFile file) {
try {
DatasetFile datasetFile = datasetFileApplicationService.uploadFile(datasetId, file);
return ResponseEntity.status(HttpStatus.CREATED).body(Response.ok(DatasetConverter.INSTANCE.convertToResponse(datasetFile)));
} catch (IllegalArgumentException e) {
return ResponseEntity.badRequest().body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
} catch (Exception e) {
log.error("upload fail", e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
}
}
@GetMapping("/{fileId}")
public ResponseEntity<Response<DatasetFileResponse>> getDatasetFileById(
@PathVariable("datasetId") String datasetId,
@PathVariable("fileId") String fileId) {
try {
DatasetFile datasetFile = datasetFileApplicationService.getDatasetFile(datasetId, fileId);
return ResponseEntity.ok(Response.ok(DatasetConverter.INSTANCE.convertToResponse(datasetFile)));
} catch (IllegalArgumentException e) {
return ResponseEntity.status(HttpStatus.NOT_FOUND).body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
}
}
@DeleteMapping("/{fileId}")
public ResponseEntity<Response<Void>> deleteDatasetFile(
@PathVariable("datasetId") String datasetId,
@PathVariable("fileId") String fileId) {
try {
datasetFileApplicationService.deleteDatasetFile(datasetId, fileId);
return ResponseEntity.ok().build();
} catch (IllegalArgumentException e) {
return ResponseEntity.status(HttpStatus.NOT_FOUND).body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
}
}
@IgnoreResponseWrap
@GetMapping(value = "/{fileId}/download", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE)
public ResponseEntity<Resource> downloadDatasetFileById(
@PathVariable("datasetId") String datasetId,
@PathVariable("fileId") String fileId) {
try {
DatasetFile datasetFile = datasetFileApplicationService.getDatasetFile(datasetId, fileId);
Resource resource = datasetFileApplicationService.downloadFile(datasetId, fileId);
return ResponseEntity.ok()
.contentType(MediaType.APPLICATION_OCTET_STREAM)
.header(HttpHeaders.CONTENT_DISPOSITION,
"attachment; filename=\"" + datasetFile.getFileName() + "\"")
.body(resource);
} catch (IllegalArgumentException e) {
return ResponseEntity.status(HttpStatus.NOT_FOUND).build();
} catch (Exception e) {
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
}
}
@IgnoreResponseWrap
@GetMapping(value = "/download", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE)
public void downloadDatasetFileAsZip(@PathVariable("datasetId") String datasetId, HttpServletResponse response) {
datasetFileApplicationService.downloadDatasetFileAsZip(datasetId, response);
}
/**
* 文件上传请求
*
* @param request 批量文件上传请求
* @return 批量上传请求id
*/
@PostMapping("/upload/pre-upload")
public ResponseEntity<Response<String>> preUpload(@PathVariable("datasetId") String datasetId, @RequestBody @Valid UploadFilesPreRequest request) {
return ResponseEntity.ok(Response.ok(datasetFileApplicationService.preUpload(request, datasetId)));
}
/**
* 分块上传
*
* @param uploadFileRequest 上传文件请求
*/
@PostMapping("/upload/chunk")
public ResponseEntity<Void> chunkUpload(@PathVariable("datasetId") String datasetId, UploadFileRequest uploadFileRequest) {
log.info("file upload reqId:{}, fileNo:{}, total chunk num:{}, current chunkNo:{}",
uploadFileRequest.getReqId(), uploadFileRequest.getFileNo(), uploadFileRequest.getTotalChunkNum(),
uploadFileRequest.getChunkNo());
datasetFileApplicationService.chunkUpload(datasetId, uploadFileRequest);
return ResponseEntity.ok().build();
}
}

View File

@@ -0,0 +1,53 @@
package com.datamate.datamanagement.interfaces.rest;
import com.datamate.datamanagement.interfaces.dto.DatasetTypeResponse;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.Arrays;
import java.util.List;
/**
* 数据集类型 REST 控制器
*/
@RestController
@RequestMapping("/data-management/dataset-types")
public class DatasetTypeController {
/**
* 获取所有支持的数据集类型
* @return 数据集类型列表
*/
@GetMapping
public List<DatasetTypeResponse> getDatasetTypes() {
return Arrays.asList(
createDatasetType("IMAGE", "图像数据集", "用于机器学习的图像数据集", Arrays.asList("jpg", "jpeg", "png", "bmp", "gif")),
createDatasetType("TEXT", "文本数据集", "用于文本分析的文本数据集", Arrays.asList("txt", "csv", "json", "xml")),
createDatasetType("AUDIO", "音频数据集", "用于音频处理的音频数据集", Arrays.asList("wav", "mp3", "flac", "aac")),
createDatasetType("VIDEO", "视频数据集", "用于视频分析的视频数据集", Arrays.asList("mp4", "avi", "mov", "mkv")),
createDatasetType("MULTIMODAL", "多模态数据集", "包含多种数据类型的数据集", List.of("*"))
);
}
private DatasetTypeResponse createDatasetType(String code, String name, String description, List<String> supportedFormats) {
DatasetTypeResponse response = new DatasetTypeResponse();
response.setCode(code);
response.setName(name);
response.setDescription(description);
response.setSupportedFormats(supportedFormats);
response.setIcon(getIconForType(code));
return response;
}
private String getIconForType(String typeCode) {
return switch (typeCode) {
case "IMAGE" -> "🖼️";
case "TEXT" -> "📄";
case "AUDIO" -> "🎵";
case "VIDEO" -> "🎬";
case "MULTIMODAL" -> "📊";
default -> "📁";
};
}
}

View File

@@ -0,0 +1,85 @@
package com.datamate.datamanagement.interfaces.rest;
import com.datamate.common.infrastructure.common.Response;
import com.datamate.common.infrastructure.exception.SystemErrorCode;
import com.datamate.datamanagement.application.TagApplicationService;
import com.datamate.datamanagement.domain.model.dataset.Tag;
import com.datamate.datamanagement.interfaces.converter.TagConverter;
import com.datamate.datamanagement.interfaces.dto.CreateTagRequest;
import com.datamate.datamanagement.interfaces.dto.TagResponse;
import com.datamate.datamanagement.interfaces.dto.UpdateTagRequest;
import jakarta.validation.Valid;
import jakarta.validation.constraints.Size;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import java.util.List;
import java.util.stream.Collectors;
/**
* 标签 REST 控制器(UUID 模式)
*/
@RestController
@RequestMapping("/data-management/tags")
public class TagController {
private final TagApplicationService tagApplicationService;
@Autowired
public TagController(TagApplicationService tagApplicationService) {
this.tagApplicationService = tagApplicationService;
}
/**
* 查询标签列表
*/
@GetMapping
public ResponseEntity<Response<List<TagResponse>>> getTags(@RequestParam(name = "keyword", required = false) String keyword) {
List<Tag> tags = tagApplicationService.searchTags(keyword);
List<TagResponse> response = tags.stream()
.map(TagConverter.INSTANCE::convertToResponse)
.collect(Collectors.toList());
return ResponseEntity.ok(Response.ok(response));
}
/**
* 创建标签
*/
@PostMapping
public ResponseEntity<Response<TagResponse>> createTag(@RequestBody CreateTagRequest createTagRequest) {
try {
Tag tag = tagApplicationService.createTag(
createTagRequest.getName(),
createTagRequest.getColor(),
createTagRequest.getDescription()
);
return ResponseEntity.ok(Response.ok(TagConverter.INSTANCE.convertToResponse(tag)));
} catch (IllegalArgumentException e) {
return ResponseEntity.badRequest().body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
}
}
/**
* 更新标签
*
* @param updateTagRequest 更新参数
* @return 更新结果
*/
@PutMapping
public ResponseEntity<Response<TagResponse>> updateTag(@RequestBody @Valid UpdateTagRequest updateTagRequest) {
Tag tag = tagApplicationService.updateTag(TagConverter.INSTANCE.updateRequestToTag(updateTagRequest));
return ResponseEntity.ok(Response.ok(TagConverter.INSTANCE.convertToResponse(tag)));
}
@DeleteMapping
public ResponseEntity<Response<Valid>> deleteTag(@RequestParam(value = "ids") @Valid @Size(max = 10) List<String> ids) {
try {
tagApplicationService.deleteTag(ids.stream().filter(StringUtils::isNoneBlank).distinct().toList());
return ResponseEntity.ok(Response.ok(null));
} catch (IllegalArgumentException e) {
return ResponseEntity.badRequest().body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
}
}
}

View File

@@ -0,0 +1,11 @@
dataMate:
datamanagement:
file-storage:
upload-dir: ${FILE_UPLOAD_DIR:./uploads}
max-file-size: 10485760 # 10MB
max-request-size: 52428800 # 50MB
cache:
ttl: 3600
max-size: 1000
# MyBatis is configured centrally in main-application (mapper-locations & aliases)
# to avoid list overriding issues when importing multiple module configs.

View File

@@ -0,0 +1,98 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.datamate.datamanagement.infrastructure.persistence.mapper.DatasetFileMapper">
<sql id="Base_Column_List">
id, dataset_id, file_name, file_path, file_type, file_size, check_sum, tags, metadata, status,
upload_time, last_access_time, created_at, updated_at
</sql>
<select id="findById" parameterType="string"
resultType="com.datamate.datamanagement.domain.model.dataset.DatasetFile">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_dataset_files
WHERE id = #{id}
</select>
<select id="findByDatasetId" parameterType="string"
resultType="com.datamate.datamanagement.domain.model.dataset.DatasetFile">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_dataset_files
WHERE dataset_id = #{datasetId}
ORDER BY upload_time DESC
</select>
<select id="findByDatasetIdAndStatus" resultType="com.datamate.datamanagement.domain.model.dataset.DatasetFile">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_dataset_files
WHERE dataset_id = #{datasetId}
AND status = #{status}
ORDER BY upload_time DESC
</select>
<select id="findByDatasetIdAndFileType" resultType="com.datamate.datamanagement.domain.model.dataset.DatasetFile">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_dataset_files
WHERE dataset_id = #{datasetId}
AND file_type = #{fileType}
ORDER BY upload_time DESC
</select>
<select id="countByDatasetId" parameterType="string" resultType="long">
SELECT COUNT(*) FROM t_dm_dataset_files WHERE dataset_id = #{datasetId}
</select>
<select id="countCompletedByDatasetId" parameterType="string" resultType="long">
SELECT COUNT(*) FROM t_dm_dataset_files WHERE dataset_id = #{datasetId} AND status = 'COMPLETED'
</select>
<select id="sumSizeByDatasetId" parameterType="string" resultType="long">
SELECT COALESCE(SUM(file_size), 0) FROM t_dm_dataset_files WHERE dataset_id = #{datasetId}
</select>
<select id="findByDatasetIdAndFileName" resultType="com.datamate.datamanagement.domain.model.dataset.DatasetFile">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_dataset_files
WHERE dataset_id = #{datasetId} AND file_name = #{fileName}
LIMIT 1
</select>
<select id="findAllByDatasetId" parameterType="string"
resultType="com.datamate.datamanagement.domain.model.dataset.DatasetFile">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_dataset_files
WHERE dataset_id = #{datasetId}
ORDER BY upload_time DESC
</select>
<select id="findByCriteria" resultType="com.datamate.datamanagement.domain.model.dataset.DatasetFile">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_dataset_files
WHERE dataset_id = #{datasetId}
<!-- Replace invalid XML '&&' with 'and' for MyBatis OGNL -->
<if test="fileType != null and fileType != ''">
AND file_type = #{fileType}
</if>
<if test="status != null and status != ''">
AND status = #{status}
</if>
ORDER BY upload_time DESC
</select>
<update id="update" parameterType="com.datamate.datamanagement.domain.model.dataset.DatasetFile">
UPDATE t_dm_dataset_files
SET file_name = #{fileName},
file_path = #{filePath},
file_type = #{fileType},
file_size = #{fileSize},
upload_time = #{uploadTime},
last_access_time = #{lastAccessTime},
status = #{status}
WHERE id = #{id}
</update>
<delete id="deleteById" parameterType="string">
DELETE FROM t_dm_dataset_files WHERE id = #{id}
</delete>
</mapper>

View File

@@ -0,0 +1,152 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.datamate.datamanagement.infrastructure.persistence.mapper.DatasetMapper">
<sql id="Base_Column_List">
id, name, description, dataset_type, category, path, format, schema_info, size_bytes, file_count, record_count,
retention_days, tags, metadata, status, is_public, is_featured, version, created_at, updated_at, created_by, updated_by
</sql>
<sql id="Alias_D_Column_List">
d.id AS id,
d.name AS name,
d.description AS description,
d.dataset_type AS dataset_type,
d.category AS category,
d.path AS path,
d.format AS format,
d.schema_info AS schema_info,
d.size_bytes AS size_bytes,
d.file_count AS file_count,
d.record_count AS record_count,
d.retention_days AS retention_days,
d.tags AS tags,
d.metadata AS metadata,
d.status AS status,
d.is_public AS is_public,
d.is_featured AS is_featured,
d.version AS version,
d.created_at AS created_at,
d.updated_at AS updated_at,
d.created_by AS created_by,
d.updated_by AS updated_by
</sql>
<select id="findById" parameterType="string" resultType="com.datamate.datamanagement.domain.model.dataset.Dataset">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_datasets
WHERE id = #{id}
</select>
<select id="findByName" parameterType="string"
resultType="com.datamate.datamanagement.domain.model.dataset.Dataset">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_datasets
WHERE name = #{name}
LIMIT 1
</select>
<select id="findByStatus" parameterType="string"
resultType="com.datamate.datamanagement.domain.model.dataset.Dataset">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_datasets
WHERE status = #{status}
ORDER BY updated_at DESC
</select>
<select id="findByCreatedBy" resultType="com.datamate.datamanagement.domain.model.dataset.Dataset">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_datasets
WHERE created_by = #{createdBy}
ORDER BY created_at DESC
</select>
<select id="findByTypeCode" resultType="com.datamate.datamanagement.domain.model.dataset.Dataset">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_datasets
WHERE dataset_type = #{typeCode}
ORDER BY created_at DESC
</select>
<select id="findByTagNames" resultType="com.datamate.datamanagement.domain.model.dataset.Dataset">
SELECT DISTINCT <include refid="Alias_D_Column_List"/>
FROM t_dm_datasets d
JOIN t_dm_dataset_tags dt ON d.id = dt.dataset_id
JOIN t_dm_tags t ON t.id = dt.tag_id
WHERE t.name IN
<foreach collection="tagNames" item="name" open="(" separator="," close=")">
#{name}
</foreach>
ORDER BY d.created_at DESC
</select>
<select id="findByKeyword" resultType="com.datamate.datamanagement.domain.model.dataset.Dataset">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_datasets
WHERE name LIKE CONCAT('%', #{keyword}, '%')
OR description LIKE CONCAT('%', #{keyword}, '%')
ORDER BY created_at DESC
</select>
<select id="findByCriteria" resultType="com.datamate.datamanagement.domain.model.dataset.Dataset">
SELECT DISTINCT <include refid="Alias_D_Column_List"/>
FROM t_dm_datasets d
LEFT JOIN t_dm_dataset_tags dt ON d.id = dt.dataset_id
LEFT JOIN t_dm_tags t ON t.id = dt.tag_id
<where>
<if test="typeCode != null and typeCode != ''">
AND d.dataset_type = #{typeCode}
</if>
<if test="status != null and status != ''">
AND d.status = #{status}
</if>
<if test="keyword != null and keyword != ''">
AND (d.name LIKE CONCAT('%', #{keyword}, '%') OR d.description LIKE CONCAT('%', #{keyword}, '%'))
</if>
<if test="tagNames != null and tagNames.size > 0">
AND t.name IN
<foreach collection="tagNames" item="n" open="(" separator="," close=")">
#{n}
</foreach>
</if>
</where>
ORDER BY d.created_at DESC
</select>
<select id="countByCriteria" resultType="long">
SELECT COUNT(DISTINCT d.id)
FROM t_dm_datasets d
LEFT JOIN t_dm_dataset_tags dt ON d.id = dt.dataset_id
LEFT JOIN t_dm_tags t ON t.id = dt.tag_id
<where>
<if test="typeCode != null and typeCode != ''">
AND d.dataset_type = #{typeCode}
</if>
<if test="status != null and status != ''">
AND d.status = #{status}
</if>
<if test="keyword != null and keyword != ''">
AND (d.name LIKE CONCAT('%', #{keyword}, '%') OR d.description LIKE CONCAT('%', #{keyword}, '%'))
</if>
<if test="tagNames != null and tagNames.size > 0">
AND t.name IN
<foreach collection="tagNames" item="n" open="(" separator="," close=")">
#{n}
</foreach>
</if>
</where>
</select>
<delete id="deleteById" parameterType="string">
DELETE FROM t_dm_datasets WHERE id = #{id}
</delete>
<select id="getAllDatasetStatistics" resultType="com.datamate.datamanagement.interfaces.dto.AllDatasetStatisticsResponse">
SELECT
COUNT(*) AS total_datasets,
SUM(size_bytes) AS total_size,
SUM(file_count) AS total_files
FROM t_dm_datasets;
</select>
</mapper>

View File

@@ -0,0 +1,111 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.datamate.datamanagement.infrastructure.persistence.mapper.TagMapper">
<resultMap id="TagResultMap" type="com.datamate.datamanagement.domain.model.dataset.Tag">
<id column="id" property="id"/>
<result column="name" property="name"/>
<result column="description" property="description"/>
<result column="category" property="category"/>
<result column="color" property="color"/>
<result column="usage_count" property="usageCount"/>
<result column="created_at" property="createdAt"/>
<result column="updated_at" property="updatedAt"/>
</resultMap>
<sql id="Base_Column_List">
id, name, description, category, color, usage_count, created_at, updated_at
</sql>
<select id="findById" parameterType="string" resultMap="TagResultMap">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_tags
WHERE id = #{id}
</select>
<select id="findByName" parameterType="string" resultMap="TagResultMap">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_tags
WHERE name = #{name}
LIMIT 1
</select>
<select id="findByNameIn" parameterType="list" resultMap="TagResultMap">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_tags
WHERE name IN
<foreach collection="list" item="n" open="(" separator="," close=")">
#{n}
</foreach>
</select>
<select id="findByKeyword" parameterType="string" resultMap="TagResultMap">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_tags
WHERE name LIKE CONCAT('%', #{keyword}, '%')
ORDER BY usage_count DESC, name ASC
</select>
<select id="findAllByOrderByUsageCountDesc" resultMap="TagResultMap">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_tags
ORDER BY usage_count DESC, name ASC
</select>
<insert id="insert" parameterType="com.datamate.datamanagement.domain.model.dataset.Tag">
INSERT INTO t_dm_tags (id, name, description, category, color, usage_count)
VALUES (#{id}, #{name}, #{description}, #{category}, #{color}, #{usageCount})
</insert>
<update id="update" parameterType="com.datamate.datamanagement.domain.model.dataset.Tag">
UPDATE t_dm_tags
SET name = #{name},
description = #{description},
category = #{category},
color = #{color},
usage_count = #{usageCount}
WHERE id = #{id}
</update>
<update id="updateUsageCount">
UPDATE t_dm_tags
SET usage_count = #{usageCount}
WHERE id = #{id}
</update>
<!-- Dataset & Tag relations -->
<insert id="insertDatasetTag">
INSERT INTO t_dm_dataset_tags (dataset_id, tag_id)
VALUES (#{datasetId}, #{tagId})
</insert>
<delete id="deleteDatasetTagsByDatasetId">
DELETE FROM t_dm_dataset_tags WHERE dataset_id = #{datasetId}
</delete>
<select id="findByDatasetId" parameterType="string" resultMap="TagResultMap">
SELECT t.id, t.name, t.description, t.category, t.color, t.usage_count, t.created_at, t.updated_at
FROM t_dm_tags t
JOIN t_dm_dataset_tags dt ON dt.tag_id = t.id
WHERE dt.dataset_id = #{datasetId}
ORDER BY t.usage_count DESC, t.name ASC
</select>
<delete id="deleteTagsById">
DELETE FROM t_dm_tags WHERE
id IN
<foreach collection="ids" item="id" open="(" separator="," close=")">
#{id}
</foreach>
</delete>
<select id="findByIdIn" resultMap="TagResultMap">
SELECT <include refid="Base_Column_List"/>
FROM t_dm_tags
WHERE id IN
<foreach collection="ids" item="id" open="(" separator="," close=")">
#{id}
</foreach>
</select>
</mapper>