q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 修复知识库同步的并发控制、数据清理、文件事务和COCO导出问题

Browse Source 
问题1 - 并发控制缺失：
- 在 _ensure_knowledge_set 方法中添加数据库行锁（with_for_update）
- 修改 _update_project_config 方法，使用行锁保护配置更新

问题3 - 数据清理机制缺失：
- 添加 _cleanup_knowledge_set_for_project 方法，项目删除时清理知识集
- 添加 _cleanup_knowledge_item_for_file 方法，文件删除时清理知识条目
- 在 delete_mapping 接口中调用清理方法

问题4 - 文件操作事务问题：
- 修改 uploadKnowledgeItems，添加事务失败后的文件清理逻辑
- 修改 deleteKnowledgeItem，删除记录前先删除关联文件
- 新增 deleteKnowledgeItemFile 辅助方法

问题5 - COCO导出格式问题：
- 添加 _get_image_dimensions 方法读取图片实际宽高
- 将百分比坐标转换为像素坐标
- 在 AnnotationExportItem 中添加 file_path 字段

涉及文件：
- knowledge_sync.py
- project.py
- KnowledgeItemApplicationService.java
- export.py
- export schema.py
This commit is contained in:
Jerry Yan
2026-02-05 03:55:01 +08:00
parent c03bdf1a24
commit 99bd83d312
5 changed files with 513 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/KnowledgeItemApplicationService.java
View File

@@ -126,41 +126,53 @@ public class KnowledgeItemApplicationService {
createDirectories(setDir);
List<KnowledgeItem> items = new ArrayList<>();
List<Path> savedFilePaths = new ArrayList<>();
for (MultipartFile file : files) {
BusinessAssert.notNull(file, CommonErrorCode.PARAM_ERROR);
BusinessAssert.isTrue(!file.isEmpty(), CommonErrorCode.PARAM_ERROR);
try {
for (MultipartFile file : files) {
BusinessAssert.notNull(file, CommonErrorCode.PARAM_ERROR);
BusinessAssert.isTrue(!file.isEmpty(), CommonErrorCode.PARAM_ERROR);
String originalName = resolveOriginalFileName(file);
String safeOriginalName = sanitizeFileName(originalName);
if (StringUtils.isBlank(safeOriginalName)) {
safeOriginalName = "file";
String originalName = resolveOriginalFileName(file);
String safeOriginalName = sanitizeFileName(originalName);
if (StringUtils.isBlank(safeOriginalName)) {
safeOriginalName = "file";
}
String extension = getFileExtension(safeOriginalName);
String storedName = UUID.randomUUID().toString() +
(StringUtils.isBlank(extension) ? "" : "." + extension);
Path targetPath = setDir.resolve(storedName).normalize();
BusinessAssert.isTrue(targetPath.startsWith(setDir), CommonErrorCode.PARAM_ERROR);
saveMultipartFile(file, targetPath);
savedFilePaths.add(targetPath);
KnowledgeItem knowledgeItem = new KnowledgeItem();
knowledgeItem.setId(UUID.randomUUID().toString());
knowledgeItem.setSetId(setId);
knowledgeItem.setContent(buildRelativeFilePath(setId, storedName));
knowledgeItem.setContentType(KnowledgeContentType.FILE);
knowledgeItem.setSourceType(KnowledgeSourceType.FILE_UPLOAD);
knowledgeItem.setSourceFileId(trimToLength(safeOriginalName, MAX_TITLE_LENGTH));
knowledgeItem.setRelativePath(buildRelativePath(parentPrefix, safeOriginalName));
items.add(knowledgeItem);
}
String extension = getFileExtension(safeOriginalName);
String storedName = UUID.randomUUID().toString() +
(StringUtils.isBlank(extension) ? "" : "." + extension);
Path targetPath = setDir.resolve(storedName).normalize();
BusinessAssert.isTrue(targetPath.startsWith(setDir), CommonErrorCode.PARAM_ERROR);
saveMultipartFile(file, targetPath);
KnowledgeItem knowledgeItem = new KnowledgeItem();
knowledgeItem.setId(UUID.randomUUID().toString());
knowledgeItem.setSetId(setId);
knowledgeItem.setContent(buildRelativeFilePath(setId, storedName));
knowledgeItem.setContentType(KnowledgeContentType.FILE);
knowledgeItem.setSourceType(KnowledgeSourceType.FILE_UPLOAD);
knowledgeItem.setSourceFileId(trimToLength(safeOriginalName, MAX_TITLE_LENGTH));
knowledgeItem.setRelativePath(buildRelativePath(parentPrefix, safeOriginalName));
items.add(knowledgeItem);
if (CollectionUtils.isNotEmpty(items)) {
knowledgeItemRepository.saveBatch(items, items.size());
}
return items;
} catch (Exception e) {
for (Path filePath : savedFilePaths) {
deleteFileQuietly(filePath);
}
if (e instanceof BusinessException) {
throw (BusinessException) e;
}
throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR);
}
if (CollectionUtils.isNotEmpty(items)) {
knowledgeItemRepository.saveBatch(items, items.size());
}
return items;
}
public KnowledgeItem updateKnowledgeItem(String setId, String itemId, UpdateKnowledgeItemRequest request) {
@@ -190,6 +202,9 @@ public class KnowledgeItemApplicationService {
KnowledgeItem knowledgeItem = knowledgeItemRepository.getById(itemId);
BusinessAssert.notNull(knowledgeItem, DataManagementErrorCode.KNOWLEDGE_ITEM_NOT_FOUND);
BusinessAssert.isTrue(Objects.equals(knowledgeItem.getSetId(), setId), CommonErrorCode.PARAM_ERROR);
deleteKnowledgeItemFile(knowledgeItem);
knowledgeItemPreviewService.deletePreviewFileQuietly(setId, itemId);
knowledgeItemRepository.removeById(itemId);
}
@@ -205,6 +220,11 @@ public class KnowledgeItemApplicationService {
boolean allMatch = items.stream().allMatch(item -> Objects.equals(item.getSetId(), setId));
BusinessAssert.isTrue(allMatch, CommonErrorCode.PARAM_ERROR);
for (KnowledgeItem item : items) {
deleteKnowledgeItemFile(item);
knowledgeItemPreviewService.deletePreviewFileQuietly(setId, item.getId());
}
List<String> deleteIds = items.stream().map(KnowledgeItem::getId).toList();
knowledgeItemRepository.removeByIds(deleteIds);
}
@@ -785,6 +805,24 @@ public class KnowledgeItemApplicationService {
}
}
private void deleteKnowledgeItemFile(KnowledgeItem knowledgeItem) {
if (knowledgeItem == null) {
return;
}
if (knowledgeItem.getSourceType() == KnowledgeSourceType.FILE_UPLOAD
|| knowledgeItem.getContentType() == KnowledgeContentType.FILE) {
String relativePath = knowledgeItem.getContent();
if (StringUtils.isNotBlank(relativePath)) {
try {
Path filePath = resolveKnowledgeItemStoragePath(relativePath);
deleteFileQuietly(filePath);
} catch (Exception e) {
log.warn("delete knowledge item file error, itemId: {}, path: {}", knowledgeItem.getId(), relativePath, e);
}
}
}
}
private String resolveOriginalFileName(MultipartFile file) {
String originalName = file.getOriginalFilename();
if (StringUtils.isBlank(originalName)) {