You've already forked DataMate
fix(upload): 修复流式上传中的文件名处理逻辑
- 修正预上传接口调用时传递正确的文件总数而非固定值-1 - 移除导入配置中文件分割时的文件扩展名保留逻辑 - 删除流式上传选项中的fileExtension参数定义 - 移除流式上传实现中的文件扩展名处理相关代码 - 简化新文件名生成逻辑,不再附加扩展名后缀
This commit is contained in:
@@ -103,9 +103,6 @@ public class DatasetApplicationService {
|
|||||||
Dataset dataset = datasetRepository.getById(datasetId);
|
Dataset dataset = datasetRepository.getById(datasetId);
|
||||||
BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND);
|
BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND);
|
||||||
|
|
||||||
// 保存原始的 parentDatasetId 值,用于比较是否发生了变化
|
|
||||||
String originalParentDatasetId = dataset.getParentDatasetId();
|
|
||||||
|
|
||||||
if (StringUtils.hasText(updateDatasetRequest.getName())) {
|
if (StringUtils.hasText(updateDatasetRequest.getName())) {
|
||||||
dataset.setName(updateDatasetRequest.getName());
|
dataset.setName(updateDatasetRequest.getName());
|
||||||
}
|
}
|
||||||
@@ -118,18 +115,23 @@ public class DatasetApplicationService {
|
|||||||
if (Objects.nonNull(updateDatasetRequest.getStatus())) {
|
if (Objects.nonNull(updateDatasetRequest.getStatus())) {
|
||||||
dataset.setStatus(updateDatasetRequest.getStatus());
|
dataset.setStatus(updateDatasetRequest.getStatus());
|
||||||
}
|
}
|
||||||
// 处理父数据集变更:始终调用 handleParentChange,以支持设置新的关联或清除关联
|
if (updateDatasetRequest.isParentDatasetIdProvided()) {
|
||||||
// handleParentChange 内部通过 normalizeParentId 方法将空字符串和 null 都转换为 null
|
// 保存原始的 parentDatasetId 值,用于比较是否发生了变化
|
||||||
// 这样既支持设置新的父数据集,也支持清除关联
|
String originalParentDatasetId = dataset.getParentDatasetId();
|
||||||
handleParentChange(dataset, updateDatasetRequest.getParentDatasetId());
|
|
||||||
|
// 处理父数据集变更:仅当请求显式包含 parentDatasetId 时处理
|
||||||
// 检查 parentDatasetId 是否发生了变化
|
// handleParentChange 内部通过 normalizeParentId 方法将空字符串和 null 都转换为 null
|
||||||
if (!Objects.equals(originalParentDatasetId, dataset.getParentDatasetId())) {
|
// 这样既支持设置新的父数据集,也支持清除关联
|
||||||
// 使用 LambdaUpdateWrapper 显式地更新 parentDatasetId 字段
|
handleParentChange(dataset, updateDatasetRequest.getParentDatasetId());
|
||||||
// 这样即使值为 null 也能被正确更新到数据库
|
|
||||||
datasetRepository.update(null, new LambdaUpdateWrapper<Dataset>()
|
// 检查 parentDatasetId 是否发生了变化
|
||||||
.eq(Dataset::getId, datasetId)
|
if (!Objects.equals(originalParentDatasetId, dataset.getParentDatasetId())) {
|
||||||
.set(Dataset::getParentDatasetId, dataset.getParentDatasetId()));
|
// 使用 LambdaUpdateWrapper 显式地更新 parentDatasetId 字段
|
||||||
|
// 这样即使值为 null 也能被正确更新到数据库
|
||||||
|
datasetRepository.update(null, new LambdaUpdateWrapper<Dataset>()
|
||||||
|
.eq(Dataset::getId, datasetId)
|
||||||
|
.set(Dataset::getParentDatasetId, dataset.getParentDatasetId()));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (StringUtils.hasText(updateDatasetRequest.getDataSource())) {
|
if (StringUtils.hasText(updateDatasetRequest.getDataSource())) {
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
package com.datamate.datamanagement.interfaces.dto;
|
package com.datamate.datamanagement.interfaces.dto;
|
||||||
|
|
||||||
import com.datamate.datamanagement.common.enums.DatasetStatusType;
|
import com.datamate.datamanagement.common.enums.DatasetStatusType;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||||
import jakarta.validation.constraints.NotBlank;
|
import jakarta.validation.constraints.NotBlank;
|
||||||
import jakarta.validation.constraints.Size;
|
import jakarta.validation.constraints.Size;
|
||||||
|
import lombok.AccessLevel;
|
||||||
import lombok.Getter;
|
import lombok.Getter;
|
||||||
import lombok.Setter;
|
import lombok.Setter;
|
||||||
|
|
||||||
@@ -24,9 +26,18 @@ public class UpdateDatasetRequest {
|
|||||||
/** 归集任务id */
|
/** 归集任务id */
|
||||||
private String dataSource;
|
private String dataSource;
|
||||||
/** 父数据集ID */
|
/** 父数据集ID */
|
||||||
|
@Setter(AccessLevel.NONE)
|
||||||
private String parentDatasetId;
|
private String parentDatasetId;
|
||||||
|
@JsonIgnore
|
||||||
|
@Setter(AccessLevel.NONE)
|
||||||
|
private boolean parentDatasetIdProvided;
|
||||||
/** 标签列表 */
|
/** 标签列表 */
|
||||||
private List<String> tags;
|
private List<String> tags;
|
||||||
/** 数据集状态 */
|
/** 数据集状态 */
|
||||||
private DatasetStatusType status;
|
private DatasetStatusType status;
|
||||||
|
|
||||||
|
public void setParentDatasetId(String parentDatasetId) {
|
||||||
|
this.parentDatasetIdProvided = true;
|
||||||
|
this.parentDatasetId = parentDatasetId;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -205,7 +205,7 @@ export function useFileSliceUpload(
|
|||||||
// 预上传,获取 reqId
|
// 预上传,获取 reqId
|
||||||
const totalSize = files.reduce((acc, file) => acc + file.size, 0);
|
const totalSize = files.reduce((acc, file) => acc + file.size, 0);
|
||||||
const { data: reqId } = await preUpload(task.key, {
|
const { data: reqId } = await preUpload(task.key, {
|
||||||
totalFileNum: -1, // 流式上传,文件数量不确定
|
totalFileNum: files.length,
|
||||||
totalSize,
|
totalSize,
|
||||||
datasetId: task.key,
|
datasetId: task.key,
|
||||||
hasArchive: task.hasArchive,
|
hasArchive: task.hasArchive,
|
||||||
|
|||||||
@@ -90,14 +90,16 @@ async function splitFileByLines(file: UploadFile): Promise<UploadFile[]> {
|
|||||||
const lines = text.split(/\r?\n/).filter((line: string) => line.trim() !== "");
|
const lines = text.split(/\r?\n/).filter((line: string) => line.trim() !== "");
|
||||||
if (lines.length === 0) return [];
|
if (lines.length === 0) return [];
|
||||||
|
|
||||||
// 生成文件名:原文件名_序号.扩展名
|
// 生成文件名:原文件名_序号(不保留后缀)
|
||||||
const nameParts = file.name.split(".");
|
const nameParts = file.name.split(".");
|
||||||
const ext = nameParts.length > 1 ? "." + nameParts.pop() : "";
|
if (nameParts.length > 1) {
|
||||||
|
nameParts.pop();
|
||||||
|
}
|
||||||
const baseName = nameParts.join(".");
|
const baseName = nameParts.join(".");
|
||||||
const padLength = String(lines.length).length;
|
const padLength = String(lines.length).length;
|
||||||
|
|
||||||
return lines.map((line: string, index: number) => {
|
return lines.map((line: string, index: number) => {
|
||||||
const newFileName = `${baseName}_${String(index + 1).padStart(padLength, "0")}${ext}`;
|
const newFileName = `${baseName}_${String(index + 1).padStart(padLength, "0")}`;
|
||||||
const blob = new Blob([line], { type: "text/plain" });
|
const blob = new Blob([line], { type: "text/plain" });
|
||||||
const newFile = new File([blob], newFileName, { type: "text/plain" });
|
const newFile = new File([blob], newFileName, { type: "text/plain" });
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -403,7 +403,6 @@ export function readFileAsText(
|
|||||||
export interface StreamUploadOptions {
|
export interface StreamUploadOptions {
|
||||||
reqId: number;
|
reqId: number;
|
||||||
fileNamePrefix?: string;
|
fileNamePrefix?: string;
|
||||||
fileExtension?: string;
|
|
||||||
hasArchive?: boolean;
|
hasArchive?: boolean;
|
||||||
prefix?: string;
|
prefix?: string;
|
||||||
signal?: AbortSignal;
|
signal?: AbortSignal;
|
||||||
@@ -423,7 +422,7 @@ export async function streamSplitAndUpload(
|
|||||||
chunkSize: number = 1024 * 1024, // 1MB
|
chunkSize: number = 1024 * 1024, // 1MB
|
||||||
options: StreamUploadOptions
|
options: StreamUploadOptions
|
||||||
): Promise<StreamUploadResult> {
|
): Promise<StreamUploadResult> {
|
||||||
const { reqId, fileNamePrefix, fileExtension = ".txt", prefix, signal, maxConcurrency = 3 } = options;
|
const { reqId, fileNamePrefix, prefix, signal, maxConcurrency = 3 } = options;
|
||||||
|
|
||||||
const fileSize = file.size;
|
const fileSize = file.size;
|
||||||
let offset = 0;
|
let offset = 0;
|
||||||
@@ -434,7 +433,6 @@ export async function streamSplitAndUpload(
|
|||||||
|
|
||||||
// 获取文件名基础部分
|
// 获取文件名基础部分
|
||||||
const baseName = fileNamePrefix || file.name.replace(/\.[^/.]+$/, "");
|
const baseName = fileNamePrefix || file.name.replace(/\.[^/.]+$/, "");
|
||||||
const ext = fileExtension.startsWith(".") ? fileExtension : `.${fileExtension}`;
|
|
||||||
|
|
||||||
// 用于并发控制的队列
|
// 用于并发控制的队列
|
||||||
const uploadQueue: Promise<void>[] = [];
|
const uploadQueue: Promise<void>[] = [];
|
||||||
@@ -449,7 +447,7 @@ export async function streamSplitAndUpload(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const newFileName = `${baseName}_${String(index + 1).padStart(6, "0")}${ext}`;
|
const newFileName = `${baseName}_${String(index + 1).padStart(6, "0")}`;
|
||||||
const blob = new Blob([line], { type: "text/plain" });
|
const blob = new Blob([line], { type: "text/plain" });
|
||||||
const lineFile = new File([blob], newFileName, { type: "text/plain" });
|
const lineFile = new File([blob], newFileName, { type: "text/plain" });
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user