fix(upload): 修复流式上传中的文件名处理逻辑

- 修正预上传接口调用时传递正确的文件总数而非固定值-1
- 移除导入配置中文件分割时的文件扩展名保留逻辑
- 删除流式上传选项中的fileExtension参数定义
- 移除流式上传实现中的文件扩展名处理相关代码
- 简化新文件名生成逻辑,不再附加扩展名后缀
This commit is contained in:
2026-02-04 07:47:19 +08:00
parent c8611d29ff
commit f381d641ab
5 changed files with 36 additions and 23 deletions

View File

@@ -103,9 +103,6 @@ public class DatasetApplicationService {
Dataset dataset = datasetRepository.getById(datasetId); Dataset dataset = datasetRepository.getById(datasetId);
BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND); BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND);
// 保存原始的 parentDatasetId 值,用于比较是否发生了变化
String originalParentDatasetId = dataset.getParentDatasetId();
if (StringUtils.hasText(updateDatasetRequest.getName())) { if (StringUtils.hasText(updateDatasetRequest.getName())) {
dataset.setName(updateDatasetRequest.getName()); dataset.setName(updateDatasetRequest.getName());
} }
@@ -118,7 +115,11 @@ public class DatasetApplicationService {
if (Objects.nonNull(updateDatasetRequest.getStatus())) { if (Objects.nonNull(updateDatasetRequest.getStatus())) {
dataset.setStatus(updateDatasetRequest.getStatus()); dataset.setStatus(updateDatasetRequest.getStatus());
} }
// 处理父数据集变更:始终调用 handleParentChange,以支持设置新的关联或清除关联 if (updateDatasetRequest.isParentDatasetIdProvided()) {
// 保存原始的 parentDatasetId 值,用于比较是否发生了变化
String originalParentDatasetId = dataset.getParentDatasetId();
// 处理父数据集变更:仅当请求显式包含 parentDatasetId 时处理
// handleParentChange 内部通过 normalizeParentId 方法将空字符串和 null 都转换为 null // handleParentChange 内部通过 normalizeParentId 方法将空字符串和 null 都转换为 null
// 这样既支持设置新的父数据集,也支持清除关联 // 这样既支持设置新的父数据集,也支持清除关联
handleParentChange(dataset, updateDatasetRequest.getParentDatasetId()); handleParentChange(dataset, updateDatasetRequest.getParentDatasetId());
@@ -131,6 +132,7 @@ public class DatasetApplicationService {
.eq(Dataset::getId, datasetId) .eq(Dataset::getId, datasetId)
.set(Dataset::getParentDatasetId, dataset.getParentDatasetId())); .set(Dataset::getParentDatasetId, dataset.getParentDatasetId()));
} }
}
if (StringUtils.hasText(updateDatasetRequest.getDataSource())) { if (StringUtils.hasText(updateDatasetRequest.getDataSource())) {
// 数据源id不为空,使用异步线程进行文件扫盘落库 // 数据源id不为空,使用异步线程进行文件扫盘落库

View File

@@ -1,8 +1,10 @@
package com.datamate.datamanagement.interfaces.dto; package com.datamate.datamanagement.interfaces.dto;
import com.datamate.datamanagement.common.enums.DatasetStatusType; import com.datamate.datamanagement.common.enums.DatasetStatusType;
import com.fasterxml.jackson.annotation.JsonIgnore;
import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.Size; import jakarta.validation.constraints.Size;
import lombok.AccessLevel;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
@@ -24,9 +26,18 @@ public class UpdateDatasetRequest {
/** 归集任务id */ /** 归集任务id */
private String dataSource; private String dataSource;
/** 父数据集ID */ /** 父数据集ID */
@Setter(AccessLevel.NONE)
private String parentDatasetId; private String parentDatasetId;
@JsonIgnore
@Setter(AccessLevel.NONE)
private boolean parentDatasetIdProvided;
/** 标签列表 */ /** 标签列表 */
private List<String> tags; private List<String> tags;
/** 数据集状态 */ /** 数据集状态 */
private DatasetStatusType status; private DatasetStatusType status;
public void setParentDatasetId(String parentDatasetId) {
this.parentDatasetIdProvided = true;
this.parentDatasetId = parentDatasetId;
}
} }

View File

@@ -205,7 +205,7 @@ export function useFileSliceUpload(
// 预上传,获取 reqId // 预上传,获取 reqId
const totalSize = files.reduce((acc, file) => acc + file.size, 0); const totalSize = files.reduce((acc, file) => acc + file.size, 0);
const { data: reqId } = await preUpload(task.key, { const { data: reqId } = await preUpload(task.key, {
totalFileNum: -1, // 流式上传,文件数量不确定 totalFileNum: files.length,
totalSize, totalSize,
datasetId: task.key, datasetId: task.key,
hasArchive: task.hasArchive, hasArchive: task.hasArchive,

View File

@@ -90,14 +90,16 @@ async function splitFileByLines(file: UploadFile): Promise<UploadFile[]> {
const lines = text.split(/\r?\n/).filter((line: string) => line.trim() !== ""); const lines = text.split(/\r?\n/).filter((line: string) => line.trim() !== "");
if (lines.length === 0) return []; if (lines.length === 0) return [];
// 生成文件名:原文件名_序号.扩展名 // 生成文件名:原文件名_序号(不保留后缀)
const nameParts = file.name.split("."); const nameParts = file.name.split(".");
const ext = nameParts.length > 1 ? "." + nameParts.pop() : ""; if (nameParts.length > 1) {
nameParts.pop();
}
const baseName = nameParts.join("."); const baseName = nameParts.join(".");
const padLength = String(lines.length).length; const padLength = String(lines.length).length;
return lines.map((line: string, index: number) => { return lines.map((line: string, index: number) => {
const newFileName = `${baseName}_${String(index + 1).padStart(padLength, "0")}${ext}`; const newFileName = `${baseName}_${String(index + 1).padStart(padLength, "0")}`;
const blob = new Blob([line], { type: "text/plain" }); const blob = new Blob([line], { type: "text/plain" });
const newFile = new File([blob], newFileName, { type: "text/plain" }); const newFile = new File([blob], newFileName, { type: "text/plain" });
return { return {

View File

@@ -403,7 +403,6 @@ export function readFileAsText(
export interface StreamUploadOptions { export interface StreamUploadOptions {
reqId: number; reqId: number;
fileNamePrefix?: string; fileNamePrefix?: string;
fileExtension?: string;
hasArchive?: boolean; hasArchive?: boolean;
prefix?: string; prefix?: string;
signal?: AbortSignal; signal?: AbortSignal;
@@ -423,7 +422,7 @@ export async function streamSplitAndUpload(
chunkSize: number = 1024 * 1024, // 1MB chunkSize: number = 1024 * 1024, // 1MB
options: StreamUploadOptions options: StreamUploadOptions
): Promise<StreamUploadResult> { ): Promise<StreamUploadResult> {
const { reqId, fileNamePrefix, fileExtension = ".txt", prefix, signal, maxConcurrency = 3 } = options; const { reqId, fileNamePrefix, prefix, signal, maxConcurrency = 3 } = options;
const fileSize = file.size; const fileSize = file.size;
let offset = 0; let offset = 0;
@@ -434,7 +433,6 @@ export async function streamSplitAndUpload(
// 获取文件名基础部分 // 获取文件名基础部分
const baseName = fileNamePrefix || file.name.replace(/\.[^/.]+$/, ""); const baseName = fileNamePrefix || file.name.replace(/\.[^/.]+$/, "");
const ext = fileExtension.startsWith(".") ? fileExtension : `.${fileExtension}`;
// 用于并发控制的队列 // 用于并发控制的队列
const uploadQueue: Promise<void>[] = []; const uploadQueue: Promise<void>[] = [];
@@ -449,7 +447,7 @@ export async function streamSplitAndUpload(
return; return;
} }
const newFileName = `${baseName}_${String(index + 1).padStart(6, "0")}${ext}`; const newFileName = `${baseName}_${String(index + 1).padStart(6, "0")}`;
const blob = new Blob([line], { type: "text/plain" }); const blob = new Blob([line], { type: "text/plain" });
const lineFile = new File([blob], newFileName, { type: "text/plain" }); const lineFile = new File([blob], newFileName, { type: "text/plain" });