You've already forked DataMate
fix(upload): 修复流式上传中的文件名处理逻辑
- 修正预上传接口调用时传递正确的文件总数而非固定值-1 - 移除导入配置中文件分割时的文件扩展名保留逻辑 - 删除流式上传选项中的fileExtension参数定义 - 移除流式上传实现中的文件扩展名处理相关代码 - 简化新文件名生成逻辑,不再附加扩展名后缀
This commit is contained in:
@@ -103,9 +103,6 @@ public class DatasetApplicationService {
|
||||
Dataset dataset = datasetRepository.getById(datasetId);
|
||||
BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND);
|
||||
|
||||
// 保存原始的 parentDatasetId 值,用于比较是否发生了变化
|
||||
String originalParentDatasetId = dataset.getParentDatasetId();
|
||||
|
||||
if (StringUtils.hasText(updateDatasetRequest.getName())) {
|
||||
dataset.setName(updateDatasetRequest.getName());
|
||||
}
|
||||
@@ -118,7 +115,11 @@ public class DatasetApplicationService {
|
||||
if (Objects.nonNull(updateDatasetRequest.getStatus())) {
|
||||
dataset.setStatus(updateDatasetRequest.getStatus());
|
||||
}
|
||||
// 处理父数据集变更:始终调用 handleParentChange,以支持设置新的关联或清除关联
|
||||
if (updateDatasetRequest.isParentDatasetIdProvided()) {
|
||||
// 保存原始的 parentDatasetId 值,用于比较是否发生了变化
|
||||
String originalParentDatasetId = dataset.getParentDatasetId();
|
||||
|
||||
// 处理父数据集变更:仅当请求显式包含 parentDatasetId 时处理
|
||||
// handleParentChange 内部通过 normalizeParentId 方法将空字符串和 null 都转换为 null
|
||||
// 这样既支持设置新的父数据集,也支持清除关联
|
||||
handleParentChange(dataset, updateDatasetRequest.getParentDatasetId());
|
||||
@@ -131,6 +132,7 @@ public class DatasetApplicationService {
|
||||
.eq(Dataset::getId, datasetId)
|
||||
.set(Dataset::getParentDatasetId, dataset.getParentDatasetId()));
|
||||
}
|
||||
}
|
||||
|
||||
if (StringUtils.hasText(updateDatasetRequest.getDataSource())) {
|
||||
// 数据源id不为空,使用异步线程进行文件扫盘落库
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package com.datamate.datamanagement.interfaces.dto;
|
||||
|
||||
import com.datamate.datamanagement.common.enums.DatasetStatusType;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.AccessLevel;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
@@ -24,9 +26,18 @@ public class UpdateDatasetRequest {
|
||||
/** 归集任务id */
|
||||
private String dataSource;
|
||||
/** 父数据集ID */
|
||||
@Setter(AccessLevel.NONE)
|
||||
private String parentDatasetId;
|
||||
@JsonIgnore
|
||||
@Setter(AccessLevel.NONE)
|
||||
private boolean parentDatasetIdProvided;
|
||||
/** 标签列表 */
|
||||
private List<String> tags;
|
||||
/** 数据集状态 */
|
||||
private DatasetStatusType status;
|
||||
|
||||
public void setParentDatasetId(String parentDatasetId) {
|
||||
this.parentDatasetIdProvided = true;
|
||||
this.parentDatasetId = parentDatasetId;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -205,7 +205,7 @@ export function useFileSliceUpload(
|
||||
// 预上传,获取 reqId
|
||||
const totalSize = files.reduce((acc, file) => acc + file.size, 0);
|
||||
const { data: reqId } = await preUpload(task.key, {
|
||||
totalFileNum: -1, // 流式上传,文件数量不确定
|
||||
totalFileNum: files.length,
|
||||
totalSize,
|
||||
datasetId: task.key,
|
||||
hasArchive: task.hasArchive,
|
||||
|
||||
@@ -90,14 +90,16 @@ async function splitFileByLines(file: UploadFile): Promise<UploadFile[]> {
|
||||
const lines = text.split(/\r?\n/).filter((line: string) => line.trim() !== "");
|
||||
if (lines.length === 0) return [];
|
||||
|
||||
// 生成文件名:原文件名_序号.扩展名
|
||||
// 生成文件名:原文件名_序号(不保留后缀)
|
||||
const nameParts = file.name.split(".");
|
||||
const ext = nameParts.length > 1 ? "." + nameParts.pop() : "";
|
||||
if (nameParts.length > 1) {
|
||||
nameParts.pop();
|
||||
}
|
||||
const baseName = nameParts.join(".");
|
||||
const padLength = String(lines.length).length;
|
||||
|
||||
return lines.map((line: string, index: number) => {
|
||||
const newFileName = `${baseName}_${String(index + 1).padStart(padLength, "0")}${ext}`;
|
||||
const newFileName = `${baseName}_${String(index + 1).padStart(padLength, "0")}`;
|
||||
const blob = new Blob([line], { type: "text/plain" });
|
||||
const newFile = new File([blob], newFileName, { type: "text/plain" });
|
||||
return {
|
||||
|
||||
@@ -403,7 +403,6 @@ export function readFileAsText(
|
||||
export interface StreamUploadOptions {
|
||||
reqId: number;
|
||||
fileNamePrefix?: string;
|
||||
fileExtension?: string;
|
||||
hasArchive?: boolean;
|
||||
prefix?: string;
|
||||
signal?: AbortSignal;
|
||||
@@ -423,7 +422,7 @@ export async function streamSplitAndUpload(
|
||||
chunkSize: number = 1024 * 1024, // 1MB
|
||||
options: StreamUploadOptions
|
||||
): Promise<StreamUploadResult> {
|
||||
const { reqId, fileNamePrefix, fileExtension = ".txt", prefix, signal, maxConcurrency = 3 } = options;
|
||||
const { reqId, fileNamePrefix, prefix, signal, maxConcurrency = 3 } = options;
|
||||
|
||||
const fileSize = file.size;
|
||||
let offset = 0;
|
||||
@@ -434,7 +433,6 @@ export async function streamSplitAndUpload(
|
||||
|
||||
// 获取文件名基础部分
|
||||
const baseName = fileNamePrefix || file.name.replace(/\.[^/.]+$/, "");
|
||||
const ext = fileExtension.startsWith(".") ? fileExtension : `.${fileExtension}`;
|
||||
|
||||
// 用于并发控制的队列
|
||||
const uploadQueue: Promise<void>[] = [];
|
||||
@@ -449,7 +447,7 @@ export async function streamSplitAndUpload(
|
||||
return;
|
||||
}
|
||||
|
||||
const newFileName = `${baseName}_${String(index + 1).padStart(6, "0")}${ext}`;
|
||||
const newFileName = `${baseName}_${String(index + 1).padStart(6, "0")}`;
|
||||
const blob = new Blob([line], { type: "text/plain" });
|
||||
const lineFile = new File([blob], newFileName, { type: "text/plain" });
|
||||
|
||||
|
||||
Reference in New Issue
Block a user