You've already forked DataMate
feat(rag): 添加文件相对路径支持功能
- 在FileInfo DTO中新增relativePath字段 - 实现文件相对路径的规范化处理逻辑 - 将文件相对路径存储到元数据中 - 前端添加文件路径解析和显示功能 - 优化路径分隔符统一处理机制 - 更新文件列表展示逻辑以支持路径层级结构
This commit is contained in:
@@ -35,7 +35,9 @@ import org.springframework.transaction.annotation.Transactional;
|
|||||||
import org.springframework.util.StringUtils;
|
import org.springframework.util.StringUtils;
|
||||||
|
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -47,6 +49,8 @@ import java.util.Optional;
|
|||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class KnowledgeBaseService {
|
public class KnowledgeBaseService {
|
||||||
|
private static final String RELATIVE_PATH_KEY = "relativePath";
|
||||||
|
private static final String PATH_SEPARATOR = "/";
|
||||||
private final KnowledgeBaseRepository knowledgeBaseRepository;
|
private final KnowledgeBaseRepository knowledgeBaseRepository;
|
||||||
private final RagFileRepository ragFileRepository;
|
private final RagFileRepository ragFileRepository;
|
||||||
private final ApplicationEventPublisher eventPublisher;
|
private final ApplicationEventPublisher eventPublisher;
|
||||||
@@ -146,6 +150,12 @@ public class KnowledgeBaseService {
|
|||||||
ragFile.setKnowledgeBaseId(knowledgeBase.getId());
|
ragFile.setKnowledgeBaseId(knowledgeBase.getId());
|
||||||
ragFile.setFileId(fileInfo.id());
|
ragFile.setFileId(fileInfo.id());
|
||||||
ragFile.setFileName(fileInfo.fileName());
|
ragFile.setFileName(fileInfo.fileName());
|
||||||
|
String relativePath = normalizeRelativePath(fileInfo.relativePath());
|
||||||
|
if (StringUtils.hasText(relativePath)) {
|
||||||
|
Map<String, Object> metadata = new HashMap<>();
|
||||||
|
metadata.put(RELATIVE_PATH_KEY, relativePath);
|
||||||
|
ragFile.setMetadata(metadata);
|
||||||
|
}
|
||||||
ragFile.setStatus(FileStatus.UNPROCESSED);
|
ragFile.setStatus(FileStatus.UNPROCESSED);
|
||||||
return ragFile;
|
return ragFile;
|
||||||
}).toList();
|
}).toList();
|
||||||
@@ -153,6 +163,17 @@ public class KnowledgeBaseService {
|
|||||||
eventPublisher.publishEvent(new DataInsertedEvent(knowledgeBase, request));
|
eventPublisher.publishEvent(new DataInsertedEvent(knowledgeBase, request));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private String normalizeRelativePath(String relativePath) {
|
||||||
|
if (!StringUtils.hasText(relativePath)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
String normalized = relativePath.replace("\\", PATH_SEPARATOR).trim();
|
||||||
|
while (normalized.startsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(1);
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
public PagedResponse<RagFile> listFiles(String knowledgeBaseId, RagFileReq request) {
|
public PagedResponse<RagFile> listFiles(String knowledgeBaseId, RagFileReq request) {
|
||||||
IPage<RagFile> page = new Page<>(request.getPage(), request.getSize());
|
IPage<RagFile> page = new Page<>(request.getPage(), request.getSize());
|
||||||
request.setKnowledgeBaseId(knowledgeBaseId);
|
request.setKnowledgeBaseId(knowledgeBaseId);
|
||||||
|
|||||||
@@ -21,6 +21,6 @@ public class AddFilesReq {
|
|||||||
private String delimiter;
|
private String delimiter;
|
||||||
private List<FileInfo> files;
|
private List<FileInfo> files;
|
||||||
|
|
||||||
public record FileInfo(String id, String fileName) {
|
public record FileInfo(String id, String fileName, String relativePath) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -57,14 +57,19 @@ type KBFileRow = KBFile & {
|
|||||||
fileCount?: number;
|
fileCount?: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
const normalizePath = (value?: string) => (value ?? "").replace(/\\/g, "/");
|
const PATH_SEPARATOR = "/";
|
||||||
|
const RELATIVE_PATH_KEY = "relativePath";
|
||||||
|
const normalizePath = (value?: string) =>
|
||||||
|
(value ?? "").replace(/\\/g, PATH_SEPARATOR);
|
||||||
|
|
||||||
const normalizePrefix = (value?: string) => {
|
const normalizePrefix = (value?: string) => {
|
||||||
const trimmed = normalizePath(value).replace(/^\/+/, "").trim();
|
const trimmed = normalizePath(value).replace(/^\/+/, "").trim();
|
||||||
if (!trimmed) {
|
if (!trimmed) {
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
return trimmed.endsWith("/") ? trimmed : `${trimmed}/`;
|
return trimmed.endsWith(PATH_SEPARATOR)
|
||||||
|
? trimmed
|
||||||
|
: `${trimmed}${PATH_SEPARATOR}`;
|
||||||
};
|
};
|
||||||
|
|
||||||
const splitRelativePath = (fullPath: string, prefix: string) => {
|
const splitRelativePath = (fullPath: string, prefix: string) => {
|
||||||
@@ -72,7 +77,17 @@ const splitRelativePath = (fullPath: string, prefix: string) => {
|
|||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
const remainder = fullPath.slice(prefix.length);
|
const remainder = fullPath.slice(prefix.length);
|
||||||
return remainder.split("/").filter(Boolean);
|
return remainder.split(PATH_SEPARATOR).filter(Boolean);
|
||||||
|
};
|
||||||
|
|
||||||
|
const resolveFileRelativePath = (file: KBFile) => {
|
||||||
|
const metadata = file?.metadata as Record<string, unknown> | undefined;
|
||||||
|
const metadataPath =
|
||||||
|
metadata && typeof metadata[RELATIVE_PATH_KEY] === "string"
|
||||||
|
? String(metadata[RELATIVE_PATH_KEY])
|
||||||
|
: "";
|
||||||
|
const rawPath = metadataPath || file.fileName || file.name || "";
|
||||||
|
return normalizePath(rawPath).replace(/^\/+/, "");
|
||||||
};
|
};
|
||||||
|
|
||||||
const KnowledgeBaseDetailPage: React.FC = () => {
|
const KnowledgeBaseDetailPage: React.FC = () => {
|
||||||
@@ -196,9 +211,11 @@ const KnowledgeBaseDetailPage: React.FC = () => {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const trimmed = currentPrefix.replace(/\/$/, "");
|
const trimmed = currentPrefix.replace(/\/$/, "");
|
||||||
const parts = trimmed.split("/").filter(Boolean);
|
const parts = trimmed.split(PATH_SEPARATOR).filter(Boolean);
|
||||||
parts.pop();
|
parts.pop();
|
||||||
const parentPrefix = parts.length ? `${parts.join("/")}/` : "";
|
const parentPrefix = parts.length
|
||||||
|
? `${parts.join(PATH_SEPARATOR)}${PATH_SEPARATOR}`
|
||||||
|
: "";
|
||||||
setFilePrefix(parentPrefix);
|
setFilePrefix(parentPrefix);
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -210,7 +227,7 @@ const KnowledgeBaseDetailPage: React.FC = () => {
|
|||||||
const directoryPrefix = normalizePrefix(`${currentPrefix}${directoryName}`);
|
const directoryPrefix = normalizePrefix(`${currentPrefix}${directoryName}`);
|
||||||
const targetIds = allFiles
|
const targetIds = allFiles
|
||||||
.filter((file) => {
|
.filter((file) => {
|
||||||
const fullPath = normalizePath(file.fileName || file.name);
|
const fullPath = resolveFileRelativePath(file);
|
||||||
return fullPath.startsWith(directoryPrefix);
|
return fullPath.startsWith(directoryPrefix);
|
||||||
})
|
})
|
||||||
.map((file) => file.id);
|
.map((file) => file.id);
|
||||||
@@ -245,7 +262,7 @@ const KnowledgeBaseDetailPage: React.FC = () => {
|
|||||||
const fileItems: KBFileRow[] = [];
|
const fileItems: KBFileRow[] = [];
|
||||||
|
|
||||||
allFiles.forEach((file) => {
|
allFiles.forEach((file) => {
|
||||||
const fullPath = normalizePath(file.fileName || file.name);
|
const fullPath = resolveFileRelativePath(file);
|
||||||
if (!fullPath) {
|
if (!fullPath) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -278,10 +295,11 @@ const KnowledgeBaseDetailPage: React.FC = () => {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const displayName = file.fileName || leafName;
|
||||||
fileItems.push({
|
fileItems.push({
|
||||||
...file,
|
...file,
|
||||||
name: leafName,
|
name: displayName,
|
||||||
displayName: leafName,
|
displayName,
|
||||||
fullPath,
|
fullPath,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -26,19 +26,20 @@ export default function AddDataDialog({ knowledgeBase, onDataAdded }) {
|
|||||||
|
|
||||||
const [selectedFilesMap, setSelectedFilesMap] = useState({});
|
const [selectedFilesMap, setSelectedFilesMap] = useState({});
|
||||||
|
|
||||||
|
const PATH_SEPARATOR = "/";
|
||||||
const normalizePath = (value?: string) =>
|
const normalizePath = (value?: string) =>
|
||||||
(value ?? "").replace(/\\/g, "/");
|
(value ?? "").replace(/\\/g, PATH_SEPARATOR);
|
||||||
|
|
||||||
const resolveRelativeFileName = (file: DatasetFile) => {
|
const resolveRelativePath = (file: DatasetFile) => {
|
||||||
const normalizedName = normalizePath(file.fileName);
|
const normalizedName = normalizePath(file.fileName);
|
||||||
if (normalizedName.includes("/")) {
|
if (normalizedName.includes(PATH_SEPARATOR)) {
|
||||||
return normalizedName.replace(/^\/+/, "");
|
return normalizedName.replace(/^\/+/, "");
|
||||||
}
|
}
|
||||||
|
|
||||||
const rawPath = normalizePath(file.path || file.filePath);
|
const rawPath = normalizePath(file.path || file.filePath);
|
||||||
const datasetId = String(file.datasetId || "");
|
const datasetId = String(file.datasetId || "");
|
||||||
if (rawPath && datasetId) {
|
if (rawPath && datasetId) {
|
||||||
const marker = `/${datasetId}/`;
|
const marker = `${PATH_SEPARATOR}${datasetId}${PATH_SEPARATOR}`;
|
||||||
const index = rawPath.lastIndexOf(marker);
|
const index = rawPath.lastIndexOf(marker);
|
||||||
if (index >= 0) {
|
if (index >= 0) {
|
||||||
const relative = rawPath
|
const relative = rawPath
|
||||||
@@ -50,7 +51,7 @@ export default function AddDataDialog({ knowledgeBase, onDataAdded }) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const fallbackName = rawPath.split("/").pop();
|
const fallbackName = rawPath.split(PATH_SEPARATOR).pop();
|
||||||
return fallbackName || file.fileName;
|
return fallbackName || file.fileName;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -158,7 +159,8 @@ export default function AddDataDialog({ knowledgeBase, onDataAdded }) {
|
|||||||
const requestData = {
|
const requestData = {
|
||||||
files: Object.values(selectedFilesMap).map((file) => ({
|
files: Object.values(selectedFilesMap).map((file) => ({
|
||||||
id: String(file.id),
|
id: String(file.id),
|
||||||
fileName: resolveRelativeFileName(file as DatasetFile),
|
fileName: (file as DatasetFile).fileName,
|
||||||
|
relativePath: resolveRelativePath(file as DatasetFile),
|
||||||
})),
|
})),
|
||||||
processType: newKB.processType,
|
processType: newKB.processType,
|
||||||
chunkSize: Number(newKB.chunkSize), // 确保是数字类型
|
chunkSize: Number(newKB.chunkSize), // 确保是数字类型
|
||||||
|
|||||||
Reference in New Issue
Block a user