You've already forked DataMate
feat(annotation): 添加文本项目源文档过滤功能
- 在 EditorProjectInfo 中增加 datasetType 字段 - 移除前端硬编码的源文档扩展名列表 - 添加 isTextProject 判断逻辑 - 实现 prefetch 和 loadTasks 中的源文档排除参数 - 在后端接口中添加 excludeSourceDocuments 参数 - 实现源文档类型的数据库查询过滤逻辑 - 优化任务列表加载性能,避免不必要的源文档加载
This commit is contained in:
@@ -13,6 +13,7 @@ import {
|
||||
type EditorProjectInfo = {
|
||||
projectId: string;
|
||||
datasetId: string;
|
||||
datasetType?: string | null;
|
||||
templateId?: string | null;
|
||||
labelConfig?: string | null;
|
||||
supported: boolean;
|
||||
@@ -87,7 +88,6 @@ type SwitchDecision = "save" | "discard" | "cancel";
|
||||
const LSF_IFRAME_SRC = "/lsf/lsf.html";
|
||||
const TASK_PAGE_START = 0;
|
||||
const TASK_PAGE_SIZE = 200;
|
||||
const SOURCE_DOCUMENT_EXTENSIONS = [".pdf", ".doc", ".docx"];
|
||||
|
||||
type NormalizedTaskList = {
|
||||
items: EditorTaskListItem[];
|
||||
@@ -167,20 +167,12 @@ const mergeTaskItems = (base: EditorTaskListItem[], next: EditorTaskListItem[])
|
||||
return merged;
|
||||
};
|
||||
|
||||
const isSourceDocumentFile = (item: EditorTaskListItem) => {
|
||||
const fileName = item.fileName?.toLowerCase() ?? "";
|
||||
return SOURCE_DOCUMENT_EXTENSIONS.some((ext) => fileName.endsWith(ext));
|
||||
};
|
||||
|
||||
const filterSourceDocumentTasks = (items: EditorTaskListItem[]) =>
|
||||
items.filter((item) => !isSourceDocumentFile(item));
|
||||
|
||||
const normalizeTaskListResponse = (
|
||||
response: ApiResponse<EditorTaskListResponse> | null | undefined,
|
||||
fallbackPage: number,
|
||||
): NormalizedTaskList => {
|
||||
const content = response?.data?.content;
|
||||
const items = filterSourceDocumentTasks(Array.isArray(content) ? content : []);
|
||||
const items = Array.isArray(content) ? content : [];
|
||||
const size = response?.data?.size ?? TASK_PAGE_SIZE;
|
||||
const total = response?.data?.totalElements ?? items.length;
|
||||
const totalPages =
|
||||
@@ -237,6 +229,10 @@ export default function LabelStudioTextEditor() {
|
||||
const [segmented, setSegmented] = useState(false);
|
||||
const [segments, setSegments] = useState<SegmentInfo[]>([]);
|
||||
const [currentSegmentIndex, setCurrentSegmentIndex] = useState(0);
|
||||
const isTextProject = useMemo(
|
||||
() => (project?.datasetType || "").toUpperCase() === "TEXT",
|
||||
[project?.datasetType],
|
||||
);
|
||||
|
||||
const focusIframe = useCallback(() => {
|
||||
const iframe = iframeRef.current;
|
||||
@@ -292,9 +288,13 @@ export default function LabelStudioTextEditor() {
|
||||
for (let page = startPage; page < totalPages; page += 1) {
|
||||
if (prefetchSeqRef.current !== seq) return;
|
||||
try {
|
||||
const resp = (await listEditorTasksUsingGet(projectId, {
|
||||
const params = {
|
||||
page,
|
||||
size: TASK_PAGE_SIZE,
|
||||
...(isTextProject ? { excludeSourceDocuments: true } : {}),
|
||||
};
|
||||
const resp = (await listEditorTasksUsingGet(projectId, {
|
||||
...params,
|
||||
})) as ApiResponse<EditorTaskListResponse>;
|
||||
const normalized = normalizeTaskListResponse(resp, page);
|
||||
setTasks((prev) => mergeTaskItems(prev, normalized.items));
|
||||
@@ -311,7 +311,7 @@ export default function LabelStudioTextEditor() {
|
||||
}
|
||||
};
|
||||
void run();
|
||||
}, [projectId]);
|
||||
}, [isTextProject, projectId]);
|
||||
|
||||
const loadTasks = useCallback(async (options?: {
|
||||
mode?: "reset" | "append";
|
||||
@@ -334,9 +334,13 @@ export default function LabelStudioTextEditor() {
|
||||
}
|
||||
try {
|
||||
const nextPage = mode === "append" ? taskPage + 1 : TASK_PAGE_START;
|
||||
const resp = (await listEditorTasksUsingGet(projectId, {
|
||||
const params = {
|
||||
page: nextPage,
|
||||
size: TASK_PAGE_SIZE,
|
||||
...(isTextProject ? { excludeSourceDocuments: true } : {}),
|
||||
};
|
||||
const resp = (await listEditorTasksUsingGet(projectId, {
|
||||
...params,
|
||||
})) as ApiResponse<EditorTaskListResponse>;
|
||||
const normalized = normalizeTaskListResponse(resp, nextPage);
|
||||
if (mode === "append") {
|
||||
@@ -366,7 +370,15 @@ export default function LabelStudioTextEditor() {
|
||||
setLoadingTasks(false);
|
||||
}
|
||||
}
|
||||
}, [message, projectId, startPrefetchTasks, taskPage, taskTotalPages, updateTaskSelection]);
|
||||
}, [
|
||||
isTextProject,
|
||||
message,
|
||||
projectId,
|
||||
startPrefetchTasks,
|
||||
taskPage,
|
||||
taskTotalPages,
|
||||
updateTaskSelection,
|
||||
]);
|
||||
|
||||
const initEditorForFile = useCallback(async (fileId: string, segmentIdx?: number) => {
|
||||
if (!project?.supported) return;
|
||||
|
||||
Reference in New Issue
Block a user