feat(repository): 添加查询衍生文件源文件ID功能

- 在 DatasetFileRepository 接口中添加 findSourceFileIdsWithDerivedFiles 方法定义 - 在 DatasetFileRepositoryImpl 实现类中实现该方法 - 添加查询 metadata 中包含 derived_from_file_id 记录的源文件ID逻辑 - 提供完整的 JavaDoc 文档注释说明方法用途和参数
2026-01-29 14:38:16 +08:00
parent 0dba604cd3
commit 6eb7507adf
8 changed files with 300 additions and 214 deletions
--- a/frontend/src/components/business/DatasetFileTransfer.tsx
+++ b/frontend/src/components/business/DatasetFileTransfer.tsx
@@ -14,15 +14,20 @@ import {
 import { formatBytes } from "@/utils/unit";
 import { useDebouncedEffect } from "@/hooks/useDebouncedEffect";

-interface DatasetFileTransferProps
-  extends React.HTMLAttributes<HTMLDivElement> {
-  open: boolean;
-  selectedFilesMap: { [key: string]: DatasetFile };
-  onSelectedFilesChange: (filesMap: { [key: string]: DatasetFile }) => void;
-  onDatasetSelect?: (dataset: Dataset | null) => void;
-  datasetTypeFilter?: DatasetType;
-  hasAnnotationFilter?: boolean;
-}
+interface DatasetFileTransferProps
+  extends React.HTMLAttributes<HTMLDivElement> {
+  open: boolean;
+  selectedFilesMap: { [key: string]: DatasetFile };
+  onSelectedFilesChange: (filesMap: { [key: string]: DatasetFile }) => void;
+  onDatasetSelect?: (dataset: Dataset | null) => void;
+  datasetTypeFilter?: DatasetType;
+  hasAnnotationFilter?: boolean;
+  /**
+   * 是否排除已被转换为TXT的源文档文件（PDF/DOC/DOCX）
+   * 默认为 true，当 datasetTypeFilter 为 TEXT 时自动启用
+   */
+  excludeSourceDocuments?: boolean;
+}

 const fileCols = [
  {
@@ -47,15 +52,18 @@ const fileCols = [
 ];

 // Customize Table Transfer
-const DatasetFileTransfer: React.FC<DatasetFileTransferProps> = ({
-  open,
-  selectedFilesMap,
-  onSelectedFilesChange,
-  onDatasetSelect,
-  datasetTypeFilter = DatasetType.TEXT,
-  hasAnnotationFilter,
-  ...props
-}) => {
+const DatasetFileTransfer: React.FC<DatasetFileTransferProps> = ({
+  open,
+  selectedFilesMap,
+  onSelectedFilesChange,
+  onDatasetSelect,
+  datasetTypeFilter = DatasetType.TEXT,
+  hasAnnotationFilter,
+  excludeSourceDocuments,
+  ...props
+}) => {
+  // 当未指定时，根据数据集类型自动决定是否排除源文档
+  const shouldExcludeSourceDocuments = excludeSourceDocuments ?? (datasetTypeFilter === DatasetType.TEXT);
  const [datasets, setDatasets] = React.useState<Dataset[]>([]);
  const [datasetSearch, setDatasetSearch] = React.useState<string>("");
  const [datasetPagination, setDatasetPagination] = React.useState<{
@@ -66,13 +74,13 @@ const DatasetFileTransfer: React.FC<DatasetFileTransferProps> = ({

  const [files, setFiles] = React.useState<DatasetFile[]>([]);
  const [filesSearch, setFilesSearch] = React.useState<string>("");
-  const [filesPagination, setFilesPagination] = React.useState<{
-    current: number;
-    pageSize: number;
-    total: number;
-  }>({ current: 1, pageSize: 10, total: 0 });
-  const filesPage = filesPagination.current;
-  const filesPageSize = filesPagination.pageSize;
+  const [filesPagination, setFilesPagination] = React.useState<{
+    current: number;
+    pageSize: number;
+    total: number;
+  }>({ current: 1, pageSize: 10, total: 0 });
+  const filesPage = filesPagination.current;
+  const filesPageSize = filesPagination.pageSize;

  const [showFiles, setShowFiles] = React.useState<boolean>(false);
  const [selectedDataset, setSelectedDataset] = React.useState<Dataset | null>(
@@ -108,19 +116,20 @@ const DatasetFileTransfer: React.FC<DatasetFileTransferProps> = ({

  const fetchFiles = useCallback(
    async (
-      options?: Partial<{ page: number; pageSize: number; keyword: string }>
-    ) => {
-      if (!selectedDataset) return;
-      const page = options?.page ?? filesPage;
-      const pageSize = options?.pageSize ?? filesPageSize;
-      const keyword = options?.keyword ?? filesSearch;
-
-      const { data } = await queryDatasetFilesUsingGet(selectedDataset.id, {
-        page,
-        size: pageSize,
-        keyword,
-        ...(hasAnnotationFilter ? { hasAnnotation: true } : {}),
-      });
+      options?: Partial<{ page: number; pageSize: number; keyword: string }>
+    ) => {
+      if (!selectedDataset) return;
+      const page = options?.page ?? filesPage;
+      const pageSize = options?.pageSize ?? filesPageSize;
+      const keyword = options?.keyword ?? filesSearch;
+
+      const { data } = await queryDatasetFilesUsingGet(selectedDataset.id, {
+        page,
+        size: pageSize,
+        keyword,
+        ...(hasAnnotationFilter ? { hasAnnotation: true } : {}),
+        ...(shouldExcludeSourceDocuments ? { excludeSourceDocuments: true } : {}),
+      });
      setFiles(
        (data.content || []).map((item: DatasetFile) => ({
          ...item,
@@ -135,15 +144,16 @@ const DatasetFileTransfer: React.FC<DatasetFileTransferProps> = ({
        pageSize,
        total: data.totalElements,
      }));
-    },
-    [
-      selectedDataset,
-      filesPage,
-      filesPageSize,
-      filesSearch,
-      hasAnnotationFilter,
-    ]
-  );
+    },
+    [
+      selectedDataset,
+      filesPage,
+      filesPageSize,
+      filesSearch,
+      hasAnnotationFilter,
+      shouldExcludeSourceDocuments,
+    ]
+  );

  useEffect(() => {
    // 当数据集变化时，重置文件分页并拉取第一页文件，避免额外的循环请求
@@ -178,11 +188,12 @@ const DatasetFileTransfer: React.FC<DatasetFileTransferProps> = ({
      const allFiles: DatasetFile[] = [];

      while (true) {
-        const { data } = await queryDatasetFilesUsingGet(selectedDataset.id, {
-          page,
-          size: pageSize,
-          ...(hasAnnotationFilter ? { hasAnnotation: true } : {}),
-        });
+        const { data } = await queryDatasetFilesUsingGet(selectedDataset.id, {
+          page,
+          size: pageSize,
+          ...(hasAnnotationFilter ? { hasAnnotation: true } : {}),
+          ...(shouldExcludeSourceDocuments ? { excludeSourceDocuments: true } : {}),
+        });

        const content: DatasetFile[] = (data.content || []).map(
          (item: DatasetFile) => ({
@@ -229,7 +240,7 @@ const DatasetFileTransfer: React.FC<DatasetFileTransferProps> = ({
    } finally {
      setSelectingAll(false);
    }
-  }, [selectedDataset, selectedFilesMap, onSelectedFilesChange, hasAnnotationFilter]);
+  }, [selectedDataset, selectedFilesMap, onSelectedFilesChange, hasAnnotationFilter, shouldExcludeSourceDocuments]);

  const toggleSelectFile = (record: DatasetFile) => {
    if (!selectedFilesMap[record.id]) {
@@ -400,10 +411,10 @@ const DatasetFileTransfer: React.FC<DatasetFileTransferProps> = ({
              },

              // 全选 - 改为全选整个数据集而不是当前页
-              onSelectAll: (selected) => {
-                if (selected) {
-                  // 点击表头“全选”时，改为一键全选当前数据集的全部文件
-                  // 而不是只选中当前页
+              onSelectAll: (selected) => {
+                if (selected) {
+                  // 点击表头“全选”时，改为一键全选当前数据集的全部文件
+                  // 而不是只选中当前页
                  handleSelectAllInDataset();
                } else {
                  // 取消表头“全选”时，清空当前已选文件
--- a/frontend/src/pages/DataAnnotation/Create/components/CreateAnnotationTaskDialog.tsx
+++ b/frontend/src/pages/DataAnnotation/Create/components/CreateAnnotationTaskDialog.tsx
@@ -275,7 +275,12 @@ export default function CreateAnnotationTask({
    }
    setDatasetPreviewLoading(true);
    try {
-      const res = await queryDatasetFilesUsingGet(selectedDatasetId, { page: 0, size: 10 });
+      // 对于文本数据集，排除已被转换为TXT的源文档文件（PDF/DOC/DOCX）
+      const params: { page: number; size: number; excludeSourceDocuments?: boolean } = { page: 0, size: 10 };
+      if (isTextDataset) {
+        params.excludeSourceDocuments = true;
+      }
+      const res = await queryDatasetFilesUsingGet(selectedDatasetId, params);
      if (res.code === '0' && res.data) {
        setDatasetPreviewData((res.data.content || []) as DatasetPreviewFile[]);
        setDatasetPreviewVisible(true);