feat(repository): 添加查询衍生文件源文件ID功能

- 在 DatasetFileRepository 接口中添加 findSourceFileIdsWithDerivedFiles 方法定义
- 在 DatasetFileRepositoryImpl 实现类中实现该方法
- 添加查询 metadata 中包含 derived_from_file_id 记录的源文件ID逻辑
- 提供完整的 JavaDoc 文档注释说明方法用途和参数

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java

@@ -100,8 +100,43 @@ public class DatasetFileApplicationService {
     @Transactional(readOnly = true)
     public PagedResponse<DatasetFile> getDatasetFiles(String datasetId, String fileType, String status, String name,
                                                       Boolean hasAnnotation, PagingQuery pagingQuery) {
+        return getDatasetFiles(datasetId, fileType, status, name, hasAnnotation, false, pagingQuery);
+    }
+
+    /**
+     * 获取数据集文件列表，支持排除已被转换为TXT的源文档文件
+     *
+     * @param datasetId             数据集ID
+     * @param fileType              文件类型过滤
+     * @param status                状态过滤
+     * @param name                  文件名模糊查询
+     * @param hasAnnotation         是否有标注
+     * @param excludeSourceDocuments 是否排除已被转换为TXT的源文档（PDF/DOC/DOCX）
+     * @param pagingQuery           分页参数
+     * @return 分页文件列表
+     */
+    @Transactional(readOnly = true)
+    public PagedResponse<DatasetFile> getDatasetFiles(String datasetId, String fileType, String status, String name,
+                                                      Boolean hasAnnotation, boolean excludeSourceDocuments, PagingQuery pagingQuery) {
         IPage<DatasetFile> page = new Page<>(pagingQuery.getPage(), pagingQuery.getSize());
         IPage<DatasetFile> files = datasetFileRepository.findByCriteria(datasetId, fileType, status, name, hasAnnotation, page);
+        
+        if (excludeSourceDocuments) {
+            // 查询所有作为衍生TXT文件源的文档文件ID
+            List<String> sourceFileIds = datasetFileRepository.findSourceFileIdsWithDerivedFiles(datasetId);
+            if (!sourceFileIds.isEmpty()) {
+                // 过滤掉源文件
+                List<DatasetFile> filteredRecords = files.getRecords().stream()
+                        .filter(file -> !sourceFileIds.contains(file.getId()))
+                        .collect(Collectors.toList());
+                
+                // 重新构建分页结果
+                Page<DatasetFile> filteredPage = new Page<>(files.getCurrent(), files.getSize(), files.getTotal());
+                filteredPage.setRecords(filteredRecords);
+                return PagedResponse.of(filteredPage);
+            }
+        }
+        
         return PagedResponse.of(files);
     }
 

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/mapper/DatasetFileMapper.java

@@ -29,4 +29,13 @@ public interface DatasetFileMapper extends BaseMapper<DatasetFile> {
     int updateFilePathPrefix(@Param("datasetId") String datasetId,
                              @Param("oldPrefix") String oldPrefix,
                              @Param("newPrefix") String newPrefix);
+
+    /**
+     * 查询数据集中所有作为衍生文件源文件的ID列表
+     * 通过查询 metadata 中包含 derived_from_file_id 的字段值
+     *
+     * @param datasetId 数据集ID
+     * @return 源文件ID列表
+     */
+    List<String> findSourceFileIdsWithDerivedFiles(@Param("datasetId") String datasetId);
 }

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/repository/DatasetFileRepository.java

@@ -27,4 +27,13 @@ public interface DatasetFileRepository extends IRepository<DatasetFile> {
                                       Boolean hasAnnotation, IPage<DatasetFile> page);
 
     int updateFilePathPrefix(String datasetId, String oldPrefix, String newPrefix);
+
+    /**
+     * 查询数据集中所有作为衍生文件源文件的ID列表
+     * 通过查询 metadata 中包含 derived_from_file_id 的记录，返回所有源文件ID
+     *
+     * @param datasetId 数据集ID
+     * @return 源文件ID列表
+     */
+    List<String> findSourceFileIdsWithDerivedFiles(String datasetId);
 }

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/repository/impl/DatasetFileRepositoryImpl.java

@@ -64,4 +64,11 @@ public class DatasetFileRepositoryImpl extends CrudRepository<DatasetFileMapper,
     public int updateFilePathPrefix(String datasetId, String oldPrefix, String newPrefix) {
         return datasetFileMapper.updateFilePathPrefix(datasetId, oldPrefix, newPrefix);
     }
+
+    @Override
+    public List<String> findSourceFileIdsWithDerivedFiles(String datasetId) {
+        // 查询 metadata 中包含 derived_from_file_id 的记录的源文件ID
+        // 使用 MyBatis 的 @Select 注解或直接调用 mapper 方法
+        return datasetFileMapper.findSourceFileIdsWithDerivedFiles(datasetId);
+    }
 }

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/rest/DatasetFileController.java

@@ -51,13 +51,15 @@ public class DatasetFileController {
             @RequestParam(value = "size", required = false, defaultValue = "20") Integer size,
             @RequestParam(value = "prefix", required = false, defaultValue = "") String prefix,
             @RequestParam(value = "status", required = false) String status,
-            @RequestParam(value = "hasAnnotation", required = false) Boolean hasAnnotation) {
+            @RequestParam(value = "hasAnnotation", required = false) Boolean hasAnnotation,
+            @RequestParam(value = "excludeSourceDocuments", required = false, defaultValue = "false") Boolean excludeSourceDocuments) {
         PagingQuery pagingQuery = new PagingQuery(page, size);
         PagedResponse<DatasetFile> filesPage;
         if (isWithDirectory) {
             filesPage = datasetFileApplicationService.getDatasetFilesWithDirectory(datasetId, prefix, pagingQuery);
         } else {
-            filesPage = datasetFileApplicationService.getDatasetFiles(datasetId, null, status, null, hasAnnotation, pagingQuery);
+            filesPage = datasetFileApplicationService.getDatasetFiles(datasetId, null, status, null, hasAnnotation, 
+                    Boolean.TRUE.equals(excludeSourceDocuments), pagingQuery);
         }
         return Response.ok(filesPage);
     }

backend/services/data-management-service/src/main/resources/mappers/DatasetFileMapper.xml

@@ -102,4 +102,12 @@
         WHERE dataset_id = #{datasetId}
           AND file_path LIKE CONCAT(#{oldPrefix}, '%')
     </update>
+
+    <select id="findSourceFileIdsWithDerivedFiles" resultType="java.lang.String">
+        SELECT DISTINCT JSON_UNQUOTE(JSON_EXTRACT(metadata, '$.derived_from_file_id')) AS source_file_id
+        FROM t_dm_dataset_files
+        WHERE dataset_id = #{datasetId}
+          AND metadata IS NOT NULL
+          AND JSON_EXTRACT(metadata, '$.derived_from_file_id') IS NOT NULL
+    </select>
 </mapper>

frontend/src/components/business/DatasetFileTransfer.tsx

@@ -22,6 +22,11 @@ interface DatasetFileTransferProps
   onDatasetSelect?: (dataset: Dataset | null) => void;
   datasetTypeFilter?: DatasetType;
   hasAnnotationFilter?: boolean;
+  /**
+   * 是否排除已被转换为TXT的源文档文件（PDF/DOC/DOCX）
+   * 默认为 true，当 datasetTypeFilter 为 TEXT 时自动启用
+   */
+  excludeSourceDocuments?: boolean;
 }
 
 const fileCols = [
@@ -54,8 +59,11 @@ const DatasetFileTransfer: React.FC<DatasetFileTransferProps> = ({
   onDatasetSelect,
   datasetTypeFilter = DatasetType.TEXT,
   hasAnnotationFilter,
+  excludeSourceDocuments,
   ...props
 }) => {
+  // 当未指定时，根据数据集类型自动决定是否排除源文档
+  const shouldExcludeSourceDocuments = excludeSourceDocuments ?? (datasetTypeFilter === DatasetType.TEXT);
   const [datasets, setDatasets] = React.useState<Dataset[]>([]);
   const [datasetSearch, setDatasetSearch] = React.useState<string>("");
   const [datasetPagination, setDatasetPagination] = React.useState<{
@@ -120,6 +128,7 @@ const DatasetFileTransfer: React.FC<DatasetFileTransferProps> = ({
         size: pageSize,
         keyword,
         ...(hasAnnotationFilter ? { hasAnnotation: true } : {}),
+        ...(shouldExcludeSourceDocuments ? { excludeSourceDocuments: true } : {}),
       });
       setFiles(
         (data.content || []).map((item: DatasetFile) => ({
@@ -142,6 +151,7 @@ const DatasetFileTransfer: React.FC<DatasetFileTransferProps> = ({
       filesPageSize,
       filesSearch,
       hasAnnotationFilter,
+      shouldExcludeSourceDocuments,
     ]
   );
 
@@ -182,6 +192,7 @@ const DatasetFileTransfer: React.FC<DatasetFileTransferProps> = ({
           page,
           size: pageSize,
           ...(hasAnnotationFilter ? { hasAnnotation: true } : {}),
+          ...(shouldExcludeSourceDocuments ? { excludeSourceDocuments: true } : {}),
         });
 
         const content: DatasetFile[] = (data.content || []).map(
@@ -229,7 +240,7 @@ const DatasetFileTransfer: React.FC<DatasetFileTransferProps> = ({
     } finally {
       setSelectingAll(false);
     }
-  }, [selectedDataset, selectedFilesMap, onSelectedFilesChange, hasAnnotationFilter]);
+  }, [selectedDataset, selectedFilesMap, onSelectedFilesChange, hasAnnotationFilter, shouldExcludeSourceDocuments]);
 
   const toggleSelectFile = (record: DatasetFile) => {
     if (!selectedFilesMap[record.id]) {

frontend/src/pages/DataAnnotation/Create/components/CreateAnnotationTaskDialog.tsx

@@ -275,7 +275,12 @@ export default function CreateAnnotationTask({
     }
     setDatasetPreviewLoading(true);
     try {
-      const res = await queryDatasetFilesUsingGet(selectedDatasetId, { page: 0, size: 10 });
+      // 对于文本数据集，排除已被转换为TXT的源文档文件（PDF/DOC/DOCX）
+      const params: { page: number; size: number; excludeSourceDocuments?: boolean } = { page: 0, size: 10 };
+      if (isTextDataset) {
+        params.excludeSourceDocuments = true;
+      }
+      const res = await queryDatasetFilesUsingGet(selectedDatasetId, params);
       if (res.code === '0' && res.data) {
         setDatasetPreviewData((res.data.content || []) as DatasetPreviewFile[]);
         setDatasetPreviewVisible(true);