From 3a93098b57364b18aa689267097191ae7631a5b6 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Tue, 27 Jan 2026 18:11:30 +0800 Subject: [PATCH] =?UTF-8?q?feat(data-management):=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=9B=86=E6=96=87=E4=BB=B6=E6=A0=87=E6=B3=A8?= =?UTF-8?q?=E7=BB=93=E6=9E=9C=E8=BF=87=E6=BB=A4=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 在 OpenAPI 规范中添加 hasAnnotation 查询参数用于过滤存在标注结果的文件 - 修改后端服务层 DatasetFileApplicationService 支持 hasAnnotation 参数 - 更新数据访问层 DatasetFileRepositoryImpl 实现基于标注结果的存在性查询 - 调整前端 DatasetFileTransfer 组件支持标注过滤功能 - 移除无用的分块选项配置并优化全选逻辑 - 修复文件查询时的参数传递和依赖追踪问题 --- backend/openapi/specs/data-management.yaml | 19 ++-- .../DatasetFileApplicationService.java | 11 +- .../repository/DatasetFileRepository.java | 2 +- .../impl/DatasetFileRepositoryImpl.java | 7 +- .../rest/DatasetFileController.java | 34 +++--- .../business/DatasetFileTransfer.tsx | 102 ++++++++++-------- .../components/AddDataDialog.tsx | 9 +- 7 files changed, 100 insertions(+), 84 deletions(-) diff --git a/backend/openapi/specs/data-management.yaml b/backend/openapi/specs/data-management.yaml index 2e38bc7..81c9531 100644 --- a/backend/openapi/specs/data-management.yaml +++ b/backend/openapi/specs/data-management.yaml @@ -222,13 +222,18 @@ paths: schema: type: string description: 文件类型过滤 - - name: status - in: query - schema: - type: string - enum: [UPLOADED, PROCESSING, COMPLETED, ERROR] - description: 文件状态过滤 - responses: + - name: status + in: query + schema: + type: string + enum: [UPLOADED, PROCESSING, COMPLETED, ERROR] + description: 文件状态过滤 + - name: hasAnnotation + in: query + schema: + type: boolean + description: 是否仅返回存在标注结果的文件 + responses: '200': description: 成功 content: diff --git a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java index 90b06c6..c4da21e 100644 --- a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java +++ b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java @@ -89,11 +89,12 @@ public class DatasetFileApplicationService { * 获取数据集文件列表 */ @Transactional(readOnly = true) - public PagedResponse getDatasetFiles(String datasetId, String fileType, String status, String name, PagingQuery pagingQuery) { - IPage page = new Page<>(pagingQuery.getPage(), pagingQuery.getSize()); - IPage files = datasetFileRepository.findByCriteria(datasetId, fileType, status, name, page); - return PagedResponse.of(files); - } + public PagedResponse getDatasetFiles(String datasetId, String fileType, String status, String name, + Boolean hasAnnotation, PagingQuery pagingQuery) { + IPage page = new Page<>(pagingQuery.getPage(), pagingQuery.getSize()); + IPage files = datasetFileRepository.findByCriteria(datasetId, fileType, status, name, hasAnnotation, page); + return PagedResponse.of(files); + } /** * 获取数据集文件列表 diff --git a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/repository/DatasetFileRepository.java b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/repository/DatasetFileRepository.java index a4adcd0..76ccca4 100644 --- a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/repository/DatasetFileRepository.java +++ b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/repository/DatasetFileRepository.java @@ -24,7 +24,7 @@ public interface DatasetFileRepository extends IRepository { DatasetFile findByDatasetIdAndFileName(String datasetId, String fileName); IPage findByCriteria(String datasetId, String fileType, String status, String name, - IPage page); + Boolean hasAnnotation, IPage page); int updateFilePathPrefix(String datasetId, String oldPrefix, String newPrefix); } diff --git a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/repository/impl/DatasetFileRepositoryImpl.java b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/repository/impl/DatasetFileRepositoryImpl.java index cd9041b..1143059 100644 --- a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/repository/impl/DatasetFileRepositoryImpl.java +++ b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/infrastructure/persistence/repository/impl/DatasetFileRepositoryImpl.java @@ -22,6 +22,8 @@ import java.util.List; @RequiredArgsConstructor public class DatasetFileRepositoryImpl extends CrudRepository implements DatasetFileRepository { private final DatasetFileMapper datasetFileMapper; + private static final String ANNOTATION_EXISTS_SQL = + "SELECT 1 FROM t_dm_annotation_results ar WHERE ar.file_id = t_dm_dataset_files.id"; @Override public Long countByDatasetId(String datasetId) { @@ -49,12 +51,13 @@ public class DatasetFileRepositoryImpl extends CrudRepository findByCriteria(String datasetId, String fileType, String status, String name, - IPage page) { + Boolean hasAnnotation, IPage page) { return datasetFileMapper.selectPage(page, new LambdaQueryWrapper() .eq(DatasetFile::getDatasetId, datasetId) .eq(StringUtils.hasText(fileType), DatasetFile::getFileType, fileType) .eq(StringUtils.hasText(status), DatasetFile::getStatus, status) - .like(StringUtils.hasText(name), DatasetFile::getFileName, name)); + .like(StringUtils.hasText(name), DatasetFile::getFileName, name) + .exists(Boolean.TRUE.equals(hasAnnotation), ANNOTATION_EXISTS_SQL)); } @Override diff --git a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/rest/DatasetFileController.java b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/rest/DatasetFileController.java index bdd41f4..67914b0 100644 --- a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/rest/DatasetFileController.java +++ b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/rest/DatasetFileController.java @@ -43,22 +43,24 @@ public class DatasetFileController { this.datasetFileApplicationService = datasetFileApplicationService; } - @GetMapping - public Response> getDatasetFiles( - @PathVariable("datasetId") String datasetId, - @RequestParam(value = "isWithDirectory", required = false) boolean isWithDirectory, - @RequestParam(value = "page", required = false, defaultValue = "0") Integer page, - @RequestParam(value = "size", required = false, defaultValue = "20") Integer size, - @RequestParam(value = "prefix", required = false, defaultValue = "") String prefix) { - PagingQuery pagingQuery = new PagingQuery(page, size); - PagedResponse filesPage; - if (isWithDirectory) { - filesPage = datasetFileApplicationService.getDatasetFilesWithDirectory(datasetId, prefix, pagingQuery); - } else { - filesPage = datasetFileApplicationService.getDatasetFiles(datasetId, null, null, null, pagingQuery); - } - return Response.ok(filesPage); - } + @GetMapping + public Response> getDatasetFiles( + @PathVariable("datasetId") String datasetId, + @RequestParam(value = "isWithDirectory", required = false) boolean isWithDirectory, + @RequestParam(value = "page", required = false, defaultValue = "0") Integer page, + @RequestParam(value = "size", required = false, defaultValue = "20") Integer size, + @RequestParam(value = "prefix", required = false, defaultValue = "") String prefix, + @RequestParam(value = "status", required = false) String status, + @RequestParam(value = "hasAnnotation", required = false) Boolean hasAnnotation) { + PagingQuery pagingQuery = new PagingQuery(page, size); + PagedResponse filesPage; + if (isWithDirectory) { + filesPage = datasetFileApplicationService.getDatasetFilesWithDirectory(datasetId, prefix, pagingQuery); + } else { + filesPage = datasetFileApplicationService.getDatasetFiles(datasetId, null, status, null, hasAnnotation, pagingQuery); + } + return Response.ok(filesPage); + } @GetMapping("/{fileId}") public ResponseEntity> getDatasetFileById( diff --git a/frontend/src/components/business/DatasetFileTransfer.tsx b/frontend/src/components/business/DatasetFileTransfer.tsx index 1908d14..6b73b03 100644 --- a/frontend/src/components/business/DatasetFileTransfer.tsx +++ b/frontend/src/components/business/DatasetFileTransfer.tsx @@ -14,14 +14,15 @@ import { import { formatBytes } from "@/utils/unit"; import { useDebouncedEffect } from "@/hooks/useDebouncedEffect"; -interface DatasetFileTransferProps - extends React.HTMLAttributes { - open: boolean; - selectedFilesMap: { [key: string]: DatasetFile }; - onSelectedFilesChange: (filesMap: { [key: string]: DatasetFile }) => void; - onDatasetSelect?: (dataset: Dataset | null) => void; - datasetTypeFilter?: DatasetType; -} +interface DatasetFileTransferProps + extends React.HTMLAttributes { + open: boolean; + selectedFilesMap: { [key: string]: DatasetFile }; + onSelectedFilesChange: (filesMap: { [key: string]: DatasetFile }) => void; + onDatasetSelect?: (dataset: Dataset | null) => void; + datasetTypeFilter?: DatasetType; + hasAnnotationFilter?: boolean; +} const fileCols = [ { @@ -46,14 +47,15 @@ const fileCols = [ ]; // Customize Table Transfer -const DatasetFileTransfer: React.FC = ({ - open, - selectedFilesMap, - onSelectedFilesChange, - onDatasetSelect, - datasetTypeFilter = DatasetType.TEXT, - ...props -}) => { +const DatasetFileTransfer: React.FC = ({ + open, + selectedFilesMap, + onSelectedFilesChange, + onDatasetSelect, + datasetTypeFilter = DatasetType.TEXT, + hasAnnotationFilter, + ...props +}) => { const [datasets, setDatasets] = React.useState([]); const [datasetSearch, setDatasetSearch] = React.useState(""); const [datasetPagination, setDatasetPagination] = React.useState<{ @@ -64,11 +66,13 @@ const DatasetFileTransfer: React.FC = ({ const [files, setFiles] = React.useState([]); const [filesSearch, setFilesSearch] = React.useState(""); - const [filesPagination, setFilesPagination] = React.useState<{ - current: number; - pageSize: number; - total: number; - }>({ current: 1, pageSize: 10, total: 0 }); + const [filesPagination, setFilesPagination] = React.useState<{ + current: number; + pageSize: number; + total: number; + }>({ current: 1, pageSize: 10, total: 0 }); + const filesPage = filesPagination.current; + const filesPageSize = filesPagination.pageSize; const [showFiles, setShowFiles] = React.useState(false); const [selectedDataset, setSelectedDataset] = React.useState( @@ -104,18 +108,19 @@ const DatasetFileTransfer: React.FC = ({ const fetchFiles = useCallback( async ( - options?: Partial<{ page: number; pageSize: number; keyword: string }> - ) => { - if (!selectedDataset) return; - const page = options?.page ?? filesPagination.current; - const pageSize = options?.pageSize ?? filesPagination.pageSize; - const keyword = options?.keyword ?? filesSearch; - - const { data } = await queryDatasetFilesUsingGet(selectedDataset.id, { - page, - size: pageSize, - keyword, - }); + options?: Partial<{ page: number; pageSize: number; keyword: string }> + ) => { + if (!selectedDataset) return; + const page = options?.page ?? filesPage; + const pageSize = options?.pageSize ?? filesPageSize; + const keyword = options?.keyword ?? filesSearch; + + const { data } = await queryDatasetFilesUsingGet(selectedDataset.id, { + page, + size: pageSize, + keyword, + ...(hasAnnotationFilter ? { hasAnnotation: true } : {}), + }); setFiles( (data.content || []).map((item: DatasetFile) => ({ ...item, @@ -130,9 +135,15 @@ const DatasetFileTransfer: React.FC = ({ pageSize, total: data.totalElements, })); - }, - [selectedDataset, filesPagination.current, filesPagination.pageSize, filesSearch] - ); + }, + [ + selectedDataset, + filesPage, + filesPageSize, + filesSearch, + hasAnnotationFilter, + ] + ); useEffect(() => { // 当数据集变化时，重置文件分页并拉取第一页文件，避免额外的循环请求 @@ -167,10 +178,11 @@ const DatasetFileTransfer: React.FC = ({ const allFiles: DatasetFile[] = []; while (true) { - const { data } = await queryDatasetFilesUsingGet(selectedDataset.id, { - page, - size: pageSize, - }); + const { data } = await queryDatasetFilesUsingGet(selectedDataset.id, { + page, + size: pageSize, + ...(hasAnnotationFilter ? { hasAnnotation: true } : {}), + }); const content: DatasetFile[] = (data.content || []).map( (item: DatasetFile) => ({ @@ -217,7 +229,7 @@ const DatasetFileTransfer: React.FC = ({ } finally { setSelectingAll(false); } - }, [selectedDataset, selectedFilesMap, onSelectedFilesChange]); + }, [selectedDataset, selectedFilesMap, onSelectedFilesChange, hasAnnotationFilter]); const toggleSelectFile = (record: DatasetFile) => { if (!selectedFilesMap[record.id]) { @@ -388,10 +400,10 @@ const DatasetFileTransfer: React.FC = ({ }, // 全选 - 改为全选整个数据集而不是当前页 - onSelectAll: (selected, selectedRows: DatasetFile[]) => { - if (selected) { - // 点击表头“全选”时，改为一键全选当前数据集的全部文件 - // 而不是只选中当前页 + onSelectAll: (selected) => { + if (selected) { + // 点击表头“全选”时，改为一键全选当前数据集的全部文件 + // 而不是只选中当前页 handleSelectAllInDataset(); } else { // 取消表头“全选”时，清空当前已选文件 diff --git a/frontend/src/pages/KnowledgeBase/components/AddDataDialog.tsx b/frontend/src/pages/KnowledgeBase/components/AddDataDialog.tsx index b56a803..863b1f2 100644 --- a/frontend/src/pages/KnowledgeBase/components/AddDataDialog.tsx +++ b/frontend/src/pages/KnowledgeBase/components/AddDataDialog.tsx @@ -16,14 +16,6 @@ import DatasetFileTransfer from "@/components/business/DatasetFileTransfer"; import { DescriptionsItemType } from "antd/es/descriptions"; import { DatasetFileCols } from "../knowledge-base.const"; -const sliceOptions = [ - { label: "默认分块", value: "DEFAULT_CHUNK" }, - { label: "章节分块", value: "CHAPTER_CHUNK" }, - { label: "段落分块", value: "PARAGRAPH_CHUNK" }, - { label: "长度分块", value: "LENGTH_CHUNK" }, - { label: "自定义分割符分块", value: "CUSTOM_SEPARATOR_CHUNK" }, -]; - export default function AddDataDialog({ knowledgeBase, onDataAdded }) { const [open, setOpen] = useState(false); const { message } = App.useApp(); @@ -273,6 +265,7 @@ export default function AddDataDialog({ knowledgeBase, onDataAdded }) { open={open} selectedFilesMap={selectedFilesMap} onSelectedFilesChange={setSelectedFilesMap} + hasAnnotationFilter /> )}