From 45743f39f5821b5a25cbc789fc17698c0cb1bf8b Mon Sep 17 00:00:00 2001 From: Jason Wang <56037774+JasonW404@users.noreply.github.com> Date: Thu, 13 Nov 2025 15:32:30 +0800 Subject: [PATCH] feat: add labeling template. refactor: switch to Poetry, build and deploy of backend Python (#79) * feat: Enhance annotation module with template management and validation - Added DatasetMappingCreateRequest and DatasetMappingUpdateRequest schemas to handle dataset mapping requests with camelCase and snake_case support. - Introduced Annotation Template schemas including CreateAnnotationTemplateRequest, UpdateAnnotationTemplateRequest, and AnnotationTemplateResponse for managing annotation templates. - Implemented AnnotationTemplateService for creating, updating, retrieving, and deleting annotation templates, including validation of configurations and XML generation. - Added utility class LabelStudioConfigValidator for validating Label Studio configurations and XML formats. - Updated database schema for annotation templates and labeling projects to include new fields and constraints. - Seeded initial annotation templates for various use cases including image classification, object detection, and text classification. * feat: Enhance TemplateForm with improved validation and dynamic field rendering; update LabelStudio config validation for camelCase support * feat: Update docker-compose.yml to mark datamate dataset volume and network as external * feat: Add tag configuration management and related components - Introduced new components for tag selection and browsing in the frontend. - Added API endpoint to fetch tag configuration from the backend. - Implemented tag configuration management in the backend, including loading from YAML. - Enhanced template service to support dynamic tag rendering based on configuration. - Updated validation utilities to incorporate tag configuration checks. - Refactored existing code to utilize the new tag configuration structure. * feat: Refactor LabelStudioTagConfig for improved configuration loading and validation * feat: Update Makefile to include backend-python-docker-build in the build process * feat: Migrate to poetry for better deps management * Add pyyaml dependency and update Dockerfile to use Poetry for dependency management - Added pyyaml (>=6.0.3,<7.0.0) to pyproject.toml dependencies. - Updated Dockerfile to install Poetry and manage dependencies using it. - Improved layer caching by copying only dependency files before the application code. - Removed unnecessary installation of build dependencies to keep the final image size small. * feat: Remove duplicated backend-python-docker-build target from Makefile * fix: airflow is not ready for adding yet * feat: update Python version to 3.12 and remove project installation step in Dockerfile --- Makefile | 12 +- frontend/src/hooks/useFetchData.ts | 2 +- frontend/src/hooks/useTagConfig.ts | 67 + .../components/CreateAnnotationTaskDialog.tsx | 9 +- .../components/CreateAnnptationTaskDialog.tsx | 4 +- .../DataAnnotation/Home/DataAnnotation.tsx | 4 +- .../DataAnnotation/Template/TemplateForm.tsx | 33 +- .../Template/VisualTemplateBuilder.tsx | 161 +++ .../Template/components/TagBrowser.tsx | 260 ++++ .../Template/components/TagSelector.tsx | 301 +++++ .../Template/components/index.ts | 3 + .../pages/DataAnnotation/Template/index.ts | 1 + .../pages/DataAnnotation/annotation.api.ts | 29 +- .../DataAnnotation/annotation.tagconfig.ts | 187 +++ runtime/datamate-python/app/main.py | 2 +- .../annotation/client/labelstudio/client.py | 6 +- .../app/module/annotation/config/__init__.py | 4 + .../annotation/config/label_studio_tags.yaml | 467 +++++++ .../module/annotation/config/tag_config.py | 150 +++ .../module/annotation/interface/__init__.py | 2 +- .../app/module/annotation/interface/about.py | 25 - .../app/module/annotation/interface/config.py | 47 + .../module/annotation/interface/project.py | 47 +- .../app/module/annotation/interface/task.py | 111 +- .../module/annotation/interface/template.py | 11 +- .../app/module/annotation/schema/__init__.py | 24 +- .../app/module/annotation/schema/config.py | 32 +- .../app/module/annotation/schema/tag.py | 17 + .../app/module/annotation/schema/template.py | 2 +- .../app/module/annotation/service/mapping.py | 6 +- .../app/module/annotation/service/sync.py | 28 +- .../app/module/annotation/service/template.py | 15 +- .../annotation/utils/config_validator.py | 84 +- .../app/module/dataset/schema/__init__.py | 8 + .../app/module/dataset/schema/dataset_file.py | 39 + .../app/module/dataset/service/service.py | 74 +- runtime/datamate-python/poetry.lock | 1138 +++++++++++++++++ runtime/datamate-python/pyproject.toml | 30 + runtime/datamate-python/requirements.txt | Bin 990 -> 0 bytes scripts/images/datamate-python/Dockerfile | 43 +- 40 files changed, 3223 insertions(+), 262 deletions(-) create mode 100644 frontend/src/hooks/useTagConfig.ts create mode 100644 frontend/src/pages/DataAnnotation/Template/VisualTemplateBuilder.tsx create mode 100644 frontend/src/pages/DataAnnotation/Template/components/TagBrowser.tsx create mode 100644 frontend/src/pages/DataAnnotation/Template/components/TagSelector.tsx create mode 100644 frontend/src/pages/DataAnnotation/Template/components/index.ts create mode 100644 frontend/src/pages/DataAnnotation/annotation.tagconfig.ts create mode 100644 runtime/datamate-python/app/module/annotation/config/__init__.py create mode 100644 runtime/datamate-python/app/module/annotation/config/label_studio_tags.yaml create mode 100644 runtime/datamate-python/app/module/annotation/config/tag_config.py delete mode 100644 runtime/datamate-python/app/module/annotation/interface/about.py create mode 100644 runtime/datamate-python/app/module/annotation/interface/config.py create mode 100644 runtime/datamate-python/app/module/annotation/schema/tag.py create mode 100644 runtime/datamate-python/poetry.lock create mode 100644 runtime/datamate-python/pyproject.toml delete mode 100644 runtime/datamate-python/requirements.txt diff --git a/Makefile b/Makefile index bc3cc0c..6b535fc 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ build-%: $(MAKE) $*-docker-build .PHONY: build -build: database-docker-build backend-docker-build frontend-docker-build runtime-docker-build +build: backend-docker-build frontend-docker-build runtime-docker-build backend-python-docker-build .PHONY: create-namespace create-namespace: @@ -117,9 +117,9 @@ frontend-docker-build: runtime-docker-build: docker build -t datamate-runtime:$(VERSION) . -f scripts/images/runtime/Dockerfile -.PHONY: label-studio-adapter-docker-build -label-studio-adapter-docker-build: - docker build -t label-studio-adapter:$(VERSION) . -f scripts/images/label-studio-adapter/Dockerfile +.PHONY: backend-python-docker-build +backend-python-docker-build: + docker build -t datamate-backend-python:$(VERSION) . -f scripts/images/datamate-python/Dockerfile .PHONY: deer-flow-docker-build deer-flow-docker-build: @@ -132,10 +132,6 @@ deer-flow-docker-build: mineru-docker-build: docker build -t datamate-mineru:$(VERSION) . -f scripts/images/mineru/Dockerfile -.PHONY: backend-python-docker-build -backend-python-docker-build: - docker build -t datamate-backend-python:$(VERSION) . -f scripts/images/datamate-python/Dockerfile - .PHONY: backend-docker-install backend-docker-install: cd deployment/docker/datamate && docker compose up -d backend diff --git a/frontend/src/hooks/useFetchData.ts b/frontend/src/hooks/useFetchData.ts index eb27d05..6a7aad3 100644 --- a/frontend/src/hooks/useFetchData.ts +++ b/frontend/src/hooks/useFetchData.ts @@ -110,7 +110,7 @@ export default function useFetchData( status: getFirstOfArray(filter?.status) || undefined, tags: filter?.tags?.length ? filter.tags.join(",") : undefined, page: current - pageOffset, - size: pageSize, + pageSize: pageSize, // Use camelCase for HTTP query params }), ...additionalPollingFuncs.map((func) => func()), ]; diff --git a/frontend/src/hooks/useTagConfig.ts b/frontend/src/hooks/useTagConfig.ts new file mode 100644 index 0000000..d547c54 --- /dev/null +++ b/frontend/src/hooks/useTagConfig.ts @@ -0,0 +1,67 @@ +import { useState, useEffect } from "react"; +import { message } from "antd"; +import { getTagConfigUsingGet } from "../pages/DataAnnotation/annotation.api"; +import type { LabelStudioTagConfig } from "../pages/DataAnnotation/annotation.tagconfig"; +import { parseTagConfig, type TagOption } from "../pages/DataAnnotation/annotation.tagconfig"; + +interface UseTagConfigReturn { + config: LabelStudioTagConfig | null; + objectOptions: TagOption[]; + controlOptions: TagOption[]; + loading: boolean; + error: string | null; + refetch: () => Promise; +} + +/** + * Hook to fetch and manage Label Studio tag configuration + * @param includeLabelingOnly - If true, only include controls with category="labeling" (default: true) + */ +export function useTagConfig(includeLabelingOnly: boolean = true): UseTagConfigReturn { + const [config, setConfig] = useState(null); + const [objectOptions, setObjectOptions] = useState([]); + const [controlOptions, setControlOptions] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + const fetchConfig = async () => { + setLoading(true); + setError(null); + try { + const response = await getTagConfigUsingGet(); + if (response.code === 200 && response.data) { + const tagConfig: LabelStudioTagConfig = response.data; + setConfig(tagConfig); + + const { objectOptions: objects, controlOptions: controls } = + parseTagConfig(tagConfig, includeLabelingOnly); + setObjectOptions(objects); + setControlOptions(controls); + } else { + const errorMsg = response.message || "获取标签配置失败"; + setError(errorMsg); + message.error(errorMsg); + } + } catch (err: any) { + const errorMsg = err.message || "加载标签配置时出错"; + setError(errorMsg); + console.error("Failed to fetch tag config:", err); + message.error(errorMsg); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + fetchConfig(); + }, []); + + return { + config, + objectOptions, + controlOptions, + loading, + error, + refetch: fetchConfig, + }; +} diff --git a/frontend/src/pages/DataAnnotation/Create/components/CreateAnnotationTaskDialog.tsx b/frontend/src/pages/DataAnnotation/Create/components/CreateAnnotationTaskDialog.tsx index 9f9ddbe..8652fb1 100644 --- a/frontend/src/pages/DataAnnotation/Create/components/CreateAnnotationTaskDialog.tsx +++ b/frontend/src/pages/DataAnnotation/Create/components/CreateAnnotationTaskDialog.tsx @@ -29,16 +29,16 @@ export default function CreateAnnotationTask({ // Fetch datasets const { data: datasetData } = await queryDatasetsUsingGet({ page: 0, - size: 1000, + pageSize: 1000, // Use camelCase for HTTP params }); setDatasets(datasetData.content.map(mapDataset) || []); // Fetch templates const templateResponse = await queryAnnotationTemplatesUsingGet({ page: 1, - size: 100, // Backend max is 100 + size: 100, // Backend max is 100 (template API uses 'size' not 'pageSize') }); - + // The API returns: {code, message, data: {content, total, page, ...}} if (templateResponse.code === 200 && templateResponse.data) { const fetchedTemplates = templateResponse.data.content || []; @@ -68,7 +68,6 @@ export default function CreateAnnotationTask({ try { const values = await form.validateFields(); setSubmitting(true); - // Send templateId instead of labelingConfig const requestData = { name: values.name, @@ -76,7 +75,6 @@ export default function CreateAnnotationTask({ datasetId: values.datasetId, templateId: values.templateId, }; - await createAnnotationTaskUsingPost(requestData); message?.success?.("创建标注任务成功"); onClose(); @@ -154,7 +152,6 @@ export default function CreateAnnotationTask({ /> - {/* 描述变为可选 */}