You've already forked DataMate
feat(annotation): 完善文件版本管理和标注同步功能
- 将 useNewVersionUsingPost 重命名为 applyNewVersionUsingPost - 添加 fileVersionCheckSeqRef 避免版本检查竞态条件 - 移除 checkingFileVersion 状态变量的渲染依赖 - 在文件版本信息中添加 annotationVersionUnknown 字段 - 修复前端文件版本比较显示的 JSX 语法 - 添加历史标注缺少版本信息的提示显示 - 配置 Alembic 异步数据库迁移环境支持 aiomysql - 添加文件版本未知状态的后端判断逻辑 - 实现标注清除时的段落注释清理功能 - 添加知识库同步钩子到版本更新流程
This commit is contained in:
@@ -9,7 +9,7 @@ import {
|
|||||||
listEditorTasksUsingGet,
|
listEditorTasksUsingGet,
|
||||||
upsertEditorAnnotationUsingPut,
|
upsertEditorAnnotationUsingPut,
|
||||||
checkFileVersionUsingGet,
|
checkFileVersionUsingGet,
|
||||||
useNewVersionUsingPost,
|
applyNewVersionUsingPost,
|
||||||
type FileVersionCheckResponse,
|
type FileVersionCheckResponse,
|
||||||
} from "../annotation.api";
|
} from "../annotation.api";
|
||||||
import { AnnotationResultStatus } from "../annotation.model";
|
import { AnnotationResultStatus } from "../annotation.model";
|
||||||
@@ -232,6 +232,7 @@ export default function LabelStudioTextEditor() {
|
|||||||
const origin = useMemo(() => window.location.origin, []);
|
const origin = useMemo(() => window.location.origin, []);
|
||||||
const iframeRef = useRef<HTMLIFrameElement | null>(null);
|
const iframeRef = useRef<HTMLIFrameElement | null>(null);
|
||||||
const initSeqRef = useRef(0);
|
const initSeqRef = useRef(0);
|
||||||
|
const fileVersionCheckSeqRef = useRef(0);
|
||||||
const expectedTaskIdRef = useRef<number | null>(null);
|
const expectedTaskIdRef = useRef<number | null>(null);
|
||||||
const prefetchSeqRef = useRef(0);
|
const prefetchSeqRef = useRef(0);
|
||||||
const exportCheckRef = useRef<{
|
const exportCheckRef = useRef<{
|
||||||
@@ -274,7 +275,7 @@ export default function LabelStudioTextEditor() {
|
|||||||
|
|
||||||
// 文件版本相关状态
|
// 文件版本相关状态
|
||||||
const [fileVersionInfo, setFileVersionInfo] = useState<FileVersionCheckResponse | null>(null);
|
const [fileVersionInfo, setFileVersionInfo] = useState<FileVersionCheckResponse | null>(null);
|
||||||
const [checkingFileVersion, setCheckingFileVersion] = useState(false);
|
const [, setCheckingFileVersion] = useState(false);
|
||||||
const [usingNewVersion, setUsingNewVersion] = useState(false);
|
const [usingNewVersion, setUsingNewVersion] = useState(false);
|
||||||
|
|
||||||
const focusIframe = useCallback(() => {
|
const focusIframe = useCallback(() => {
|
||||||
@@ -558,9 +559,11 @@ export default function LabelStudioTextEditor() {
|
|||||||
|
|
||||||
const checkFileVersion = useCallback(async (fileId: string) => {
|
const checkFileVersion = useCallback(async (fileId: string) => {
|
||||||
if (!projectId || !fileId) return;
|
if (!projectId || !fileId) return;
|
||||||
|
const seq = ++fileVersionCheckSeqRef.current;
|
||||||
setCheckingFileVersion(true);
|
setCheckingFileVersion(true);
|
||||||
try {
|
try {
|
||||||
const resp = (await checkFileVersionUsingGet(projectId, fileId)) as ApiResponse<FileVersionCheckResponse>;
|
const resp = (await checkFileVersionUsingGet(projectId, fileId)) as ApiResponse<FileVersionCheckResponse>;
|
||||||
|
if (seq !== fileVersionCheckSeqRef.current) return;
|
||||||
const data = resp?.data;
|
const data = resp?.data;
|
||||||
if (data) {
|
if (data) {
|
||||||
setFileVersionInfo(data);
|
setFileVersionInfo(data);
|
||||||
@@ -584,9 +587,9 @@ export default function LabelStudioTextEditor() {
|
|||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.error("检查文件版本失败", e);
|
console.error("检查文件版本失败", e);
|
||||||
} finally {
|
} finally {
|
||||||
setCheckingFileVersion(false);
|
if (seq === fileVersionCheckSeqRef.current) setCheckingFileVersion(false);
|
||||||
}
|
}
|
||||||
}, [modal, message, projectId]);
|
}, [modal, projectId]);
|
||||||
|
|
||||||
const handleUseNewVersion = useCallback(async () => {
|
const handleUseNewVersion = useCallback(async () => {
|
||||||
if (!selectedFileId) return;
|
if (!selectedFileId) return;
|
||||||
@@ -612,7 +615,7 @@ export default function LabelStudioTextEditor() {
|
|||||||
if (!projectId || !selectedFileId) return;
|
if (!projectId || !selectedFileId) return;
|
||||||
setUsingNewVersion(true);
|
setUsingNewVersion(true);
|
||||||
try {
|
try {
|
||||||
await useNewVersionUsingPost(projectId, selectedFileId);
|
await applyNewVersionUsingPost(projectId, selectedFileId);
|
||||||
message.success("已使用新版本并清空标注");
|
message.success("已使用新版本并清空标注");
|
||||||
setFileVersionInfo(null);
|
setFileVersionInfo(null);
|
||||||
await loadTasks({ mode: "reset" });
|
await loadTasks({ mode: "reset" });
|
||||||
@@ -985,7 +988,7 @@ export default function LabelStudioTextEditor() {
|
|||||||
{fileVersionInfo?.hasNewVersion && (
|
{fileVersionInfo?.hasNewVersion && (
|
||||||
<div className="flex items-center gap-2 mr-4">
|
<div className="flex items-center gap-2 mr-4">
|
||||||
<Typography.Text type="warning" className="text-xs">
|
<Typography.Text type="warning" className="text-xs">
|
||||||
⚠ 文件有新版本({fileVersionInfo.currentFileVersion} > {fileVersionInfo.annotationFileVersion})
|
⚠ 文件有新版本({fileVersionInfo.currentFileVersion}{" > "}{fileVersionInfo.annotationFileVersion})
|
||||||
</Typography.Text>
|
</Typography.Text>
|
||||||
<Button
|
<Button
|
||||||
size="small"
|
size="small"
|
||||||
@@ -998,6 +1001,11 @@ export default function LabelStudioTextEditor() {
|
|||||||
</Button>
|
</Button>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
{fileVersionInfo?.annotationVersionUnknown && !fileVersionInfo?.hasNewVersion && (
|
||||||
|
<Typography.Text type="secondary" className="text-xs mr-4">
|
||||||
|
历史标注缺少文件版本信息,保存后将绑定当前版本({fileVersionInfo.currentFileVersion})
|
||||||
|
</Typography.Text>
|
||||||
|
)}
|
||||||
<Button
|
<Button
|
||||||
type="primary"
|
type="primary"
|
||||||
icon={<SaveOutlined />}
|
icon={<SaveOutlined />}
|
||||||
|
|||||||
@@ -104,6 +104,7 @@ export interface FileVersionCheckResponse {
|
|||||||
fileId: string;
|
fileId: string;
|
||||||
currentFileVersion: number;
|
currentFileVersion: number;
|
||||||
annotationFileVersion: number | null;
|
annotationFileVersion: number | null;
|
||||||
|
annotationVersionUnknown: boolean;
|
||||||
hasNewVersion: boolean;
|
hasNewVersion: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -118,7 +119,7 @@ export interface UseNewVersionResponse {
|
|||||||
message: string;
|
message: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function useNewVersionUsingPost(projectId: string, fileId: string) {
|
export function applyNewVersionUsingPost(projectId: string, fileId: string) {
|
||||||
return post(`/api/annotation/editor/projects/${projectId}/files/${fileId}/use-new-version`, {});
|
return post(`/api/annotation/editor/projects/${projectId}/files/${fileId}/use-new-version`, {});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,27 +1,47 @@
|
|||||||
"""Alembic environment configuration"""
|
"""Alembic environment configuration
|
||||||
|
|
||||||
from logging.config import fileConfig
|
说明:
|
||||||
from sqlalchemy import engine_from_config, pool
|
- DataMate Python 服务使用异步 SQLAlchemy(mysql+aiomysql)。
|
||||||
from alembic import context
|
- 为了让 `alembic upgrade head` 开箱即用,这里自动从 `settings.database_url`
|
||||||
import sys
|
注入 `sqlalchemy.url`,无需在 alembic.ini 手工维护。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import os
|
import os
|
||||||
|
import sys
|
||||||
|
from logging.config import fileConfig
|
||||||
|
|
||||||
# 添加项目路径
|
from alembic import context
|
||||||
|
from sqlalchemy import pool
|
||||||
|
from sqlalchemy.ext.asyncio import async_engine_from_config
|
||||||
|
|
||||||
|
# 添加项目路径(runtime/datamate-python)
|
||||||
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
|
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
|
||||||
|
|
||||||
|
from app.core.config import settings
|
||||||
from app.db.session import Base
|
from app.db.session import Base
|
||||||
from app.db.models import *
|
from app.db.models import * # noqa: F401,F403
|
||||||
|
|
||||||
config = context.config
|
config = context.config
|
||||||
|
|
||||||
if config.config_file_name is not None:
|
if config.config_file_name is not None:
|
||||||
fileConfig(config.config_file_name)
|
fileConfig(config.config_file_name)
|
||||||
|
|
||||||
|
# 自动注入 sqlalchemy.url(优先使用 settings.database_url)
|
||||||
|
if not config.get_main_option("sqlalchemy.url"):
|
||||||
|
config.set_main_option("sqlalchemy.url", settings.database_url)
|
||||||
|
|
||||||
target_metadata = Base.metadata
|
target_metadata = Base.metadata
|
||||||
|
|
||||||
|
|
||||||
def run_migrations_offline() -> None:
|
def run_migrations_offline() -> None:
|
||||||
"""Run migrations in 'offline' mode."""
|
"""Run migrations in 'offline' mode."""
|
||||||
url = config.get_main_option("sqlalchemy.url")
|
url = config.get_main_option("sqlalchemy.url")
|
||||||
|
if not url:
|
||||||
|
raise RuntimeError("alembic 未配置 sqlalchemy.url,且无法从 settings.database_url 推导")
|
||||||
|
|
||||||
context.configure(
|
context.configure(
|
||||||
url=url,
|
url=url,
|
||||||
target_metadata=target_metadata,
|
target_metadata=target_metadata,
|
||||||
@@ -33,19 +53,29 @@ def run_migrations_offline() -> None:
|
|||||||
context.run_migrations()
|
context.run_migrations()
|
||||||
|
|
||||||
|
|
||||||
def run_migrations_online() -> None:
|
def do_run_migrations(connection) -> None:
|
||||||
"""Run migrations in 'online' mode."""
|
context.configure(connection=connection, target_metadata=target_metadata)
|
||||||
connectable = engine_from_config(
|
|
||||||
|
with context.begin_transaction():
|
||||||
|
context.run_migrations()
|
||||||
|
|
||||||
|
|
||||||
|
async def run_async_migrations() -> None:
|
||||||
|
connectable = async_engine_from_config(
|
||||||
config.get_section(config.config_ini_section, {}),
|
config.get_section(config.config_ini_section, {}),
|
||||||
prefix="sqlalchemy.",
|
prefix="sqlalchemy.",
|
||||||
poolclass=pool.NullPool,
|
poolclass=pool.NullPool,
|
||||||
)
|
)
|
||||||
|
|
||||||
with connectable.connect() as connection:
|
async with connectable.connect() as connection:
|
||||||
context.configure(connection=connection, target_metadata=target_metadata)
|
await connection.run_sync(do_run_migrations)
|
||||||
|
|
||||||
with context.begin_transaction():
|
await connectable.dispose()
|
||||||
context.run_migrations()
|
|
||||||
|
|
||||||
|
def run_migrations_online() -> None:
|
||||||
|
"""Run migrations in 'online' mode."""
|
||||||
|
asyncio.run(run_async_migrations())
|
||||||
|
|
||||||
|
|
||||||
if context.is_offline_mode():
|
if context.is_offline_mode():
|
||||||
|
|||||||
@@ -205,6 +205,11 @@ class FileVersionCheckResponse(BaseModel):
|
|||||||
annotation_file_version: Optional[int] = Field(
|
annotation_file_version: Optional[int] = Field(
|
||||||
None, alias="annotationFileVersion", description="标注时的文件版本"
|
None, alias="annotationFileVersion", description="标注时的文件版本"
|
||||||
)
|
)
|
||||||
|
annotation_version_unknown: bool = Field(
|
||||||
|
False,
|
||||||
|
alias="annotationVersionUnknown",
|
||||||
|
description="是否缺少标注时的文件版本(历史数据)",
|
||||||
|
)
|
||||||
has_new_version: bool = Field(
|
has_new_version: bool = Field(
|
||||||
..., alias="hasNewVersion", description="是否有新版本"
|
..., alias="hasNewVersion", description="是否有新版本"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1500,6 +1500,7 @@ class AnnotationEditorService:
|
|||||||
for file_record in valid_files:
|
for file_record in valid_files:
|
||||||
file_id = str(file_record.id) # type: ignore
|
file_id = str(file_record.id) # type: ignore
|
||||||
file_name = str(getattr(file_record, "file_name", ""))
|
file_name = str(getattr(file_record, "file_name", ""))
|
||||||
|
current_file_version = getattr(file_record, "version", None)
|
||||||
|
|
||||||
for retry in range(max_retries):
|
for retry in range(max_retries):
|
||||||
try:
|
try:
|
||||||
@@ -1602,6 +1603,7 @@ class AnnotationEditorService:
|
|||||||
# 更新现有标注
|
# 更新现有标注
|
||||||
existing.annotation = final_payload # type: ignore[assignment]
|
existing.annotation = final_payload # type: ignore[assignment]
|
||||||
existing.annotation_status = ANNOTATION_STATUS_IN_PROGRESS # type: ignore[assignment]
|
existing.annotation_status = ANNOTATION_STATUS_IN_PROGRESS # type: ignore[assignment]
|
||||||
|
existing.file_version = current_file_version # type: ignore[assignment]
|
||||||
existing.updated_at = now # type: ignore[assignment]
|
existing.updated_at = now # type: ignore[assignment]
|
||||||
else:
|
else:
|
||||||
# 创建新标注记录
|
# 创建新标注记录
|
||||||
@@ -1611,6 +1613,7 @@ class AnnotationEditorService:
|
|||||||
file_id=file_id,
|
file_id=file_id,
|
||||||
annotation=final_payload,
|
annotation=final_payload,
|
||||||
annotation_status=ANNOTATION_STATUS_IN_PROGRESS,
|
annotation_status=ANNOTATION_STATUS_IN_PROGRESS,
|
||||||
|
file_version=current_file_version,
|
||||||
created_at=now,
|
created_at=now,
|
||||||
updated_at=now,
|
updated_at=now,
|
||||||
)
|
)
|
||||||
@@ -1679,18 +1682,21 @@ class AnnotationEditorService:
|
|||||||
current_file_version = file_record.version
|
current_file_version = file_record.version
|
||||||
annotation_file_version = annotation.file_version if annotation else None
|
annotation_file_version = annotation.file_version if annotation else None
|
||||||
|
|
||||||
if annotation is None:
|
annotation_version_unknown = (
|
||||||
has_new_version = False
|
annotation is not None and annotation_file_version is None
|
||||||
elif annotation_file_version is None:
|
)
|
||||||
has_new_version = True
|
has_new_version = (
|
||||||
else:
|
current_file_version > annotation_file_version
|
||||||
has_new_version = current_file_version > annotation_file_version
|
if annotation_file_version is not None
|
||||||
|
else False
|
||||||
|
)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"fileId": file_id,
|
"fileId": file_id,
|
||||||
"currentFileVersion": current_file_version,
|
"currentFileVersion": current_file_version,
|
||||||
"annotationFileVersion": annotation_file_version,
|
"annotationFileVersion": annotation_file_version,
|
||||||
"hasNewVersion": has_new_version,
|
"hasNewVersion": has_new_version,
|
||||||
|
"annotationVersionUnknown": annotation_version_unknown,
|
||||||
}
|
}
|
||||||
|
|
||||||
async def use_new_version(self, project_id: str, file_id: str) -> Dict[str, Any]:
|
async def use_new_version(self, project_id: str, file_id: str) -> Dict[str, Any]:
|
||||||
@@ -1749,24 +1755,31 @@ class AnnotationEditorService:
|
|||||||
|
|
||||||
# 清空标注并更新版本号
|
# 清空标注并更新版本号
|
||||||
now = datetime.utcnow()
|
now = datetime.utcnow()
|
||||||
if isinstance(annotation.annotation, dict):
|
cleared_payload: Dict[str, Any] = {}
|
||||||
if annotation.annotation.get(SEGMENTED_KEY):
|
if isinstance(annotation.annotation, dict) and self._is_segmented_annotation(
|
||||||
segments = annotation.annotation.get(SEGMENTS_KEY, {})
|
annotation.annotation
|
||||||
for segment_id, segment_data in segments.items():
|
):
|
||||||
if isinstance(segment_data, dict):
|
segments = self._extract_segment_annotations(annotation.annotation)
|
||||||
segment_data[SEGMENT_RESULT_KEY] = []
|
cleared_segments: Dict[str, Dict[str, Any]] = {}
|
||||||
annotation.annotation = {
|
for segment_id, segment_data in segments.items():
|
||||||
SEGMENTED_KEY: True,
|
if not isinstance(segment_data, dict):
|
||||||
"version": annotation.annotation.get("version", 1),
|
continue
|
||||||
SEGMENTS_KEY: segments,
|
normalized = dict(segment_data)
|
||||||
"total_segments": annotation.annotation.get(
|
normalized[SEGMENT_RESULT_KEY] = []
|
||||||
"total_segments", len(segments)
|
cleared_segments[str(segment_id)] = normalized
|
||||||
),
|
|
||||||
}
|
total_segments = self._resolve_segment_total(annotation.annotation)
|
||||||
else:
|
if total_segments is None:
|
||||||
annotation.annotation = {}
|
total_segments = len(cleared_segments)
|
||||||
else:
|
|
||||||
annotation.annotation = {}
|
cleared_payload = {
|
||||||
|
SEGMENTED_KEY: True,
|
||||||
|
"version": annotation.annotation.get("version", 1),
|
||||||
|
SEGMENTS_KEY: cleared_segments,
|
||||||
|
SEGMENT_TOTAL_KEY: total_segments,
|
||||||
|
}
|
||||||
|
|
||||||
|
annotation.annotation = cleared_payload
|
||||||
annotation.annotation_status = ANNOTATION_STATUS_NO_ANNOTATION
|
annotation.annotation_status = ANNOTATION_STATUS_NO_ANNOTATION
|
||||||
annotation.file_version = current_file_version
|
annotation.file_version = current_file_version
|
||||||
annotation.updated_at = now
|
annotation.updated_at = now
|
||||||
@@ -1774,6 +1787,13 @@ class AnnotationEditorService:
|
|||||||
await self.db.commit()
|
await self.db.commit()
|
||||||
await self.db.refresh(annotation)
|
await self.db.refresh(annotation)
|
||||||
|
|
||||||
|
await self._sync_annotation_to_knowledge(
|
||||||
|
project,
|
||||||
|
file_record,
|
||||||
|
cleared_payload,
|
||||||
|
annotation.updated_at or now,
|
||||||
|
)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"fileId": file_id,
|
"fileId": file_id,
|
||||||
"previousFileVersion": previous_file_version,
|
"previousFileVersion": previous_file_version,
|
||||||
|
|||||||
Reference in New Issue
Block a user