q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: Labeling Frontend adaptations + Backend build and deploy + Logging improvement (#55)

Browse Source 
* feat: Front-end data annotation page adaptation to the backend API.

* feat: Implement labeling configuration editor and enhance annotation task creation form

* feat: add python backend build and deployment; add backend configuration for Label Studio integration and improve logging setup

* refactor: remove duplicate log configuration
This commit is contained in:
Jason Wang
2025-11-05 01:55:53 +08:00
committed by GitHub
parent f3958f08d9
commit b5fe787c20
13 changed files with 190 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
runtime/datamate-python/app/core/config.py
View File

@@ -1,4 +1,5 @@
from pydantic_settings import BaseSettings
from pydantic import model_validator
from typing import Optional, List
import os
from pathlib import Path
@@ -17,124 +18,60 @@ class Settings(BaseSettings):
app_name: str = "Label Studio Adapter"
app_version: str = "1.0.0"
app_description: str = "Adapter for integrating Data Management System with Label Studio"
# 日志配置
log_level: str = "INFO"
debug: bool = True
log_file_dir: str = "/var/log/datamate"
# 服务器配置
host: str = "0.0.0.0"
port: int = 8000
# CORS配置
allowed_origins: List[str] = ["*"]
allowed_methods: List[str] = ["*"]
allowed_headers: List[str] = ["*"]
# allowed_origins: List[str] = ["*"]
# allowed_methods: List[str] = ["*"]
# allowed_headers: List[str] = ["*"]
# MySQL数据库配置 (优先级1)
mysql_host: Optional[str] = None
mysql_host: str = "datamate-database"
mysql_port: int = 3306
mysql_user: Optional[str] = None
mysql_password: Optional[str] = None
mysql_database: Optional[str] = None
# PostgreSQL数据库配置 (优先级2)
postgres_host: Optional[str] = None
postgres_port: int = 5432
postgres_user: Optional[str] = None
postgres_password: Optional[str] = None
postgres_database: Optional[str] = None
# SQLite数据库配置 (优先级3 - 兜底)
sqlite_path: str = "data/labelstudio_adapter.db"
mysql_user: str = "root"
mysql_password: str = "password"
mysql_database: str = "datamate"
# 直接数据库URL配置（如果提供，将覆盖上述配置）
database_url: Optional[str] = None
# 初始值为空字符串，在 model_validator 中会被设置为完整的 URL
database_url: str = ""
@model_validator(mode='after')
def build_database_url(self):
"""如果没有提供 database_url，则根据 MySQL 配置构建"""
if not self.database_url:
if self.mysql_password and self.mysql_user:
self.database_url = f"mysql+aiomysql://{self.mysql_user}:{self.mysql_password}@{self.mysql_host}:{self.mysql_port}/{self.mysql_database}"
else:
self.database_url = f"mysql+aiomysql://{self.mysql_host}:{self.mysql_port}/{self.mysql_database}"
return self
# 日志配置
log_level: str = "DEBUG"
# =========================
# Label Studio 服务配置
# =========================
label_studio_base_url: str = "http://label-studio:8080"
label_studio_username: Optional[str] = None # Label Studio 用户名（用于登录）
label_studio_password: Optional[str] = None # Label Studio 密码（用于登录）
label_studio_user_token: Optional[str] = None # Legacy Token
label_studio_base_url: str = "http://label-studio:8000"
label_studio_username: Optional[str] = "admin@demo.com" # Label Studio 用户名（用于登录）
label_studio_password: Optional[str] = "demoadmin" # Label Studio 密码（用于登录）
label_studio_user_token: Optional[str] = "abc123abc123" # Legacy Token
label_studio_local_storage_dataset_base_path: str = "/label-studio/local_files" # Label Studio容器中的本地存储基础路径
label_studio_local_storage_dataset_base_path: str = "/label-studio/local" # Label Studio容器中的本地存储基础路径
label_studio_file_path_prefix: str = "/data/local-files/?d=" # Label Studio本地文件服务路径前缀
ls_task_page_size: int = 1000
# =========================
# Data Management 服务配置
# =========================
dm_file_path_prefix: str = "/dataset" # DM存储文件夹前缀
@property
def computed_database_url(self) -> str:
"""
根据优先级自动选择数据库连接URL
优先级：MySQL > PostgreSQL > SQLite3
"""
# 如果直接提供了database_url，优先使用
if self.database_url:
return self.database_url
# 优先级1: MySQL
if all([self.mysql_host, self.mysql_user, self.mysql_password, self.mysql_database]):
return f"mysql+aiomysql://{self.mysql_user}:{self.mysql_password}@{self.mysql_host}:{self.mysql_port}/{self.mysql_database}"
# 优先级2: PostgreSQL
if all([self.postgres_host, self.postgres_user, self.postgres_password, self.postgres_database]):
return f"postgresql+asyncpg://{self.postgres_user}:{self.postgres_password}@{self.postgres_host}:{self.postgres_port}/{self.postgres_database}"
# 优先级3: SQLite (兜底)
sqlite_full_path = Path(self.sqlite_path).absolute()
# 确保目录存在
sqlite_full_path.parent.mkdir(parents=True, exist_ok=True)
return f"sqlite+aiosqlite:///{sqlite_full_path}"
@property
def sync_database_url(self) -> str:
"""
用于数据库迁移的同步连接URL
将异步驱动替换为同步驱动
"""
async_url = self.computed_database_url
# 替换异步驱动为同步驱动
sync_replacements = {
"mysql+aiomysql://": "mysql+pymysql://",
"postgresql+asyncpg://": "postgresql+psycopg2://",
"sqlite+aiosqlite:///": "sqlite:///"
}
for async_driver, sync_driver in sync_replacements.items():
if async_url.startswith(async_driver):
return async_url.replace(async_driver, sync_driver)
return async_url
def get_database_info(self) -> dict:
"""获取数据库配置信息"""
url = self.computed_database_url
if url.startswith("mysql"):
db_type = "MySQL"
elif url.startswith("postgresql"):
db_type = "PostgreSQL"
elif url.startswith("sqlite"):
db_type = "SQLite"
else:
db_type = "Unknown"
return {
"type": db_type,
"url": url,
"sync_url": self.sync_database_url
}
# 全局设置实例
settings = Settings()