You've already forked DataMate
Develop labeling module (#25)
* refactor: remove db table management from LS adapter (mv to scripts later); change adapter to use the same MySQL DB as other modules. * refactor: Rename LS Adapter module to datamate-python
This commit is contained in:
13
runtime/datamate-python/app/models/cleaning/__init__.py
Normal file
13
runtime/datamate-python/app/models/cleaning/__init__.py
Normal file
@@ -0,0 +1,13 @@
|
||||
# app/models/cleaning/__init__.py
|
||||
|
||||
from .clean_template import CleanTemplate
|
||||
from .clean_task import CleanTask
|
||||
from .operator_instance import OperatorInstance
|
||||
from .clean_result import CleanResult
|
||||
|
||||
__all__ = [
|
||||
"CleanTemplate",
|
||||
"CleanTask",
|
||||
"OperatorInstance",
|
||||
"CleanResult"
|
||||
]
|
||||
22
runtime/datamate-python/app/models/cleaning/clean_result.py
Normal file
22
runtime/datamate-python/app/models/cleaning/clean_result.py
Normal file
@@ -0,0 +1,22 @@
|
||||
from sqlalchemy import Column, String, BigInteger, Text
|
||||
from app.db.database import Base
|
||||
|
||||
class CleanResult(Base):
|
||||
"""清洗结果模型"""
|
||||
|
||||
__tablename__ = "t_clean_result"
|
||||
|
||||
instance_id = Column(String(64), primary_key=True, comment="实例ID")
|
||||
src_file_id = Column(String(64), nullable=True, comment="源文件ID")
|
||||
dest_file_id = Column(String(64), primary_key=True, comment="目标文件ID")
|
||||
src_name = Column(String(256), nullable=True, comment="源文件名")
|
||||
dest_name = Column(String(256), nullable=True, comment="目标文件名")
|
||||
src_type = Column(String(256), nullable=True, comment="源文件类型")
|
||||
dest_type = Column(String(256), nullable=True, comment="目标文件类型")
|
||||
src_size = Column(BigInteger, nullable=True, comment="源文件大小")
|
||||
dest_size = Column(BigInteger, nullable=True, comment="目标文件大小")
|
||||
status = Column(String(256), nullable=True, comment="处理状态")
|
||||
result = Column(Text, nullable=True, comment="处理结果")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<CleanResult(instance_id={self.instance_id}, dest_file_id={self.dest_file_id}, status={self.status})>"
|
||||
27
runtime/datamate-python/app/models/cleaning/clean_task.py
Normal file
27
runtime/datamate-python/app/models/cleaning/clean_task.py
Normal file
@@ -0,0 +1,27 @@
|
||||
from sqlalchemy import Column, String, BigInteger, Integer, TIMESTAMP
|
||||
from sqlalchemy.sql import func
|
||||
from app.db.database import Base
|
||||
|
||||
class CleanTask(Base):
|
||||
"""清洗任务模型"""
|
||||
|
||||
__tablename__ = "t_clean_task"
|
||||
|
||||
id = Column(String(64), primary_key=True, comment="任务ID")
|
||||
name = Column(String(64), nullable=True, comment="任务名称")
|
||||
description = Column(String(256), nullable=True, comment="任务描述")
|
||||
status = Column(String(256), nullable=True, comment="任务状态")
|
||||
src_dataset_id = Column(String(64), nullable=True, comment="源数据集ID")
|
||||
src_dataset_name = Column(String(64), nullable=True, comment="源数据集名称")
|
||||
dest_dataset_id = Column(String(64), nullable=True, comment="目标数据集ID")
|
||||
dest_dataset_name = Column(String(64), nullable=True, comment="目标数据集名称")
|
||||
before_size = Column(BigInteger, nullable=True, comment="清洗前大小")
|
||||
after_size = Column(BigInteger, nullable=True, comment="清洗后大小")
|
||||
file_count = Column(Integer, nullable=True, comment="文件数量")
|
||||
created_at = Column(TIMESTAMP, server_default=func.current_timestamp(), comment="创建时间")
|
||||
started_at = Column(TIMESTAMP, nullable=True, comment="开始时间")
|
||||
finished_at = Column(TIMESTAMP, nullable=True, comment="完成时间")
|
||||
created_by = Column(String(256), nullable=True, comment="创建者")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<CleanTask(id={self.id}, name={self.name}, status={self.status})>"
|
||||
@@ -0,0 +1,18 @@
|
||||
from sqlalchemy import Column, String, Text, TIMESTAMP
|
||||
from sqlalchemy.sql import func
|
||||
from app.db.database import Base
|
||||
|
||||
class CleanTemplate(Base):
|
||||
"""清洗模板模型"""
|
||||
|
||||
__tablename__ = "t_clean_template"
|
||||
|
||||
id = Column(String(64), primary_key=True, unique=True, comment="模板ID")
|
||||
name = Column(String(64), nullable=True, comment="模板名称")
|
||||
description = Column(String(256), nullable=True, comment="模板描述")
|
||||
created_at = Column(TIMESTAMP, server_default=func.current_timestamp(), comment="创建时间")
|
||||
updated_at = Column(TIMESTAMP, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), comment="更新时间")
|
||||
created_by = Column(String(256), nullable=True, comment="创建者")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<CleanTemplate(id={self.id}, name={self.name})>"
|
||||
@@ -0,0 +1,15 @@
|
||||
from sqlalchemy import Column, String, Integer, Text
|
||||
from app.db.database import Base
|
||||
|
||||
class OperatorInstance(Base):
|
||||
"""算子实例模型"""
|
||||
|
||||
__tablename__ = "t_operator_instance"
|
||||
|
||||
instance_id = Column(String(256), primary_key=True, comment="实例ID")
|
||||
operator_id = Column(String(256), primary_key=True, comment="算子ID")
|
||||
op_index = Column(Integer, primary_key=True, comment="算子索引")
|
||||
settings_override = Column(Text, nullable=True, comment="配置覆盖")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<OperatorInstance(instance_id={self.instance_id}, operator_id={self.operator_id}, index={self.op_index})>"
|
||||
Reference in New Issue
Block a user