You've already forked FrameTour-RenderWorker
feat(render): 实现渲染系统v2核心架构
- 添加v2支持的任务类型常量定义 - 更新软件版本至0.0.9 - 定义v2统一音视频编码参数 - 实现系统信息工具get_sys_info_v2方法 - 新增get_capabilities和_get_gpu_info功能 - 创建core模块及TaskHandler抽象基类 - 添加渲染系统设计文档包括集群架构、v2 PRD和Worker PRD - 实现任务处理器抽象基类及接口规范
This commit is contained in:
188
index_v2.py
Normal file
188
index_v2.py
Normal file
@@ -0,0 +1,188 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
RenderWorker v2 入口
|
||||
|
||||
支持 v2 API 协议的渲染 Worker,处理以下任务类型:
|
||||
- RENDER_SEGMENT_VIDEO: 渲染视频片段
|
||||
- PREPARE_JOB_AUDIO: 生成全局音频
|
||||
- PACKAGE_SEGMENT_TS: 封装 TS 分片
|
||||
- FINALIZE_MP4: 产出最终 MP4
|
||||
|
||||
使用方法:
|
||||
python index_v2.py
|
||||
|
||||
环境变量:
|
||||
API_ENDPOINT_V2: v2 API 端点(或使用 API_ENDPOINT)
|
||||
ACCESS_KEY: Worker 认证密钥
|
||||
WORKER_ID: Worker ID(默认 100001)
|
||||
MAX_CONCURRENCY: 最大并发数(默认 4)
|
||||
HEARTBEAT_INTERVAL: 心跳间隔秒数(默认 5)
|
||||
TEMP_DIR: 临时文件目录
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import signal
|
||||
import logging
|
||||
|
||||
# 加载配置(必须在其他模块之前)
|
||||
import config
|
||||
|
||||
from domain.config import WorkerConfig
|
||||
from services.api_client import APIClientV2
|
||||
from services.task_executor import TaskExecutor
|
||||
|
||||
# 日志配置
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s',
|
||||
datefmt='%Y-%m-%d %H:%M:%S'
|
||||
)
|
||||
logger = logging.getLogger('worker_v2')
|
||||
|
||||
|
||||
class WorkerV2:
|
||||
"""
|
||||
v2 渲染 Worker 主类
|
||||
|
||||
负责:
|
||||
- 配置加载
|
||||
- API 客户端初始化
|
||||
- 任务执行器管理
|
||||
- 主循环运行
|
||||
- 优雅退出处理
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""初始化 Worker"""
|
||||
# 加载配置
|
||||
try:
|
||||
self.config = WorkerConfig.from_env()
|
||||
except ValueError as e:
|
||||
logger.error(f"Configuration error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
# 初始化 API 客户端
|
||||
self.api_client = APIClientV2(self.config)
|
||||
|
||||
# 初始化任务执行器
|
||||
self.task_executor = TaskExecutor(self.config, self.api_client)
|
||||
|
||||
# 运行状态
|
||||
self.running = True
|
||||
|
||||
# 确保临时目录存在
|
||||
self.config.ensure_temp_dir()
|
||||
|
||||
# 注册信号处理器
|
||||
self._setup_signal_handlers()
|
||||
|
||||
def _setup_signal_handlers(self):
|
||||
"""设置信号处理器"""
|
||||
# Windows 不支持 SIGTERM
|
||||
signal.signal(signal.SIGINT, self._signal_handler)
|
||||
if hasattr(signal, 'SIGTERM'):
|
||||
signal.signal(signal.SIGTERM, self._signal_handler)
|
||||
|
||||
def _signal_handler(self, signum, frame):
|
||||
"""
|
||||
信号处理,优雅退出
|
||||
|
||||
Args:
|
||||
signum: 信号编号
|
||||
frame: 当前栈帧
|
||||
"""
|
||||
signal_name = signal.Signals(signum).name
|
||||
logger.info(f"Received signal {signal_name}, initiating shutdown...")
|
||||
self.running = False
|
||||
|
||||
def run(self):
|
||||
"""主循环"""
|
||||
logger.info("=" * 60)
|
||||
logger.info("RenderWorker v2 Starting")
|
||||
logger.info("=" * 60)
|
||||
logger.info(f"Worker ID: {self.config.worker_id}")
|
||||
logger.info(f"API Endpoint: {self.config.api_endpoint}")
|
||||
logger.info(f"Max Concurrency: {self.config.max_concurrency}")
|
||||
logger.info(f"Heartbeat Interval: {self.config.heartbeat_interval}s")
|
||||
logger.info(f"Capabilities: {', '.join(self.config.capabilities)}")
|
||||
logger.info(f"Temp Directory: {self.config.temp_dir}")
|
||||
logger.info("=" * 60)
|
||||
|
||||
consecutive_errors = 0
|
||||
max_consecutive_errors = 10
|
||||
|
||||
while self.running:
|
||||
try:
|
||||
# 心跳同步并拉取任务
|
||||
current_task_ids = self.task_executor.get_current_task_ids()
|
||||
tasks = self.api_client.sync(current_task_ids)
|
||||
|
||||
# 提交新任务
|
||||
for task in tasks:
|
||||
if self.task_executor.submit_task(task):
|
||||
logger.info(f"Submitted task: {task.task_id} ({task.task_type.value})")
|
||||
|
||||
# 重置错误计数
|
||||
consecutive_errors = 0
|
||||
|
||||
# 等待下次心跳
|
||||
time.sleep(self.config.heartbeat_interval)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Keyboard interrupt received")
|
||||
self.running = False
|
||||
except Exception as e:
|
||||
consecutive_errors += 1
|
||||
logger.error(f"Worker loop error ({consecutive_errors}/{max_consecutive_errors}): {e}", exc_info=True)
|
||||
|
||||
# 连续错误过多,增加等待时间
|
||||
if consecutive_errors >= max_consecutive_errors:
|
||||
logger.error("Too many consecutive errors, waiting 30 seconds...")
|
||||
time.sleep(30)
|
||||
consecutive_errors = 0
|
||||
else:
|
||||
time.sleep(5)
|
||||
|
||||
# 优雅关闭
|
||||
self._shutdown()
|
||||
|
||||
def _shutdown(self):
|
||||
"""优雅关闭"""
|
||||
logger.info("Shutting down...")
|
||||
|
||||
# 等待当前任务完成
|
||||
current_count = self.task_executor.get_current_task_count()
|
||||
if current_count > 0:
|
||||
logger.info(f"Waiting for {current_count} running task(s) to complete...")
|
||||
|
||||
# 关闭执行器
|
||||
self.task_executor.shutdown(wait=True)
|
||||
|
||||
# 关闭 API 客户端
|
||||
self.api_client.close()
|
||||
|
||||
logger.info("Worker stopped")
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数"""
|
||||
# 初始化 OpenTelemetry(如果可用)
|
||||
try:
|
||||
from telemetry import init_opentelemetry
|
||||
init_opentelemetry()
|
||||
logger.info("OpenTelemetry initialized")
|
||||
except ImportError:
|
||||
logger.debug("OpenTelemetry not available, skipping initialization")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to initialize OpenTelemetry: {e}")
|
||||
|
||||
# 创建并运行 Worker
|
||||
worker = WorkerV2()
|
||||
worker.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user