You've already forked FrameTour-RenderWorker
refactor(core): 移除旧版 FFmpeg 业务逻辑并重构常量配置
- 删除 biz/ffmpeg.py 和 biz/task.py 旧版业务模块 - 删除 entity/ffmpeg.py FFmpeg 任务实体类 - 删除 config/__init__.py 旧版配置初始化 - 更新 constant/__init__.py 常量定义,从 v1/v2 版本改为统一版本 - 修改 handlers/base.py 基础处理器,替换 OSS 相关导入为存储服务 - 添加 subprocess_args 工具函数支持跨平台进程参数配置 - 新增 probe_video_info 函数用于视频信息探测 - 新增 probe_duration_json 函数用于媒体时长探测
This commit is contained in:
219
index.py
219
index.py
@@ -1,52 +1,177 @@
|
||||
from time import sleep
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
RenderWorker v2 入口
|
||||
|
||||
支持 v2 API 协议的渲染 Worker,处理以下任务类型:
|
||||
- RENDER_SEGMENT_VIDEO: 渲染视频片段
|
||||
- PREPARE_JOB_AUDIO: 生成全局音频
|
||||
- PACKAGE_SEGMENT_TS: 封装 TS 分片
|
||||
- FINALIZE_MP4: 产出最终 MP4
|
||||
|
||||
使用方法:
|
||||
python index.py
|
||||
|
||||
环境变量:
|
||||
API_ENDPOINT_V2: v2 API 端点(或使用 API_ENDPOINT)
|
||||
ACCESS_KEY: Worker 认证密钥
|
||||
WORKER_ID: Worker ID(默认 100001)
|
||||
MAX_CONCURRENCY: 最大并发数(默认 4)
|
||||
HEARTBEAT_INTERVAL: 心跳间隔秒数(默认 5)
|
||||
TEMP_DIR: 临时文件目录
|
||||
"""
|
||||
|
||||
import sys
|
||||
|
||||
import config
|
||||
import biz.task
|
||||
from telemetry import init_opentelemetry
|
||||
from template import load_local_template, download_template, TEMPLATES
|
||||
from util import api
|
||||
|
||||
import os
|
||||
import glob
|
||||
|
||||
load_local_template()
|
||||
|
||||
# Check for redownload parameter
|
||||
if 'redownload' in sys.argv:
|
||||
print("Redownloading all templates...")
|
||||
for template_name in TEMPLATES.keys():
|
||||
print(f"Redownloading template: {template_name}")
|
||||
download_template(template_name)
|
||||
print("All templates redownloaded successfully!")
|
||||
sys.exit(0)
|
||||
import time
|
||||
import signal
|
||||
import logging
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
init_opentelemetry()
|
||||
from domain.config import WorkerConfig
|
||||
from services.api_client import APIClientV2
|
||||
from services.task_executor import TaskExecutor
|
||||
from constant import SOFTWARE_VERSION
|
||||
|
||||
while True:
|
||||
# print(get_sys_info())
|
||||
print("waiting for task...")
|
||||
try:
|
||||
task_list = api.sync_center()
|
||||
except Exception as e:
|
||||
LOGGER.error("sync_center error", exc_info=e)
|
||||
sleep(5)
|
||||
continue
|
||||
if len(task_list) == 0:
|
||||
# 删除当前文件夹下所有以.mp4、.ts结尾的文件
|
||||
for file_globs in ['*.mp4', '*.ts', 'tmp_concat*.txt']:
|
||||
for file_path in glob.glob(file_globs):
|
||||
try:
|
||||
os.remove(file_path)
|
||||
print(f"Deleted file: {file_path}")
|
||||
except Exception as e:
|
||||
LOGGER.error(f"Error deleting file {file_path}", exc_info=e)
|
||||
sleep(5)
|
||||
for task in task_list:
|
||||
print("start task:", task)
|
||||
# 日志配置
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s',
|
||||
datefmt='%Y-%m-%d %H:%M:%S'
|
||||
)
|
||||
logger = logging.getLogger('worker')
|
||||
|
||||
|
||||
class WorkerV2:
|
||||
"""
|
||||
v2 渲染 Worker 主类
|
||||
|
||||
负责:
|
||||
- 配置加载
|
||||
- API 客户端初始化
|
||||
- 任务执行器管理
|
||||
- 主循环运行
|
||||
- 优雅退出处理
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""初始化 Worker"""
|
||||
# 加载配置
|
||||
try:
|
||||
biz.task.start_task(task)
|
||||
except Exception as e:
|
||||
LOGGER.error("task_start error", exc_info=e)
|
||||
self.config = WorkerConfig.from_env()
|
||||
except ValueError as e:
|
||||
logger.error(f"Configuration error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
# 初始化 API 客户端
|
||||
self.api_client = APIClientV2(self.config)
|
||||
|
||||
# 初始化任务执行器
|
||||
self.task_executor = TaskExecutor(self.config, self.api_client)
|
||||
|
||||
# 运行状态
|
||||
self.running = True
|
||||
|
||||
# 确保临时目录存在
|
||||
self.config.ensure_temp_dir()
|
||||
|
||||
# 注册信号处理器
|
||||
self._setup_signal_handlers()
|
||||
|
||||
def _setup_signal_handlers(self):
|
||||
"""设置信号处理器"""
|
||||
# Windows 不支持 SIGTERM
|
||||
signal.signal(signal.SIGINT, self._signal_handler)
|
||||
if hasattr(signal, 'SIGTERM'):
|
||||
signal.signal(signal.SIGTERM, self._signal_handler)
|
||||
|
||||
def _signal_handler(self, signum, frame):
|
||||
"""
|
||||
信号处理,优雅退出
|
||||
|
||||
Args:
|
||||
signum: 信号编号
|
||||
frame: 当前栈帧
|
||||
"""
|
||||
signal_name = signal.Signals(signum).name
|
||||
logger.info(f"Received signal {signal_name}, initiating shutdown...")
|
||||
self.running = False
|
||||
|
||||
def run(self):
|
||||
"""主循环"""
|
||||
logger.info("=" * 60)
|
||||
logger.info("RenderWorker v2 Starting")
|
||||
logger.info("=" * 60)
|
||||
logger.info(f"Worker ID: {self.config.worker_id}")
|
||||
logger.info(f"API Endpoint: {self.config.api_endpoint}")
|
||||
logger.info(f"Max Concurrency: {self.config.max_concurrency}")
|
||||
logger.info(f"Heartbeat Interval: {self.config.heartbeat_interval}s")
|
||||
logger.info(f"Capabilities: {', '.join(self.config.capabilities)}")
|
||||
logger.info(f"Temp Directory: {self.config.temp_dir}")
|
||||
logger.info("=" * 60)
|
||||
|
||||
consecutive_errors = 0
|
||||
max_consecutive_errors = 10
|
||||
|
||||
while self.running:
|
||||
try:
|
||||
# 心跳同步并拉取任务
|
||||
current_task_ids = self.task_executor.get_current_task_ids()
|
||||
tasks = self.api_client.sync(current_task_ids)
|
||||
|
||||
# 提交新任务
|
||||
for task in tasks:
|
||||
if self.task_executor.submit_task(task):
|
||||
logger.info(f"Submitted task: {task.task_id} ({task.task_type.value})")
|
||||
|
||||
# 重置错误计数
|
||||
consecutive_errors = 0
|
||||
|
||||
# 等待下次心跳
|
||||
time.sleep(self.config.heartbeat_interval)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Keyboard interrupt received")
|
||||
self.running = False
|
||||
except Exception as e:
|
||||
consecutive_errors += 1
|
||||
logger.error(f"Worker loop error ({consecutive_errors}/{max_consecutive_errors}): {e}", exc_info=True)
|
||||
|
||||
# 连续错误过多,增加等待时间
|
||||
if consecutive_errors >= max_consecutive_errors:
|
||||
logger.error("Too many consecutive errors, waiting 30 seconds...")
|
||||
time.sleep(30)
|
||||
consecutive_errors = 0
|
||||
else:
|
||||
time.sleep(5)
|
||||
|
||||
# 优雅关闭
|
||||
self._shutdown()
|
||||
|
||||
def _shutdown(self):
|
||||
"""优雅关闭"""
|
||||
logger.info("Shutting down...")
|
||||
|
||||
# 等待当前任务完成
|
||||
current_count = self.task_executor.get_current_task_count()
|
||||
if current_count > 0:
|
||||
logger.info(f"Waiting for {current_count} running task(s) to complete...")
|
||||
|
||||
# 关闭执行器
|
||||
self.task_executor.shutdown(wait=True)
|
||||
|
||||
# 关闭 API 客户端
|
||||
self.api_client.close()
|
||||
|
||||
logger.info("Worker stopped")
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数"""
|
||||
logger.info(f"RenderWorker v{SOFTWARE_VERSION}")
|
||||
|
||||
# 创建并运行 Worker
|
||||
worker = WorkerV2()
|
||||
worker.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user