11 Commits

Author SHA1 Message Date
7ecc33b43c i 2025-09-24 11:38:17 +08:00
ba90778b2d l 2025-09-24 11:35:52 +08:00
501cea3888 a 2025-09-24 11:30:52 +08:00
873c89c778 u 2025-09-24 11:28:48 +08:00
ec1705769c q 2025-09-24 10:50:34 +08:00
c055a68592 mypy 2025-09-24 10:17:11 +08:00
dfb07d679f test 2025-09-24 09:21:03 +08:00
6d37e7c23c refactor 2025-09-24 04:51:12 +08:00
a54c157f9a docs: 添加 2025-09-12 15:03:38 +08:00
d496c7400d refactor(biz): 重构 FFmpeg 任务处理逻辑
-将主要处理逻辑迁移到新的 TaskService 架构中
-保持 FfmpegTask 类的接口
2025-09-12 14:59:04 +08:00
d770d84927 feat(重构): 实现新的渲染服务架构
- 新增 RenderTask
2025-09-12 14:41:58 +08:00
73 changed files with 6626 additions and 5810 deletions

View File

@@ -1,65 +1,13 @@
# =================== TEMPLATE_DIR=template/
# API 配置 API_ENDPOINT=https://zhentuai.com/task/v1
# ===================
API_ENDPOINT=http://127.0.0.1:18084/api
ACCESS_KEY=TEST_ACCESS_KEY ACCESS_KEY=TEST_ACCESS_KEY
WORKER_ID=1
# ===================
# 目录配置
# ===================
TEMP_DIR=tmp/ TEMP_DIR=tmp/
#REDIRECT_TO_URL=https://renderworker-deuvulkhes.cn-shanghai.fcapp.run/
# =================== # QSV
# 并发与调度 ENCODER_ARGS="-c:v h264_qsv -global_quality 28 -look_ahead 1"
# =================== # NVENC
#MAX_CONCURRENCY=4 # 最大并发任务数 #ENCODER_ARGS="-c:v h264_nvenc -cq:v 24 -preset:v p7 -tune:v hq -profile:v high"
#HEARTBEAT_INTERVAL=5 # 心跳间隔(秒) # HEVC
#LEASE_EXTENSION_THRESHOLD=60 # 租约续期阈值(秒),提前多久续期 #VIDEO_ARGS="-profile:v main
#LEASE_EXTENSION_DURATION=300 # 租约续期时长(秒) UPLOAD_METHOD="rclone"
RCLONE_REPLACE_MAP="https://oss.zhentuai.com|alioss://frametour-assets,https://frametour-assets.oss-cn-shanghai.aliyuncs.com|alioss://frametour-assets"
# ===================
# 能力配置
# ===================
# 支持的任务类型,逗号分隔,默认全部支持
#CAPABILITIES=RENDER_SEGMENT_VIDEO,PREPARE_JOB_AUDIO,PACKAGE_SEGMENT_TS,FINALIZE_MP4
# ===================
# 超时配置
# ===================
#FFMPEG_TIMEOUT=3600 # FFmpeg 执行超时(秒)
#DOWNLOAD_TIMEOUT=300 # 下载超时(秒)
#UPLOAD_TIMEOUT=600 # 上传超时(秒)
# ===================
# 硬件加速与多显卡
# ===================
# 硬件加速类型: none, qsv, cuda
HW_ACCEL=none
# GPU 设备列表(逗号分隔的设备索引)
# 不配置时:自动检测所有设备
# 单设备示例:GPU_DEVICES=0
# 多设备示例:GPU_DEVICES=0,1,2
#GPU_DEVICES=0,1
# ===================
# 素材缓存
# ===================
#CACHE_ENABLED=true # 是否启用素材缓存
#CACHE_DIR= # 缓存目录,默认 TEMP_DIR/cache
#CACHE_MAX_SIZE_GB=0 # 最大缓存大小(GB),0 表示不限制
# ===================
# URL 映射(内网下载加速)
# ===================
# 格式: src1|dst1,src2|dst2
#HTTP_REPLACE_MAP="https://cdcdn.zhentuai.com|http://192.168.10.254:9000"
# ===================
# 上传配置
# ===================
# 上传方式: 默认 HTTP,可选 rclone
#UPLOAD_METHOD=rclone
#RCLONE_CONFIG_FILE= # rclone 配置文件路径
#RCLONE_REPLACE_MAP="https://oss.example.com|alioss://bucket"

15
.flake8 Normal file
View File

@@ -0,0 +1,15 @@
[flake8]
max-line-length = 88
ignore =
# Line too long - handled by black
E501,
# Line break before binary operator - handled by black
W503
exclude =
.git,
__pycache__,
.venv,
venv,
tests,
.claude,
.serena

179
CLAUDE.md Normal file
View File

@@ -0,0 +1,179 @@
# CLAUDE.md
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
## Project Overview
RenderWorker is a Python-based video processing service that renders video content using FFmpeg. It operates as both a Flask web service and a standalone worker process, processing video rendering tasks with templates, effects, and various transformations.
## Architecture
The project uses a modern layered architecture with clear separation of concerns:
### Core Components
- **Services Layer** (`services/`): Main business logic with dependency injection
- `RenderService`: Handles video rendering operations
- `TemplateService`: Manages video templates and resources
- `TaskService`: Orchestrates the complete task lifecycle
- **Entity Layer** (`entity/`): Data models and domain logic
- `RenderTask`: Core data model for rendering tasks
- `FFmpegCommandBuilder`: Generates FFmpeg commands from tasks
- `effects/`: Pluggable effect processing system (zoom, speed, skip, tail, cameraShot)
- **Configuration** (`config/`): Centralized configuration management
- Environment-based configuration with dataclasses
- FFmpeg, API, Storage, and Server configurations
- **Utilities** (`util/`): Common utilities for API, OSS, FFmpeg operations
### Entry Points
- **Web Service**: `app.py` - Flask application serving HTTP endpoints
- **Worker Process**: `index.py` - Long-running task processor that polls for work
- **Legacy Compatibility**: Maintains backward compatibility with older `entity/ffmpeg.py` and `biz/` modules
## Common Commands
### Running the Application
```bash
# Web service mode (Flask API)
python app.py
# Worker mode (background task processor)
python index.py
# Worker with template redownload
python index.py redownload
```
### Development Setup
```bash
# Install dependencies
pip install -r requirements.txt
# Environment setup
cp .env.example .env
# Edit .env with your configuration
```
### Building
```bash
# Build standalone executable
pyinstaller index.spec
# Build updater
pyinstaller updater_helper.spec
```
## Key Environment Variables
Required configuration (see `.env.example`):
- `API_ENDPOINT`: Backend API endpoint
- `ACCESS_KEY`: Authentication key
- `TEMPLATE_DIR`: Directory for template storage
- `ENCODER_ARGS`: FFmpeg encoder arguments (default: "-c:v h264")
- `VIDEO_ARGS`: Video processing arguments
- `OTLP_ENDPOINT`: OpenTelemetry endpoint for monitoring
## Architecture Patterns
### Service Layer Pattern
All new code should use the service layer:
```python
from services import DefaultRenderService, DefaultTemplateService, DefaultTaskService
# Dependency injection
render_service = DefaultRenderService()
template_service = DefaultTemplateService()
task_service = DefaultTaskService(render_service, template_service)
# Process tasks
success = task_service.process_task(task_info)
```
### Effect Processing System
Video effects use a pluggable architecture:
```python
from entity.effects import registry
from entity.effects.base import EffectProcessor
# Get existing effect
processor = registry.get_processor('zoom', '0,2.0,1.5')
# Register new effect
class MyEffect(EffectProcessor):
def validate_params(self) -> bool:
return True
def generate_filter_args(self, video_input: str, effect_index: int):
return filter_args, output_stream
def get_effect_name(self) -> str:
return "myeffect"
registry.register('myeffect', MyEffect)
```
### Task Processing Flow
1. **Task Reception**: API receives task via HTTP endpoint or polling
2. **Template Resolution**: Template service downloads/loads video templates
3. **Task Creation**: Task service creates RenderTask with effects and resources
4. **Command Building**: FFmpegCommandBuilder generates FFmpeg commands
5. **Execution**: RenderService executes FFmpeg with monitoring
6. **Cleanup**: Temporary files cleaned, results uploaded
## Backward Compatibility
The codebase maintains compatibility with legacy components:
- `biz.task.start_task()` - Main task entry point
- `entity.ffmpeg.FfmpegTask` - Legacy task object (now simplified)
- `template.get_template_def()` - Template access
These should work but new development should use the service layer.
## FFmpeg Integration
The system generates complex FFmpeg commands for video processing:
- Supports effects chaining (zoom, speed changes, cropping)
- Handles multiple input files and concatenation
- Manages audio processing and overlays
- Supports LUT color grading and subtitle embedding
## Monitoring and Telemetry
- **OpenTelemetry**: Distributed tracing for request monitoring
- **Structured Logging**: Detailed logs for debugging
- **Error Handling**: Custom exception types (`RenderError`, `FFmpegError`, `TemplateError`)
## File Organization
- Configuration and environment variables → `config/`
- Business logic and services → `services/`
- Data models and domain objects → `entity/`
- HTTP API and external integrations → `util/`
- Legacy compatibility layer → `biz/`
- Application entry points → `app.py`, `index.py`
## Working with Templates
Templates define video processing workflows:
- Downloaded from remote API and cached locally
- Support video parts, effects, overlays, and audio
- Use placeholder system for dynamic content
- Managed by `TemplateService` with automatic updates
## Development Notes
- The project recently underwent major architectural refactoring from monolithic to layered design
- New features should use the services layer and effect processor system
- Legacy code is maintained for compatibility but simplified
- All video processing ultimately uses FFmpeg with generated command lines
- The system supports both synchronous (web) and asynchronous (worker) operation modes

347
Jenkinsfile vendored Normal file
View File

@@ -0,0 +1,347 @@
pipeline {
agent any
environment {
// 环境变量
PYTHON_VERSION = '3.9'
VENV_NAME = 'venv'
TEST_REPORTS_DIR = 'test-reports'
COVERAGE_DIR = 'coverage-reports'
// 设置Python模块路径
PYTHONPATH = "${WORKSPACE}"
}
stages {
stage('Preparation') {
steps {
script {
// 清理工作空间
echo 'Cleaning workspace...'
deleteDir()
// 检出代码
checkout scm
// 创建报告目录
sh """
mkdir -p ${TEST_REPORTS_DIR}
mkdir -p ${COVERAGE_DIR}
"""
}
}
}
stage('Environment Setup') {
steps {
script {
echo 'Setting up Python environment...'
sh """
# 创建虚拟环境
python${PYTHON_VERSION} -m venv ${VENV_NAME}
# 激活虚拟环境并安装依赖
. ${VENV_NAME}/bin/activate
pip config set global.index-url https://mirrors.ustc.edu.cn/pypi/simple
# 设置PYTHONPATH
export PYTHONPATH=\${PWD}:\$PYTHONPATH
# 升级pip
pip install --upgrade pip
# 安装项目依赖
pip install -r requirements.txt
# 安装测试依赖
pip install -r requirements-test.txt
# 验证FFmpeg可用性
which ffmpeg || echo "Warning: FFmpeg not found, integration tests may be skipped"
"""
}
}
}
stage('Code Quality Check') {
parallel {
stage('Linting') {
steps {
script {
echo 'Running code linting...'
sh """
. ${VENV_NAME}/bin/activate
export PYTHONPATH=\${PWD}:\$PYTHONPATH
# 运行flake8检查(使用项目配置.flake8)
flake8 entity/ services/ --output-file=${TEST_REPORTS_DIR}/flake8-report.txt --tee || true
# 运行black格式检查
black --check --diff --line-length 88 entity/ services/ > ${TEST_REPORTS_DIR}/black-report.txt || true
"""
}
}
}
stage('Type Checking') {
steps {
script {
echo 'Running type checking with mypy...'
sh """
. ${VENV_NAME}/bin/activate
export PYTHONPATH=\${PWD}:\$PYTHONPATH
# 运行mypy类型检查
mypy --explicit-package-bases services/ entity/ > ${TEST_REPORTS_DIR}/mypy-report.txt || true
"""
}
}
}
}
post {
always {
// 发布代码质量报告(等待所有并行任务完成后统一发布)
publishHTML([
allowMissing: true,
alwaysLinkToLastBuild: true,
keepAll: true,
reportDir: TEST_REPORTS_DIR,
reportFiles: 'flake8-report.txt,black-report.txt,mypy-report.txt',
reportName: 'Code Quality Report'
])
}
}
}
stage('Unit Tests') {
steps {
script {
echo 'Running unit tests...'
sh """
. ${VENV_NAME}/bin/activate
export PYTHONPATH=\${PWD}:\$PYTHONPATH
# 运行单元测试
pytest tests/test_effects/ tests/test_ffmpeg_builder/ \\
--junitxml=${TEST_REPORTS_DIR}/unit-tests.xml \\
--html=${TEST_REPORTS_DIR}/unit-tests.html \\
--self-contained-html \\
--cov=entity \\
--cov=services \\
--cov-report=xml:${COVERAGE_DIR}/unit-coverage.xml \\
--cov-branch \\
-v \\
-m "not integration"
"""
}
}
post {
always {
// 发布单元测试结果
junit "${TEST_REPORTS_DIR}/unit-tests.xml"
// 发布HTML测试报告
publishHTML([
allowMissing: false,
alwaysLinkToLastBuild: true,
keepAll: true,
reportDir: TEST_REPORTS_DIR,
reportFiles: 'unit-tests.html',
reportName: 'Unit Tests Report'
])
}
}
}
stage('Integration Tests') {
when {
// 只在有FFmpeg的环境中运行集成测试
expression {
return sh(
script: 'which ffmpeg',
returnStatus: true
) == 0
}
}
steps {
script {
echo 'Running integration tests...'
sh """
. ${VENV_NAME}/bin/activate
export PYTHONPATH=\${PWD}:\$PYTHONPATH
# 运行集成测试
pytest tests/test_integration/ \\
--junitxml=${TEST_REPORTS_DIR}/integration-tests.xml \\
--html=${TEST_REPORTS_DIR}/integration-tests.html \\
--self-contained-html \\
--cov=entity \\
--cov=services \\
--cov-report=xml:${COVERAGE_DIR}/integration-coverage.xml \\
--cov-branch \\
--timeout=300 \\
-v \\
-m "integration"
"""
}
}
post {
always {
// 发布集成测试结果
junit "${TEST_REPORTS_DIR}/integration-tests.xml"
// 发布HTML集成测试报告
publishHTML([
allowMissing: false,
alwaysLinkToLastBuild: true,
keepAll: true,
reportDir: TEST_REPORTS_DIR,
reportFiles: 'integration-tests.html',
reportName: 'Integration Tests Report'
])
}
}
}
stage('Complete Test Suite') {
steps {
script {
echo 'Running complete test suite with coverage...'
sh """
. ${VENV_NAME}/bin/activate
export PYTHONPATH=\${PWD}:\$PYTHONPATH
# 运行完整测试套件
pytest tests/ \\
--junitxml=${TEST_REPORTS_DIR}/all-tests.xml \\
--html=${TEST_REPORTS_DIR}/all-tests.html \\
--self-contained-html \\
--cov=entity \\
--cov=services \\
--cov-report=xml:${COVERAGE_DIR}/coverage.xml \\
--cov-report=term-missing \\
--cov-branch \\
--cov-fail-under=70 \\
-v
"""
}
}
post {
always {
// 发布完整测试结果
junit "${TEST_REPORTS_DIR}/all-tests.xml"
// 发布代码覆盖率报告
publishCoverage([
adapters: [
[
mergeToOneReport: true,
path: "${COVERAGE_DIR}/coverage.xml"
]
],
sourceFileResolver: [
[
level: 'NEVER_STORE'
]
]
])
// 发布完整测试HTML报告
publishHTML([
allowMissing: false,
alwaysLinkToLastBuild: true,
keepAll: true,
reportDir: TEST_REPORTS_DIR,
reportFiles: 'all-tests.html',
reportName: 'Complete Test Report'
])
}
}
}
stage('Performance Tests') {
when {
// 只在主分支或发布分支运行性能测试
anyOf {
branch 'master'
branch 'main'
branch 'release/*'
}
}
steps {
script {
echo 'Running performance tests...'
sh """
. ${VENV_NAME}/bin/activate
export PYTHONPATH=\${PWD}:\$PYTHONPATH
# 运行性能测试
RUN_STRESS_TESTS=1 pytest tests/test_integration/test_ffmpeg_execution.py::TestFFmpegExecution::test_stress_test_large_effects_chain \\
--benchmark-json=${TEST_REPORTS_DIR}/benchmark.json \\
--html=${TEST_REPORTS_DIR}/performance-tests.html \\
--self-contained-html \\
-v \\
-m "stress" || true
"""
}
}
post {
always {
// 发布性能测试报告
publishHTML([
allowMissing: true,
alwaysLinkToLastBuild: true,
keepAll: true,
reportDir: TEST_REPORTS_DIR,
reportFiles: 'performance-tests.html',
reportName: 'Performance Tests Report'
])
}
}
}
}
post {
always {
// 清理虚拟环境
sh """
rm -rf ${VENV_NAME}
"""
// 归档测试报告
archiveArtifacts artifacts: "${TEST_REPORTS_DIR}/**/*", allowEmptyArchive: true
archiveArtifacts artifacts: "${COVERAGE_DIR}/**/*", allowEmptyArchive: true
}
success {
echo 'All tests passed successfully!'
// 发送成功通知
script {
if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME == 'main') {
// 这里可以添加Slack、邮件或其他通知
echo 'Sending success notification...'
}
}
}
failure {
echo 'Tests failed!'
// 发送失败通知
script {
// 这里可以添加Slack、邮件或其他通知
echo 'Sending failure notification...'
}
}
unstable {
echo 'Tests completed with warnings!'
}
cleanup {
// 清理临时文件
deleteDir()
}
}
}

71
app.py Normal file
View File

@@ -0,0 +1,71 @@
import time
import flask
import config
import biz.task
from services import DefaultTemplateService
from telemetry import init_opentelemetry
from util import api
# 使用新的服务容器架构
from services.service_container import get_template_service, register_default_services
# 确保服务已注册
register_default_services()
template_service = get_template_service()
import logging
LOGGER = logging.getLogger(__name__)
init_opentelemetry(batch=False)
app = flask.Flask(__name__)
@app.get("/health/check")
def health_check():
return api.sync_center()
@app.post("/")
def do_nothing():
return "NOOP"
@app.post("/<task_id>")
def do_task(task_id):
try:
task_info = api.get_task_info(task_id)
if not task_info:
LOGGER.error("Failed to get task info for task: %s", task_id)
return "Failed to get task info", 400
template_id = task_info.get("templateId")
if not template_id:
LOGGER.error("Task %s missing templateId", task_id)
return "Missing templateId", 400
local_template_info = template_service.get_template(template_id)
template_info = api.get_template_info(template_id)
if not template_info:
LOGGER.error("Failed to get template info for template: %s", template_id)
return "Failed to get template info", 400
if local_template_info:
if local_template_info.get("updateTime") != template_info.get("updateTime"):
LOGGER.info("Template %s needs update, downloading...", template_id)
if not template_service.download_template(template_id):
LOGGER.error("Failed to download template: %s", template_id)
return "Failed to download template", 500
biz.task.start_task(task_info)
return "OK"
except Exception as e:
LOGGER.error("Error processing task %s: %s", task_id, e, exc_info=True)
return "Internal server error", 500
if __name__ == "__main__":
app.run(host="0.0.0.0", port=9998)

144
biz/ffmpeg.py Normal file
View File

@@ -0,0 +1,144 @@
import json
import os.path
import time
from concurrent.futures import ThreadPoolExecutor
from opentelemetry.trace import Status, StatusCode
# 使用新架构组件,保持对旧FfmpegTask的兼容
from entity.ffmpeg import FfmpegTask
from entity.render_task import RenderTask, TaskType
from services import DefaultRenderService
import logging
from util import ffmpeg, oss
from util.ffmpeg import fade_out_audio
from telemetry import get_tracer
logger = logging.getLogger("biz/ffmpeg")
_render_service = None
def _get_render_service():
"""获取渲染服务实例"""
global _render_service
if _render_service is None:
_render_service = DefaultRenderService()
return _render_service
def parse_ffmpeg_task(task_info, template_info):
"""
解析FFmpeg任务 - 保留用于向后兼容
实际处理逻辑已迁移到 services.TaskService.create_render_task
"""
logger.warning(
"parse_ffmpeg_task is deprecated, use TaskService.create_render_task instead"
)
# 使用新的任务服务创建任务
from services import (
DefaultTaskService,
DefaultRenderService,
DefaultTemplateService,
)
render_service = DefaultRenderService()
template_service = DefaultTemplateService()
task_service = DefaultTaskService(render_service, template_service)
# 创建新的渲染任务
render_task = task_service.create_render_task(task_info, template_info)
# 为了向后兼容,创建一个FfmpegTask包装器
ffmpeg_task = FfmpegTask(
render_task.input_files, output_file=render_task.output_file
)
ffmpeg_task.resolution = render_task.resolution
ffmpeg_task.frame_rate = render_task.frame_rate
ffmpeg_task.annexb = render_task.annexb
ffmpeg_task.center_cut = render_task.center_cut
ffmpeg_task.zoom_cut = render_task.zoom_cut
ffmpeg_task.ext_data = render_task.ext_data
ffmpeg_task.effects = render_task.effects
ffmpeg_task.luts = render_task.luts
ffmpeg_task.audios = render_task.audios
ffmpeg_task.overlays = render_task.overlays
return ffmpeg_task
# 以下函数已迁移到新架构,保留用于向后兼容
def parse_video(source, task_params, template_info):
"""已迁移到 TaskService._parse_video_source"""
logger.warning("parse_video is deprecated, functionality moved to TaskService")
return source, {}
def check_placeholder_exist(placeholder_id, task_params):
"""已迁移到 TaskService._check_placeholder_exist_with_count"""
logger.warning(
"check_placeholder_exist is deprecated, functionality moved to TaskService"
)
return placeholder_id in task_params
def check_placeholder_exist_with_count(placeholder_id, task_params, required_count=1):
"""已迁移到 TaskService._check_placeholder_exist_with_count"""
logger.warning(
"check_placeholder_exist_with_count is deprecated, functionality moved to TaskService"
)
if placeholder_id in task_params:
new_sources = task_params.get(placeholder_id, [])
if isinstance(new_sources, list):
return len(new_sources) >= required_count
return required_count <= 1
return False
def start_ffmpeg_task(ffmpeg_task):
"""启动FFmpeg任务 - 使用新的渲染服务"""
tracer = get_tracer(__name__)
with tracer.start_as_current_span("start_ffmpeg_task") as span:
try:
# 使用新的渲染服务
render_service = _get_render_service()
result = render_service.render(ffmpeg_task)
if result:
span.set_status(Status(StatusCode.OK))
else:
span.set_status(Status(StatusCode.ERROR))
return result
except Exception as e:
span.set_status(Status(StatusCode.ERROR))
logger.error(f"FFmpeg task failed: {e}", exc_info=True)
return False
def clear_task_tmp_file(ffmpeg_task):
"""清理临时文件 - 已迁移到 TaskService._cleanup_temp_files"""
logger.warning(
"clear_task_tmp_file is deprecated, functionality moved to TaskService"
)
try:
template_dir = os.getenv("TEMPLATE_DIR", "")
output_file = ffmpeg_task.get_output_file()
if template_dir and template_dir not in output_file:
if os.path.exists(output_file):
os.remove(output_file)
logger.info("Cleaned up temp file: %s", output_file)
else:
logger.info("Skipped cleanup of template file: %s", output_file)
return True
except OSError as e:
logger.warning("Failed to cleanup temp file %s: %s", output_file, e)
return False
def probe_video_info(ffmpeg_task):
"""获取视频长度宽度和时长 - 使用新的渲染服务"""
render_service = _get_render_service()
return render_service.get_video_info(ffmpeg_task.get_output_file())

0
biz/render.py Normal file
View File

39
biz/task.py Normal file
View File

@@ -0,0 +1,39 @@
import json
import logging
from opentelemetry.trace import Status, StatusCode
# 使用新的服务容器架构
from services.service_container import get_task_service, register_default_services
from telemetry import get_tracer
logger = logging.getLogger(__name__)
# 确保服务已注册
register_default_services()
def start_task(task_info):
"""启动任务处理(保持向后兼容的接口)"""
tracer = get_tracer(__name__)
with tracer.start_as_current_span("start_task_legacy") as span:
try:
# 使用服务容器获取任务服务
task_service = get_task_service()
# 使用新的任务服务处理
result = task_service.process_task(task_info)
if result:
span.set_status(Status(StatusCode.OK))
logger.info("Task completed successfully: %s", task_info.get("id"))
else:
span.set_status(Status(StatusCode.ERROR))
logger.error("Task failed: %s", task_info.get("id"))
return None # 保持原有返回值格式
except Exception as e:
span.set_status(Status(StatusCode.ERROR))
logger.error("Task processing failed: %s", e, exc_info=True)
return None

31
config/__init__.py Normal file
View File

@@ -0,0 +1,31 @@
import datetime
import logging
from logging.handlers import TimedRotatingFileHandler
from dotenv import load_dotenv
# 导入新的配置系统,保持向后兼容
from .settings import (
get_config,
get_ffmpeg_config,
get_api_config,
get_storage_config,
get_server_config,
)
load_dotenv()
logging.basicConfig(level=logging.INFO)
root_logger = logging.getLogger()
rf_handler = TimedRotatingFileHandler("all_log.log", when="midnight")
rf_handler.setFormatter(
logging.Formatter("[%(asctime)s][%(name)s]%(levelname)s - %(message)s")
)
rf_handler.setLevel(logging.DEBUG)
f_handler = TimedRotatingFileHandler("error.log", when="midnight")
f_handler.setLevel(logging.ERROR)
f_handler.setFormatter(
logging.Formatter(
"[%(asctime)s][%(name)s][:%(lineno)d]%(levelname)s - - %(message)s"
)
)
root_logger.addHandler(rf_handler)
root_logger.addHandler(f_handler)

181
config/settings.py Normal file
View File

@@ -0,0 +1,181 @@
import os
from dataclasses import dataclass
from typing import Dict, List, Optional, Union
import logging
from dotenv import load_dotenv
load_dotenv()
@dataclass
class FFmpegConfig:
"""FFmpeg相关配置"""
encoder_args: List[str]
video_args: List[str]
audio_args: List[str]
default_args: List[str]
old_ffmpeg: bool = False
re_encode_video_args: Optional[List[str]] = None
re_encode_encoder_args: Optional[List[str]] = None
# 新增配置选项,消除硬编码
max_download_workers: int = 8
progress_args: Optional[List[str]] = None
loglevel_args: Optional[List[str]] = None
null_audio_args: Optional[List[str]] = None
overlay_scale_mode: str = "scale2ref" # 新版本使用scale2ref,旧版本使用scale
amix_args: Optional[List[str]] = None
@classmethod
def from_env(cls) -> "FFmpegConfig":
encoder_args = os.getenv("ENCODER_ARGS", "-c:v h264").split(" ")
video_args = os.getenv("VIDEO_ARGS", "-profile:v high -level:v 4").split(" ")
audio_args = ["-c:a", "aac", "-b:a", "128k", "-ar", "48000", "-ac", "2"]
default_args = ["-shortest"]
re_encode_video_args = None
re_encode_video_env = os.getenv("RE_ENCODE_VIDEO_ARGS")
if re_encode_video_env:
re_encode_video_args = re_encode_video_env.split(" ")
re_encode_encoder_args = None
re_encode_encoder_env = os.getenv("RE_ENCODE_ENCODER_ARGS")
if re_encode_encoder_env:
re_encode_encoder_args = re_encode_encoder_env.split(" ")
# 新增配置项的默认值
progress_args = ["-progress", "-"]
loglevel_args = ["-loglevel", "error"]
null_audio_args = ["-f", "lavfi", "-i", "anullsrc=cl=stereo:r=48000"]
amix_args = ["amix=duration=shortest:dropout_transition=0:normalize=0"]
overlay_scale_mode = (
"scale" if bool(os.getenv("OLD_FFMPEG", False)) else "scale2ref"
)
return cls(
encoder_args=encoder_args,
video_args=video_args,
audio_args=audio_args,
default_args=default_args,
old_ffmpeg=bool(os.getenv("OLD_FFMPEG", False)),
re_encode_video_args=re_encode_video_args,
re_encode_encoder_args=re_encode_encoder_args,
max_download_workers=int(os.getenv("MAX_DOWNLOAD_WORKERS", "8")),
progress_args=progress_args,
loglevel_args=loglevel_args,
null_audio_args=null_audio_args,
overlay_scale_mode=overlay_scale_mode,
amix_args=amix_args,
)
@dataclass
class APIConfig:
"""API相关配置"""
endpoint: str
access_key: str
timeout: int = 10
redirect_to_url: Optional[str] = None
@classmethod
def from_env(cls) -> "APIConfig":
endpoint = os.getenv("API_ENDPOINT", "")
if not endpoint:
raise ValueError("API_ENDPOINT environment variable is required")
access_key = os.getenv("ACCESS_KEY", "")
if not access_key:
raise ValueError("ACCESS_KEY environment variable is required")
return cls(
endpoint=endpoint,
access_key=access_key,
timeout=int(os.getenv("API_TIMEOUT", "10")),
redirect_to_url=os.getenv("REDIRECT_TO_URL") or None,
)
@dataclass
class StorageConfig:
"""存储相关配置"""
template_dir: str
@classmethod
def from_env(cls) -> "StorageConfig":
template_dir = os.getenv("TEMPLATE_DIR", "./template")
return cls(template_dir=template_dir)
@dataclass
class ServerConfig:
"""服务器相关配置"""
host: str = "0.0.0.0"
port: int = 9998
debug: bool = False
@classmethod
def from_env(cls) -> "ServerConfig":
return cls(
host=os.getenv("HOST", "0.0.0.0"),
port=int(os.getenv("PORT", "9998")),
debug=bool(os.getenv("DEBUG", False)),
)
@dataclass
class AppConfig:
"""应用总配置"""
ffmpeg: FFmpegConfig
api: APIConfig
storage: StorageConfig
server: ServerConfig
@classmethod
def from_env(cls) -> "AppConfig":
return cls(
ffmpeg=FFmpegConfig.from_env(),
api=APIConfig.from_env(),
storage=StorageConfig.from_env(),
server=ServerConfig.from_env(),
)
# 全局配置实例
_config: Optional[AppConfig] = None
def get_config() -> AppConfig:
"""获取全局配置实例"""
global _config
if _config is None:
_config = AppConfig.from_env()
return _config
def reload_config() -> AppConfig:
"""重新加载配置"""
global _config
_config = AppConfig.from_env()
return _config
# 向后兼容的配置获取函数
def get_ffmpeg_config() -> FFmpegConfig:
return get_config().ffmpeg
def get_api_config() -> APIConfig:
return get_config().api
def get_storage_config() -> StorageConfig:
return get_config().storage
def get_server_config() -> ServerConfig:
return get_config().server

View File

@@ -1,99 +1,9 @@
# -*- coding: utf-8 -*- SUPPORT_FEATURE = (
""" "simple_render_algo",
常量定义 "gpu_accelerate",
"hevc_encode",
v2 版本常量,用于 Render Worker v2 API。 "rapid_download",
""" "rclone_upload",
"custom_re_encode",
# 软件版本
SOFTWARE_VERSION = '2.0.0'
# 支持的任务类型
TASK_TYPES = (
'RENDER_SEGMENT_VIDEO',
'COMPOSE_TRANSITION',
'PREPARE_JOB_AUDIO',
'PACKAGE_SEGMENT_TS',
'FINALIZE_MP4',
) )
SOFTWARE_VERSION = "0.0.5"
# 默认能力
DEFAULT_CAPABILITIES = list(TASK_TYPES)
# 支持的转场类型(对应 FFmpeg xfade 参数)
TRANSITION_TYPES = (
'fade', # 淡入淡出(默认)
'dissolve', # 溶解过渡
'wipeleft', # 向左擦除
'wiperight', # 向右擦除
'wipeup', # 向上擦除
'wipedown', # 向下擦除
'slideleft', # 向左滑动
'slideright', # 向右滑动
'slideup', # 向上滑动
'slidedown', # 向下滑动
)
# 支持的特效类型
EFFECT_TYPES = (
'cameraShot', # 相机定格效果:在指定时间点冻结画面
'zoom', # 缩放效果(预留)
'blur', # 模糊效果(预留)
)
# 硬件加速类型
HW_ACCEL_NONE = 'none' # 纯软件编解码
HW_ACCEL_QSV = 'qsv' # Intel Quick Sync Video (核显/独显)
HW_ACCEL_CUDA = 'cuda' # NVIDIA NVENC/NVDEC
HW_ACCEL_TYPES = (HW_ACCEL_NONE, HW_ACCEL_QSV, HW_ACCEL_CUDA)
# 统一视频编码参数(软件编码,来自集成文档)
VIDEO_ENCODE_PARAMS = {
'codec': 'libx264',
'preset': 'medium',
'profile': 'main',
'level': '4.0',
'crf': '23',
'pix_fmt': 'yuv420p',
}
# QSV 硬件加速视频编码参数(Intel Quick Sync)
VIDEO_ENCODE_PARAMS_QSV = {
'codec': 'h264_qsv',
'preset': 'medium', # QSV 支持: veryfast, faster, fast, medium, slow, slower, veryslow
'profile': 'main',
'level': '4.0',
'global_quality': '23', # QSV 使用 global_quality 代替 crf(1-51,值越低质量越高)
'look_ahead': '1', # 启用前瞻分析提升质量
'pix_fmt': 'nv12', # QSV 硬件表面格式
}
# CUDA 硬件加速视频编码参数(NVIDIA NVENC)
VIDEO_ENCODE_PARAMS_CUDA = {
'codec': 'h264_nvenc',
'preset': 'p4', # NVENC 预设 p1-p7(p1 最快,p7 最慢/质量最高),p4 ≈ medium
'profile': 'main',
'level': '4.0',
'rc': 'vbr', # 码率控制模式:vbr 可变码率
'cq': '23', # 恒定质量模式的质量值(0-51)
'pix_fmt': 'yuv420p', # NVENC 输入格式(会自动转换)
}
# 统一音频编码参数
AUDIO_ENCODE_PARAMS = {
'codec': 'aac',
'bitrate': '128k',
'sample_rate': '48000',
'channels': '2',
}
# 错误码
ERROR_CODES = {
'E_INPUT_UNAVAILABLE': '素材不可访问',
'E_FFMPEG_FAILED': 'FFmpeg 执行失败',
'E_UPLOAD_FAILED': '上传失败',
'E_SPEC_INVALID': '渲染规格非法',
'E_TIMEOUT': '执行超时',
'E_UNKNOWN': '未知错误',
}

View File

@@ -1,12 +0,0 @@
# -*- coding: utf-8 -*-
"""
核心抽象层
包含任务处理器抽象基类等核心接口定义。
"""
from core.handler import TaskHandler
__all__ = [
'TaskHandler',
]

View File

@@ -1,79 +0,0 @@
# -*- coding: utf-8 -*-
"""
任务处理器抽象基类
定义任务处理器的接口规范。
"""
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from domain.task import Task, TaskType
from domain.result import TaskResult
class TaskHandler(ABC):
"""
任务处理器抽象基类
所有任务处理器都必须继承此类并实现相应方法。
"""
@abstractmethod
def handle(self, task: 'Task') -> 'TaskResult':
"""
处理任务的主方法
Args:
task: 任务实体
Returns:
TaskResult: 任务结果(成功或失败)
"""
pass
@abstractmethod
def get_supported_type(self) -> 'TaskType':
"""
返回此处理器支持的任务类型
Returns:
TaskType: 支持的任务类型枚举值
"""
pass
def before_handle(self, task: 'Task') -> None:
"""
处理前钩子(可选重写)
用于任务执行前的准备工作,如日志记录、资源检查等。
Args:
task: 任务实体
"""
pass
def after_handle(self, task: 'Task', result: 'TaskResult') -> None:
"""
处理后钩子(可选重写)
用于任务执行后的清理工作,如资源释放、统计记录等。
Args:
task: 任务实体
result: 任务结果
"""
pass
def validate_task(self, task: 'Task') -> bool:
"""
验证任务是否有效(可选重写)
Args:
task: 任务实体
Returns:
bool: 任务是否有效
"""
return True

View File

@@ -1,24 +0,0 @@
# -*- coding: utf-8 -*-
"""
领域模型层
包含任务实体、结果、配置等核心数据结构。
"""
from domain.task import Task, TaskType, TaskStatus, RenderSpec, OutputSpec, AudioSpec, AudioProfile
from domain.result import TaskResult, ErrorCode, RETRY_CONFIG
from domain.config import WorkerConfig
__all__ = [
'Task',
'TaskType',
'TaskStatus',
'RenderSpec',
'OutputSpec',
'AudioSpec',
'AudioProfile',
'TaskResult',
'ErrorCode',
'RETRY_CONFIG',
'WorkerConfig',
]

View File

@@ -1,183 +0,0 @@
# -*- coding: utf-8 -*-
"""
Worker 配置模型
定义 Worker 运行时的配置参数。
"""
import logging
import os
from dataclasses import dataclass, field
from typing import List, Optional
from constant import HW_ACCEL_NONE, HW_ACCEL_QSV, HW_ACCEL_CUDA, HW_ACCEL_TYPES
logger = logging.getLogger(__name__)
# 默认支持的任务类型
DEFAULT_CAPABILITIES = [
"RENDER_SEGMENT_VIDEO",
"PREPARE_JOB_AUDIO",
"PACKAGE_SEGMENT_TS",
"FINALIZE_MP4"
]
@dataclass
class WorkerConfig:
"""
Worker 配置
包含 Worker 运行所需的所有配置参数。
"""
# API 配置
api_endpoint: str
access_key: str
worker_id: str
# 并发控制
max_concurrency: int = 4
# 心跳配置
heartbeat_interval: int = 5 # 秒
# 租约配置
lease_extension_threshold: int = 60 # 秒,提前多久续期
lease_extension_duration: int = 300 # 秒,每次续期时长
# 目录配置
temp_dir: str = "/tmp/render_worker"
# 能力配置
capabilities: List[str] = field(default_factory=lambda: DEFAULT_CAPABILITIES.copy())
# FFmpeg 配置
ffmpeg_timeout: int = 3600 # 秒,FFmpeg 执行超时
# 下载/上传配置
download_timeout: int = 300 # 秒,下载超时
upload_timeout: int = 600 # 秒,上传超时
# 硬件加速配置
hw_accel: str = HW_ACCEL_NONE # 硬件加速类型: none, qsv, cuda
# GPU 设备配置(多显卡调度)
gpu_devices: List[int] = field(default_factory=list) # 空列表表示使用默认设备
# 素材缓存配置
cache_enabled: bool = True # 是否启用素材缓存
cache_dir: str = "" # 缓存目录,默认为 temp_dir/cache
cache_max_size_gb: float = 0 # 最大缓存大小(GB),0 表示不限制
@classmethod
def from_env(cls) -> 'WorkerConfig':
"""从环境变量创建配置"""
# API 端点,优先使用 V2 版本
api_endpoint = os.getenv('API_ENDPOINT_V2') or os.getenv('API_ENDPOINT', '')
if not api_endpoint:
raise ValueError("API_ENDPOINT_V2 or API_ENDPOINT environment variable is required")
# Access Key
access_key = os.getenv('ACCESS_KEY', '')
if not access_key:
raise ValueError("ACCESS_KEY environment variable is required")
# Worker ID
worker_id = os.getenv('WORKER_ID', '100001')
# 并发数
max_concurrency = int(os.getenv('MAX_CONCURRENCY', '4'))
# 心跳间隔
heartbeat_interval = int(os.getenv('HEARTBEAT_INTERVAL', '5'))
# 租约配置
lease_extension_threshold = int(os.getenv('LEASE_EXTENSION_THRESHOLD', '60'))
lease_extension_duration = int(os.getenv('LEASE_EXTENSION_DURATION', '300'))
# 临时目录
temp_dir = os.getenv('TEMP_DIR', os.getenv('TEMP', '/tmp/render_worker'))
# 能力列表
capabilities_str = os.getenv('CAPABILITIES', '')
if capabilities_str:
capabilities = [c.strip() for c in capabilities_str.split(',') if c.strip()]
else:
capabilities = DEFAULT_CAPABILITIES.copy()
# FFmpeg 超时
ffmpeg_timeout = int(os.getenv('FFMPEG_TIMEOUT', '3600'))
# 下载/上传超时
download_timeout = int(os.getenv('DOWNLOAD_TIMEOUT', '300'))
upload_timeout = int(os.getenv('UPLOAD_TIMEOUT', '600'))
# 硬件加速配置
hw_accel = os.getenv('HW_ACCEL', HW_ACCEL_NONE).lower()
if hw_accel not in HW_ACCEL_TYPES:
hw_accel = HW_ACCEL_NONE
# GPU 设备列表(用于多显卡调度)
gpu_devices_str = os.getenv('GPU_DEVICES', '')
gpu_devices: List[int] = []
if gpu_devices_str:
try:
gpu_devices = [int(d.strip()) for d in gpu_devices_str.split(',') if d.strip()]
except ValueError:
logger.warning(f"Invalid GPU_DEVICES value: {gpu_devices_str}, using auto-detect")
gpu_devices = []
# 素材缓存配置
cache_enabled = os.getenv('CACHE_ENABLED', 'true').lower() in ('true', '1', 'yes')
cache_dir = os.getenv('CACHE_DIR', '') # 空字符串表示使用默认路径
cache_max_size_gb = float(os.getenv('CACHE_MAX_SIZE_GB', '0'))
return cls(
api_endpoint=api_endpoint,
access_key=access_key,
worker_id=worker_id,
max_concurrency=max_concurrency,
heartbeat_interval=heartbeat_interval,
lease_extension_threshold=lease_extension_threshold,
lease_extension_duration=lease_extension_duration,
temp_dir=temp_dir,
capabilities=capabilities,
ffmpeg_timeout=ffmpeg_timeout,
download_timeout=download_timeout,
upload_timeout=upload_timeout,
hw_accel=hw_accel,
gpu_devices=gpu_devices,
cache_enabled=cache_enabled,
cache_dir=cache_dir if cache_dir else os.path.join(temp_dir, 'cache'),
cache_max_size_gb=cache_max_size_gb
)
def get_work_dir_path(self, task_id: str) -> str:
"""获取任务工作目录路径"""
return os.path.join(self.temp_dir, f"task_{task_id}")
def ensure_temp_dir(self) -> None:
"""确保临时目录存在"""
os.makedirs(self.temp_dir, exist_ok=True)
def is_hw_accel_enabled(self) -> bool:
"""是否启用了硬件加速"""
return self.hw_accel != HW_ACCEL_NONE
def is_qsv(self) -> bool:
"""是否使用 QSV 硬件加速"""
return self.hw_accel == HW_ACCEL_QSV
def is_cuda(self) -> bool:
"""是否使用 CUDA 硬件加速"""
return self.hw_accel == HW_ACCEL_CUDA
def has_multi_gpu(self) -> bool:
"""是否配置了多 GPU"""
return len(self.gpu_devices) > 1
def get_gpu_devices(self) -> List[int]:
"""获取 GPU 设备列表"""
return self.gpu_devices.copy()

View File

@@ -1,31 +0,0 @@
# -*- coding: utf-8 -*-
"""
GPU 设备模型
定义 GPU 设备的数据结构。
"""
from dataclasses import dataclass
from typing import Optional
@dataclass
class GPUDevice:
"""
GPU 设备信息
Attributes:
index: 设备索引(对应 nvidia-smi 中的 GPU ID)
name: 设备名称(如 "NVIDIA GeForce RTX 3090"
memory_total: 显存总量(MB),可选
available: 设备是否可用
"""
index: int
name: str
memory_total: Optional[int] = None
available: bool = True
def __str__(self) -> str:
status = "available" if self.available else "unavailable"
mem_info = f", {self.memory_total}MB" if self.memory_total else ""
return f"GPU[{self.index}]: {self.name}{mem_info} ({status})"

View File

@@ -1,105 +0,0 @@
# -*- coding: utf-8 -*-
"""
任务结果模型
定义错误码、重试配置、任务结果等数据结构。
"""
from enum import Enum
from dataclasses import dataclass
from typing import Optional, Dict, Any, List
class ErrorCode(Enum):
"""错误码枚举"""
E_INPUT_UNAVAILABLE = "E_INPUT_UNAVAILABLE" # 素材不可访问/404
E_FFMPEG_FAILED = "E_FFMPEG_FAILED" # FFmpeg 执行失败
E_UPLOAD_FAILED = "E_UPLOAD_FAILED" # 上传失败
E_SPEC_INVALID = "E_SPEC_INVALID" # renderSpec 非法
E_TIMEOUT = "E_TIMEOUT" # 执行超时
E_UNKNOWN = "E_UNKNOWN" # 未知错误
# 重试配置
RETRY_CONFIG: Dict[ErrorCode, Dict[str, Any]] = {
ErrorCode.E_INPUT_UNAVAILABLE: {
'max_retries': 3,
'backoff': [1, 2, 5] # 重试间隔(秒)
},
ErrorCode.E_FFMPEG_FAILED: {
'max_retries': 2,
'backoff': [1, 3]
},
ErrorCode.E_UPLOAD_FAILED: {
'max_retries': 3,
'backoff': [1, 2, 5]
},
ErrorCode.E_SPEC_INVALID: {
'max_retries': 0, # 不重试
'backoff': []
},
ErrorCode.E_TIMEOUT: {
'max_retries': 2,
'backoff': [5, 10]
},
ErrorCode.E_UNKNOWN: {
'max_retries': 1,
'backoff': [2]
},
}
@dataclass
class TaskResult:
"""
任务结果
封装任务执行的结果,包括成功数据或失败信息。
"""
success: bool
data: Optional[Dict[str, Any]] = None
error_code: Optional[ErrorCode] = None
error_message: Optional[str] = None
@classmethod
def ok(cls, data: Dict[str, Any]) -> 'TaskResult':
"""创建成功结果"""
return cls(success=True, data=data)
@classmethod
def fail(cls, error_code: ErrorCode, error_message: str) -> 'TaskResult':
"""创建失败结果"""
return cls(
success=False,
error_code=error_code,
error_message=error_message
)
def to_report_dict(self) -> Dict[str, Any]:
"""
转换为上报格式
用于 API 上报时的数据格式转换。
"""
if self.success:
return {'result': self.data}
else:
return {
'errorCode': self.error_code.value if self.error_code else 'E_UNKNOWN',
'errorMessage': self.error_message or 'Unknown error'
}
def can_retry(self) -> bool:
"""是否可以重试"""
if self.success:
return False
if not self.error_code:
return True
config = RETRY_CONFIG.get(self.error_code, {})
return config.get('max_retries', 0) > 0
def get_retry_config(self) -> Dict[str, Any]:
"""获取重试配置"""
if not self.error_code:
return {'max_retries': 1, 'backoff': [2]}
return RETRY_CONFIG.get(self.error_code, {'max_retries': 1, 'backoff': [2]})

View File

@@ -1,554 +0,0 @@
# -*- coding: utf-8 -*-
"""
任务领域模型
定义任务类型、任务实体、渲染规格、输出规格等数据结构。
"""
from enum import Enum
from dataclasses import dataclass, field
from typing import Dict, Any, Optional, List
from datetime import datetime
from urllib.parse import urlparse, unquote
import os
# 支持的图片扩展名
IMAGE_EXTENSIONS = {'.jpg', '.jpeg', '.png', '.webp', '.bmp', '.gif'}
class TaskType(Enum):
"""任务类型枚举"""
RENDER_SEGMENT_VIDEO = "RENDER_SEGMENT_VIDEO" # 渲染视频片段
COMPOSE_TRANSITION = "COMPOSE_TRANSITION" # 合成转场效果
PREPARE_JOB_AUDIO = "PREPARE_JOB_AUDIO" # 生成全局音频
PACKAGE_SEGMENT_TS = "PACKAGE_SEGMENT_TS" # 封装 TS 分片
FINALIZE_MP4 = "FINALIZE_MP4" # 产出最终 MP4
# 支持的转场类型(对应 FFmpeg xfade 参数)
TRANSITION_TYPES = {
'fade': 'fade', # 淡入淡出(默认)
'dissolve': 'dissolve', # 溶解过渡
'wipeleft': 'wipeleft', # 向左擦除
'wiperight': 'wiperight', # 向右擦除
'wipeup': 'wipeup', # 向上擦除
'wipedown': 'wipedown', # 向下擦除
'slideleft': 'slideleft', # 向左滑动
'slideright': 'slideright', # 向右滑动
'slideup': 'slideup', # 向上滑动
'slidedown': 'slidedown', # 向下滑动
}
# 支持的特效类型
EFFECT_TYPES = {
'cameraShot', # 相机定格效果
'zoom', # 缩放效果(预留)
'blur', # 模糊效果(预留)
}
class TaskStatus(Enum):
"""任务状态枚举"""
PENDING = "PENDING"
RUNNING = "RUNNING"
SUCCESS = "SUCCESS"
FAILED = "FAILED"
@dataclass
class TransitionConfig:
"""
转场配置
用于 RENDER_SEGMENT_VIDEO 任务的入场/出场转场配置。
"""
type: str = "fade" # 转场类型
duration_ms: int = 500 # 转场时长(毫秒)
@classmethod
def from_dict(cls, data: Optional[Dict]) -> Optional['TransitionConfig']:
"""从字典创建 TransitionConfig"""
if not data:
return None
trans_type = data.get('type', 'fade')
# 验证转场类型是否支持
if trans_type not in TRANSITION_TYPES:
trans_type = 'fade'
return cls(
type=trans_type,
duration_ms=int(data.get('durationMs', 500))
)
def get_overlap_ms(self) -> int:
"""获取 overlap 时长(单边,为转场时长的一半)"""
return self.duration_ms // 2
def get_ffmpeg_transition(self) -> str:
"""获取 FFmpeg xfade 参数"""
return TRANSITION_TYPES.get(self.type, 'fade')
@dataclass
class Effect:
"""
特效配置
格式:type:params
例如:cameraShot:3,1 表示在第3秒定格1秒
"""
effect_type: str # 效果类型
params: str = "" # 参数字符串
@classmethod
def from_string(cls, effect_str: str) -> Optional['Effect']:
"""
从字符串解析 Effect
格式:type:params 或 type(无参数时)
"""
if not effect_str:
return None
parts = effect_str.split(':', 1)
effect_type = parts[0].strip()
if effect_type not in EFFECT_TYPES:
return None
params = parts[1].strip() if len(parts) > 1 else ""
return cls(effect_type=effect_type, params=params)
@classmethod
def parse_effects(cls, effects_str: Optional[str]) -> List['Effect']:
"""
解析效果字符串
格式:effect1|effect2|effect3
例如:cameraShot:3,1|blur:5
"""
if not effects_str:
return []
effects = []
for part in effects_str.split('|'):
effect = cls.from_string(part.strip())
if effect:
effects.append(effect)
return effects
def get_camera_shot_params(self) -> tuple:
"""
获取 cameraShot 效果参数
Returns:
(start_sec, duration_sec): 开始时间和持续时间(秒)
"""
if self.effect_type != 'cameraShot':
return (0, 0)
if not self.params:
return (3, 1) # 默认值
parts = self.params.split(',')
try:
start = int(parts[0]) if len(parts) >= 1 else 3
duration = int(parts[1]) if len(parts) >= 2 else 1
return (start, duration)
except ValueError:
return (3, 1)
@dataclass
class RenderSpec:
"""
渲染规格
用于 RENDER_SEGMENT_VIDEO 任务,定义视频渲染参数。
"""
crop_enable: bool = False
crop_size: Optional[str] = None
speed: str = "1.0"
lut_url: Optional[str] = None
overlay_url: Optional[str] = None
effects: Optional[str] = None
zoom_cut: bool = False
video_crop: Optional[str] = None
face_pos: Optional[str] = None
transitions: Optional[str] = None
# 转场配置(PRD v2 新增)
transition_in: Optional[TransitionConfig] = None # 入场转场
transition_out: Optional[TransitionConfig] = None # 出场转场
@classmethod
def from_dict(cls, data: Optional[Dict]) -> 'RenderSpec':
"""从字典创建 RenderSpec"""
if not data:
return cls()
return cls(
crop_enable=data.get('cropEnable', False),
crop_size=data.get('cropSize'),
speed=str(data.get('speed', '1.0')),
lut_url=data.get('lutUrl'),
overlay_url=data.get('overlayUrl'),
effects=data.get('effects'),
zoom_cut=data.get('zoomCut', False),
video_crop=data.get('videoCrop'),
face_pos=data.get('facePos'),
transitions=data.get('transitions'),
transition_in=TransitionConfig.from_dict(data.get('transitionIn')),
transition_out=TransitionConfig.from_dict(data.get('transitionOut'))
)
def has_transition_in(self) -> bool:
"""是否有入场转场"""
return self.transition_in is not None and self.transition_in.duration_ms > 0
def has_transition_out(self) -> bool:
"""是否有出场转场"""
return self.transition_out is not None and self.transition_out.duration_ms > 0
def get_overlap_head_ms(self) -> int:
"""获取头部 overlap 时长(毫秒)"""
if self.has_transition_in():
return self.transition_in.get_overlap_ms()
return 0
def get_overlap_tail_ms(self) -> int:
"""获取尾部 overlap 时长(毫秒)"""
if self.has_transition_out():
return self.transition_out.get_overlap_ms()
return 0
def get_effects(self) -> List['Effect']:
"""获取解析后的特效列表"""
return Effect.parse_effects(self.effects)
@dataclass
class OutputSpec:
"""
输出规格
用于 RENDER_SEGMENT_VIDEO 任务,定义视频输出参数。
"""
width: int = 1080
height: int = 1920
fps: int = 30
bitrate: int = 4000000
codec: str = "h264"
@classmethod
def from_dict(cls, data: Optional[Dict]) -> 'OutputSpec':
"""从字典创建 OutputSpec"""
if not data:
return cls()
return cls(
width=data.get('width', 1080),
height=data.get('height', 1920),
fps=data.get('fps', 30),
bitrate=data.get('bitrate', 4000000),
codec=data.get('codec', 'h264')
)
@dataclass
class AudioSpec:
"""
音频规格
用于 PREPARE_JOB_AUDIO 任务中的片段叠加音效。
"""
audio_url: Optional[str] = None
volume: float = 1.0
fade_in_ms: int = 10
fade_out_ms: int = 10
start_ms: int = 0
delay_ms: int = 0
loop_enable: bool = False
@classmethod
def from_dict(cls, data: Optional[Dict]) -> Optional['AudioSpec']:
"""从字典创建 AudioSpec"""
if not data:
return None
return cls(
audio_url=data.get('audioUrl'),
volume=float(data.get('volume', 1.0)),
fade_in_ms=int(data.get('fadeInMs', 10)),
fade_out_ms=int(data.get('fadeOutMs', 10)),
start_ms=int(data.get('startMs', 0)),
delay_ms=int(data.get('delayMs', 0)),
loop_enable=data.get('loopEnable', False)
)
@dataclass
class AudioProfile:
"""
音频配置
用于 PREPARE_JOB_AUDIO 任务的全局音频参数。
"""
sample_rate: int = 48000
channels: int = 2
codec: str = "aac"
@classmethod
def from_dict(cls, data: Optional[Dict]) -> 'AudioProfile':
"""从字典创建 AudioProfile"""
if not data:
return cls()
return cls(
sample_rate=data.get('sampleRate', 48000),
channels=data.get('channels', 2),
codec=data.get('codec', 'aac')
)
@dataclass
class Task:
"""
任务实体
表示一个待执行的渲染任务。
"""
task_id: str
task_type: TaskType
priority: int
lease_expire_time: datetime
payload: Dict[str, Any]
@classmethod
def from_dict(cls, data: Dict) -> 'Task':
"""从 API 响应字典创建 Task"""
lease_time_str = data.get('leaseExpireTime', '')
# 解析 ISO 8601 时间格式
if lease_time_str:
if lease_time_str.endswith('Z'):
lease_time_str = lease_time_str[:-1] + '+00:00'
try:
lease_expire_time = datetime.fromisoformat(lease_time_str)
except ValueError:
# 解析失败时使用当前时间 + 5分钟
lease_expire_time = datetime.now()
else:
lease_expire_time = datetime.now()
return cls(
task_id=str(data['taskId']),
task_type=TaskType(data['taskType']),
priority=data.get('priority', 0),
lease_expire_time=lease_expire_time,
payload=data.get('payload', {})
)
def get_job_id(self) -> str:
"""获取作业 ID"""
return str(self.payload.get('jobId', ''))
def get_segment_id(self) -> Optional[str]:
"""获取片段 ID(如果有)"""
segment_id = self.payload.get('segmentId')
return str(segment_id) if segment_id else None
def get_plan_segment_index(self) -> int:
"""获取计划片段索引"""
return int(self.payload.get('planSegmentIndex', 0))
def get_duration_ms(self) -> int:
"""获取时长(毫秒)"""
return int(self.payload.get('durationMs', 5000))
def get_material_url(self) -> Optional[str]:
"""
获取素材 URL
优先使用 boundMaterialUrl(实际可下载的 HTTP URL),
如果不存在则回退到 sourceRef(可能是 slot 引用)。
Returns:
素材 URL,如果都不存在返回 None
"""
return self.payload.get('boundMaterialUrl') or self.payload.get('sourceRef')
def get_source_ref(self) -> Optional[str]:
"""获取素材源引用(slot 标识符,如 device:xxx)"""
return self.payload.get('sourceRef')
def get_bound_material_url(self) -> Optional[str]:
"""获取绑定的素材 URL(实际可下载的 HTTP URL)"""
return self.payload.get('boundMaterialUrl')
def get_material_type(self) -> str:
"""
获取素材类型
优先使用服务端下发的 materialType 字段,
如果不存在则根据 URL 后缀自动推断。
Returns:
素材类型:"video""image"
"""
# 优先使用服务端下发的类型
material_type = self.payload.get('materialType')
if material_type in ('video', 'image'):
return material_type
# 降级:根据 URL 后缀推断
material_url = self.get_material_url()
if material_url:
parsed = urlparse(material_url)
path = unquote(parsed.path)
_, ext = os.path.splitext(path)
if ext.lower() in IMAGE_EXTENSIONS:
return 'image'
# 默认视频类型
return 'video'
def is_image_material(self) -> bool:
"""判断素材是否为图片类型"""
return self.get_material_type() == 'image'
def get_render_spec(self) -> RenderSpec:
"""获取渲染规格"""
return RenderSpec.from_dict(self.payload.get('renderSpec'))
def get_output_spec(self) -> OutputSpec:
"""获取输出规格"""
return OutputSpec.from_dict(self.payload.get('output'))
def get_transition_type(self) -> Optional[str]:
"""获取转场类型(来自 TaskPayload 顶层)"""
return self.payload.get('transitionType')
def get_transition_ms(self) -> int:
"""获取转场时长(毫秒,来自 TaskPayload 顶层)"""
return int(self.payload.get('transitionMs', 0))
def has_transition(self) -> bool:
"""是否有转场效果"""
return self.get_transition_ms() > 0
def get_overlap_tail_ms(self) -> int:
"""
获取尾部 overlap 时长(毫秒)
转场发生在当前片段与下一片段之间,当前片段需要在尾部多渲染 overlap 帧。
overlap = transitionMs / 2
"""
return self.get_transition_ms() // 2
def get_transition_in_type(self) -> Optional[str]:
"""获取入场转场类型(来自前一片段的出场转场)"""
return self.payload.get('transitionInType')
def get_transition_in_ms(self) -> int:
"""获取入场转场时长(毫秒)"""
return int(self.payload.get('transitionInMs', 0))
def get_transition_out_type(self) -> Optional[str]:
"""获取出场转场类型(当前片段的转场配置)"""
return self.payload.get('transitionOutType')
def get_transition_out_ms(self) -> int:
"""获取出场转场时长(毫秒)"""
return int(self.payload.get('transitionOutMs', 0))
def has_transition_in(self) -> bool:
"""是否有入场转场"""
return self.get_transition_in_ms() > 0
def has_transition_out(self) -> bool:
"""是否有出场转场"""
return self.get_transition_out_ms() > 0
def get_overlap_head_ms(self) -> int:
"""
获取头部 overlap 时长(毫秒)
入场转场来自前一个片段,当前片段需要在头部多渲染 overlap 帧。
overlap = transitionInMs / 2
"""
return self.get_transition_in_ms() // 2
def get_overlap_tail_ms_v2(self) -> int:
"""
获取尾部 overlap 时长(毫秒)- 使用新的字段名
出场转场用于当前片段与下一片段之间,当前片段需要在尾部多渲染 overlap 帧。
overlap = transitionOutMs / 2
"""
return self.get_transition_out_ms() // 2
def get_bgm_url(self) -> Optional[str]:
"""获取 BGM URL"""
return self.payload.get('bgmUrl')
def get_total_duration_ms(self) -> int:
"""获取总时长(毫秒)"""
return int(self.payload.get('totalDurationMs', 0))
def get_segments(self) -> List[Dict]:
"""获取片段列表"""
return self.payload.get('segments', [])
def get_audio_profile(self) -> AudioProfile:
"""获取音频配置"""
return AudioProfile.from_dict(self.payload.get('audioProfile'))
def get_video_url(self) -> Optional[str]:
"""获取视频 URL(用于 PACKAGE_SEGMENT_TS)"""
return self.payload.get('videoUrl')
def get_audio_url(self) -> Optional[str]:
"""获取音频 URL(用于 PACKAGE_SEGMENT_TS)"""
return self.payload.get('audioUrl')
def get_start_time_ms(self) -> int:
"""获取开始时间(毫秒)"""
return int(self.payload.get('startTimeMs', 0))
def get_m3u8_url(self) -> Optional[str]:
"""获取 m3u8 URL(用于 FINALIZE_MP4)"""
return self.payload.get('m3u8Url')
def get_ts_list(self) -> List[str]:
"""获取 TS 列表(用于 FINALIZE_MP4)"""
return self.payload.get('tsList', [])
# ========== COMPOSE_TRANSITION 相关方法 ==========
def get_transition_id(self) -> Optional[str]:
"""获取转场 ID(用于 COMPOSE_TRANSITION)"""
return self.payload.get('transitionId')
def get_prev_segment(self) -> Optional[Dict]:
"""获取前一个片段信息(用于 COMPOSE_TRANSITION)"""
return self.payload.get('prevSegment')
def get_next_segment(self) -> Optional[Dict]:
"""获取后一个片段信息(用于 COMPOSE_TRANSITION)"""
return self.payload.get('nextSegment')
def get_transition_config(self) -> Optional[TransitionConfig]:
"""获取转场配置(用于 COMPOSE_TRANSITION)"""
return TransitionConfig.from_dict(self.payload.get('transition'))
# ========== PACKAGE_SEGMENT_TS 转场相关方法 ==========
def is_transition_segment(self) -> bool:
"""是否为转场分片(用于 PACKAGE_SEGMENT_TS)"""
return self.payload.get('isTransitionSegment', False)
def should_trim_head(self) -> bool:
"""是否需要裁剪头部 overlap(用于 PACKAGE_SEGMENT_TS)"""
return self.payload.get('trimHead', False)
def should_trim_tail(self) -> bool:
"""是否需要裁剪尾部 overlap(用于 PACKAGE_SEGMENT_TS)"""
return self.payload.get('trimTail', False)
def get_trim_head_ms(self) -> int:
"""获取头部裁剪时长(毫秒)"""
return int(self.payload.get('trimHeadMs', 0))
def get_trim_tail_ms(self) -> int:
"""获取尾部裁剪时长(毫秒)"""
return int(self.payload.get('trimTailMs', 0))

0
entity/__init__.py Normal file
View File

View File

@@ -0,0 +1,25 @@
from .base import EffectProcessor, EffectRegistry
from .camera_shot import CameraShotEffect
from .speed import SpeedEffect
from .zoom import ZoomEffect
from .skip import SkipEffect
from .tail import TailEffect
# 注册所有效果处理器
registry = EffectRegistry()
registry.register("cameraShot", CameraShotEffect)
registry.register("ospeed", SpeedEffect)
registry.register("zoom", ZoomEffect)
registry.register("skip", SkipEffect)
registry.register("tail", TailEffect)
__all__ = [
"EffectProcessor",
"EffectRegistry",
"registry",
"CameraShotEffect",
"SpeedEffect",
"ZoomEffect",
"SkipEffect",
"TailEffect",
]

103
entity/effects/base.py Normal file
View File

@@ -0,0 +1,103 @@
from abc import ABC, abstractmethod
from typing import Dict, List, Type, Any, Optional
import json
import logging
logger = logging.getLogger(__name__)
class EffectProcessor(ABC):
"""效果处理器抽象基类"""
def __init__(self, params: str = "", ext_data: Optional[Dict[str, Any]] = None):
self.params = params
self.ext_data = ext_data or {}
self.frame_rate = 25 # 默认帧率
@abstractmethod
def validate_params(self) -> bool:
"""验证参数是否有效"""
pass
@abstractmethod
def generate_filter_args(
self, video_input: str, effect_index: int
) -> tuple[List[str], str]:
"""
生成FFmpeg滤镜参数
Args:
video_input: 输入视频流标识符 (例如: "[0:v]", "[v_eff1]")
effect_index: 效果索引,用于生成唯一的输出标识符
Returns:
tuple: (filter_args_list, output_stream_identifier)
"""
pass
@abstractmethod
def get_effect_name(self) -> str:
"""获取效果名称"""
pass
def parse_params(self) -> List[str]:
"""解析参数字符串为列表"""
if not self.params:
return []
return self.params.split(",")
def get_pos_json(self) -> Dict[str, Any]:
"""获取位置JSON数据"""
pos_json_str = self.ext_data.get("posJson", "{}")
try:
return json.loads(pos_json_str) if pos_json_str != "{}" else {}
except Exception as e:
logger.warning(f"Failed to parse posJson: {e}")
return {}
class EffectRegistry:
"""效果处理器注册表"""
def __init__(self):
self._processors: Dict[str, Type[EffectProcessor]] = {}
def register(self, name: str, processor_class: Type[EffectProcessor]):
"""注册效果处理器"""
if not issubclass(processor_class, EffectProcessor):
raise ValueError(f"{processor_class} must be a subclass of EffectProcessor")
self._processors[name] = processor_class
logger.debug(f"Registered effect processor: {name}")
def get_processor(
self,
effect_name: str,
params: str = "",
ext_data: Optional[Dict[str, Any]] = None,
) -> Optional[EffectProcessor]:
"""获取效果处理器实例"""
if effect_name not in self._processors:
logger.warning(f"Unknown effect: {effect_name}")
return None
processor_class = self._processors[effect_name]
return processor_class(params, ext_data)
def list_effects(self) -> List[str]:
"""列出所有注册的效果"""
return list(self._processors.keys())
def parse_effect_string(self, effect_string: str) -> tuple[str, str]:
"""
解析效果字符串
Args:
effect_string: 效果字符串,格式为 "effect_name:params"
Returns:
tuple: (effect_name, params)
"""
if ":" in effect_string:
parts = effect_string.split(":", 2)
return parts[0], parts[1] if len(parts) > 1 else ""
return effect_string, ""

View File

@@ -0,0 +1,99 @@
from typing import List
from .base import EffectProcessor
class CameraShotEffect(EffectProcessor):
"""相机镜头效果处理器"""
def validate_params(self) -> bool:
"""验证参数:start_time,duration,rotate_deg"""
params = self.parse_params()
if not params:
return True # 使用默认参数
# 参数格式: "start_time,duration,rotate_deg"
if len(params) > 3:
return False
try:
for i, param in enumerate(params):
if param == "":
continue
if i == 2: # rotate_deg
int(param)
else: # start_time, duration
float(param)
return True
except ValueError:
return False
def generate_filter_args(
self, video_input: str, effect_index: int
) -> tuple[List[str], str]:
"""生成相机镜头效果的滤镜参数"""
if not self.validate_params():
return [], video_input
params = self.parse_params()
# 设置默认值
start = 3.0
duration = 1.0
rotate_deg = 0
if len(params) >= 1 and params[0] != "":
start = float(params[0])
if len(params) >= 2 and params[1] != "":
duration = float(params[1])
if len(params) >= 3 and params[2] != "":
rotate_deg = int(params[2])
filter_args = []
# 生成输出流标识符
start_out_str = "[eff_s]"
mid_out_str = "[eff_m]"
end_out_str = "[eff_e]"
final_output = f"[v_eff{effect_index}]"
# 分割视频流为三部分
filter_args.append(
f"{video_input}split=3{start_out_str}{mid_out_str}{end_out_str}"
)
# 选择开始部分帧
filter_args.append(
f"{start_out_str}select=lt(n\\,"
f"{int(start * self.frame_rate)}){start_out_str}"
)
# 选择结束部分帧
filter_args.append(
f"{end_out_str}select=gt(n\\,"
f"{int(start * self.frame_rate)}){end_out_str}"
)
# 选择中间特定帧并扩展
filter_args.append(
f"{mid_out_str}select=eq(n\\,"
f"{int(start * self.frame_rate)}){mid_out_str}"
)
filter_args.append(
f"{mid_out_str}tpad=start_mode=clone:"
f"start_duration={duration:.4f}{mid_out_str}"
)
# 如果需要旋转
if rotate_deg != 0:
filter_args.append(f"{mid_out_str}rotate=PI*{rotate_deg}/180{mid_out_str}")
# 连接三部分
filter_args.append(
f"{start_out_str}{mid_out_str}{end_out_str}concat=n=3:v=1:a=0,"
f"setpts=N/{self.frame_rate}/TB{final_output}"
)
return filter_args, final_output
def get_effect_name(self) -> str:
return "cameraShot"

41
entity/effects/skip.py Normal file
View File

@@ -0,0 +1,41 @@
from typing import List
from .base import EffectProcessor
class SkipEffect(EffectProcessor):
"""跳过开头效果处理器"""
def validate_params(self) -> bool:
"""验证参数:跳过的秒数"""
if not self.params:
return True # 默认不跳过
try:
skip_seconds = float(self.params)
return skip_seconds >= 0
except ValueError:
return False
def generate_filter_args(
self, video_input: str, effect_index: int
) -> tuple[List[str], str]:
"""生成跳过开头效果的滤镜参数"""
if not self.validate_params():
return [], video_input
if not self.params:
return [], video_input
skip_seconds = float(self.params)
if skip_seconds <= 0:
return [], video_input
output_stream = f"[v_eff{effect_index}]"
# 使用trim滤镜跳过开头
filter_args = [f"{video_input}trim=start={skip_seconds}{output_stream}"]
return filter_args, output_stream
def get_effect_name(self) -> str:
return "skip"

38
entity/effects/speed.py Normal file
View File

@@ -0,0 +1,38 @@
from typing import List
from .base import EffectProcessor
class SpeedEffect(EffectProcessor):
"""视频变速效果处理器"""
def validate_params(self) -> bool:
"""验证参数:速度倍数"""
if not self.params:
return True # 默认不变速
try:
speed = float(self.params)
return speed > 0
except ValueError:
return False
def generate_filter_args(
self, video_input: str, effect_index: int
) -> tuple[List[str], str]:
"""生成变速效果的滤镜参数"""
if not self.validate_params():
return [], video_input
if not self.params or self.params == "1":
return [], video_input # 不需要变速
speed = float(self.params)
output_stream = f"[v_eff{effect_index}]"
# 使用setpts进行变速
filter_args = [f"{video_input}setpts={speed}*PTS{output_stream}"]
return filter_args, output_stream
def get_effect_name(self) -> str:
return "ospeed"

46
entity/effects/tail.py Normal file
View File

@@ -0,0 +1,46 @@
from typing import List
from .base import EffectProcessor
class TailEffect(EffectProcessor):
"""保留末尾效果处理器"""
def validate_params(self) -> bool:
"""验证参数:保留的秒数"""
if not self.params:
return True # 默认不截取
try:
tail_seconds = float(self.params)
return tail_seconds >= 0
except ValueError:
return False
def generate_filter_args(
self, video_input: str, effect_index: int
) -> tuple[List[str], str]:
"""生成保留末尾效果的滤镜参数"""
if not self.validate_params():
return [], video_input
if not self.params:
return [], video_input
tail_seconds = float(self.params)
if tail_seconds <= 0:
return [], video_input
output_stream = f"[v_eff{effect_index}]"
# 使用reverse+trim+reverse的方法来精确获取最后N秒
filter_args = [
f"{video_input}reverse[v_rev{effect_index}]",
f"[v_rev{effect_index}]trim=duration={tail_seconds}"
f"[v_trim{effect_index}]",
f"[v_trim{effect_index}]reverse{output_stream}",
]
return filter_args, output_stream
def get_effect_name(self) -> str:
return "tail"

89
entity/effects/zoom.py Normal file
View File

@@ -0,0 +1,89 @@
from typing import List
from .base import EffectProcessor
class ZoomEffect(EffectProcessor):
"""缩放效果处理器"""
def validate_params(self) -> bool:
"""验证参数:start_time,zoom_factor,duration"""
params = self.parse_params()
if len(params) < 3:
return False
try:
start_time = float(params[0])
zoom_factor = float(params[1])
duration = float(params[2])
return start_time >= 0 and zoom_factor > 0 and duration >= 0
except (ValueError, IndexError):
return False
def generate_filter_args(
self, video_input: str, effect_index: int
) -> tuple[List[str], str]:
"""生成缩放效果的滤镜参数"""
if not self.validate_params():
return [], video_input
params = self.parse_params()
start_time = float(params[0])
zoom_factor = float(params[1])
duration = float(params[2])
if zoom_factor == 1:
return [], video_input # 不需要缩放
output_stream = f"[v_eff{effect_index}]"
# 获取缩放中心点
center_x, center_y = self._get_zoom_center()
filter_args = []
if duration == 0:
# 静态缩放(整个视频时长)
x_expr = f"({center_x})-(ow*zoom)/2"
y_expr = f"({center_y})-(oh*zoom)/2"
filter_args.append(
f"{video_input}trim=start={start_time},zoompan=z={zoom_factor}:x={x_expr}:y={y_expr}:d=1{output_stream}"
)
else:
# 动态缩放(指定时间段内)
zoom_expr = f"if(between(t\\,{start_time}\\,{start_time + duration})\\,{zoom_factor}\\,1)"
x_expr = f"({center_x})-(ow*zoom)/2"
y_expr = f"({center_y})-(oh*zoom)/2"
filter_args.append(
f"{video_input}zoompan=z={zoom_expr}:x={x_expr}:y={y_expr}:d=1{output_stream}"
)
return filter_args, output_stream
def _get_zoom_center(self) -> tuple[str, str]:
"""获取缩放中心点坐标表达式"""
# 默认中心点
center_x = "iw/2"
center_y = "ih/2"
pos_json = self.get_pos_json()
if pos_json:
_f_x = pos_json.get("ltX", 0)
_f_x2 = pos_json.get("rbX", 0)
_f_y = pos_json.get("ltY", 0)
_f_y2 = pos_json.get("rbY", 0)
_v_w = pos_json.get("imgWidth", 1)
_v_h = pos_json.get("imgHeight", 1)
if _v_w > 0 and _v_h > 0:
# 计算坐标系统中的中心点
center_x_ratio = (_f_x + _f_x2) / (2 * _v_w)
center_y_ratio = (_f_y + _f_y2) / (2 * _v_h)
# 转换为视频坐标系统
center_x = f"iw*{center_x_ratio:.6f}"
center_y = f"ih*{center_y_ratio:.6f}"
return center_x, center_y
def get_effect_name(self) -> str:
return "zoom"

216
entity/ffmpeg.py Normal file
View File

@@ -0,0 +1,216 @@
# 保留用于向后兼容的常量定义
import os
DEFAULT_ARGS = ("-shortest",)
ENCODER_ARGS = (
(
"-c:v",
"h264",
)
if not os.getenv("ENCODER_ARGS", False)
else os.getenv("ENCODER_ARGS", "").split(" ")
)
VIDEO_ARGS = (
(
"-profile:v",
"high",
"-level:v",
"4",
)
if not os.getenv("VIDEO_ARGS", False)
else os.getenv("VIDEO_ARGS", "").split(" ")
)
AUDIO_ARGS = (
"-c:a",
"aac",
"-b:a",
"128k",
"-ar",
"48000",
"-ac",
"2",
)
MUTE_AUDIO_INPUT = (
"-f",
"lavfi",
"-i",
"anullsrc=cl=stereo:r=48000",
)
def get_mp4toannexb_filter():
"""
Determine which mp4toannexb filter to use based on ENCODER_ARGS.
Returns 'hevc_mp4toannexb' if ENCODER_ARGS contains 'hevc', otherwise 'h264_mp4toannexb'.
"""
encoder_args_str = os.getenv("ENCODER_ARGS", "").lower()
if "hevc" in encoder_args_str:
return "hevc_mp4toannexb"
return "h264_mp4toannexb"
class FfmpegTask(object):
"""
兼容类:保留原有FfmpegTask接口用于向后兼容
实际处理逻辑已迁移到新架构,该类主要用作数据载体
"""
def __init__(self, input_file, task_type="copy", output_file=""):
"""保持原有构造函数签名"""
self.annexb = False
if type(input_file) is str:
if input_file.endswith(".ts"):
self.annexb = True
self.input_file = [input_file]
elif type(input_file) is list:
self.input_file = input_file
else:
self.input_file = []
self.zoom_cut = None
self.center_cut = None
self.ext_data = {}
self.task_type = task_type
self.output_file = output_file or ""
self.mute = True
self.speed = 1
self.frame_rate = 25
self.resolution = None
self.subtitles = []
self.luts = []
self.audios = []
self.overlays = []
self.effects = []
def __repr__(self):
return f"FfmpegTask(input_file={self.input_file}, task_type={self.task_type})"
def analyze_input_render_tasks(self):
"""分析输入中的子任务"""
for i in self.input_file:
if isinstance(i, FfmpegTask) and i.need_run():
yield i
def need_run(self):
"""判断是否需要运行"""
if self.annexb:
return True
return not self.check_can_copy()
def add_inputs(self, *inputs):
"""添加输入文件"""
self.input_file.extend(inputs)
def add_overlay(self, *overlays):
"""添加覆盖层"""
for overlay in overlays:
if str(overlay).endswith(".ass"):
self.subtitles.append(overlay)
else:
self.overlays.append(overlay)
self.correct_task_type()
def add_audios(self, *audios):
"""添加音频"""
self.audios.extend(audios)
self.correct_task_type()
def add_lut(self, *luts):
"""添加LUT"""
self.luts.extend(luts)
self.correct_task_type()
def add_effect(self, *effects):
"""添加效果"""
self.effects.extend(effects)
self.correct_task_type()
def get_output_file(self):
"""获取输出文件"""
if self.task_type == "copy":
return self.input_file[0] if self.input_file else ""
if not self.output_file:
self.set_output_file()
return self.output_file
def correct_task_type(self):
"""校正任务类型"""
if self.check_can_copy():
self.task_type = "copy"
elif self.check_can_concat():
self.task_type = "concat"
else:
self.task_type = "encode"
def check_can_concat(self):
"""检查是否可以连接"""
return (
len(self.luts) == 0
and len(self.overlays) == 0
and len(self.subtitles) == 0
and len(self.effects) == 0
and self.speed == 1
and self.zoom_cut is None
and self.center_cut is None
)
def check_can_copy(self):
"""检查是否可以复制"""
return (
len(self.luts) == 0
and len(self.overlays) == 0
and len(self.subtitles) == 0
and len(self.effects) == 0
and self.speed == 1
and len(self.audios) == 0
and len(self.input_file) <= 1
and self.zoom_cut is None
and self.center_cut is None
)
def set_output_file(self, file=None):
"""设置输出文件"""
if file is None:
import uuid
if self.annexb:
self.output_file = f"rand_{uuid.uuid4()}.ts"
else:
self.output_file = f"rand_{uuid.uuid4()}.mp4"
else:
if isinstance(file, FfmpegTask):
if file != self:
self.output_file = file.get_output_file()
elif isinstance(file, str):
self.output_file = file
def check_annexb(self):
"""检查annexb格式"""
return self.annexb
def get_ffmpeg_args(self):
"""
保留用于向后兼容,但实际逻辑已迁移到新架构
建议使用新的 FFmpegCommandBuilder 来生成命令
"""
# 简化版本,主要用于向后兼容
if self.task_type == "copy" and len(self.input_file) == 1:
if isinstance(self.input_file[0], str):
if self.input_file[0] == self.get_output_file():
return []
return [
"-y",
"-hide_banner",
"-i",
self.input_file[0],
"-c",
"copy",
self.get_output_file(),
]
# 对于复杂情况,返回基础命令结构
# 实际处理会在新的服务架构中完成
return (
["-y", "-hide_banner", "-i"]
+ self.input_file
+ ["-c", "copy", self.get_output_file()]
)

View File

@@ -0,0 +1,314 @@
import time
from typing import List, Optional
from config.settings import get_ffmpeg_config
from entity.render_task import RenderTask, TaskType
from entity.effects import registry as effect_registry
from util.exceptions import FFmpegError
from util.ffmpeg import probe_video_info, probe_video_audio
from util.ffmpeg_utils import (
build_base_ffmpeg_args,
build_null_audio_input,
get_annexb_filter,
build_standard_output_args,
)
from util.json_utils import safe_json_loads
import logging
logger = logging.getLogger(__name__)
class FFmpegCommandBuilder:
"""FFmpeg命令构建器"""
def __init__(self, task: RenderTask):
self.task = task
self.config = get_ffmpeg_config()
def build_command(self) -> List[str]:
"""构建FFmpeg命令"""
self.task.update_task_type()
if self.task.task_type == TaskType.COPY:
return self._build_copy_command()
elif self.task.task_type == TaskType.CONCAT:
return self._build_concat_command()
elif self.task.task_type == TaskType.ENCODE:
return self._build_encode_command()
else:
raise FFmpegError(f"Unsupported task type: {self.task.task_type}")
def _build_copy_command(self) -> List[str]:
"""构建复制命令"""
if len(self.task.input_files) == 1:
input_file = self.task.input_files[0]
if input_file == self.task.output_file:
return [] # 不需要处理
return [
"ffmpeg",
"-y",
"-hide_banner",
"-i",
self.task.input_files[0],
"-c",
"copy",
self.task.output_file,
]
def _build_concat_command(self) -> List[str]:
"""构建拼接命令"""
args = ["ffmpeg", "-y", "-hide_banner"]
input_args = []
output_args = [*self.config.default_args]
filter_args = []
if len(self.task.input_files) == 1:
# 单个文件
file = self.task.input_files[0]
input_args.extend(["-i", file])
self.task.mute = not probe_video_audio(file)
else:
# 多个文件使用concat协议
tmp_file = f"tmp_concat_{time.time()}.txt"
with open(tmp_file, "w", encoding="utf-8") as f:
for input_file in self.task.input_files:
f.write(f"file '{input_file}'\n")
input_args.extend(["-f", "concat", "-safe", "0", "-i", tmp_file])
self.task.mute = not probe_video_audio(tmp_file, "concat")
# 视频流映射
output_args.extend(["-map", "0:v", "-c:v", "copy"])
# 音频处理
audio_output_str = self._handle_audio_concat(input_args, filter_args)
if audio_output_str:
output_args.extend(["-map", audio_output_str])
output_args.extend(self.config.audio_args)
# annexb处理
if self.task.annexb:
output_args.extend(["-bsf:v", self._get_mp4toannexb_filter()])
output_args.extend(["-bsf:a", "setts=pts=DTS"])
output_args.extend(["-f", "mpegts"])
else:
output_args.extend(["-f", "mp4"])
filter_complex = (
["-filter_complex", ";".join(filter_args)] if filter_args else []
)
return (
args + input_args + filter_complex + output_args + [self.task.output_file]
)
def _build_encode_command(self) -> List[str]:
"""构建编码命令"""
args = build_base_ffmpeg_args()
input_args = []
filter_args = []
output_args = build_standard_output_args()
# annexb处理
if self.task.annexb:
output_args.extend(["-bsf:v", get_annexb_filter()])
output_args.extend(["-reset_timestamps", "1"])
# 处理输入文件
for input_file in self.task.input_files:
input_args.extend(["-i", input_file])
# 处理视频流
video_output_str = "[0:v]"
effect_index = 0
# 处理中心裁剪
if self.task.center_cut == 1:
video_output_str, effect_index = self._add_center_cut(
filter_args, video_output_str, effect_index
)
# 处理缩放裁剪
if self.task.zoom_cut == 1 and self.task.resolution:
video_output_str, effect_index = self._add_zoom_cut(
filter_args, video_output_str, effect_index
)
# 处理效果
video_output_str, effect_index = self._add_effects(
filter_args, video_output_str, effect_index
)
# 处理分辨率
if self.task.resolution:
filter_args.append(
f"{video_output_str}scale={self.task.resolution.replace('x', ':')}[v]"
)
video_output_str = "[v]"
# 处理LUT
for lut in self.task.luts:
filter_args.append(f"{video_output_str}lut3d=file={lut}{video_output_str}")
# 处理覆盖层
video_output_str = self._add_overlays(input_args, filter_args, video_output_str)
# 处理字幕
for subtitle in self.task.subtitles:
filter_args.append(f"{video_output_str}ass={subtitle}[v]")
video_output_str = "[v]"
# 映射视频流
output_args.extend(["-map", video_output_str])
output_args.extend(["-r", str(self.task.frame_rate)])
output_args.extend(["-fps_mode", "cfr"])
# 处理音频
audio_output_str = self._handle_audio_encode(input_args, filter_args)
if audio_output_str:
output_args.extend(["-map", audio_output_str])
filter_complex = (
["-filter_complex", ";".join(filter_args)] if filter_args else []
)
return (
args + input_args + filter_complex + output_args + [self.task.output_file]
)
def _add_center_cut(
self, filter_args: List[str], video_input: str, effect_index: int
) -> tuple[str, int]:
"""添加中心裁剪"""
pos_json = self.task.ext_data.get("posJson", "{}")
pos_data = safe_json_loads(pos_json, {})
_v_w = pos_data.get("imgWidth", 1)
_f_x = pos_data.get("ltX", 0)
_f_x2 = pos_data.get("rbX", 0)
_x = f"{float((_f_x2 + _f_x)/(2 * _v_w)):.4f}*iw-ih*ih/(2*iw)"
filter_args.append(
f"{video_input}crop=x={_x}:y=0:w=ih*ih/iw:h=ih[v_cut{effect_index}]"
)
return f"[v_cut{effect_index}]", effect_index + 1
def _add_zoom_cut(
self, filter_args: List[str], video_input: str, effect_index: int
) -> tuple[str, int]:
"""添加缩放裁剪"""
# 获取输入视频尺寸
input_file = self.task.input_files[0]
_iw, _ih, _ = probe_video_info(input_file)
_w, _h = self.task.resolution.split("x", 1)
pos_json = self.task.ext_data.get("posJson", "{}")
pos_data = safe_json_loads(pos_json, {})
_f_x = pos_data.get("ltX", 0)
_f_x2 = pos_data.get("rbX", 0)
_f_y = pos_data.get("ltY", 0)
_f_y2 = pos_data.get("rbY", 0)
_x = min(max(0, int((_f_x + _f_x2) / 2 - int(_w) / 2)), _iw - int(_w))
_y = min(max(0, int((_f_y + _f_y2) / 2 - int(_h) / 2)), _ih - int(_h))
filter_args.append(
f"{video_input}crop=x={_x}:y={_y}:w={_w}:h={_h}[vz_cut{effect_index}]"
)
return f"[vz_cut{effect_index}]", effect_index + 1
def _add_effects(
self, filter_args: List[str], video_input: str, effect_index: int
) -> tuple[str, int]:
"""添加效果处理"""
current_input = video_input
for effect_str in self.task.effects:
effect_name, params = effect_registry.parse_effect_string(effect_str)
processor = effect_registry.get_processor(
effect_name, params, self.task.ext_data
)
if processor:
processor.frame_rate = self.task.frame_rate
effect_filters, output_stream = processor.generate_filter_args(
current_input, effect_index
)
if effect_filters:
filter_args.extend(effect_filters)
current_input = output_stream
effect_index += 1
return current_input, effect_index
def _add_overlays(
self, input_args: List[str], filter_args: List[str], video_input: str
) -> str:
"""添加覆盖层"""
current_input = video_input
for overlay in self.task.overlays:
input_index = input_args.count("-i") // 2 # 每个输入占两个参数 -i filename
input_args.extend(["-i", overlay])
if self.config.overlay_scale_mode == "scale":
filter_args.append(f"{current_input}[{input_index}:v]scale=iw:ih[v]")
else:
filter_args.append(
f"{current_input}[{input_index}:v]{self.config.overlay_scale_mode}=iw:ih[v]"
)
filter_args.append(f"[v][{input_index}:v]overlay=1:eof_action=endall[v]")
current_input = "[v]"
return current_input
def _handle_audio_concat(
self, input_args: List[str], filter_args: List[str]
) -> Optional[str]:
"""处理concat模式的音频"""
audio_output_str = ""
if self.task.mute:
input_index = input_args.count("-i") // 2
input_args.extend(build_null_audio_input())
audio_output_str = f"[{input_index}:a]"
else:
audio_output_str = "[0:a]"
for audio in self.task.audios:
input_index = input_args.count("-i") // 2
input_args.extend(["-i", audio.replace("\\", "/")])
filter_args.append(
f"{audio_output_str}[{input_index}:a]{self.config.amix_args[0]}[a]"
)
audio_output_str = "[a]"
return audio_output_str.strip("[]") if audio_output_str else None
def _handle_audio_encode(
self, input_args: List[str], filter_args: List[str]
) -> Optional[str]:
"""处理encode模式的音频"""
audio_output_str = ""
if self.task.mute:
input_index = input_args.count("-i") // 2
input_args.extend(["-f", "lavfi", "-i", "anullsrc=cl=stereo:r=48000"])
filter_args.append(f"[{input_index}:a]acopy[a]")
audio_output_str = "[a]"
else:
audio_output_str = "[0:a]"
for audio in self.task.audios:
input_index = input_args.count("-i") // 2
input_args.extend(["-i", audio.replace("\\", "/")])
filter_args.append(
f"{audio_output_str}[{input_index}:a]{self.config.amix_args[0]}[a]"
)
audio_output_str = "[a]"
return audio_output_str if audio_output_str else None

157
entity/render_task.py Normal file
View File

@@ -0,0 +1,157 @@
import uuid
from typing import List, Optional, Dict, Any
from dataclasses import dataclass, field
from enum import Enum
from util.exceptions import TaskValidationError, EffectError
from entity.effects import registry as effect_registry
class TaskType(Enum):
COPY = "copy"
CONCAT = "concat"
ENCODE = "encode"
@dataclass
class RenderTask:
"""渲染任务数据类,只包含任务数据,不包含处理逻辑"""
input_files: List[str] = field(default_factory=list)
output_file: str = ""
task_type: TaskType = TaskType.COPY
# 视频参数
resolution: Optional[str] = None
frame_rate: int = 25
speed: float = 1.0
mute: bool = True
annexb: bool = False
# 裁剪参数
zoom_cut: Optional[int] = None
center_cut: Optional[int] = None
# 资源列表
subtitles: List[str] = field(default_factory=list)
luts: List[str] = field(default_factory=list)
audios: List[str] = field(default_factory=list)
overlays: List[str] = field(default_factory=list)
effects: List[str] = field(default_factory=list)
# 扩展数据
ext_data: Dict[str, Any] = field(default_factory=dict)
def __post_init__(self):
"""初始化后处理"""
# 检测annexb格式
for input_file in self.input_files:
if isinstance(input_file, str) and input_file.endswith(".ts"):
self.annexb = True
break
# 自动生成输出文件名
if not self.output_file:
self._generate_output_filename()
def _generate_output_filename(self):
"""生成输出文件名"""
if self.annexb:
self.output_file = f"rand_{uuid.uuid4()}.ts"
else:
self.output_file = f"rand_{uuid.uuid4()}.mp4"
def add_input_file(self, file_path: str):
"""添加输入文件"""
self.input_files.append(file_path)
if file_path.endswith(".ts"):
self.annexb = True
def add_overlay(self, *overlays: str):
"""添加覆盖层"""
for overlay in overlays:
if overlay.endswith(".ass"):
self.subtitles.append(overlay)
else:
self.overlays.append(overlay)
def add_audios(self, *audios: str):
"""添加音频"""
self.audios.extend(audios)
def add_lut(self, *luts: str):
"""添加LUT"""
self.luts.extend(luts)
def add_effect(self, *effects: str):
"""添加效果"""
self.effects.extend(effects)
def validate(self) -> bool:
"""验证任务参数"""
if not self.input_files:
raise TaskValidationError("No input files specified")
# 验证所有效果
for effect_str in self.effects:
effect_name, params = effect_registry.parse_effect_string(effect_str)
processor = effect_registry.get_processor(
effect_name, params, self.ext_data
)
if processor and not processor.validate_params():
raise EffectError(
f"Invalid parameters for effect {effect_name}: {params}",
effect_name,
params,
)
return True
def can_copy(self) -> bool:
"""检查是否可以直接复制"""
return (
len(self.luts) == 0
and len(self.overlays) == 0
and len(self.subtitles) == 0
and len(self.effects) == 0
and self.speed == 1
and len(self.audios) == 0
and len(self.input_files) == 1
and self.zoom_cut is None
and self.center_cut is None
)
def can_concat(self) -> bool:
"""检查是否可以使用concat模式"""
return (
len(self.luts) == 0
and len(self.overlays) == 0
and len(self.subtitles) == 0
and len(self.effects) == 0
and self.speed == 1
and self.zoom_cut is None
and self.center_cut is None
)
def determine_task_type(self) -> TaskType:
"""自动确定任务类型"""
if self.can_copy():
return TaskType.COPY
elif self.can_concat():
return TaskType.CONCAT
else:
return TaskType.ENCODE
def update_task_type(self):
"""更新任务类型"""
self.task_type = self.determine_task_type()
def need_processing(self) -> bool:
"""检查是否需要处理"""
if self.annexb:
return True
return not self.can_copy()
def get_output_extension(self) -> str:
"""获取输出文件扩展名"""
return ".ts" if self.annexb else ".mp4"

View File

@@ -1,22 +0,0 @@
# -*- coding: utf-8 -*-
"""
任务处理器层
包含各种任务类型的具体处理器实现。
"""
from handlers.base import BaseHandler
from handlers.render_video import RenderSegmentVideoHandler
from handlers.compose_transition import ComposeTransitionHandler
from handlers.prepare_audio import PrepareJobAudioHandler
from handlers.package_ts import PackageSegmentTsHandler
from handlers.finalize_mp4 import FinalizeMp4Handler
__all__ = [
'BaseHandler',
'RenderSegmentVideoHandler',
'ComposeTransitionHandler',
'PrepareJobAudioHandler',
'PackageSegmentTsHandler',
'FinalizeMp4Handler',
]

View File

@@ -1,585 +0,0 @@
# -*- coding: utf-8 -*-
"""
任务处理器基类
提供所有处理器共用的基础功能。
"""
import os
import json
import logging
import shutil
import tempfile
import subprocess
import threading
from abc import ABC
from typing import Optional, List, Dict, Any, Tuple, TYPE_CHECKING
from core.handler import TaskHandler
from domain.task import Task
from domain.result import TaskResult, ErrorCode
from domain.config import WorkerConfig
from services import storage
from services.cache import MaterialCache
from constant import (
HW_ACCEL_NONE, HW_ACCEL_QSV, HW_ACCEL_CUDA,
VIDEO_ENCODE_PARAMS, VIDEO_ENCODE_PARAMS_QSV, VIDEO_ENCODE_PARAMS_CUDA
)
if TYPE_CHECKING:
from services.api_client import APIClientV2
logger = logging.getLogger(__name__)
def get_video_encode_args(hw_accel: str = HW_ACCEL_NONE) -> List[str]:
"""
根据硬件加速配置获取视频编码参数
Args:
hw_accel: 硬件加速类型 (none, qsv, cuda)
Returns:
FFmpeg 视频编码参数列表
"""
if hw_accel == HW_ACCEL_QSV:
params = VIDEO_ENCODE_PARAMS_QSV
return [
'-c:v', params['codec'],
'-preset', params['preset'],
'-profile:v', params['profile'],
'-level', params['level'],
'-global_quality', params['global_quality'],
'-look_ahead', params['look_ahead'],
]
elif hw_accel == HW_ACCEL_CUDA:
params = VIDEO_ENCODE_PARAMS_CUDA
return [
'-c:v', params['codec'],
'-preset', params['preset'],
'-profile:v', params['profile'],
'-level', params['level'],
'-rc', params['rc'],
'-cq', params['cq'],
'-b:v', '0', # 配合 vbr 模式使用 cq
]
else:
# 软件编码(默认)
params = VIDEO_ENCODE_PARAMS
return [
'-c:v', params['codec'],
'-preset', params['preset'],
'-profile:v', params['profile'],
'-level', params['level'],
'-crf', params['crf'],
'-pix_fmt', params['pix_fmt'],
]
def get_hwaccel_decode_args(hw_accel: str = HW_ACCEL_NONE, device_index: Optional[int] = None) -> List[str]:
"""
获取硬件加速解码参数(输入文件之前使用)
Args:
hw_accel: 硬件加速类型 (none, qsv, cuda)
device_index: GPU 设备索引,用于多显卡调度
Returns:
FFmpeg 硬件加速解码参数列表
"""
if hw_accel == HW_ACCEL_CUDA:
# CUDA 硬件加速解码
args = ['-hwaccel', 'cuda']
# 多显卡模式下指定设备
if device_index is not None:
args.extend(['-hwaccel_device', str(device_index)])
args.extend(['-hwaccel_output_format', 'cuda'])
return args
elif hw_accel == HW_ACCEL_QSV:
# QSV 硬件加速解码
args = ['-hwaccel', 'qsv']
# QSV 在 Windows 上使用 -qsv_device
if device_index is not None:
args.extend(['-qsv_device', str(device_index)])
args.extend(['-hwaccel_output_format', 'qsv'])
return args
else:
return []
def get_hwaccel_filter_prefix(hw_accel: str = HW_ACCEL_NONE) -> str:
"""
获取硬件加速滤镜前缀(用于 hwdownload 从 GPU 到 CPU)
注意:由于大多数复杂滤镜(如 lut3d, overlay, crop 等)不支持硬件表面,
我们需要在滤镜链开始时将硬件表面下载到系统内存。
CUDA/QSV hwdownload 只支持 nv12 格式输出,因此需要两步转换:
1. hwdownload,format=nv12 - 从 GPU 下载到 CPU
2. format=yuv420p - 转换为标准格式(确保与 RGBA/YUVA overlay 混合时颜色正确)
Args:
hw_accel: 硬件加速类型
Returns:
需要添加到滤镜链开头的 hwdownload 滤镜字符串
"""
if hw_accel == HW_ACCEL_CUDA:
return 'hwdownload,format=nv12,format=yuv420p,'
elif hw_accel == HW_ACCEL_QSV:
return 'hwdownload,format=nv12,format=yuv420p,'
else:
return ''
# v2 统一视频编码参数(兼容旧代码,使用软件编码)
VIDEO_ENCODE_ARGS = get_video_encode_args(HW_ACCEL_NONE)
# v2 统一音频编码参数
AUDIO_ENCODE_ARGS = [
'-c:a', 'aac',
'-b:a', '128k',
'-ar', '48000',
'-ac', '2',
]
FFMPEG_LOGLEVEL = 'error'
def subprocess_args(include_stdout: bool = True) -> Dict[str, Any]:
"""
创建跨平台的 subprocess 参数
在 Windows 上使用 Pyinstaller --noconsole 打包时,需要特殊处理以避免弹出命令行窗口。
Args:
include_stdout: 是否包含 stdout 捕获
Returns:
subprocess.run 使用的参数字典
"""
ret: Dict[str, Any] = {}
# Windows 特殊处理
if hasattr(subprocess, 'STARTUPINFO'):
si = subprocess.STARTUPINFO()
si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
ret['startupinfo'] = si
ret['env'] = os.environ
# 重定向 stdin 避免 "handle is invalid" 错误
ret['stdin'] = subprocess.PIPE
if include_stdout:
ret['stdout'] = subprocess.PIPE
return ret
def probe_video_info(video_file: str) -> Tuple[int, int, float]:
"""
探测视频信息(宽度、高度、时长)
Args:
video_file: 视频文件路径
Returns:
(width, height, duration) 元组,失败返回 (0, 0, 0)
"""
try:
result = subprocess.run(
[
'ffprobe', '-v', 'error',
'-select_streams', 'v:0',
'-show_entries', 'stream=width,height:format=duration',
'-of', 'csv=s=x:p=0',
video_file
],
capture_output=True,
timeout=30,
**subprocess_args(False)
)
if result.returncode != 0:
logger.warning(f"ffprobe failed for {video_file}")
return 0, 0, 0
output = result.stdout.decode('utf-8').strip()
if not output:
return 0, 0, 0
lines = output.split('\n')
if len(lines) >= 2:
wh = lines[0].strip()
duration_str = lines[1].strip()
width, height = wh.split('x')
return int(width), int(height), float(duration_str)
return 0, 0, 0
except Exception as e:
logger.warning(f"probe_video_info error: {e}")
return 0, 0, 0
def probe_duration_json(file_path: str) -> Optional[float]:
"""
使用 ffprobe JSON 输出探测媒体时长
Args:
file_path: 媒体文件路径
Returns:
时长(秒),失败返回 None
"""
try:
result = subprocess.run(
[
'ffprobe', '-v', 'error',
'-show_entries', 'format=duration',
'-of', 'json',
file_path
],
capture_output=True,
timeout=30,
**subprocess_args(False)
)
if result.returncode != 0:
return None
data = json.loads(result.stdout.decode('utf-8'))
duration = data.get('format', {}).get('duration')
return float(duration) if duration else None
except Exception as e:
logger.warning(f"probe_duration_json error: {e}")
return None
class BaseHandler(TaskHandler, ABC):
"""
任务处理器基类
提供所有处理器共用的基础功能,包括:
- 临时目录管理
- 文件下载/上传
- FFmpeg 命令执行
- GPU 设备管理(多显卡调度)
- 日志记录
"""
# 线程本地存储:用于存储当前线程的 GPU 设备索引
_thread_local = threading.local()
def __init__(self, config: WorkerConfig, api_client: 'APIClientV2'):
"""
初始化处理器
Args:
config: Worker 配置
api_client: API 客户端
"""
self.config = config
self.api_client = api_client
self.material_cache = MaterialCache(
cache_dir=config.cache_dir,
enabled=config.cache_enabled,
max_size_gb=config.cache_max_size_gb
)
# ========== GPU 设备管理 ==========
def set_gpu_device(self, device_index: int) -> None:
"""
设置当前线程的 GPU 设备索引
由 TaskExecutor 在任务执行前调用。
Args:
device_index: GPU 设备索引
"""
self._thread_local.gpu_device = device_index
def get_gpu_device(self) -> Optional[int]:
"""
获取当前线程的 GPU 设备索引
Returns:
GPU 设备索引,未设置则返回 None
"""
return getattr(self._thread_local, 'gpu_device', None)
def clear_gpu_device(self) -> None:
"""
清除当前线程的 GPU 设备索引
由 TaskExecutor 在任务执行后调用。
"""
if hasattr(self._thread_local, 'gpu_device'):
del self._thread_local.gpu_device
# ========== FFmpeg 参数生成 ==========
def get_video_encode_args(self) -> List[str]:
"""
获取当前配置的视频编码参数
Returns:
FFmpeg 视频编码参数列表
"""
return get_video_encode_args(self.config.hw_accel)
def get_hwaccel_decode_args(self) -> List[str]:
"""
获取硬件加速解码参数(支持设备指定)
Returns:
FFmpeg 硬件加速解码参数列表
"""
device_index = self.get_gpu_device()
return get_hwaccel_decode_args(self.config.hw_accel, device_index)
def get_hwaccel_filter_prefix(self) -> str:
"""
获取硬件加速滤镜前缀
Returns:
需要添加到滤镜链开头的 hwdownload 滤镜字符串
"""
return get_hwaccel_filter_prefix(self.config.hw_accel)
def before_handle(self, task: Task) -> None:
"""处理前钩子"""
logger.debug(f"[task:{task.task_id}] Before handle: {task.task_type.value}")
def after_handle(self, task: Task, result: TaskResult) -> None:
"""处理后钩子"""
status = "success" if result.success else "failed"
logger.debug(f"[task:{task.task_id}] After handle: {status}")
def create_work_dir(self, task_id: str = None) -> str:
"""
创建临时工作目录
Args:
task_id: 任务 ID(用于目录命名)
Returns:
工作目录路径
"""
# 确保临时根目录存在
os.makedirs(self.config.temp_dir, exist_ok=True)
# 创建唯一的工作目录
prefix = f"task_{task_id}_" if task_id else "task_"
work_dir = tempfile.mkdtemp(dir=self.config.temp_dir, prefix=prefix)
logger.debug(f"Created work directory: {work_dir}")
return work_dir
def cleanup_work_dir(self, work_dir: str) -> None:
"""
清理临时工作目录
Args:
work_dir: 工作目录路径
"""
if not work_dir or not os.path.exists(work_dir):
return
try:
shutil.rmtree(work_dir)
logger.debug(f"Cleaned up work directory: {work_dir}")
except Exception as e:
logger.warning(f"Failed to cleanup work directory {work_dir}: {e}")
def download_file(self, url: str, dest: str, timeout: int = None, use_cache: bool = True) -> bool:
"""
下载文件(支持缓存)
Args:
url: 文件 URL
dest: 目标路径
timeout: 超时时间(秒)
use_cache: 是否使用缓存(默认 True)
Returns:
是否成功
"""
if timeout is None:
timeout = self.config.download_timeout
try:
if use_cache:
# 使用缓存下载
result = self.material_cache.get_or_download(url, dest, timeout=timeout)
else:
# 直接下载(不走缓存)
result = storage.download_file(url, dest, timeout=timeout)
if result:
file_size = os.path.getsize(dest) if os.path.exists(dest) else 0
logger.debug(f"Downloaded: {url} -> {dest} ({file_size} bytes)")
return result
except Exception as e:
logger.error(f"Download failed: {url} -> {e}")
return False
def upload_file(
self,
task_id: str,
file_type: str,
file_path: str,
file_name: str = None
) -> Optional[str]:
"""
上传文件并返回访问 URL
Args:
task_id: 任务 ID
file_type: 文件类型(video/audio/ts/mp4)
file_path: 本地文件路径
file_name: 文件名(可选)
Returns:
访问 URL,失败返回 None
"""
# 获取上传 URL
upload_info = self.api_client.get_upload_url(task_id, file_type, file_name)
if not upload_info:
logger.error(f"[task:{task_id}] Failed to get upload URL")
return None
upload_url = upload_info.get('uploadUrl')
access_url = upload_info.get('accessUrl')
if not upload_url:
logger.error(f"[task:{task_id}] Invalid upload URL response")
return None
# 上传文件
try:
result = storage.upload_file(upload_url, file_path, timeout=self.config.upload_timeout)
if result:
file_size = os.path.getsize(file_path)
logger.info(f"[task:{task_id}] Uploaded: {file_path} ({file_size} bytes)")
# 将上传成功的文件加入缓存
if access_url:
self.material_cache.add_to_cache(access_url, file_path)
return access_url
else:
logger.error(f"[task:{task_id}] Upload failed: {file_path}")
return None
except Exception as e:
logger.error(f"[task:{task_id}] Upload error: {e}")
return None
def run_ffmpeg(
self,
cmd: List[str],
task_id: str,
timeout: int = None
) -> bool:
"""
执行 FFmpeg 命令
Args:
cmd: FFmpeg 命令参数列表
task_id: 任务 ID(用于日志)
timeout: 超时时间(秒)
Returns:
是否成功
"""
if timeout is None:
timeout = self.config.ffmpeg_timeout
cmd_to_run = list(cmd)
if cmd_to_run and cmd_to_run[0] == 'ffmpeg' and '-loglevel' not in cmd_to_run:
cmd_to_run[1:1] = ['-loglevel', FFMPEG_LOGLEVEL]
# 日志记录命令(限制长度)
cmd_str = ' '.join(cmd_to_run)
if len(cmd_str) > 500:
cmd_str = cmd_str[:500] + '...'
logger.info(f"[task:{task_id}] FFmpeg: {cmd_str}")
try:
run_args = subprocess_args(False)
run_args['stdout'] = subprocess.DEVNULL
run_args['stderr'] = subprocess.PIPE
result = subprocess.run(
cmd_to_run,
timeout=timeout,
**run_args
)
if result.returncode != 0:
stderr = (result.stderr or b'').decode('utf-8', errors='replace')[:1000]
logger.error(f"[task:{task_id}] FFmpeg failed (code={result.returncode}): {stderr}")
return False
return True
except subprocess.TimeoutExpired:
logger.error(f"[task:{task_id}] FFmpeg timeout after {timeout}s")
return False
except Exception as e:
logger.error(f"[task:{task_id}] FFmpeg error: {e}")
return False
def probe_duration(self, file_path: str) -> Optional[float]:
"""
探测媒体文件时长
Args:
file_path: 文件路径
Returns:
时长(秒),失败返回 None
"""
# 首先尝试 JSON 输出方式
duration = probe_duration_json(file_path)
if duration is not None:
return duration
# 回退到旧方式
try:
_, _, duration = probe_video_info(file_path)
return float(duration) if duration else None
except Exception as e:
logger.warning(f"Failed to probe duration: {file_path} -> {e}")
return None
def get_file_size(self, file_path: str) -> int:
"""
获取文件大小
Args:
file_path: 文件路径
Returns:
文件大小(字节)
"""
try:
return os.path.getsize(file_path)
except Exception:
return 0
def ensure_file_exists(self, file_path: str, min_size: int = 0) -> bool:
"""
确保文件存在且大小满足要求
Args:
file_path: 文件路径
min_size: 最小大小(字节)
Returns:
是否满足要求
"""
if not os.path.exists(file_path):
return False
return os.path.getsize(file_path) >= min_size

View File

@@ -1,273 +0,0 @@
# -*- coding: utf-8 -*-
"""
转场合成处理器
处理 COMPOSE_TRANSITION 任务,将相邻两个片段的 overlap 区域进行混合,生成转场效果。
使用 FFmpeg xfade 滤镜实现多种转场效果。
"""
import os
import logging
from typing import List, Optional
from handlers.base import BaseHandler
from domain.task import Task, TaskType, TransitionConfig, TRANSITION_TYPES
from domain.result import TaskResult, ErrorCode
logger = logging.getLogger(__name__)
class ComposeTransitionHandler(BaseHandler):
"""
转场合成处理器
职责:
- 下载前一个片段的视频(含尾部 overlap)
- 下载后一个片段的视频(含头部 overlap)
- 使用 xfade 滤镜合成转场效果
- 上传转场视频产物
关键约束:
- 转场任务必须等待前后两个片段的 RENDER_SEGMENT_VIDEO 都完成后才能执行
- 输出编码参数必须与片段视频一致,确保后续 TS 封装兼容
- 转场视频不含音频轨道(音频由 PREPARE_JOB_AUDIO 统一处理)
"""
def get_supported_type(self) -> TaskType:
return TaskType.COMPOSE_TRANSITION
def handle(self, task: Task) -> TaskResult:
"""处理转场合成任务"""
work_dir = self.create_work_dir(task.task_id)
try:
# 解析参数
transition_id = task.get_transition_id()
prev_segment = task.get_prev_segment()
next_segment = task.get_next_segment()
transition_config = task.get_transition_config()
output_spec = task.get_output_spec()
# 参数验证
if not transition_id:
return TaskResult.fail(
ErrorCode.E_SPEC_INVALID,
"Missing transitionId"
)
if not prev_segment or not prev_segment.get('videoUrl'):
return TaskResult.fail(
ErrorCode.E_SPEC_INVALID,
"Missing prevSegment.videoUrl"
)
if not next_segment or not next_segment.get('videoUrl'):
return TaskResult.fail(
ErrorCode.E_SPEC_INVALID,
"Missing nextSegment.videoUrl"
)
if not transition_config:
return TaskResult.fail(
ErrorCode.E_SPEC_INVALID,
"Missing transition config"
)
# 获取 overlap 时长
overlap_tail_ms = prev_segment.get('overlapTailMs', 0)
overlap_head_ms = next_segment.get('overlapHeadMs', 0)
transition_duration_ms = transition_config.duration_ms
# 验证 overlap 时长
if overlap_tail_ms <= 0 or overlap_head_ms <= 0:
return TaskResult.fail(
ErrorCode.E_SPEC_INVALID,
f"Invalid overlap duration: tail={overlap_tail_ms}ms, head={overlap_head_ms}ms"
)
logger.info(
f"[task:{task.task_id}] Composing transition: {transition_config.type}, "
f"duration={transition_duration_ms}ms, "
f"overlap_tail={overlap_tail_ms}ms, overlap_head={overlap_head_ms}ms"
)
# 1. 下载前一个片段视频
prev_video_file = os.path.join(work_dir, 'prev_segment.mp4')
if not self.download_file(prev_segment['videoUrl'], prev_video_file):
return TaskResult.fail(
ErrorCode.E_INPUT_UNAVAILABLE,
f"Failed to download prev segment video: {prev_segment['videoUrl']}"
)
# 2. 下载后一个片段视频
next_video_file = os.path.join(work_dir, 'next_segment.mp4')
if not self.download_file(next_segment['videoUrl'], next_video_file):
return TaskResult.fail(
ErrorCode.E_INPUT_UNAVAILABLE,
f"Failed to download next segment video: {next_segment['videoUrl']}"
)
# 3. 获取前一个片段的实际时长
prev_duration = self.probe_duration(prev_video_file)
if not prev_duration:
return TaskResult.fail(
ErrorCode.E_FFMPEG_FAILED,
"Failed to probe prev segment duration"
)
# 4. 构建转场合成命令
output_file = os.path.join(work_dir, 'transition.mp4')
cmd = self._build_command(
prev_video_file=prev_video_file,
next_video_file=next_video_file,
output_file=output_file,
prev_duration_sec=prev_duration,
overlap_tail_ms=overlap_tail_ms,
overlap_head_ms=overlap_head_ms,
transition_config=transition_config,
output_spec=output_spec
)
# 5. 执行 FFmpeg
if not self.run_ffmpeg(cmd, task.task_id):
return TaskResult.fail(
ErrorCode.E_FFMPEG_FAILED,
"FFmpeg transition composition failed"
)
# 6. 验证输出文件
if not self.ensure_file_exists(output_file, min_size=1024):
return TaskResult.fail(
ErrorCode.E_FFMPEG_FAILED,
"Transition output file is missing or too small"
)
# 7. 获取实际时长
actual_duration = self.probe_duration(output_file)
actual_duration_ms = int(actual_duration * 1000) if actual_duration else transition_duration_ms
# 8. 上传产物
transition_video_url = self.upload_file(task.task_id, 'video', output_file)
if not transition_video_url:
return TaskResult.fail(
ErrorCode.E_UPLOAD_FAILED,
"Failed to upload transition video"
)
return TaskResult.ok({
'transitionVideoUrl': transition_video_url,
'actualDurationMs': actual_duration_ms
})
except Exception as e:
logger.error(f"[task:{task.task_id}] Unexpected error: {e}", exc_info=True)
return TaskResult.fail(ErrorCode.E_UNKNOWN, str(e))
finally:
self.cleanup_work_dir(work_dir)
def _build_command(
self,
prev_video_file: str,
next_video_file: str,
output_file: str,
prev_duration_sec: float,
overlap_tail_ms: int,
overlap_head_ms: int,
transition_config: TransitionConfig,
output_spec
) -> List[str]:
"""
构建转场合成命令
使用 xfade 滤镜合成转场效果:
1. 从前一个片段截取尾部 overlap 区域
2. 从后一个片段截取头部 overlap 区域
3. 使用 xfade 进行混合
注意:
- 转场视频时长很短,需要特别处理 GOP 大小
- 确保第一帧是关键帧以便后续 TS 封装
Args:
prev_video_file: 前一个片段视频路径
next_video_file: 后一个片段视频路径
output_file: 输出文件路径
prev_duration_sec: 前一个片段总时长(秒)
overlap_tail_ms: 尾部 overlap 时长(毫秒)
overlap_head_ms: 头部 overlap 时长(毫秒)
transition_config: 转场配置
output_spec: 输出规格
Returns:
FFmpeg 命令参数列表
"""
# 计算时间参数
overlap_tail_sec = overlap_tail_ms / 1000.0
overlap_head_sec = overlap_head_ms / 1000.0
# 前一个片段的尾部 overlap 起始位置
tail_start_sec = prev_duration_sec - overlap_tail_sec
# 转场时长(使用两个 overlap 区域的总和,xfade 会将两段合成为此时长)
# 注意:xfade 的输出时长 = overlap_tail + overlap_head - duration
# 当 duration = overlap_tail + overlap_head 时,输出时长约等于 duration
transition_duration_sec = min(overlap_tail_sec, overlap_head_sec)
# 获取 xfade 转场类型
xfade_transition = transition_config.get_ffmpeg_transition()
# 构建滤镜
# [0:v] trim 截取前一个片段的尾部 overlap
# [1:v] trim 截取后一个片段的头部 overlap
# xfade 混合两段视频
filter_complex = (
f"[0:v]trim=start={tail_start_sec},setpts=PTS-STARTPTS[v0];"
f"[1:v]trim=end={overlap_head_sec},setpts=PTS-STARTPTS[v1];"
f"[v0][v1]xfade=transition={xfade_transition}:duration={transition_duration_sec}:offset=0[outv]"
)
cmd = [
'ffmpeg', '-y', '-hide_banner',
'-i', prev_video_file,
'-i', next_video_file,
'-filter_complex', filter_complex,
'-map', '[outv]',
]
# 编码参数(根据硬件加速配置动态获取)
cmd.extend(self.get_video_encode_args())
# 帧率
fps = output_spec.fps
# 计算输出视频的预估帧数
# xfade 输出时长 ≈ overlap_tail + overlap_head - transition_duration
output_duration_sec = overlap_tail_sec + overlap_head_sec - transition_duration_sec
total_frames = int(output_duration_sec * fps)
# 动态调整 GOP 大小:对于短视频,GOP 不能大于总帧数
# 确保至少有 1 个关键帧(第一帧),最小 GOP = 1
if total_frames <= 1:
gop_size = 1
elif total_frames < fps:
# 短于 1 秒的视频,使用全部帧数作为 GOP(整个视频只有开头一个关键帧)
gop_size = total_frames
else:
# 正常情况,每秒一个关键帧(比标准的 2 秒更密集,适合短视频)
gop_size = fps
cmd.extend(['-r', str(fps)])
cmd.extend(['-g', str(gop_size)])
cmd.extend(['-keyint_min', str(min(gop_size, fps // 2 or 1))])
# 强制第一帧为关键帧
cmd.extend(['-force_key_frames', 'expr:eq(n,0)'])
# 无音频
cmd.append('-an')
# 输出文件
cmd.append(output_file)
return cmd

View File

@@ -1,190 +0,0 @@
# -*- coding: utf-8 -*-
"""
最终 MP4 合并处理器
处理 FINALIZE_MP4 任务,将所有 TS 分片合并为最终可下载的 MP4 文件。
"""
import os
import logging
from typing import List
from handlers.base import BaseHandler
from domain.task import Task, TaskType
from domain.result import TaskResult, ErrorCode
logger = logging.getLogger(__name__)
class FinalizeMp4Handler(BaseHandler):
"""
最终 MP4 合并处理器
职责:
- 下载所有 TS 分片
- 使用 concat demuxer 合并
- 产出最终 MP4(remux,不重编码)
- 上传 MP4 产物
关键约束:
- 优先使用 remux(复制流,不重新编码)
- 使用 aac_adtstoasc bitstream filter 处理音频
"""
def get_supported_type(self) -> TaskType:
return TaskType.FINALIZE_MP4
def handle(self, task: Task) -> TaskResult:
"""处理 MP4 合并任务"""
work_dir = self.create_work_dir(task.task_id)
try:
# 获取 TS 列表
ts_list = task.get_ts_list()
m3u8_url = task.get_m3u8_url()
if not ts_list and not m3u8_url:
return TaskResult.fail(
ErrorCode.E_SPEC_INVALID,
"Missing tsList or m3u8Url"
)
output_file = os.path.join(work_dir, 'final.mp4')
if ts_list:
# 方式1:使用 TS 列表
result = self._process_ts_list(task, work_dir, ts_list, output_file)
else:
# 方式2:使用 m3u8 URL
result = self._process_m3u8(task, work_dir, m3u8_url, output_file)
if not result.success:
return result
# 验证输出文件
if not self.ensure_file_exists(output_file, min_size=4096):
return TaskResult.fail(
ErrorCode.E_FFMPEG_FAILED,
"MP4 output file is missing or too small"
)
# 获取文件大小
file_size = self.get_file_size(output_file)
# 上传产物
mp4_url = self.upload_file(task.task_id, 'mp4', output_file)
if not mp4_url:
return TaskResult.fail(
ErrorCode.E_UPLOAD_FAILED,
"Failed to upload MP4"
)
return TaskResult.ok({
'mp4Url': mp4_url,
'fileSizeBytes': file_size
})
except Exception as e:
logger.error(f"[task:{task.task_id}] Unexpected error: {e}", exc_info=True)
return TaskResult.fail(ErrorCode.E_UNKNOWN, str(e))
finally:
self.cleanup_work_dir(work_dir)
def _process_ts_list(
self,
task: Task,
work_dir: str,
ts_list: List[str],
output_file: str
) -> TaskResult:
"""
使用 TS 列表处理
Args:
task: 任务实体
work_dir: 工作目录
ts_list: TS URL 列表
output_file: 输出文件路径
Returns:
TaskResult
"""
# 1. 下载所有 TS 分片
ts_files = []
for i, ts_url in enumerate(ts_list):
ts_file = os.path.join(work_dir, f'seg_{i}.ts')
if not self.download_file(ts_url, ts_file):
return TaskResult.fail(
ErrorCode.E_INPUT_UNAVAILABLE,
f"Failed to download TS segment {i}: {ts_url}"
)
ts_files.append(ts_file)
logger.info(f"[task:{task.task_id}] Downloaded {len(ts_files)} TS segments")
# 2. 创建 concat 文件列表
concat_file = os.path.join(work_dir, 'concat.txt')
with open(concat_file, 'w', encoding='utf-8') as f:
for ts_file in ts_files:
# FFmpeg concat 路径相对于 concat.txt 所在目录,只需写文件名
ts_filename = os.path.basename(ts_file)
f.write(f"file '{ts_filename}'\n")
# 3. 构建合并命令(remux,不重编码)
cmd = [
'ffmpeg', '-y', '-hide_banner',
'-f', 'concat',
'-safe', '0',
'-i', concat_file,
'-c', 'copy', # 复制流,不重编码
'-bsf:a', 'aac_adtstoasc', # 音频 bitstream filter
output_file
]
# 4. 执行 FFmpeg
if not self.run_ffmpeg(cmd, task.task_id):
return TaskResult.fail(
ErrorCode.E_FFMPEG_FAILED,
"MP4 concatenation failed"
)
return TaskResult.ok({})
def _process_m3u8(
self,
task: Task,
work_dir: str,
m3u8_url: str,
output_file: str
) -> TaskResult:
"""
使用 m3u8 URL 处理
Args:
task: 任务实体
work_dir: 工作目录
m3u8_url: m3u8 URL
output_file: 输出文件路径
Returns:
TaskResult
"""
# 构建命令
cmd = [
'ffmpeg', '-y', '-hide_banner',
'-protocol_whitelist', 'file,http,https,tcp,tls',
'-i', m3u8_url,
'-c', 'copy',
'-bsf:a', 'aac_adtstoasc',
output_file
]
# 执行 FFmpeg
if not self.run_ffmpeg(cmd, task.task_id):
return TaskResult.fail(
ErrorCode.E_FFMPEG_FAILED,
"MP4 conversion from m3u8 failed"
)
return TaskResult.ok({})

View File

@@ -1,304 +0,0 @@
# -*- coding: utf-8 -*-
"""
TS 分片封装处理器
处理 PACKAGE_SEGMENT_TS 任务,将视频片段和对应时间区间的音频封装为 TS 分片。
支持转场相关的 overlap 裁剪和转场分片封装。
"""
import os
import logging
from typing import List, Optional
from handlers.base import BaseHandler, VIDEO_ENCODE_ARGS
from domain.task import Task, TaskType
from domain.result import TaskResult, ErrorCode
logger = logging.getLogger(__name__)
class PackageSegmentTsHandler(BaseHandler):
"""
TS 分片封装处理器
职责:
- 下载视频片段
- 下载全局音频
- 截取对应时间区间的音频
- 封装为 TS 分片
- 上传 TS 产物
关键约束:
- TS 必须包含音视频同轨
- 使用 output_ts_offset 保证时间戳连续
- 输出 extinfDurationSec 供 m3u8 使用
转场相关:
- 普通片段 TS:需要裁剪掉 overlap 区域(已被转场分片使用)
- 转场分片 TS:直接封装转场视频产物,无需裁剪
- 无转场时:走原有逻辑,不做裁剪
精确裁剪:
- 当需要裁剪 overlap 区域时,必须使用重编码方式(-vf trim)才能精确切割
- 使用 -c copy 只能从关键帧切割,会导致不精确
"""
def get_supported_type(self) -> TaskType:
return TaskType.PACKAGE_SEGMENT_TS
def handle(self, task: Task) -> TaskResult:
"""处理 TS 封装任务"""
work_dir = self.create_work_dir(task.task_id)
try:
# 解析参数
video_url = task.get_video_url()
audio_url = task.get_audio_url()
start_time_ms = task.get_start_time_ms()
duration_ms = task.get_duration_ms()
output_spec = task.get_output_spec()
# 转场相关参数
is_transition_segment = task.is_transition_segment()
trim_head = task.should_trim_head()
trim_tail = task.should_trim_tail()
trim_head_ms = task.get_trim_head_ms()
trim_tail_ms = task.get_trim_tail_ms()
if not video_url:
return TaskResult.fail(
ErrorCode.E_SPEC_INVALID,
"Missing videoUrl"
)
if not audio_url:
return TaskResult.fail(
ErrorCode.E_SPEC_INVALID,
"Missing audioUrl"
)
# 计算时间参数
start_sec = start_time_ms / 1000.0
duration_sec = duration_ms / 1000.0
# 1. 下载视频片段
video_file = os.path.join(work_dir, 'video.mp4')
if not self.download_file(video_url, video_file):
return TaskResult.fail(
ErrorCode.E_INPUT_UNAVAILABLE,
f"Failed to download video: {video_url}"
)
# 2. 下载全局音频
audio_file = os.path.join(work_dir, 'audio.aac')
if not self.download_file(audio_url, audio_file):
return TaskResult.fail(
ErrorCode.E_INPUT_UNAVAILABLE,
f"Failed to download audio: {audio_url}"
)
# 3. 判断是否需要精确裁剪视频
needs_video_trim = not is_transition_segment and (
(trim_head and trim_head_ms > 0) or
(trim_tail and trim_tail_ms > 0)
)
# 4. 如果需要裁剪,先重编码裁剪视频
processed_video_file = video_file
if needs_video_trim:
processed_video_file = os.path.join(work_dir, 'trimmed_video.mp4')
trim_cmd = self._build_trim_command(
video_file=video_file,
output_file=processed_video_file,
trim_head_ms=trim_head_ms if trim_head else 0,
trim_tail_ms=trim_tail_ms if trim_tail else 0,
output_spec=output_spec
)
logger.info(f"[task:{task.task_id}] Trimming video: head={trim_head_ms}ms, tail={trim_tail_ms}ms")
if not self.run_ffmpeg(trim_cmd, task.task_id):
return TaskResult.fail(
ErrorCode.E_FFMPEG_FAILED,
"Video trim failed"
)
if not self.ensure_file_exists(processed_video_file, min_size=1024):
return TaskResult.fail(
ErrorCode.E_FFMPEG_FAILED,
"Trimmed video file is missing or too small"
)
# 5. 构建 TS 封装命令
output_file = os.path.join(work_dir, 'segment.ts')
cmd = self._build_package_command(
video_file=processed_video_file,
audio_file=audio_file,
output_file=output_file,
start_sec=start_sec,
duration_sec=duration_sec
)
# 6. 执行 FFmpeg
if not self.run_ffmpeg(cmd, task.task_id):
return TaskResult.fail(
ErrorCode.E_FFMPEG_FAILED,
"TS packaging failed"
)
# 7. 验证输出文件
if not self.ensure_file_exists(output_file, min_size=1024):
return TaskResult.fail(
ErrorCode.E_FFMPEG_FAILED,
"TS output file is missing or too small"
)
# 8. 获取实际时长(用于 EXTINF)
actual_duration = self.probe_duration(output_file)
extinf_duration = actual_duration if actual_duration else duration_sec
# 9. 上传产物
ts_url = self.upload_file(task.task_id, 'ts', output_file)
if not ts_url:
return TaskResult.fail(
ErrorCode.E_UPLOAD_FAILED,
"Failed to upload TS"
)
return TaskResult.ok({
'tsUrl': ts_url,
'extinfDurationSec': extinf_duration
})
except Exception as e:
logger.error(f"[task:{task.task_id}] Unexpected error: {e}", exc_info=True)
return TaskResult.fail(ErrorCode.E_UNKNOWN, str(e))
finally:
self.cleanup_work_dir(work_dir)
def _build_trim_command(
self,
video_file: str,
output_file: str,
trim_head_ms: int,
trim_tail_ms: int,
output_spec
) -> List[str]:
"""
构建视频精确裁剪命令(重编码方式)
使用 trim 滤镜进行精确帧级裁剪,而非 -ss/-t 参数的关键帧裁剪。
Args:
video_file: 输入视频路径
output_file: 输出视频路径
trim_head_ms: 头部裁剪时长(毫秒)
trim_tail_ms: 尾部裁剪时长(毫秒)
output_spec: 输出规格
Returns:
FFmpeg 命令参数列表
"""
# 获取原视频时长
original_duration = self.probe_duration(video_file)
if not original_duration:
original_duration = 10.0 # 默认值,避免除零
trim_head_sec = trim_head_ms / 1000.0
trim_tail_sec = trim_tail_ms / 1000.0
# 计算裁剪后的起止时间
start_time = trim_head_sec
end_time = original_duration - trim_tail_sec
# 构建 trim 滤镜
vf_filter = f"trim=start={start_time}:end={end_time},setpts=PTS-STARTPTS"
cmd = [
'ffmpeg', '-y', '-hide_banner',
'-i', video_file,
'-vf', vf_filter,
]
# 编码参数
cmd.extend(VIDEO_ENCODE_ARGS)
# 帧率
fps = output_spec.fps
cmd.extend(['-r', str(fps)])
# 计算输出视频帧数,动态调整 GOP
output_duration_sec = end_time - start_time
total_frames = int(output_duration_sec * fps)
# 动态 GOP:短视频使用较小的 GOP
if total_frames <= 1:
gop_size = 1
elif total_frames < fps:
gop_size = total_frames
else:
gop_size = fps # 每秒一个关键帧
cmd.extend(['-g', str(gop_size)])
cmd.extend(['-keyint_min', str(min(gop_size, fps // 2 or 1))])
# 强制第一帧为关键帧
cmd.extend(['-force_key_frames', 'expr:eq(n,0)'])
# 无音频(音频单独处理)
cmd.append('-an')
cmd.append(output_file)
return cmd
def _build_package_command(
self,
video_file: str,
audio_file: str,
output_file: str,
start_sec: float,
duration_sec: float
) -> List[str]:
"""
构建 TS 封装命令
将视频和对应时间区间的音频封装为 TS 分片。
视频使用 copy 模式(已经过精确裁剪或无需裁剪)。
Args:
video_file: 视频文件路径(已处理)
audio_file: 音频文件路径
output_file: 输出文件路径
start_sec: 音频开始时间(秒)
duration_sec: 音频时长(秒)
Returns:
FFmpeg 命令参数列表
"""
cmd = [
'ffmpeg', '-y', '-hide_banner',
# 视频输入
'-i', video_file,
# 音频输入(从 start_sec 开始截取 duration_sec)
'-ss', str(start_sec),
'-t', str(duration_sec),
'-i', audio_file,
# 映射流
'-map', '0:v:0', # 使用第一个输入的视频流
'-map', '1:a:0', # 使用第二个输入的音频流
# 复制编码(视频已处理,无需重编码)
'-c:v', 'copy',
'-c:a', 'copy',
# 关键:时间戳偏移,保证整体连续
'-output_ts_offset', str(start_sec),
# 复用参数
'-muxdelay', '0',
'-muxpreload', '0',
# 输出格式
'-f', 'mpegts',
output_file
]
return cmd

View File

@@ -1,251 +0,0 @@
# -*- coding: utf-8 -*-
"""
全局音频准备处理器
处理 PREPARE_JOB_AUDIO 任务,生成整个视频的连续音频轨道。
"""
import os
import logging
from typing import List, Dict, Optional
from handlers.base import BaseHandler, AUDIO_ENCODE_ARGS
from domain.task import Task, TaskType, AudioSpec, AudioProfile
from domain.result import TaskResult, ErrorCode
logger = logging.getLogger(__name__)
class PrepareJobAudioHandler(BaseHandler):
"""
全局音频准备处理器
职责:
- 下载全局 BGM
- 下载各片段叠加音效
- 构建复杂混音命令
- 执行混音
- 上传音频产物
关键约束:
- 全局 BGM 连续生成一次,贯穿整个时长
- 禁止使用 amix normalize=1
- 只对叠加音轨做极短淡入淡出(5-20ms)
- 不对 BGM 做边界 fade
"""
def get_supported_type(self) -> TaskType:
return TaskType.PREPARE_JOB_AUDIO
def handle(self, task: Task) -> TaskResult:
"""处理音频准备任务"""
work_dir = self.create_work_dir(task.task_id)
try:
# 解析参数
total_duration_ms = task.get_total_duration_ms()
if total_duration_ms <= 0:
return TaskResult.fail(
ErrorCode.E_SPEC_INVALID,
"Invalid totalDurationMs"
)
total_duration_sec = total_duration_ms / 1000.0
audio_profile = task.get_audio_profile()
bgm_url = task.get_bgm_url()
segments = task.get_segments()
# 1. 下载 BGM(如有)
bgm_file = None
if bgm_url:
bgm_file = os.path.join(work_dir, 'bgm.mp3')
if not self.download_file(bgm_url, bgm_file):
logger.warning(f"[task:{task.task_id}] Failed to download BGM")
bgm_file = None
# 2. 下载叠加音效
sfx_files = []
for i, seg in enumerate(segments):
audio_spec_data = seg.get('audioSpecJson')
if audio_spec_data:
audio_spec = AudioSpec.from_dict(audio_spec_data)
if audio_spec and audio_spec.audio_url:
sfx_file = os.path.join(work_dir, f'sfx_{i}.mp3')
if self.download_file(audio_spec.audio_url, sfx_file):
sfx_files.append({
'file': sfx_file,
'spec': audio_spec,
'segment': seg
})
else:
logger.warning(f"[task:{task.task_id}] Failed to download SFX {i}")
# 3. 构建音频混音命令
output_file = os.path.join(work_dir, 'audio_full.aac')
cmd = self._build_audio_command(
bgm_file=bgm_file,
sfx_files=sfx_files,
output_file=output_file,
total_duration_sec=total_duration_sec,
audio_profile=audio_profile
)
# 4. 执行 FFmpeg
if not self.run_ffmpeg(cmd, task.task_id):
return TaskResult.fail(
ErrorCode.E_FFMPEG_FAILED,
"Audio mixing failed"
)
# 5. 验证输出文件
if not self.ensure_file_exists(output_file, min_size=1024):
return TaskResult.fail(
ErrorCode.E_FFMPEG_FAILED,
"Audio output file is missing or too small"
)
# 6. 上传产物
audio_url = self.upload_file(task.task_id, 'audio', output_file)
if not audio_url:
return TaskResult.fail(
ErrorCode.E_UPLOAD_FAILED,
"Failed to upload audio"
)
return TaskResult.ok({
'audioUrl': audio_url
})
except Exception as e:
logger.error(f"[task:{task.task_id}] Unexpected error: {e}", exc_info=True)
return TaskResult.fail(ErrorCode.E_UNKNOWN, str(e))
finally:
self.cleanup_work_dir(work_dir)
def _build_audio_command(
self,
bgm_file: Optional[str],
sfx_files: List[Dict],
output_file: str,
total_duration_sec: float,
audio_profile: AudioProfile
) -> List[str]:
"""
构建音频混音命令
Args:
bgm_file: BGM 文件路径(可选)
sfx_files: 叠加音效列表
output_file: 输出文件路径
total_duration_sec: 总时长(秒)
audio_profile: 音频配置
Returns:
FFmpeg 命令参数列表
"""
sample_rate = audio_profile.sample_rate
channels = audio_profile.channels
# 情况1:无 BGM 也无叠加音效 -> 生成静音
if not bgm_file and not sfx_files:
return [
'ffmpeg', '-y', '-hide_banner',
'-f', 'lavfi',
'-i', f'anullsrc=r={sample_rate}:cl=stereo',
'-t', str(total_duration_sec),
'-c:a', 'aac', '-b:a', '128k',
output_file
]
# 情况2:仅 BGM,无叠加音效
if not sfx_files:
return [
'ffmpeg', '-y', '-hide_banner',
'-i', bgm_file,
'-t', str(total_duration_sec),
'-c:a', 'aac', '-b:a', '128k',
'-ar', str(sample_rate), '-ac', str(channels),
output_file
]
# 情况3:BGM + 叠加音效 -> 复杂滤镜
inputs = []
if bgm_file:
inputs.extend(['-i', bgm_file])
for sfx in sfx_files:
inputs.extend(['-i', sfx['file']])
filter_parts = []
input_idx = 0
# BGM 处理(或生成静音底轨)
if bgm_file:
filter_parts.append(
f"[0:a]atrim=0:{total_duration_sec},asetpts=PTS-STARTPTS,"
f"apad=whole_dur={total_duration_sec}[bgm]"
)
input_idx = 1
else:
filter_parts.append(
f"anullsrc=r={sample_rate}:cl=stereo,"
f"atrim=0:{total_duration_sec}[bgm]"
)
input_idx = 0
# 叠加音效处理
sfx_labels = []
for i, sfx in enumerate(sfx_files):
idx = input_idx + i
spec = sfx['spec']
seg = sfx['segment']
# 计算时间参数
start_time_ms = seg.get('startTimeMs', 0)
duration_ms = seg.get('durationMs', 5000)
delay_ms = start_time_ms + spec.delay_ms
delay_sec = delay_ms / 1000.0
duration_sec = duration_ms / 1000.0
# 淡入淡出参数(极短,5-20ms)
fade_in_sec = spec.fade_in_ms / 1000.0
fade_out_sec = spec.fade_out_ms / 1000.0
# 音量
volume = spec.volume
label = f"sfx{i}"
sfx_labels.append(f"[{label}]")
# 构建滤镜:延迟 + 淡入淡出 + 音量
# 注意:只对叠加音轨做淡入淡出,不对 BGM 做
sfx_filter = (
f"[{idx}:a]"
f"adelay={int(delay_ms)}|{int(delay_ms)},"
f"afade=t=in:st={delay_sec}:d={fade_in_sec},"
f"afade=t=out:st={delay_sec + duration_sec - fade_out_sec}:d={fade_out_sec},"
f"volume={volume}"
f"[{label}]"
)
filter_parts.append(sfx_filter)
# 混音(关键:normalize=0,禁止归一化)
# dropout_transition=0 表示输入结束时不做渐变
mix_inputs = "[bgm]" + "".join(sfx_labels)
num_inputs = 1 + len(sfx_files)
filter_parts.append(
f"{mix_inputs}amix=inputs={num_inputs}:duration=first:"
f"dropout_transition=0:normalize=0[out]"
)
filter_complex = ';'.join(filter_parts)
cmd = ['ffmpeg', '-y', '-hide_banner'] + inputs + [
'-filter_complex', filter_complex,
'-map', '[out]',
'-c:a', 'aac', '-b:a', '128k',
'-ar', str(sample_rate), '-ac', str(channels),
output_file
]
return cmd

View File

@@ -1,726 +0,0 @@
# -*- coding: utf-8 -*-
"""
视频片段渲染处理器
处理 RENDER_SEGMENT_VIDEO 任务,将原素材渲染为符合输出规格的视频片段。
支持转场 overlap 区域的帧冻结生成。
"""
import os
import logging
from typing import List, Optional, Tuple
from urllib.parse import urlparse, unquote
from handlers.base import BaseHandler
from domain.task import Task, TaskType, RenderSpec, OutputSpec, Effect, IMAGE_EXTENSIONS
from domain.result import TaskResult, ErrorCode
logger = logging.getLogger(__name__)
def _get_extension_from_url(url: str) -> str:
"""从 URL 提取文件扩展名"""
parsed = urlparse(url)
path = unquote(parsed.path)
_, ext = os.path.splitext(path)
return ext.lower() if ext else ''
class RenderSegmentVideoHandler(BaseHandler):
"""
视频片段渲染处理器
职责:
- 下载素材文件
- 下载 LUT 文件(如有)
- 下载叠加层(如有)
- 构建 FFmpeg 渲染命令
- 执行渲染(支持帧冻结生成 overlap 区域)
- 上传产物
"""
def get_supported_type(self) -> TaskType:
return TaskType.RENDER_SEGMENT_VIDEO
def handle(self, task: Task) -> TaskResult:
"""处理视频渲染任务"""
work_dir = self.create_work_dir(task.task_id)
try:
# 解析参数
material_url = task.get_material_url()
if not material_url:
return TaskResult.fail(
ErrorCode.E_SPEC_INVALID,
"Missing material URL (boundMaterialUrl or sourceRef)"
)
# 检查 URL 格式:必须是 HTTP 或 HTTPS 协议
if not material_url.startswith(('http://', 'https://')):
source_ref = task.get_source_ref()
bound_url = task.get_bound_material_url()
logger.error(
f"[task:{task.task_id}] Invalid material URL format: '{material_url}'. "
f"boundMaterialUrl={bound_url}, sourceRef={source_ref}. "
f"Server should provide boundMaterialUrl with HTTP/HTTPS URL."
)
return TaskResult.fail(
ErrorCode.E_SPEC_INVALID,
f"Invalid material URL: '{material_url}' is not a valid HTTP/HTTPS URL. "
f"Server must provide boundMaterialUrl."
)
render_spec = task.get_render_spec()
output_spec = task.get_output_spec()
duration_ms = task.get_duration_ms()
# 1. 检测素材类型并确定输入文件扩展名
is_image = task.is_image_material()
if is_image:
# 图片素材:根据 URL 确定扩展名
ext = _get_extension_from_url(material_url)
if not ext or ext not in IMAGE_EXTENSIONS:
ext = '.jpg' # 默认扩展名
input_file = os.path.join(work_dir, f'input{ext}')
else:
input_file = os.path.join(work_dir, 'input.mp4')
# 2. 下载素材
if not self.download_file(material_url, input_file):
return TaskResult.fail(
ErrorCode.E_INPUT_UNAVAILABLE,
f"Failed to download material: {material_url}"
)
# 3. 图片素材转换为视频
if is_image:
video_input_file = os.path.join(work_dir, 'input_video.mp4')
if not self._convert_image_to_video(
image_file=input_file,
output_file=video_input_file,
duration_ms=duration_ms,
output_spec=output_spec,
render_spec=render_spec,
task_id=task.task_id
):
return TaskResult.fail(
ErrorCode.E_FFMPEG_FAILED,
"Failed to convert image to video"
)
# 使用转换后的视频作为输入
input_file = video_input_file
logger.info(f"[task:{task.task_id}] Image converted to video successfully")
# 4. 下载 LUT(如有)
lut_file = None
if render_spec.lut_url:
lut_file = os.path.join(work_dir, 'lut.cube')
if not self.download_file(render_spec.lut_url, lut_file):
logger.warning(f"[task:{task.task_id}] Failed to download LUT, continuing without it")
lut_file = None
# 5. 下载叠加层(如有)
overlay_file = None
if render_spec.overlay_url:
# 根据 URL 后缀确定文件扩展名
url_lower = render_spec.overlay_url.lower()
if url_lower.endswith('.jpg') or url_lower.endswith('.jpeg'):
ext = '.jpg'
elif url_lower.endswith('.mov'):
ext = '.mov'
else:
ext = '.png' # 默认
overlay_file = os.path.join(work_dir, f'overlay{ext}')
if not self.download_file(render_spec.overlay_url, overlay_file):
logger.warning(f"[task:{task.task_id}] Failed to download overlay, continuing without it")
overlay_file = None
# 6. 探测源视频时长(仅对视频素材)
# 用于检测时长不足并通过冻结最后一帧补足
source_duration_sec = None
if not is_image:
source_duration = self.probe_duration(input_file)
if source_duration:
source_duration_sec = source_duration
speed = float(render_spec.speed) if render_spec.speed else 1.0
if speed > 0:
# 计算变速后的有效时长
effective_duration_sec = source_duration_sec / speed
required_duration_sec = duration_ms / 1000.0
# 如果源视频时长不足,记录日志
if effective_duration_sec < required_duration_sec:
shortage_sec = required_duration_sec - effective_duration_sec
logger.warning(
f"[task:{task.task_id}] Source video duration insufficient: "
f"effective={effective_duration_sec:.2f}s (speed={speed}), "
f"required={required_duration_sec:.2f}s, "
f"will freeze last frame for {shortage_sec:.2f}s"
)
# 7. 计算 overlap 时长(用于转场帧冻结)
# 头部 overlap: 来自前一片段的出场转场
overlap_head_ms = task.get_overlap_head_ms()
# 尾部 overlap: 当前片段的出场转场
overlap_tail_ms = task.get_overlap_tail_ms_v2()
# 8. 构建 FFmpeg 命令
output_file = os.path.join(work_dir, 'output.mp4')
cmd = self._build_command(
input_file=input_file,
output_file=output_file,
render_spec=render_spec,
output_spec=output_spec,
duration_ms=duration_ms,
lut_file=lut_file,
overlay_file=overlay_file,
overlap_head_ms=overlap_head_ms,
overlap_tail_ms=overlap_tail_ms,
source_duration_sec=source_duration_sec
)
# 9. 执行 FFmpeg
if not self.run_ffmpeg(cmd, task.task_id):
return TaskResult.fail(
ErrorCode.E_FFMPEG_FAILED,
"FFmpeg rendering failed"
)
# 10. 验证输出文件
if not self.ensure_file_exists(output_file, min_size=4096):
return TaskResult.fail(
ErrorCode.E_FFMPEG_FAILED,
"Output file is missing or too small"
)
# 11. 获取实际时长
actual_duration = self.probe_duration(output_file)
actual_duration_ms = int(actual_duration * 1000) if actual_duration else duration_ms
# 12. 上传产物
video_url = self.upload_file(task.task_id, 'video', output_file)
if not video_url:
return TaskResult.fail(
ErrorCode.E_UPLOAD_FAILED,
"Failed to upload video"
)
# 13. 构建结果(包含 overlap 信息)
result_data = {
'videoUrl': video_url,
'actualDurationMs': actual_duration_ms,
'overlapHeadMs': overlap_head_ms,
'overlapTailMs': overlap_tail_ms
}
return TaskResult.ok(result_data)
except Exception as e:
logger.error(f"[task:{task.task_id}] Unexpected error: {e}", exc_info=True)
return TaskResult.fail(ErrorCode.E_UNKNOWN, str(e))
finally:
self.cleanup_work_dir(work_dir)
def _convert_image_to_video(
self,
image_file: str,
output_file: str,
duration_ms: int,
output_spec: OutputSpec,
render_spec: RenderSpec,
task_id: str
) -> bool:
"""
将图片转换为视频
使用 FFmpeg 将静态图片转换为指定时长的视频,
同时应用缩放填充和变速处理。
Args:
image_file: 输入图片文件路径
output_file: 输出视频文件路径
duration_ms: 目标时长(毫秒)
output_spec: 输出规格
render_spec: 渲染规格
task_id: 任务 ID(用于日志)
Returns:
是否成功
"""
width = output_spec.width
height = output_spec.height
fps = output_spec.fps
# 计算实际时长(考虑变速)
speed = float(render_spec.speed) if render_spec.speed else 1.0
if speed <= 0:
speed = 1.0
# 变速后的实际播放时长
actual_duration_sec = (duration_ms / 1000.0) / speed
# 构建 FFmpeg 命令
cmd = [
'ffmpeg', '-y', '-hide_banner',
'-loop', '1', # 循环输入图片
'-i', image_file,
'-t', str(actual_duration_sec), # 输出时长
]
# 构建滤镜:缩放填充到目标尺寸
filters = []
# 裁切处理(与视频相同逻辑)
if render_spec.crop_enable and render_spec.face_pos:
try:
fx, fy = map(float, render_spec.face_pos.split(','))
target_ratio = width / height
filters.append(
f"crop='min(iw,ih*{target_ratio})':'min(ih,iw/{target_ratio})':"
f"'(iw-min(iw,ih*{target_ratio}))*{fx}':"
f"'(ih-min(ih,iw/{target_ratio}))*{fy}'"
)
except (ValueError, ZeroDivisionError):
logger.warning(f"[task:{task_id}] Invalid face position: {render_spec.face_pos}")
elif render_spec.zoom_cut:
target_ratio = width / height
filters.append(
f"crop='min(iw,ih*{target_ratio})':'min(ih,iw/{target_ratio})'"
)
# 缩放填充
filters.append(
f"scale={width}:{height}:force_original_aspect_ratio=decrease,"
f"pad={width}:{height}:(ow-iw)/2:(oh-ih)/2:black"
)
# 格式转换(确保兼容性)
filters.append("format=yuv420p")
cmd.extend(['-vf', ','.join(filters)])
# 计算总帧数,动态调整 GOP
total_frames = int(actual_duration_sec * fps)
if total_frames <= 1:
gop_size = 1
elif total_frames < fps:
gop_size = total_frames
else:
gop_size = fps * 2 # 正常情况,2 秒一个关键帧
# 编码参数
cmd.extend([
'-c:v', 'libx264',
'-preset', 'fast',
'-crf', '18',
'-r', str(fps),
'-g', str(gop_size),
'-keyint_min', str(min(gop_size, fps // 2 or 1)),
'-force_key_frames', 'expr:eq(n,0)',
'-an', # 无音频
output_file
])
logger.info(f"[task:{task_id}] Converting image to video: {actual_duration_sec:.2f}s at {fps}fps")
return self.run_ffmpeg(cmd, task_id)
def _build_command(
self,
input_file: str,
output_file: str,
render_spec: RenderSpec,
output_spec: OutputSpec,
duration_ms: int,
lut_file: Optional[str] = None,
overlay_file: Optional[str] = None,
overlap_head_ms: int = 0,
overlap_tail_ms: int = 0,
source_duration_sec: Optional[float] = None
) -> List[str]:
"""
构建 FFmpeg 渲染命令
Args:
input_file: 输入文件路径
output_file: 输出文件路径
render_spec: 渲染规格
output_spec: 输出规格
duration_ms: 目标时长(毫秒)
lut_file: LUT 文件路径(可选)
overlay_file: 叠加层文件路径(可选)
overlap_head_ms: 头部 overlap 时长(毫秒)
overlap_tail_ms: 尾部 overlap 时长(毫秒)
source_duration_sec: 源视频实际时长(秒),用于检测时长不足
Returns:
FFmpeg 命令参数列表
"""
cmd = ['ffmpeg', '-y', '-hide_banner']
# 硬件加速解码参数(在输入文件之前)
hwaccel_args = self.get_hwaccel_decode_args()
if hwaccel_args:
cmd.extend(hwaccel_args)
# 输入文件
cmd.extend(['-i', input_file])
# 叠加层输入
if overlay_file:
cmd.extend(['-i', overlay_file])
# 构建视频滤镜链
filters = self._build_video_filters(
render_spec=render_spec,
output_spec=output_spec,
duration_ms=duration_ms,
lut_file=lut_file,
overlay_file=overlay_file,
overlap_head_ms=overlap_head_ms,
overlap_tail_ms=overlap_tail_ms,
source_duration_sec=source_duration_sec
)
# 应用滤镜
# 检测是否为 filter_complex 格式(包含分号或方括号标签)
is_filter_complex = ';' in filters or (filters.startswith('[') and ']' in filters)
if is_filter_complex or overlay_file:
# 使用 filter_complex 处理
cmd.extend(['-filter_complex', filters])
elif filters:
cmd.extend(['-vf', filters])
# 编码参数(根据硬件加速配置动态获取)
cmd.extend(self.get_video_encode_args())
# 帧率
fps = output_spec.fps
cmd.extend(['-r', str(fps)])
# 时长(包含 overlap 区域)
total_duration_ms = duration_ms + overlap_head_ms + overlap_tail_ms
duration_sec = total_duration_ms / 1000.0
cmd.extend(['-t', str(duration_sec)])
# 动态调整 GOP 大小:对于短视频,GOP 不能大于总帧数
total_frames = int(duration_sec * fps)
if total_frames <= 1:
gop_size = 1
elif total_frames < fps:
# 短于 1 秒的视频,使用全部帧数作为 GOP(整个视频只有开头一个关键帧)
gop_size = total_frames
else:
# 正常情况,2 秒一个关键帧
gop_size = fps * 2
cmd.extend(['-g', str(gop_size)])
cmd.extend(['-keyint_min', str(min(gop_size, fps // 2 or 1))])
# 强制第一帧为关键帧
cmd.extend(['-force_key_frames', 'expr:eq(n,0)'])
# 无音频(视频片段不包含音频)
cmd.append('-an')
# 输出文件
cmd.append(output_file)
return cmd
def _build_video_filters(
self,
render_spec: RenderSpec,
output_spec: OutputSpec,
duration_ms: int,
lut_file: Optional[str] = None,
overlay_file: Optional[str] = None,
overlap_head_ms: int = 0,
overlap_tail_ms: int = 0,
source_duration_sec: Optional[float] = None
) -> str:
"""
构建视频滤镜链
Args:
render_spec: 渲染规格
output_spec: 输出规格
duration_ms: 目标时长(毫秒)
lut_file: LUT 文件路径
overlay_file: 叠加层文件路径(支持图片 png/jpg 和视频 mov)
overlap_head_ms: 头部 overlap 时长(毫秒)
overlap_tail_ms: 尾部 overlap 时长(毫秒)
source_duration_sec: 源视频实际时长(秒),用于检测时长不足
Returns:
滤镜字符串
"""
filters = []
width = output_spec.width
height = output_spec.height
fps = output_spec.fps
# 判断 overlay 类型
has_overlay = overlay_file is not None
is_video_overlay = has_overlay and overlay_file.lower().endswith('.mov')
# 解析 effects
effects = render_spec.get_effects()
has_camera_shot = any(e.effect_type == 'cameraShot' for e in effects)
# 硬件加速时需要先 hwdownload(将 GPU 表面下载到系统内存)
hwaccel_prefix = self.get_hwaccel_filter_prefix()
if hwaccel_prefix:
# 去掉末尾的逗号,作为第一个滤镜
filters.append(hwaccel_prefix.rstrip(','))
# 1. 变速处理
speed = float(render_spec.speed) if render_spec.speed else 1.0
if speed != 1.0 and speed > 0:
# setpts 公式:PTS / speed
pts_factor = 1.0 / speed
filters.append(f"setpts={pts_factor}*PTS")
# 2. LUT 调色
if lut_file:
# 路径中的反斜杠需要转换,冒号需要转义(FFmpeg filter语法中冒号是特殊字符)
lut_path = lut_file.replace('\\', '/').replace(':', r'\:')
filters.append(f"lut3d='{lut_path}'")
# 3. 裁切处理
if render_spec.crop_enable and render_spec.face_pos:
# 根据人脸位置进行智能裁切
try:
fx, fy = map(float, render_spec.face_pos.split(','))
# 计算裁切区域(保持输出比例)
target_ratio = width / height
# 假设裁切到目标比例
filters.append(
f"crop='min(iw,ih*{target_ratio})':'min(ih,iw/{target_ratio})':"
f"'(iw-min(iw,ih*{target_ratio}))*{fx}':"
f"'(ih-min(ih,iw/{target_ratio}))*{fy}'"
)
except (ValueError, ZeroDivisionError):
logger.warning(f"Invalid face position: {render_spec.face_pos}")
elif render_spec.zoom_cut:
# 中心缩放裁切
target_ratio = width / height
filters.append(
f"crop='min(iw,ih*{target_ratio})':'min(ih,iw/{target_ratio})'"
)
# 4. 缩放和填充
scale_filter = (
f"scale={width}:{height}:force_original_aspect_ratio=decrease,"
f"pad={width}:{height}:(ow-iw)/2:(oh-ih)/2:black"
)
filters.append(scale_filter)
# 5. 特效处理(cameraShot 需要特殊处理)
if has_camera_shot:
# cameraShot 需要使用 filter_complex 格式
return self._build_filter_complex_with_effects(
base_filters=filters,
effects=effects,
fps=fps,
width=width,
height=height,
has_overlay=has_overlay,
is_video_overlay=is_video_overlay,
overlap_head_ms=overlap_head_ms,
overlap_tail_ms=overlap_tail_ms,
use_hwdownload=bool(hwaccel_prefix),
duration_ms=duration_ms,
render_spec=render_spec,
source_duration_sec=source_duration_sec
)
# 6. 帧冻结(tpad)- 用于转场 overlap 区域和时长不足补足
# 注意:tpad 必须在缩放之后应用
tpad_parts = []
# 计算是否需要额外的尾部冻结(源视频时长不足)
extra_tail_freeze_sec = 0.0
if source_duration_sec is not None:
speed = float(render_spec.speed) if render_spec.speed else 1.0
if speed > 0:
# 计算变速后的有效时长
effective_duration_sec = source_duration_sec / speed
required_duration_sec = duration_ms / 1000.0
# 如果源视频时长不足,需要冻结最后一帧来补足
if effective_duration_sec < required_duration_sec:
extra_tail_freeze_sec = required_duration_sec - effective_duration_sec
if overlap_head_ms > 0:
# 头部冻结:将第一帧冻结指定时长
head_duration_sec = overlap_head_ms / 1000.0
tpad_parts.append(f"start_mode=clone:start_duration={head_duration_sec}")
# 尾部冻结:合并 overlap 和时长不足的冻结
total_tail_freeze_sec = (overlap_tail_ms / 1000.0) + extra_tail_freeze_sec
if total_tail_freeze_sec > 0:
# 将最后一帧冻结指定时长
tpad_parts.append(f"stop_mode=clone:stop_duration={total_tail_freeze_sec}")
if tpad_parts:
filters.append(f"tpad={':'.join(tpad_parts)}")
# 7. 构建最终滤镜
if has_overlay:
# 使用 filter_complex 格式
base_filters = ','.join(filters) if filters else 'copy'
overlay_scale = f"scale={width}:{height}"
# 视频 overlay 使用 eof_action=pass(结束后消失),图片 overlay 使用默认行为(保持显示)
overlay_params = 'eof_action=pass' if is_video_overlay else ''
overlay_filter = f"overlay=0:0:{overlay_params}" if overlay_params else 'overlay=0:0'
# 视频 overlay 需要在末尾统一颜色范围,避免 overlay 结束后 range 从 tv 变为 pc
range_fix = ',format=yuv420p,setrange=tv' if is_video_overlay else ''
return f"[0:v]{base_filters}[base];[1:v]{overlay_scale}[overlay];[base][overlay]{overlay_filter}{range_fix}"
else:
return ','.join(filters) if filters else ''
def _build_filter_complex_with_effects(
self,
base_filters: List[str],
effects: List[Effect],
fps: int,
width: int,
height: int,
has_overlay: bool = False,
is_video_overlay: bool = False,
overlap_head_ms: int = 0,
overlap_tail_ms: int = 0,
use_hwdownload: bool = False,
duration_ms: int = 0,
render_spec: Optional[RenderSpec] = None,
source_duration_sec: Optional[float] = None
) -> str:
"""
构建包含特效的 filter_complex 滤镜图
cameraShot 效果需要使用 split/freezeframes/concat 滤镜组合。
Args:
base_filters: 基础滤镜列表
effects: 特效列表
fps: 帧率
width: 输出宽度
height: 输出高度
has_overlay: 是否有叠加层
is_video_overlay: 叠加层是否为视频格式(如 .mov)
overlap_head_ms: 头部 overlap 时长
overlap_tail_ms: 尾部 overlap 时长
use_hwdownload: 是否使用了硬件加速解码(已在 base_filters 中包含 hwdownload)
duration_ms: 目标时长(毫秒)
render_spec: 渲染规格(用于获取变速参数)
source_duration_sec: 源视频实际时长(秒),用于检测时长不足
Returns:
filter_complex 格式的滤镜字符串
"""
filter_parts = []
# 基础滤镜链
base_chain = ','.join(base_filters) if base_filters else 'copy'
# 当前输出标签
current_output = '[v_base]'
filter_parts.append(f"[0:v]{base_chain}{current_output}")
# 处理每个特效
effect_idx = 0
for effect in effects:
if effect.effect_type == 'cameraShot':
start_sec, duration_sec = effect.get_camera_shot_params()
if start_sec <= 0 or duration_sec <= 0:
continue
# cameraShot 实现(定格效果):
# 1. fps + split 分割
# 2. 第一路:trim(0, start) + tpad冻结duration秒
# 3. 第二路:trim(start, end)
# 4. concat 拼接
split_out_a = f'[eff{effect_idx}_a]'
split_out_b = f'[eff{effect_idx}_b]'
frozen_out = f'[eff{effect_idx}_frozen]'
rest_out = f'[eff{effect_idx}_rest]'
effect_output = f'[v_eff{effect_idx}]'
# fps + split
filter_parts.append(
f"{current_output}fps=fps={fps},split{split_out_a}{split_out_b}"
)
# 第一路:trim(0, start) + tpad冻结
# tpad=stop_mode=clone 将最后一帧冻结指定时长
filter_parts.append(
f"{split_out_a}trim=start=0:end={start_sec},setpts=PTS-STARTPTS,"
f"tpad=stop_mode=clone:stop_duration={duration_sec}{frozen_out}"
)
# 第二路:trim 从 start 开始
filter_parts.append(
f"{split_out_b}trim=start={start_sec},setpts=PTS-STARTPTS{rest_out}"
)
# concat 拼接
filter_parts.append(
f"{frozen_out}{rest_out}concat=n=2:v=1:a=0{effect_output}"
)
current_output = effect_output
effect_idx += 1
# 帧冻结(tpad)- 用于转场 overlap 区域和时长不足补足
tpad_parts = []
# 计算是否需要额外的尾部冻结(源视频时长不足)
extra_tail_freeze_sec = 0.0
if source_duration_sec is not None and render_spec is not None and duration_ms > 0:
speed = float(render_spec.speed) if render_spec.speed else 1.0
if speed > 0:
# 计算变速后的有效时长
effective_duration_sec = source_duration_sec / speed
required_duration_sec = duration_ms / 1000.0
# 如果源视频时长不足,需要冻结最后一帧来补足
if effective_duration_sec < required_duration_sec:
extra_tail_freeze_sec = required_duration_sec - effective_duration_sec
if overlap_head_ms > 0:
head_duration_sec = overlap_head_ms / 1000.0
tpad_parts.append(f"start_mode=clone:start_duration={head_duration_sec}")
# 尾部冻结:合并 overlap 和时长不足的冻结
total_tail_freeze_sec = (overlap_tail_ms / 1000.0) + extra_tail_freeze_sec
if total_tail_freeze_sec > 0:
tpad_parts.append(f"stop_mode=clone:stop_duration={total_tail_freeze_sec}")
if tpad_parts:
tpad_output = '[v_tpad]'
filter_parts.append(f"{current_output}tpad={':'.join(tpad_parts)}{tpad_output}")
current_output = tpad_output
# 最终输出
if has_overlay:
# 叠加层处理
# 视频 overlay 使用 eof_action=pass(结束后消失),图片 overlay 使用默认行为(保持显示)
overlay_params = 'eof_action=pass' if is_video_overlay else ''
overlay_filter = f"overlay=0:0:{overlay_params}" if overlay_params else 'overlay=0:0'
overlay_scale = f"scale={width}:{height}"
overlay_output = '[v_overlay]'
# 视频 overlay 需要在末尾统一颜色范围,避免 overlay 结束后 range 从 tv 变为 pc
range_fix = ',format=yuv420p,setrange=tv' if is_video_overlay else ''
filter_parts.append(f"[1:v]{overlay_scale}{overlay_output}")
filter_parts.append(f"{current_output}{overlay_output}{overlay_filter}{range_fix}")
else:
# 移除最后一个标签,直接输出
# 将最后一个滤镜的输出标签替换为空(直接输出)
if filter_parts:
last_filter = filter_parts[-1]
# 移除末尾的输出标签
if last_filter.endswith(current_output):
filter_parts[-1] = last_filter[:-len(current_output)]
return ';'.join(filter_parts)

282
index.py
View File

@@ -1,228 +1,94 @@
#!/usr/bin/env python3 from time import sleep
# -*- coding: utf-8 -*-
"""
RenderWorker v2 入口
支持 v2 API 协议的渲染 Worker,处理以下任务类型:
- RENDER_SEGMENT_VIDEO: 渲染视频片段
- PREPARE_JOB_AUDIO: 生成全局音频
- PACKAGE_SEGMENT_TS: 封装 TS 分片
- FINALIZE_MP4: 产出最终 MP4
使用方法:
python index.py
环境变量:
API_ENDPOINT_V2: v2 API 端点(或使用 API_ENDPOINT)
ACCESS_KEY: Worker 认证密钥
WORKER_ID: Worker ID(默认 100001)
MAX_CONCURRENCY: 最大并发数(默认 4)
HEARTBEAT_INTERVAL: 心跳间隔秒数(默认 5)
TEMP_DIR: 临时文件目录
"""
import sys import sys
import time
import signal import config
import logging import biz.task
from telemetry import init_opentelemetry
from services import DefaultTemplateService
from util import api
import os import os
from logging.handlers import RotatingFileHandler import glob
from dotenv import load_dotenv # 使用新的服务容器架构
from services.service_container import get_template_service, register_default_services
from domain.config import WorkerConfig # 确保服务已注册
from services.api_client import APIClientV2 register_default_services()
from services.task_executor import TaskExecutor template_service = get_template_service()
from constant import SOFTWARE_VERSION
# 日志配置 # Check for redownload parameter
def setup_logging(): if "redownload" in sys.argv:
"""配置日志系统,输出到控制台和文件""" print("Redownloading all templates...")
# 日志格式
log_format = '[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s'
date_format = '%Y-%m-%d %H:%M:%S'
formatter = logging.Formatter(log_format, date_format)
# 获取根logger
root_logger = logging.getLogger()
root_logger.setLevel(logging.INFO)
# 清除已有的handlers(避免重复)
root_logger.handlers.clear()
# 1. 控制台handler(只输出WARNING及以上级别)
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.WARNING)
console_handler.setFormatter(formatter)
root_logger.addHandler(console_handler)
# 确保日志文件所在目录存在
log_dir = os.path.dirname(os.path.abspath(__file__))
# 2. 所有日志文件handler(all_log.log)
all_log_path = os.path.join(log_dir, 'all_log.log')
all_log_handler = RotatingFileHandler(
all_log_path,
maxBytes=10*1024*1024, # 10MB
backupCount=5,
encoding='utf-8'
)
all_log_handler.setLevel(logging.DEBUG) # 记录所有级别
all_log_handler.setFormatter(formatter)
root_logger.addHandler(all_log_handler)
# 3. 错误日志文件handler(error.log)
error_log_path = os.path.join(log_dir, 'error.log')
error_log_handler = RotatingFileHandler(
error_log_path,
maxBytes=10*1024*1024, # 10MB
backupCount=5,
encoding='utf-8'
)
error_log_handler.setLevel(logging.ERROR) # 只记录ERROR及以上
error_log_handler.setFormatter(formatter)
root_logger.addHandler(error_log_handler)
# 初始化日志系统
setup_logging()
logger = logging.getLogger('worker')
class WorkerV2:
"""
v2 渲染 Worker 主类
负责:
- 配置加载
- API 客户端初始化
- 任务执行器管理
- 主循环运行
- 优雅退出处理
"""
def __init__(self):
"""初始化 Worker"""
# 加载配置
try: try:
self.config = WorkerConfig.from_env() for template_name in template_service.get_all_templates().keys():
except ValueError as e: print(f"Redownloading template: {template_name}")
logger.error(f"Configuration error: {e}") if not template_service.download_template(template_name):
print(f"Failed to download template: {template_name}")
print("Template redownload process completed!")
except Exception as e:
print(f"Error during template redownload: {e}")
sys.exit(1) sys.exit(1)
sys.exit(0)
import logging
# 初始化 API 客户端 LOGGER = logging.getLogger(__name__)
self.api_client = APIClientV2(self.config) init_opentelemetry()
# 初始化任务执行器
self.task_executor = TaskExecutor(self.config, self.api_client)
# 运行状态 def cleanup_temp_files():
self.running = True """清理临时文件 - 异步执行避免阻塞主循环"""
import threading
# 确保临时目录存在 def _cleanup():
self.config.ensure_temp_dir() for file_globs in ["*.mp4", "*.ts", "tmp_concat*.txt"]:
for file_path in glob.glob(file_globs):
# 注册信号处理器
self._setup_signal_handlers()
def _setup_signal_handlers(self):
"""设置信号处理器"""
# Windows 不支持 SIGTERM
signal.signal(signal.SIGINT, self._signal_handler)
if hasattr(signal, 'SIGTERM'):
signal.signal(signal.SIGTERM, self._signal_handler)
def _signal_handler(self, signum, frame):
"""
信号处理,优雅退出
Args:
signum: 信号编号
frame: 当前栈帧
"""
signal_name = signal.Signals(signum).name
logger.info(f"Received signal {signal_name}, initiating shutdown...")
self.running = False
def run(self):
"""主循环"""
logger.info("=" * 60)
logger.info("RenderWorker v2 Starting")
logger.info("=" * 60)
logger.info(f"Worker ID: {self.config.worker_id}")
logger.info(f"API Endpoint: {self.config.api_endpoint}")
logger.info(f"Max Concurrency: {self.config.max_concurrency}")
logger.info(f"Heartbeat Interval: {self.config.heartbeat_interval}s")
logger.info(f"Capabilities: {', '.join(self.config.capabilities)}")
logger.info(f"Temp Directory: {self.config.temp_dir}")
logger.info("=" * 60)
consecutive_errors = 0
max_consecutive_errors = 10
while self.running:
try: try:
# 心跳同步并拉取任务 if os.path.exists(file_path):
current_task_ids = self.task_executor.get_current_task_ids() os.remove(file_path)
tasks = self.api_client.sync(current_task_ids) LOGGER.debug(f"Deleted temp file: {file_path}")
except Exception as e:
LOGGER.warning(f"Error deleting file {file_path}: {e}")
# 提交新任务 # 在后台线程中执行清理
for task in tasks: threading.Thread(target=_cleanup, daemon=True).start()
if self.task_executor.submit_task(task):
logger.info(f"Submitted task: {task.task_id} ({task.task_type.value})")
# 重置错误计数
consecutive_errors = 0
# 等待下次心跳 def main_loop():
time.sleep(self.config.heartbeat_interval) """主处理循环"""
while True:
try:
print("waiting for task...")
task_list = api.sync_center()
if len(task_list) == 0:
# 异步清理临时文件
cleanup_temp_files()
sleep(5)
continue
for task in task_list:
task_id = task.get("id", "unknown")
print(f"Processing task: {task_id}")
try:
biz.task.start_task(task)
LOGGER.info(f"Task {task_id} completed successfully")
except Exception as e:
LOGGER.error(f"Task {task_id} failed: {e}", exc_info=True)
# 继续处理下一个任务而不是崩溃
except KeyboardInterrupt: except KeyboardInterrupt:
logger.info("Keyboard interrupt received") LOGGER.info("Received shutdown signal, exiting...")
self.running = False break
except Exception as e: except Exception as e:
consecutive_errors += 1 LOGGER.error("Unexpected error in main loop", exc_info=e)
logger.error(f"Worker loop error ({consecutive_errors}/{max_consecutive_errors}): {e}", exc_info=True) sleep(5) # 避免快速循环消耗CPU
# 连续错误过多,增加等待时间
if consecutive_errors >= max_consecutive_errors:
logger.error("Too many consecutive errors, waiting 30 seconds...")
time.sleep(30)
consecutive_errors = 0
else:
time.sleep(5)
# 优雅关闭
self._shutdown()
def _shutdown(self):
"""优雅关闭"""
logger.info("Shutting down...")
# 等待当前任务完成
current_count = self.task_executor.get_current_task_count()
if current_count > 0:
logger.info(f"Waiting for {current_count} running task(s) to complete...")
# 关闭执行器
self.task_executor.shutdown(wait=True)
# 关闭 API 客户端
self.api_client.close()
logger.info("Worker stopped")
def main(): if __name__ == "__main__":
"""主函数""" try:
# 加载 .env 文件(如果存在) main_loop()
load_dotenv() except Exception as e:
LOGGER.critical("Critical error in main process", exc_info=e)
logger.info(f"RenderWorker v{SOFTWARE_VERSION}") sys.exit(1)
# 创建并运行 Worker
worker = WorkerV2()
worker.run()
if __name__ == '__main__':
main()

42
mypy.ini Normal file
View File

@@ -0,0 +1,42 @@
[mypy]
python_version = 3.9
warn_return_any = False
warn_unused_configs = False
disallow_untyped_defs = False
disallow_incomplete_defs = False
check_untyped_defs = False
disallow_untyped_decorators = False
no_implicit_optional = False
warn_redundant_casts = False
warn_unused_ignores = False
warn_no_return = False
warn_unreachable = False
strict_equality = False
namespace_packages = True
explicit_package_bases = True
show_error_codes = True
# Exclude tests code
exclude = tests/
# Ignore missing type annotations for third-party libraries
[mypy-requests.*]
ignore_missing_imports = True
[mypy-flask.*]
ignore_missing_imports = True
[mypy-pytest.*]
ignore_missing_imports = True
[mypy-PIL.*]
ignore_missing_imports = True
[mypy-psutil.*]
ignore_missing_imports = True
[mypy-opentelemetry.*]
ignore_missing_imports = True
[mypy-dotenv.*]
ignore_missing_imports = True

53
pytest.ini Normal file
View File

@@ -0,0 +1,53 @@
[tool:pytest]
testpaths = tests
python_files = test_*.py
python_classes = Test*
python_functions = test_*
# 标记定义
markers =
integration: marks tests as integration tests (may be slow)
unit: marks tests as unit tests (fast)
slow: marks tests as slow running
stress: marks tests as stress tests
# 输出配置
addopts =
-v
--tb=short
--strict-markers
--strict-config
--cov=entity
--cov=services
--cov-report=xml:coverage.xml
--cov-report=html:htmlcov
--cov-report=term-missing
--cov-branch
# 过滤警告
filterwarnings =
ignore::DeprecationWarning
ignore::PendingDeprecationWarning
# 最小覆盖率要求
[tool:coverage:run]
source = entity,services
omit =
*/tests/*
*/test_*
*/__pycache__/*
*/venv/*
*/env/*
[tool:coverage:report]
exclude_lines =
pragma: no cover
def __repr__
if self.debug:
if settings.DEBUG
raise AssertionError
raise NotImplementedError
if 0:
if __name__ == .__main__.:
class .*\bProtocol\):
@(abc\.)?abstractmethod

23
requirements-test.txt Normal file
View File

@@ -0,0 +1,23 @@
# 测试依赖
pytest>=7.0.0
pytest-cov>=4.0.0
pytest-mock>=3.10.0
pytest-xdist>=3.0.0 # 并行测试
pytest-timeout>=2.1.0
pytest-html>=3.1.0 # HTML报告
# 代码质量
flake8>=5.0.0
black>=22.0.0
mypy>=1.0.0
# 覆盖率报告
coverage[toml]>=6.0.0
# 性能测试
pytest-benchmark>=4.0.0
# 测试辅助
factory-boy>=3.2.0 # 测试数据工厂
freezegun>=1.2.0 # 时间模拟
responses>=0.22.0 # HTTP模拟

294
run_tests.py Normal file
View File

@@ -0,0 +1,294 @@
#!/usr/bin/env python
"""
测试运行器脚本
支持不同类型的测试运行和报告生成
"""
import os
import sys
import subprocess
import argparse
import tempfile
from pathlib import Path
def run_command(cmd, cwd=None):
"""运行命令并返回结果"""
print(f"Running: {' '.join(cmd)}")
try:
result = subprocess.run(
cmd, cwd=cwd, capture_output=True, text=True, check=False
)
if result.stdout:
print(result.stdout)
if result.stderr:
print(result.stderr, file=sys.stderr)
return result.returncode == 0
except Exception as e:
print(f"Error running command: {e}", file=sys.stderr)
return False
def check_dependencies():
"""检查依赖是否安装"""
print("Checking dependencies...")
# 检查pytest
if not run_command([sys.executable, "-c", "import pytest"]):
print("pytest not found. Installing test dependencies...")
if not run_command(
[sys.executable, "-m", "pip", "install", "-r", "requirements-test.txt"]
):
print("Failed to install test dependencies", file=sys.stderr)
return False
# 检查FFmpeg
ffmpeg_available = run_command(["ffmpeg", "-version"])
if not ffmpeg_available:
print("Warning: FFmpeg not found. Integration tests may be skipped.")
return True
def run_unit_tests(args):
"""运行单元测试"""
print("\n=== Running Unit Tests ===")
cmd = [
sys.executable,
"-m",
"pytest",
"tests/test_effects/",
"tests/test_ffmpeg_builder/",
"-v",
"-m",
"not integration",
]
if args.coverage:
cmd.extend(
[
"--cov=entity",
"--cov=services",
"--cov-report=xml:coverage.xml",
"--cov-report=html:htmlcov",
"--cov-report=term-missing",
"--cov-branch",
]
)
if args.xml_report:
cmd.extend(["--junitxml=unit-tests.xml"])
if args.html_report:
cmd.extend(["--html=unit-tests.html", "--self-contained-html"])
return run_command(cmd)
def run_integration_tests(args):
"""运行集成测试"""
print("\n=== Running Integration Tests ===")
# 检查FFmpeg
if not run_command(["ffmpeg", "-version"]):
print("FFmpeg not available, skipping integration tests")
return True
cmd = [
sys.executable,
"-m",
"pytest",
"tests/test_integration/",
"-v",
"-m",
"integration",
"--timeout=300",
]
if args.coverage:
cmd.extend(
[
"--cov=entity",
"--cov=services",
"--cov-report=xml:integration-coverage.xml",
"--cov-report=html:integration-htmlcov",
"--cov-branch",
]
)
if args.xml_report:
cmd.extend(["--junitxml=integration-tests.xml"])
if args.html_report:
cmd.extend(["--html=integration-tests.html", "--self-contained-html"])
return run_command(cmd)
def run_all_tests(args):
"""运行所有测试"""
print("\n=== Running All Tests ===")
cmd = [sys.executable, "-m", "pytest", "tests/", "-v"]
if args.coverage:
cmd.extend(
[
"--cov=entity",
"--cov=services",
"--cov-report=xml:coverage.xml",
"--cov-report=html:htmlcov",
"--cov-report=term-missing",
"--cov-branch",
]
)
if args.fail_under:
cmd.extend([f"--cov-fail-under={args.fail_under}"])
if args.xml_report:
cmd.extend(["--junitxml=all-tests.xml"])
if args.html_report:
cmd.extend(["--html=all-tests.html", "--self-contained-html"])
return run_command(cmd)
def run_effect_tests(effect_name=None):
"""运行特定特效测试"""
if effect_name:
print(f"\n=== Running {effect_name} Effect Tests ===")
cmd = [
sys.executable,
"-m",
"pytest",
f"tests/test_effects/test_{effect_name}_effect.py",
"-v",
]
else:
print("\n=== Running All Effect Tests ===")
cmd = [sys.executable, "-m", "pytest", "tests/test_effects/", "-v"]
return run_command(cmd)
def run_stress_tests():
"""运行压力测试"""
print("\n=== Running Stress Tests ===")
env = os.environ.copy()
env["RUN_STRESS_TESTS"] = "1"
cmd = [
sys.executable,
"-m",
"pytest",
"tests/test_integration/",
"-v",
"-m",
"stress",
"--timeout=600",
]
return subprocess.run(cmd, env=env).returncode == 0
def create_test_video():
"""创建测试视频文件"""
print("\n=== Creating Test Video Files ===")
if not run_command(["ffmpeg", "-version"]):
print("FFmpeg not available, cannot create test videos")
return False
test_data_dir = Path("tests/test_data/videos")
test_data_dir.mkdir(parents=True, exist_ok=True)
# 创建短视频文件
video_path = test_data_dir / "sample.mp4"
cmd = [
"ffmpeg",
"-y",
"-f",
"lavfi",
"-i",
"testsrc=duration=5:size=640x480:rate=25",
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-crf",
"23",
str(video_path),
]
if run_command(cmd):
print(f"Created test video: {video_path}")
return True
else:
print("Failed to create test video")
return False
def main():
parser = argparse.ArgumentParser(description="RenderWorker Test Runner")
parser.add_argument(
"test_type",
choices=["unit", "integration", "all", "effects", "stress", "setup"],
help="Type of tests to run",
)
parser.add_argument(
"--effect", help="Specific effect to test (for effects command)"
)
parser.add_argument(
"--coverage", action="store_true", help="Generate coverage report"
)
parser.add_argument(
"--xml-report", action="store_true", help="Generate XML test report"
)
parser.add_argument(
"--html-report", action="store_true", help="Generate HTML test report"
)
parser.add_argument(
"--fail-under", type=int, default=70, help="Minimum coverage percentage"
)
parser.add_argument(
"--no-deps-check", action="store_true", help="Skip dependency check"
)
args = parser.parse_args()
# 检查依赖
if not args.no_deps_check and not check_dependencies():
sys.exit(1)
# 运行相应的测试
success = True
if args.test_type == "unit":
success = run_unit_tests(args)
elif args.test_type == "integration":
success = run_integration_tests(args)
elif args.test_type == "all":
success = run_all_tests(args)
elif args.test_type == "effects":
success = run_effect_tests(args.effect)
elif args.test_type == "stress":
success = run_stress_tests()
elif args.test_type == "setup":
success = create_test_video()
if success:
print("\n✅ Tests completed successfully!")
if args.coverage:
print("📊 Coverage report generated in htmlcov/")
sys.exit(0)
else:
print("\n❌ Tests failed!")
sys.exit(1)
if __name__ == "__main__":
main()

View File

@@ -1,18 +1,26 @@
# -*- coding: utf-8 -*- from .render_service import RenderService, DefaultRenderService
""" from .task_service import TaskService, DefaultTaskService
服务层 from .template_service import TemplateService, DefaultTemplateService
from .service_container import (
包含 API 客户端、任务执行器、租约服务、存储服务等组件。 ServiceContainer,
""" get_container,
register_default_services,
from services.api_client import APIClientV2 get_render_service,
from services.lease_service import LeaseService get_template_service,
from services.task_executor import TaskExecutor get_task_service,
from services import storage )
__all__ = [ __all__ = [
'APIClientV2', "RenderService",
'LeaseService', "DefaultRenderService",
'TaskExecutor', "TaskService",
'storage', "DefaultTaskService",
"TemplateService",
"DefaultTemplateService",
"ServiceContainer",
"get_container",
"register_default_services",
"get_render_service",
"get_template_service",
"get_task_service",
] ]

View File

@@ -1,417 +0,0 @@
# -*- coding: utf-8 -*-
"""
v2 API 客户端
实现与渲染服务端 v2 接口的通信。
"""
import logging
import subprocess
import time
import requests
from typing import Dict, List, Optional, Any
from domain.task import Task
from domain.config import WorkerConfig
from util.system import get_hw_accel_info_str
logger = logging.getLogger(__name__)
class APIClientV2:
"""
v2 API 客户端
负责与渲染服务端的所有 HTTP 通信。
"""
SYSTEM_INFO_TTL_SECONDS = 30
def __init__(self, config: WorkerConfig):
"""
初始化 API 客户端
Args:
config: Worker 配置
"""
self.config = config
self.base_url = config.api_endpoint.rstrip('/')
self.access_key = config.access_key
self.worker_id = config.worker_id
self.session = requests.Session()
self._ffmpeg_version: Optional[str] = None
self._codec_info: Optional[str] = None
self._hw_accel_info: Optional[str] = None
self._gpu_info: Optional[str] = None
self._gpu_info_checked = False
self._static_system_info: Optional[Dict[str, Any]] = None
self._system_info_cache: Optional[Dict[str, Any]] = None
self._system_info_cache_ts = 0.0
# 设置默认请求头
self.session.headers.update({
'Content-Type': 'application/json',
'Accept': 'application/json'
})
def sync(self, current_task_ids: List[str]) -> List[Task]:
"""
心跳同步并拉取任务
Args:
current_task_ids: 当前正在执行的任务 ID 列表
Returns:
List[Task]: 新分配的任务列表
"""
url = f"{self.base_url}/render/v2/worker/sync"
# 将 task_id 转换为整数(服务端期望 []int64)
task_ids_int = [int(tid) for tid in current_task_ids if tid.isdigit()]
payload = {
'accessKey': self.access_key,
'workerId': self.worker_id,
'capabilities': self.config.capabilities,
'maxConcurrency': self.config.max_concurrency,
'currentTaskCount': len(current_task_ids),
'currentTaskIds': task_ids_int,
'ffmpegVersion': self._get_ffmpeg_version(),
'codecInfo': self._get_codec_info(),
'systemInfo': self._get_system_info()
}
try:
resp = self.session.post(url, json=payload, timeout=10)
resp.raise_for_status()
data = resp.json()
if data.get('code') != 200:
logger.warning(f"Sync failed: {data.get('message')}")
return []
# 解析任务列表
tasks = []
for task_data in data.get('data', {}).get('tasks') or []:
try:
task = Task.from_dict(task_data)
tasks.append(task)
except Exception as e:
logger.error(f"Failed to parse task: {e}")
if tasks:
logger.info(f"Received {len(tasks)} new tasks")
return tasks
except requests.exceptions.Timeout:
logger.warning("Sync timeout")
return []
except requests.exceptions.RequestException as e:
logger.error(f"Sync request error: {e}")
return []
except Exception as e:
logger.error(f"Sync error: {e}")
return []
def report_start(self, task_id: str) -> bool:
"""
报告任务开始
Args:
task_id: 任务 ID
Returns:
bool: 是否成功
"""
url = f"{self.base_url}/render/v2/task/{task_id}/start"
try:
resp = self.session.post(
url,
json={'workerId': self.worker_id},
timeout=10
)
if resp.status_code == 200:
logger.debug(f"[task:{task_id}] Start reported")
return True
else:
logger.warning(f"[task:{task_id}] Report start failed: {resp.status_code}")
return False
except Exception as e:
logger.error(f"[task:{task_id}] Report start error: {e}")
return False
def report_success(self, task_id: str, result: Dict[str, Any]) -> bool:
"""
报告任务成功
Args:
task_id: 任务 ID
result: 任务结果数据
Returns:
bool: 是否成功
"""
url = f"{self.base_url}/render/v2/task/{task_id}/success"
try:
resp = self.session.post(
url,
json={
'workerId': self.worker_id,
'result': result
},
timeout=10
)
if resp.status_code == 200:
logger.debug(f"[task:{task_id}] Success reported")
return True
else:
logger.warning(f"[task:{task_id}] Report success failed: {resp.status_code}")
return False
except Exception as e:
logger.error(f"[task:{task_id}] Report success error: {e}")
return False
def report_fail(self, task_id: str, error_code: str, error_message: str) -> bool:
"""
报告任务失败
Args:
task_id: 任务 ID
error_code: 错误码
error_message: 错误信息
Returns:
bool: 是否成功
"""
url = f"{self.base_url}/render/v2/task/{task_id}/fail"
try:
resp = self.session.post(
url,
json={
'workerId': self.worker_id,
'errorCode': error_code,
'errorMessage': error_message[:1000] # 限制长度
},
timeout=10
)
if resp.status_code == 200:
logger.debug(f"[task:{task_id}] Failure reported")
return True
else:
logger.warning(f"[task:{task_id}] Report fail failed: {resp.status_code}")
return False
except Exception as e:
logger.error(f"[task:{task_id}] Report fail error: {e}")
return False
def get_upload_url(self, task_id: str, file_type: str, file_name: str = None) -> Optional[Dict[str, str]]:
"""
获取上传 URL
Args:
task_id: 任务 ID
file_type: 文件类型(video/audio/ts/mp4)
file_name: 文件名(可选)
Returns:
Dict 包含 uploadUrl 和 accessUrl,失败返回 None
"""
url = f"{self.base_url}/render/v2/task/{task_id}/uploadUrl"
payload = {'fileType': file_type}
if file_name:
payload['fileName'] = file_name
try:
resp = self.session.post(url, json=payload, timeout=10)
if resp.status_code == 200:
data = resp.json()
if data.get('code') == 200:
return data.get('data')
logger.warning(f"[task:{task_id}] Get upload URL failed: {resp.status_code}")
return None
except Exception as e:
logger.error(f"[task:{task_id}] Get upload URL error: {e}")
return None
def extend_lease(self, task_id: str, extension: int = None) -> bool:
"""
延长租约
Args:
task_id: 任务 ID
extension: 延长秒数(默认使用配置值)
Returns:
bool: 是否成功
"""
if extension is None:
extension = self.config.lease_extension_duration
url = f"{self.base_url}/render/v2/task/{task_id}/extend-lease"
try:
resp = self.session.post(
url,
params={
'workerId': self.worker_id,
'extension': extension
},
timeout=10
)
if resp.status_code == 200:
logger.debug(f"[task:{task_id}] Lease extended by {extension}s")
return True
else:
logger.warning(f"[task:{task_id}] Extend lease failed: {resp.status_code}")
return False
except Exception as e:
logger.error(f"[task:{task_id}] Extend lease error: {e}")
return False
def get_task_info(self, task_id: str) -> Optional[Dict]:
"""
获取任务详情
Args:
task_id: 任务 ID
Returns:
任务详情字典,失败返回 None
"""
url = f"{self.base_url}/render/v2/task/{task_id}"
try:
resp = self.session.get(url, timeout=10)
if resp.status_code == 200:
data = resp.json()
if data.get('code') == 200:
return data.get('data')
return None
except Exception as e:
logger.error(f"[task:{task_id}] Get task info error: {e}")
return None
def _get_ffmpeg_version(self) -> str:
"""获取 FFmpeg 版本"""
if self._ffmpeg_version is not None:
return self._ffmpeg_version
try:
result = subprocess.run(
['ffmpeg', '-version'],
capture_output=True,
text=True,
timeout=5
)
first_line = result.stdout.split('\n')[0]
if 'version' in first_line:
parts = first_line.split()
for i, part in enumerate(parts):
if part == 'version' and i + 1 < len(parts):
self._ffmpeg_version = parts[i + 1]
return self._ffmpeg_version
self._ffmpeg_version = 'unknown'
return self._ffmpeg_version
except Exception:
self._ffmpeg_version = 'unknown'
return self._ffmpeg_version
def _get_codec_info(self) -> str:
"""获取支持的编解码器信息"""
if self._codec_info is not None:
return self._codec_info
try:
result = subprocess.run(
['ffmpeg', '-codecs'],
capture_output=True,
text=True,
timeout=5
)
# 检查常用编解码器
codecs = []
output = result.stdout
if 'libx264' in output:
codecs.append('libx264')
if 'libx265' in output or 'hevc' in output:
codecs.append('libx265')
if 'aac' in output:
codecs.append('aac')
if 'libfdk_aac' in output:
codecs.append('libfdk_aac')
self._codec_info = ', '.join(codecs) if codecs else 'unknown'
return self._codec_info
except Exception:
self._codec_info = 'unknown'
return self._codec_info
def _get_system_info(self) -> Dict[str, Any]:
"""获取系统信息"""
try:
now = time.monotonic()
if (
self._system_info_cache
and now - self._system_info_cache_ts < self.SYSTEM_INFO_TTL_SECONDS
):
return self._system_info_cache
import platform
import psutil
if self._hw_accel_info is None:
self._hw_accel_info = get_hw_accel_info_str()
if self._static_system_info is None:
self._static_system_info = {
'os': platform.system(),
'cpu': f"{psutil.cpu_count()} cores",
'memory': f"{psutil.virtual_memory().total // (1024**3)}GB",
'hwAccelConfig': self.config.hw_accel, # 当前配置的硬件加速
'hwAccelSupport': self._hw_accel_info, # 系统支持的硬件加速
}
info = dict(self._static_system_info)
info.update({
'cpuUsage': f"{psutil.cpu_percent()}%",
'memoryAvailable': f"{psutil.virtual_memory().available // (1024**3)}GB",
})
# 尝试获取 GPU 信息
gpu_info = self._get_gpu_info()
if gpu_info:
info['gpu'] = gpu_info
self._system_info_cache = info
self._system_info_cache_ts = now
return info
except Exception:
return {}
def _get_gpu_info(self) -> Optional[str]:
"""获取 GPU 信息"""
if self._gpu_info_checked:
return self._gpu_info
self._gpu_info_checked = True
try:
result = subprocess.run(
['nvidia-smi', '--query-gpu=name', '--format=csv,noheader'],
capture_output=True,
text=True,
timeout=5
)
if result.returncode == 0:
gpu_name = result.stdout.strip().split('\n')[0]
self._gpu_info = gpu_name
except Exception:
self._gpu_info = None
return self._gpu_info
def close(self):
"""关闭会话"""
self.session.close()

View File

@@ -1,513 +0,0 @@
# -*- coding: utf-8 -*-
"""
素材缓存服务
提供素材下载缓存功能,避免相同素材重复下载。
"""
import json
import os
import hashlib
import logging
import shutil
import time
import uuid
from typing import Optional, Tuple
from urllib.parse import urlparse, unquote
import psutil
from services import storage
logger = logging.getLogger(__name__)
def _extract_cache_key(url: str) -> str:
"""
从 URL 提取缓存键
去除签名等查询参数,保留路径作为唯一标识。
Args:
url: 完整的素材 URL
Returns:
缓存键(URL 路径的 MD5 哈希)
"""
parsed = urlparse(url)
# 使用 scheme + host + path 作为唯一标识(忽略签名等查询参数)
cache_key_source = f"{parsed.scheme}://{parsed.netloc}{unquote(parsed.path)}"
return hashlib.md5(cache_key_source.encode('utf-8')).hexdigest()
def _get_file_extension(url: str) -> str:
"""
从 URL 提取文件扩展名
Args:
url: 素材 URL
Returns:
文件扩展名(如 .mp4, .png),无法识别时返回空字符串
"""
parsed = urlparse(url)
path = unquote(parsed.path)
_, ext = os.path.splitext(path)
return ext.lower() if ext else ''
class MaterialCache:
"""
素材缓存管理器
负责素材文件的缓存存储和检索。
"""
LOCK_TIMEOUT_SEC = 30.0
LOCK_POLL_INTERVAL_SEC = 0.1
LOCK_STALE_SECONDS = 24 * 60 * 60
def __init__(self, cache_dir: str, enabled: bool = True, max_size_gb: float = 0):
"""
初始化缓存管理器
Args:
cache_dir: 缓存目录路径
enabled: 是否启用缓存
max_size_gb: 最大缓存大小(GB),0 表示不限制
"""
self.cache_dir = cache_dir
self.enabled = enabled
self.max_size_bytes = int(max_size_gb * 1024 * 1024 * 1024) if max_size_gb > 0 else 0
if self.enabled:
os.makedirs(self.cache_dir, exist_ok=True)
logger.info(f"Material cache initialized: {cache_dir}")
def get_cache_path(self, url: str) -> str:
"""
获取素材的缓存文件路径
Args:
url: 素材 URL
Returns:
缓存文件的完整路径
"""
cache_key = _extract_cache_key(url)
ext = _get_file_extension(url)
filename = f"{cache_key}{ext}"
return os.path.join(self.cache_dir, filename)
def _get_lock_path(self, cache_key: str) -> str:
"""获取缓存锁文件路径"""
assert self.cache_dir
return os.path.join(self.cache_dir, f"{cache_key}.lock")
def _write_lock_metadata(self, lock_fd: int, lock_path: str) -> bool:
"""写入锁元数据,失败则清理锁文件"""
try:
try:
process_start_time = psutil.Process(os.getpid()).create_time()
except Exception as e:
process_start_time = None
logger.warning(f"Cache lock process start time error: {e}")
metadata = {
'pid': os.getpid(),
'process_start_time': process_start_time,
'created_at': time.time()
}
with os.fdopen(lock_fd, 'w', encoding='utf-8') as lock_file:
json.dump(metadata, lock_file)
return True
except Exception as e:
try:
os.close(lock_fd)
except Exception:
pass
self._remove_lock_file(lock_path, f"write metadata failed: {e}")
return False
def _read_lock_metadata(self, lock_path: str) -> Optional[dict]:
"""读取锁元数据,失败返回 None(兼容历史空锁文件)"""
try:
with open(lock_path, 'r', encoding='utf-8') as lock_file:
data = json.load(lock_file)
return data if isinstance(data, dict) else None
except Exception:
return None
def _is_process_alive(self, pid: int, expected_start_time: Optional[float]) -> bool:
"""判断进程是否存活并校验启动时间(防止 PID 复用)"""
try:
process = psutil.Process(pid)
if expected_start_time is None:
return process.is_running()
actual_start_time = process.create_time()
return abs(actual_start_time - expected_start_time) < 1.0
except psutil.NoSuchProcess:
return False
except Exception as e:
logger.warning(f"Cache lock process check error: {e}")
return True
def _is_lock_stale(self, lock_path: str) -> bool:
"""判断锁是否过期(进程已退出或超过最大存活时长)"""
if not os.path.exists(lock_path):
return False
now = time.time()
metadata = self._read_lock_metadata(lock_path)
if metadata:
created_at = metadata.get('created_at')
if isinstance(created_at, (int, float)) and now - created_at > self.LOCK_STALE_SECONDS:
return True
pid = metadata.get('pid')
pid_value = int(pid) if isinstance(pid, int) or (isinstance(pid, str) and pid.isdigit()) else None
expected_start_time = metadata.get('process_start_time')
expected_start_time_value = (
expected_start_time if isinstance(expected_start_time, (int, float)) else None
)
if pid_value is not None and not self._is_process_alive(pid_value, expected_start_time_value):
return True
return self._is_lock_stale_by_mtime(lock_path, now)
return self._is_lock_stale_by_mtime(lock_path, now)
def _is_lock_stale_by_mtime(self, lock_path: str, now: float) -> bool:
"""基于文件时间判断锁是否过期"""
try:
mtime = os.path.getmtime(lock_path)
return now - mtime > self.LOCK_STALE_SECONDS
except Exception as e:
logger.warning(f"Cache lock stat error: {e}")
return False
def _remove_lock_file(self, lock_path: str, reason: str = "") -> bool:
"""删除锁文件"""
try:
os.remove(lock_path)
if reason:
logger.info(f"Cache lock removed: {lock_path} ({reason})")
return True
except FileNotFoundError:
return True
except Exception as e:
logger.warning(f"Cache lock remove error: {e}")
return False
def _acquire_lock(self, cache_key: str) -> Optional[str]:
"""获取缓存锁(跨进程安全)"""
if not self.enabled:
return None
lock_path = self._get_lock_path(cache_key)
deadline = time.monotonic() + self.LOCK_TIMEOUT_SEC
while True:
try:
fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
if not self._write_lock_metadata(fd, lock_path):
return None
return lock_path
except FileExistsError:
if self._is_lock_stale(lock_path):
removed = self._remove_lock_file(lock_path, "stale lock")
if removed:
continue
if time.monotonic() >= deadline:
logger.warning(f"Cache lock timeout: {lock_path}")
return None
time.sleep(self.LOCK_POLL_INTERVAL_SEC)
except Exception as e:
logger.warning(f"Cache lock error: {e}")
return None
def _release_lock(self, lock_path: Optional[str]) -> None:
"""释放缓存锁"""
if not lock_path:
return
self._remove_lock_file(lock_path)
def is_cached(self, url: str) -> Tuple[bool, str]:
"""
检查素材是否已缓存
Args:
url: 素材 URL
Returns:
(是否已缓存, 缓存文件路径)
"""
if not self.enabled:
return False, ''
cache_path = self.get_cache_path(url)
exists = os.path.exists(cache_path) and os.path.getsize(cache_path) > 0
return exists, cache_path
def get_or_download(
self,
url: str,
dest: str,
timeout: int = 300,
max_retries: int = 5
) -> bool:
"""
从缓存获取素材,若未缓存则下载并缓存
Args:
url: 素材 URL
dest: 目标文件路径(任务工作目录中的路径)
timeout: 下载超时时间(秒)
max_retries: 最大重试次数
Returns:
是否成功
"""
# 确保目标目录存在
dest_dir = os.path.dirname(dest)
if dest_dir:
os.makedirs(dest_dir, exist_ok=True)
# 缓存未启用时直接下载
if not self.enabled:
return storage.download_file(url, dest, max_retries=max_retries, timeout=timeout)
cache_key = _extract_cache_key(url)
lock_path = self._acquire_lock(cache_key)
if not lock_path:
logger.warning(f"Cache lock unavailable, downloading without cache: {url[:80]}...")
return storage.download_file(url, dest, max_retries=max_retries, timeout=timeout)
try:
cache_path = self.get_cache_path(url)
cached = os.path.exists(cache_path) and os.path.getsize(cache_path) > 0
if cached:
# 命中缓存,复制到目标路径
try:
shutil.copy2(cache_path, dest)
# 更新访问时间(用于 LRU 清理)
os.utime(cache_path, None)
file_size = os.path.getsize(dest)
logger.info(f"Cache hit: {url[:80]}... -> {dest} ({file_size} bytes)")
return True
except Exception as e:
logger.warning(f"Failed to copy from cache: {e}, will re-download")
# 缓存复制失败,删除可能损坏的缓存文件
try:
os.remove(cache_path)
except Exception:
pass
# 未命中缓存,下载到缓存目录
logger.debug(f"Cache miss: {url[:80]}...")
# 先下载到临时文件(唯一文件名,避免并发覆盖)
temp_cache_path = os.path.join(
self.cache_dir,
f"{cache_key}.{uuid.uuid4().hex}.downloading"
)
try:
if not storage.download_file(url, temp_cache_path, max_retries=max_retries, timeout=timeout):
# 下载失败,清理临时文件
if os.path.exists(temp_cache_path):
os.remove(temp_cache_path)
return False
if not os.path.exists(temp_cache_path) or os.path.getsize(temp_cache_path) <= 0:
if os.path.exists(temp_cache_path):
os.remove(temp_cache_path)
return False
# 下载成功,原子替换缓存文件
os.replace(temp_cache_path, cache_path)
# 复制到目标路径
shutil.copy2(cache_path, dest)
file_size = os.path.getsize(dest)
logger.info(f"Downloaded and cached: {url[:80]}... ({file_size} bytes)")
# 检查是否需要清理缓存
if self.max_size_bytes > 0:
self._cleanup_if_needed()
return True
except Exception as e:
logger.error(f"Cache download error: {e}")
# 清理临时文件
if os.path.exists(temp_cache_path):
try:
os.remove(temp_cache_path)
except Exception:
pass
return False
finally:
self._release_lock(lock_path)
def add_to_cache(self, url: str, source_path: str) -> bool:
"""
将本地文件添加到缓存
Args:
url: 对应的 URL(用于生成缓存键)
source_path: 本地文件路径
Returns:
是否成功
"""
if not self.enabled:
return False
if not os.path.exists(source_path):
logger.warning(f"Source file not found for cache: {source_path}")
return False
cache_key = _extract_cache_key(url)
lock_path = self._acquire_lock(cache_key)
if not lock_path:
logger.warning(f"Cache lock unavailable for adding: {url[:80]}...")
return False
try:
cache_path = self.get_cache_path(url)
# 先复制到临时文件
temp_cache_path = os.path.join(
self.cache_dir,
f"{cache_key}.{uuid.uuid4().hex}.adding"
)
shutil.copy2(source_path, temp_cache_path)
# 原子替换
os.replace(temp_cache_path, cache_path)
# 更新访问时间
os.utime(cache_path, None)
logger.info(f"Added to cache: {url[:80]}... <- {source_path}")
# 检查清理
if self.max_size_bytes > 0:
self._cleanup_if_needed()
return True
except Exception as e:
logger.error(f"Failed to add to cache: {e}")
if 'temp_cache_path' in locals() and os.path.exists(temp_cache_path):
try:
os.remove(temp_cache_path)
except Exception:
pass
return False
finally:
self._release_lock(lock_path)
def _cleanup_if_needed(self) -> None:
"""
检查并清理缓存(LRU 策略)
当缓存大小超过限制时,删除最久未访问的文件。
"""
if self.max_size_bytes <= 0:
return
try:
# 获取所有缓存文件及其信息
cache_files = []
total_size = 0
for filename in os.listdir(self.cache_dir):
if filename.endswith('.downloading') or filename.endswith('.lock'):
continue
file_path = os.path.join(self.cache_dir, filename)
if os.path.isfile(file_path):
stat = os.stat(file_path)
cache_files.append({
'path': file_path,
'size': stat.st_size,
'atime': stat.st_atime
})
total_size += stat.st_size
# 如果未超过限制,无需清理
if total_size <= self.max_size_bytes:
return
# 按访问时间排序(最久未访问的在前)
cache_files.sort(key=lambda x: x['atime'])
# 删除文件直到低于限制的 80%
target_size = int(self.max_size_bytes * 0.8)
deleted_count = 0
for file_info in cache_files:
if total_size <= target_size:
break
# 从文件名提取 cache_key,检查是否有锁(说明正在被使用)
filename = os.path.basename(file_info['path'])
cache_key = os.path.splitext(filename)[0]
lock_path = self._get_lock_path(cache_key)
if os.path.exists(lock_path):
if self._is_lock_stale(lock_path):
self._remove_lock_file(lock_path, "cleanup stale lock")
else:
# 该文件正在被其他任务使用,跳过删除
logger.debug(f"Cache cleanup: skipping locked file {filename}")
continue
try:
os.remove(file_info['path'])
total_size -= file_info['size']
deleted_count += 1
except Exception as e:
logger.warning(f"Failed to delete cache file: {e}")
if deleted_count > 0:
logger.info(f"Cache cleanup: deleted {deleted_count} files, current size: {total_size / (1024*1024*1024):.2f} GB")
except Exception as e:
logger.warning(f"Cache cleanup error: {e}")
def clear(self) -> None:
"""清空所有缓存"""
if not self.enabled:
return
try:
if os.path.exists(self.cache_dir):
shutil.rmtree(self.cache_dir)
os.makedirs(self.cache_dir, exist_ok=True)
logger.info("Cache cleared")
except Exception as e:
logger.error(f"Failed to clear cache: {e}")
def get_stats(self) -> dict:
"""
获取缓存统计信息
Returns:
包含缓存统计的字典
"""
if not self.enabled or not os.path.exists(self.cache_dir):
return {'enabled': False, 'file_count': 0, 'total_size_mb': 0}
file_count = 0
total_size = 0
for filename in os.listdir(self.cache_dir):
if filename.endswith('.downloading') or filename.endswith('.lock'):
continue
file_path = os.path.join(self.cache_dir, filename)
if os.path.isfile(file_path):
file_count += 1
total_size += os.path.getsize(file_path)
return {
'enabled': True,
'cache_dir': self.cache_dir,
'file_count': file_count,
'total_size_mb': round(total_size / (1024 * 1024), 2),
'max_size_gb': self.max_size_bytes / (1024 * 1024 * 1024) if self.max_size_bytes > 0 else 0
}

View File

@@ -1,164 +0,0 @@
# -*- coding: utf-8 -*-
"""
GPU 调度器
提供多 GPU 设备的轮询调度功能。
"""
import logging
import threading
from typing import List, Optional
from domain.config import WorkerConfig
from domain.gpu import GPUDevice
from util.system import get_all_gpu_info, validate_gpu_device
from constant import HW_ACCEL_CUDA, HW_ACCEL_QSV
logger = logging.getLogger(__name__)
class GPUScheduler:
"""
GPU 调度器
实现多 GPU 设备的轮询(Round Robin)调度。
线程安全,支持并发任务执行。
使用方式:
scheduler = GPUScheduler(config)
# 在任务执行时
device_index = scheduler.acquire()
try:
# 执行任务
pass
finally:
scheduler.release(device_index)
"""
def __init__(self, config: WorkerConfig):
"""
初始化调度器
Args:
config: Worker 配置
"""
self._config = config
self._devices: List[GPUDevice] = []
self._next_index: int = 0
self._lock = threading.Lock()
self._enabled = False
# 初始化设备列表
self._init_devices()
def _init_devices(self) -> None:
"""初始化 GPU 设备列表"""
# 仅在启用硬件加速时才初始化
if self._config.hw_accel not in (HW_ACCEL_CUDA, HW_ACCEL_QSV):
logger.info("Hardware acceleration not enabled, GPU scheduler disabled")
return
configured_devices = self._config.gpu_devices
if configured_devices:
# 使用配置指定的设备
self._devices = self._validate_configured_devices(configured_devices)
else:
# 自动检测所有设备
self._devices = self._auto_detect_devices()
if self._devices:
self._enabled = True
device_info = ', '.join(str(d) for d in self._devices)
logger.info(f"GPU scheduler initialized with {len(self._devices)} device(s): {device_info}")
else:
logger.warning("No GPU devices available, scheduler disabled")
def _validate_configured_devices(self, indices: List[int]) -> List[GPUDevice]:
"""
验证配置的设备列表
Args:
indices: 配置的设备索引列表
Returns:
验证通过的设备列表
"""
devices = []
for index in indices:
if validate_gpu_device(index):
devices.append(GPUDevice(
index=index,
name=f"GPU-{index}",
available=True
))
else:
logger.warning(f"GPU device {index} is not available, skipping")
return devices
def _auto_detect_devices(self) -> List[GPUDevice]:
"""
自动检测所有可用 GPU
Returns:
检测到的设备列表
"""
all_devices = get_all_gpu_info()
# 过滤不可用设备
return [d for d in all_devices if d.available]
@property
def enabled(self) -> bool:
"""调度器是否启用"""
return self._enabled
@property
def device_count(self) -> int:
"""设备数量"""
return len(self._devices)
def acquire(self) -> Optional[int]:
"""
获取下一个可用的 GPU 设备(轮询调度)
Returns:
GPU 设备索引,如果调度器未启用或无设备则返回 None
"""
if not self._enabled or not self._devices:
return None
with self._lock:
device = self._devices[self._next_index]
self._next_index = (self._next_index + 1) % len(self._devices)
logger.debug(f"Acquired GPU device: {device.index}")
return device.index
def release(self, device_index: Optional[int]) -> None:
"""
释放 GPU 设备
当前实现为无状态轮询,此方法仅用于日志记录。
Args:
device_index: 设备索引
"""
if device_index is not None:
logger.debug(f"Released GPU device: {device_index}")
def get_status(self) -> dict:
"""
获取调度器状态信息
Returns:
状态字典
"""
return {
'enabled': self._enabled,
'device_count': len(self._devices),
'devices': [
{'index': d.index, 'name': d.name, 'available': d.available}
for d in self._devices
],
'hw_accel': self._config.hw_accel,
}

View File

@@ -1,110 +0,0 @@
# -*- coding: utf-8 -*-
"""
租约续期服务
后台线程定期为正在执行的任务续期租约。
"""
import logging
import threading
import time
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from services.api_client import APIClientV2
logger = logging.getLogger(__name__)
class LeaseService:
"""
租约续期服务
在后台线程中定期调用 API 延长任务租约,
防止长时间任务因租约过期被回收。
"""
def __init__(
self,
api_client: 'APIClientV2',
task_id: str,
interval: int = 60,
extension: int = 300
):
"""
初始化租约服务
Args:
api_client: API 客户端
task_id: 任务 ID
interval: 续期间隔(秒),默认 60 秒
extension: 每次续期时长(秒),默认 300 秒
"""
self.api_client = api_client
self.task_id = task_id
self.interval = interval
self.extension = extension
self.running = False
self.thread: threading.Thread = None
self._stop_event = threading.Event()
def start(self):
"""启动租约续期线程"""
if self.running:
logger.warning(f"[task:{self.task_id}] Lease service already running")
return
self.running = True
self._stop_event.clear()
self.thread = threading.Thread(
target=self._run,
name=f"LeaseService-{self.task_id}",
daemon=True
)
self.thread.start()
logger.debug(f"[task:{self.task_id}] Lease service started (interval={self.interval}s)")
def stop(self):
"""停止租约续期线程"""
if not self.running:
return
self.running = False
self._stop_event.set()
if self.thread and self.thread.is_alive():
self.thread.join(timeout=5)
logger.debug(f"[task:{self.task_id}] Lease service stopped")
def _run(self):
"""续期线程主循环"""
while self.running:
# 等待指定间隔或收到停止信号
if self._stop_event.wait(timeout=self.interval):
# 收到停止信号
break
if self.running:
self._extend_lease()
def _extend_lease(self):
"""执行租约续期"""
try:
success = self.api_client.extend_lease(self.task_id, self.extension)
if success:
logger.debug(f"[task:{self.task_id}] Lease extended by {self.extension}s")
else:
logger.warning(f"[task:{self.task_id}] Failed to extend lease")
except Exception as e:
logger.warning(f"[task:{self.task_id}] Lease extension error: {e}")
def __enter__(self):
"""上下文管理器入口"""
self.start()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
"""上下文管理器出口"""
self.stop()
return False

214
services/render_service.py Normal file
View File

@@ -0,0 +1,214 @@
import subprocess
import os
import logging
from abc import ABC, abstractmethod
from typing import Union
from opentelemetry.trace import Status, StatusCode
from entity.render_task import RenderTask
from entity.ffmpeg_command_builder import FFmpegCommandBuilder
from entity.ffmpeg import FfmpegTask
from util.exceptions import RenderError, FFmpegError
from util.ffmpeg import (
probe_video_info,
fade_out_audio,
handle_ffmpeg_output,
subprocess_args,
)
from telemetry import get_tracer
logger = logging.getLogger(__name__)
# 向后兼容层 - 处理旧的FfmpegTask对象
class RenderService(ABC):
"""渲染服务抽象接口"""
@abstractmethod
def render(self, task: Union[RenderTask, FfmpegTask]) -> bool:
"""
执行渲染任务
Args:
task: 渲染任务
Returns:
bool: 渲染是否成功
"""
pass
@abstractmethod
def get_video_info(self, file_path: str) -> tuple[int, int, float]:
"""
获取视频信息
Args:
file_path: 视频文件路径
Returns:
tuple: (width, height, duration)
"""
pass
@abstractmethod
def fade_out_audio(
self, file_path: str, duration: float, fade_seconds: float = 2.0
) -> str:
"""
音频淡出处理
Args:
file_path: 音频文件路径
duration: 音频总时长
fade_seconds: 淡出时长
Returns:
str: 处理后的文件路径
"""
pass
class DefaultRenderService(RenderService):
"""默认渲染服务实现"""
def render(self, task: Union[RenderTask, FfmpegTask]) -> bool:
"""执行渲染任务"""
# 兼容旧的FfmpegTask
if hasattr(task, "get_ffmpeg_args"): # 这是FfmpegTask
# 使用旧的方式执行
return self._render_legacy_ffmpeg_task(task)
tracer = get_tracer(__name__)
with tracer.start_as_current_span("render_task") as span:
try:
# 验证任务
task.validate()
span.set_attribute("task.type", task.task_type.value)
span.set_attribute("task.input_files", len(task.input_files))
span.set_attribute("task.output_file", task.output_file)
# 检查是否需要处理
if not task.need_processing():
if len(task.input_files) == 1:
task.output_file = task.input_files[0]
span.set_status(Status(StatusCode.OK))
return True
# 构建FFmpeg命令
builder = FFmpegCommandBuilder(task)
ffmpeg_args = builder.build_command()
if not ffmpeg_args:
# 不需要处理,直接返回
if len(task.input_files) == 1:
task.output_file = task.input_files[0]
span.set_status(Status(StatusCode.OK))
return True
# 执行FFmpeg命令
return self._execute_ffmpeg(ffmpeg_args, span)
except Exception as e:
span.set_status(Status(StatusCode.ERROR))
logger.error(f"Render failed: {e}", exc_info=True)
raise RenderError(f"Render failed: {e}") from e
def _execute_ffmpeg(self, args: list[str], span) -> bool:
"""执行FFmpeg命令"""
span.set_attribute("ffmpeg.args", " ".join(args))
logger.info("Executing FFmpeg: %s", " ".join(args))
try:
# 执行FFmpeg进程 (使用构建器已经包含的参数)
process = subprocess.run(
args, stderr=subprocess.PIPE, **subprocess_args(True)
)
span.set_attribute("ffmpeg.return_code", process.returncode)
# 处理输出
if process.stdout:
output = handle_ffmpeg_output(process.stdout)
span.set_attribute("ffmpeg.output", output)
logger.info("FFmpeg output: %s", output)
# 检查返回码
if process.returncode != 0:
error_msg = (
process.stderr.decode() if process.stderr else "Unknown error"
)
span.set_attribute("ffmpeg.error", error_msg)
span.set_status(Status(StatusCode.ERROR))
logger.error(
"FFmpeg failed with return code %d: %s",
process.returncode,
error_msg,
)
raise FFmpegError(
"FFmpeg execution failed",
command=args,
return_code=process.returncode,
stderr=error_msg,
)
# 检查输出文件
output_file = args[-1] # 输出文件总是最后一个参数
if not os.path.exists(output_file):
span.set_status(Status(StatusCode.ERROR))
raise RenderError(f"Output file not created: {output_file}")
# 检查文件大小
file_size = os.path.getsize(output_file)
span.set_attribute("output.file_size", file_size)
if file_size < 4096: # 文件过小
span.set_status(Status(StatusCode.ERROR))
raise RenderError(f"Output file too small: {file_size} bytes")
span.set_status(Status(StatusCode.OK))
logger.info("FFmpeg execution completed successfully")
return True
except subprocess.SubprocessError as e:
span.set_status(Status(StatusCode.ERROR))
logger.error("Subprocess error: %s", e)
raise FFmpegError(f"Subprocess error: {e}") from e
def get_video_info(self, file_path: str) -> tuple[int, int, float]:
"""获取视频信息"""
return probe_video_info(file_path)
def fade_out_audio(
self, file_path: str, duration: float, fade_seconds: float = 2.0
) -> str:
"""音频淡出处理"""
return fade_out_audio(file_path, duration, fade_seconds)
def _render_legacy_ffmpeg_task(self, ffmpeg_task) -> bool:
"""兼容处理旧的FfmpegTask"""
tracer = get_tracer(__name__)
with tracer.start_as_current_span("render_legacy_ffmpeg_task") as span:
try:
# 处理依赖任务
for sub_task in ffmpeg_task.analyze_input_render_tasks():
if not self.render(sub_task):
span.set_status(Status(StatusCode.ERROR))
return False
# 获取FFmpeg参数
ffmpeg_args = ffmpeg_task.get_ffmpeg_args()
if not ffmpeg_args:
# 不需要处理,直接返回
span.set_status(Status(StatusCode.OK))
return True
# 执行FFmpeg命令
return self._execute_ffmpeg(ffmpeg_args, span)
except Exception as e:
span.set_status(Status(StatusCode.ERROR))
logger.error(f"Legacy FFmpeg task render failed: {e}", exc_info=True)
raise RenderError(f"Legacy render failed: {e}") from e

View File

@@ -0,0 +1,139 @@
"""
服务容器模块 - 提供线程安全的服务实例管理
"""
import threading
from typing import Dict, Type, TypeVar, Optional, TYPE_CHECKING
import logging
if TYPE_CHECKING:
from .render_service import RenderService
from .template_service import TemplateService
from .task_service import TaskService
logger = logging.getLogger(__name__)
T = TypeVar("T")
class ServiceContainer:
"""线程安全的服务容器,实现依赖注入和单例管理"""
def __init__(self):
self._services: Dict[Type, object] = {}
self._factories: Dict[Type, callable] = {}
self._lock = threading.RLock()
def register_singleton(self, service_type: Type[T], factory: callable) -> None:
"""注册单例服务工厂"""
with self._lock:
self._factories[service_type] = factory
logger.debug(f"Registered singleton factory for {service_type.__name__}")
def get_service(self, service_type: Type[T]) -> T:
"""获取服务实例(懒加载单例)"""
with self._lock:
# 检查是否已存在实例
if service_type in self._services:
return self._services[service_type]
# 检查是否有工厂方法
if service_type not in self._factories:
raise ValueError(
f"No factory registered for service type: {service_type}"
)
# 创建新实例
factory = self._factories[service_type]
try:
instance = factory()
self._services[service_type] = instance
logger.debug(f"Created new instance of {service_type.__name__}")
return instance
except Exception as e:
logger.error(
f"Failed to create instance of {service_type.__name__}: {e}"
)
raise
def has_service(self, service_type: Type[T]) -> bool:
"""检查是否有服务注册"""
with self._lock:
return service_type in self._factories
def clear_cache(self, service_type: Optional[Type[T]] = None) -> None:
"""清理服务缓存"""
with self._lock:
if service_type:
self._services.pop(service_type, None)
logger.debug(f"Cleared cache for {service_type.__name__}")
else:
self._services.clear()
logger.debug("Cleared all service cache")
# 全局服务容器实例
_container: Optional[ServiceContainer] = None
_container_lock = threading.Lock()
def get_container() -> ServiceContainer:
"""获取全局服务容器实例"""
global _container
if _container is None:
with _container_lock:
if _container is None:
_container = ServiceContainer()
return _container
def register_default_services():
"""注册默认的服务实现"""
from .render_service import DefaultRenderService, RenderService
from .template_service import DefaultTemplateService, TemplateService
from .task_service import DefaultTaskService, TaskService
container = get_container()
# 注册渲染服务
container.register_singleton(RenderService, lambda: DefaultRenderService())
# 注册模板服务
def create_template_service():
service = DefaultTemplateService()
service.load_local_templates()
return service
container.register_singleton(TemplateService, create_template_service)
# 注册任务服务(依赖其他服务)
def create_task_service():
render_service = container.get_service(RenderService)
template_service = container.get_service(TemplateService)
return DefaultTaskService(render_service, template_service)
container.register_singleton(TaskService, create_task_service)
logger.info("Default services registered successfully")
# 便捷函数
def get_render_service() -> "RenderService":
"""获取渲染服务实例"""
from .render_service import RenderService
return get_container().get_service(RenderService)
def get_template_service() -> "TemplateService":
"""获取模板服务实例"""
from .template_service import TemplateService
return get_container().get_service(TemplateService)
def get_task_service() -> "TaskService":
"""获取任务服务实例"""
from .task_service import TaskService
return get_container().get_service(TaskService)

View File

@@ -1,239 +0,0 @@
# -*- coding: utf-8 -*-
"""
存储服务
提供文件上传/下载功能,支持 OSS 签名 URL 和 HTTP_REPLACE_MAP 环境变量。
"""
import os
import logging
import subprocess
from typing import Optional
from urllib.parse import unquote
import requests
logger = logging.getLogger(__name__)
# 文件扩展名到 Content-Type 的映射
_CONTENT_TYPE_MAP = {
'.mp4': 'video/mp4',
'.aac': 'audio/aac',
'.ts': 'video/mp2t',
'.m4a': 'audio/mp4',
}
def _get_content_type(file_path: str) -> str:
"""
根据文件扩展名获取 Content-Type
Args:
file_path: 文件路径
Returns:
Content-Type 字符串
"""
ext = os.path.splitext(file_path)[1].lower()
return _CONTENT_TYPE_MAP.get(ext, 'application/octet-stream')
def _apply_http_replace_map(url: str) -> str:
"""
应用 HTTP_REPLACE_MAP 环境变量替换 URL
Args:
url: 原始 URL
Returns:
替换后的 URL
"""
replace_map = os.getenv("HTTP_REPLACE_MAP", "")
if not replace_map:
return url
new_url = url
replace_list = [i.split("|", 1) for i in replace_map.split(",") if "|" in i]
for src, dst in replace_list:
new_url = new_url.replace(src, dst)
if new_url != url:
logger.debug(f"HTTP_REPLACE_MAP: {url} -> {new_url}")
return new_url
def upload_file(url: str, file_path: str, max_retries: int = 5, timeout: int = 60) -> bool:
"""
使用签名 URL 上传文件到 OSS
Args:
url: 签名 URL
file_path: 本地文件路径
max_retries: 最大重试次数
timeout: 超时时间(秒)
Returns:
是否成功
"""
if not os.path.exists(file_path):
logger.error(f"File not found: {file_path}")
return False
file_size = os.path.getsize(file_path)
logger.info(f"Uploading: {file_path} ({file_size} bytes)")
# 检查是否使用 rclone 上传
if os.getenv("UPLOAD_METHOD") == "rclone":
logger.info(f"Uploading to: {url}")
result = _upload_with_rclone(url, file_path)
if result:
return True
# rclone 失败时回退到 HTTP
# 应用 HTTP_REPLACE_MAP 替换 URL
http_url = _apply_http_replace_map(url)
content_type = _get_content_type(file_path)
logger.info(f"Uploading to: {http_url} (Content-Type: {content_type})")
retries = 0
while retries < max_retries:
try:
with open(file_path, 'rb') as f:
with requests.put(
http_url,
data=f,
stream=True,
timeout=timeout,
headers={"Content-Type": content_type}
) as response:
response.raise_for_status()
logger.info(f"Upload succeeded: {file_path}")
return True
except requests.exceptions.Timeout:
retries += 1
logger.warning(f"Upload timed out. Retrying {retries}/{max_retries}...")
except requests.exceptions.RequestException as e:
retries += 1
logger.warning(f"Upload failed ({e}). Retrying {retries}/{max_retries}...")
logger.error(f"Upload failed after {max_retries} retries: {file_path}")
return False
def _upload_with_rclone(url: str, file_path: str) -> bool:
"""
使用 rclone 上传文件
Args:
url: 目标 URL
file_path: 本地文件路径
Returns:
是否成功
"""
replace_map = os.getenv("RCLONE_REPLACE_MAP", "")
if not replace_map:
return False
config_file = os.getenv("RCLONE_CONFIG_FILE", "")
# 替换 URL
new_url = url
replace_list = [i.split("|", 1) for i in replace_map.split(",") if "|" in i]
for src, dst in replace_list:
new_url = new_url.replace(src, dst)
new_url = new_url.split("?", 1)[0] # 移除查询参数
new_url = unquote(new_url) # 解码 URL 编码的字符(如 %2F -> /)
if new_url == url:
return False
cmd = [
"rclone",
"copyto",
"--no-check-dest",
"--ignore-existing",
"--multi-thread-chunk-size",
"8M",
"--multi-thread-streams",
"8",
]
if config_file:
cmd.extend(["--config", config_file])
cmd.extend([file_path, new_url])
logger.debug(f"rclone command: {' '.join(cmd)}")
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode == 0:
logger.info(f"rclone upload succeeded: {file_path}")
return True
stderr = (result.stderr or '').strip()
stderr = stderr[:500] if stderr else ""
logger.warning(f"rclone upload failed (code={result.returncode}): {file_path} {stderr}")
return False
def download_file(
url: str,
file_path: str,
max_retries: int = 5,
timeout: int = 30,
skip_if_exist: bool = False
) -> bool:
"""
使用签名 URL 下载文件
Args:
url: 签名 URL
file_path: 本地文件路径
max_retries: 最大重试次数
timeout: 超时时间(秒)
skip_if_exist: 如果文件存在则跳过
Returns:
是否成功
"""
# 如果文件已存在且跳过
if skip_if_exist and os.path.exists(file_path):
logger.debug(f"File exists, skipping download: {file_path}")
return True
logger.info(f"Downloading: {url}")
# 确保目标目录存在
file_dir = os.path.dirname(file_path)
if file_dir:
os.makedirs(file_dir, exist_ok=True)
# 应用 HTTP_REPLACE_MAP 替换 URL
http_url = _apply_http_replace_map(url)
retries = 0
while retries < max_retries:
try:
with requests.get(http_url, timeout=timeout, stream=True) as response:
response.raise_for_status()
with open(file_path, 'wb') as f:
for chunk in response.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
file_size = os.path.getsize(file_path)
logger.info(f"Download succeeded: {file_path} ({file_size} bytes)")
return True
except requests.exceptions.Timeout:
retries += 1
logger.warning(f"Download timed out. Retrying {retries}/{max_retries}...")
except requests.exceptions.RequestException as e:
retries += 1
logger.warning(f"Download failed ({e}). Retrying {retries}/{max_retries}...")
logger.error(f"Download failed after {max_retries} retries: {url}")
return False

View File

@@ -1,278 +0,0 @@
# -*- coding: utf-8 -*-
"""
任务执行器
管理任务的并发执行,协调处理器、租约服务等组件。
"""
import logging
import threading
from concurrent.futures import ThreadPoolExecutor, Future
from typing import Dict, Optional, TYPE_CHECKING
from domain.task import Task, TaskType
from domain.result import TaskResult, ErrorCode
# 需要 GPU 加速的任务类型
GPU_REQUIRED_TASK_TYPES = {
TaskType.RENDER_SEGMENT_VIDEO,
TaskType.COMPOSE_TRANSITION,
}
from domain.config import WorkerConfig
from core.handler import TaskHandler
from services.lease_service import LeaseService
from services.gpu_scheduler import GPUScheduler
if TYPE_CHECKING:
from services.api_client import APIClientV2
logger = logging.getLogger(__name__)
class TaskExecutor:
"""
任务执行器
负责任务的并发调度和执行,包括:
- 注册和管理任务处理器
- 维护任务执行状态
- 协调租约续期
- 上报执行结果
"""
def __init__(self, config: WorkerConfig, api_client: 'APIClientV2'):
"""
初始化任务执行器
Args:
config: Worker 配置
api_client: API 客户端
"""
self.config = config
self.api_client = api_client
# 任务处理器注册表
self.handlers: Dict[TaskType, TaskHandler] = {}
# 当前任务跟踪
self.current_tasks: Dict[str, Task] = {}
self.current_futures: Dict[str, Future] = {}
# 线程池
self.executor = ThreadPoolExecutor(
max_workers=config.max_concurrency,
thread_name_prefix="TaskWorker"
)
# 线程安全锁
self.lock = threading.Lock()
# GPU 调度器(如果启用硬件加速)
self.gpu_scheduler = GPUScheduler(config)
if self.gpu_scheduler.enabled:
logger.info(f"GPU scheduler enabled with {self.gpu_scheduler.device_count} device(s)")
# 注册处理器
self._register_handlers()
def _register_handlers(self):
"""注册所有任务处理器"""
# 延迟导入以避免循环依赖
from handlers.render_video import RenderSegmentVideoHandler
from handlers.compose_transition import ComposeTransitionHandler
from handlers.prepare_audio import PrepareJobAudioHandler
from handlers.package_ts import PackageSegmentTsHandler
from handlers.finalize_mp4 import FinalizeMp4Handler
handlers = [
RenderSegmentVideoHandler(self.config, self.api_client),
ComposeTransitionHandler(self.config, self.api_client),
PrepareJobAudioHandler(self.config, self.api_client),
PackageSegmentTsHandler(self.config, self.api_client),
FinalizeMp4Handler(self.config, self.api_client),
]
for handler in handlers:
task_type = handler.get_supported_type()
self.handlers[task_type] = handler
logger.debug(f"Registered handler for {task_type.value}")
def get_current_task_ids(self) -> list:
"""
获取当前正在执行的任务 ID 列表
Returns:
任务 ID 列表
"""
with self.lock:
return list(self.current_tasks.keys())
def get_current_task_count(self) -> int:
"""
获取当前正在执行的任务数量
Returns:
任务数量
"""
with self.lock:
return len(self.current_tasks)
def can_accept_task(self) -> bool:
"""
检查是否可以接受新任务
Returns:
是否可以接受
"""
return self.get_current_task_count() < self.config.max_concurrency
def submit_task(self, task: Task) -> bool:
"""
提交任务到线程池
Args:
task: 任务实体
Returns:
是否提交成功
"""
with self.lock:
# 检查任务是否已在执行
if task.task_id in self.current_tasks:
logger.warning(f"[task:{task.task_id}] Task already running, skipping")
return False
# 检查并发上限
if len(self.current_tasks) >= self.config.max_concurrency:
logger.info(
f"[task:{task.task_id}] Max concurrency reached "
f"({self.config.max_concurrency}), rejecting task"
)
return False
# 检查是否有对应的处理器
if task.task_type not in self.handlers:
logger.error(f"[task:{task.task_id}] No handler for type: {task.task_type.value}")
return False
# 记录任务
self.current_tasks[task.task_id] = task
# 提交到线程池
future = self.executor.submit(self._process_task, task)
self.current_futures[task.task_id] = future
logger.info(f"[task:{task.task_id}] Submitted ({task.task_type.value})")
return True
def _process_task(self, task: Task):
"""
处理单个任务(在线程池中执行)
Args:
task: 任务实体
"""
task_id = task.task_id
logger.info(f"[task:{task_id}] Starting {task.task_type.value}")
# 启动租约续期服务
lease_service = LeaseService(
self.api_client,
task_id,
interval=self.config.lease_extension_threshold,
extension=self.config.lease_extension_duration
)
lease_service.start()
# 获取 GPU 设备(仅对需要 GPU 的任务类型)
device_index = None
needs_gpu = task.task_type in GPU_REQUIRED_TASK_TYPES
if needs_gpu and self.gpu_scheduler.enabled:
device_index = self.gpu_scheduler.acquire()
if device_index is not None:
logger.info(f"[task:{task_id}] Assigned to GPU device {device_index}")
# 获取处理器(需要在设置 GPU 设备前获取)
handler = self.handlers.get(task.task_type)
try:
# 报告任务开始
self.api_client.report_start(task_id)
if not handler:
raise ValueError(f"No handler for task type: {task.task_type}")
# 设置 GPU 设备(线程本地存储)
if device_index is not None:
handler.set_gpu_device(device_index)
# 执行前钩子
handler.before_handle(task)
# 执行任务
result = handler.handle(task)
# 执行后钩子
handler.after_handle(task, result)
# 上报结果
if result.success:
self.api_client.report_success(task_id, result.data)
logger.info(f"[task:{task_id}] Completed successfully")
else:
error_code = result.error_code.value if result.error_code else 'E_UNKNOWN'
self.api_client.report_fail(task_id, error_code, result.error_message or '')
logger.error(f"[task:{task_id}] Failed: {result.error_message}")
except Exception as e:
logger.error(f"[task:{task_id}] Exception: {e}", exc_info=True)
self.api_client.report_fail(task_id, 'E_UNKNOWN', str(e))
finally:
# 清除 GPU 设备设置
if handler:
handler.clear_gpu_device()
# 释放 GPU 设备(仅当实际分配了设备时)
if device_index is not None:
self.gpu_scheduler.release(device_index)
# 停止租约续期
lease_service.stop()
# 从当前任务中移除
with self.lock:
self.current_tasks.pop(task_id, None)
self.current_futures.pop(task_id, None)
def shutdown(self, wait: bool = True):
"""
关闭执行器
Args:
wait: 是否等待所有任务完成
"""
logger.info("Shutting down task executor...")
# 关闭线程池
self.executor.shutdown(wait=wait)
# 清理状态
with self.lock:
self.current_tasks.clear()
self.current_futures.clear()
logger.info("Task executor shutdown complete")
def get_handler(self, task_type: TaskType) -> Optional[TaskHandler]:
"""
获取指定类型的处理器
Args:
task_type: 任务类型
Returns:
处理器实例,不存在则返回 None
"""
return self.handlers.get(task_type)

357
services/task_service.py Normal file
View File

@@ -0,0 +1,357 @@
import logging
import os
from abc import ABC, abstractmethod
from concurrent.futures import ThreadPoolExecutor
from typing import Dict, Any, Optional
from opentelemetry.trace import Status, StatusCode
from entity.render_task import RenderTask
from services.render_service import RenderService
from services.template_service import TemplateService
from util.exceptions import TaskError, TaskValidationError
from util import api, oss
from util.json_utils import safe_json_loads
from telemetry import get_tracer
logger = logging.getLogger(__name__)
class TaskService(ABC):
"""任务服务抽象接口"""
@abstractmethod
def process_task(self, task_info: Dict[str, Any]) -> bool:
"""
处理任务
Args:
task_info: 任务信息
Returns:
bool: 处理是否成功
"""
pass
@abstractmethod
def create_render_task(
self, task_info: Dict[str, Any], template_info: Dict[str, Any]
) -> RenderTask:
"""
创建渲染任务
Args:
task_info: 任务信息
template_info: 模板信息
Returns:
RenderTask: 渲染任务对象
"""
pass
class DefaultTaskService(TaskService):
"""默认任务服务实现"""
def __init__(
self, render_service: RenderService, template_service: TemplateService
):
self.render_service = render_service
self.template_service = template_service
def process_task(self, task_info: Dict[str, Any]) -> bool:
"""处理任务"""
tracer = get_tracer(__name__)
with tracer.start_as_current_span("process_task") as span:
try:
# 标准化任务信息
task_info = api.normalize_task(task_info)
span.set_attribute("task.id", task_info.get("id", "unknown"))
span.set_attribute(
"task.template_id", task_info.get("templateId", "unknown")
)
# 获取模板信息
template_id = task_info.get("templateId")
template_info = self.template_service.get_template(template_id)
if not template_info:
raise TaskError(f"Template not found: {template_id}")
# 报告任务开始
api.report_task_start(task_info)
# 创建渲染任务
render_task = self.create_render_task(task_info, template_info)
# 执行渲染
success = self.render_service.render(render_task)
if not success:
span.set_status(Status(StatusCode.ERROR))
api.report_task_failed(task_info, "Render failed")
return False
# 获取视频信息
width, height, duration = self.render_service.get_video_info(
render_task.output_file
)
span.set_attribute("video.width", width)
span.set_attribute("video.height", height)
span.set_attribute("video.duration", duration)
# 音频淡出
new_file = self.render_service.fade_out_audio(
render_task.output_file, duration
)
render_task.output_file = new_file
# 上传文件 - 创建一个兼容对象
class TaskCompat:
def __init__(self, output_file):
self.output_file = output_file
def get_output_file(self):
return self.output_file
task_compat = TaskCompat(render_task.output_file)
upload_success = api.upload_task_file(task_info, task_compat)
if not upload_success:
span.set_status(Status(StatusCode.ERROR))
api.report_task_failed(task_info, "Upload failed")
return False
# 清理临时文件
self._cleanup_temp_files(render_task)
# 报告任务成功
api.report_task_success(
task_info,
videoInfo={
"width": width,
"height": height,
"duration": duration,
},
)
span.set_status(Status(StatusCode.OK))
return True
except Exception as e:
span.set_status(Status(StatusCode.ERROR))
logger.error(f"Task processing failed: {e}", exc_info=True)
api.report_task_failed(task_info, str(e))
return False
def create_render_task(
self, task_info: Dict[str, Any], template_info: Dict[str, Any]
) -> RenderTask:
"""创建渲染任务"""
tracer = get_tracer(__name__)
with tracer.start_as_current_span("create_render_task") as span:
# 解析任务参数
task_params_str = task_info.get("taskParams", "{}")
span.set_attribute("task_params", task_params_str)
task_params = safe_json_loads(task_params_str, {})
task_params_orig = safe_json_loads(task_params_str, {})
if not task_params:
raise TaskValidationError("Invalid or empty task params JSON")
# 并行下载资源
self._download_resources(task_params)
# 创建子任务列表
sub_tasks = []
only_if_usage_count = {}
for part in template_info.get("video_parts", []):
source, ext_data = self._parse_video_source(
part.get("source"), task_params, template_info
)
if not source:
logger.warning("No video found for part: %s", part)
continue
# 检查only_if条件
only_if = part.get("only_if", "")
if only_if:
only_if_usage_count[only_if] = (
only_if_usage_count.get(only_if, 0) + 1
)
required_count = only_if_usage_count[only_if]
if not self._check_placeholder_exist_with_count(
only_if, task_params_orig, required_count
):
logger.info(
"Skipping part due to only_if condition: %s (need %d)",
only_if,
required_count,
)
continue
# 创建子任务
sub_task = self._create_sub_task(part, source, ext_data, template_info)
sub_tasks.append(sub_task)
# 创建主任务
output_file = f"out_{task_info.get('id', 'unknown')}.mp4"
main_task = RenderTask(
input_files=[task.output_file for task in sub_tasks],
output_file=output_file,
resolution=template_info.get("video_size", ""),
frame_rate=template_info.get("frame_rate", 25),
center_cut=template_info.get("crop_mode"),
zoom_cut=template_info.get("zoom_cut"),
)
# 应用整体模板设置
overall_template = template_info.get("overall_template", {})
self._apply_template_settings(main_task, overall_template, template_info)
# 设置扩展数据
main_task.ext_data = task_info
span.set_attribute("render_task.sub_tasks", len(sub_tasks))
span.set_attribute("render_task.effects", len(main_task.effects))
return main_task
def _download_resources(self, task_params: Dict[str, Any]):
"""并行下载资源"""
from config.settings import get_ffmpeg_config
config = get_ffmpeg_config()
download_futures = []
with ThreadPoolExecutor(max_workers=config.max_download_workers) as executor:
for param_list in task_params.values():
if isinstance(param_list, list):
for param in param_list:
url = param.get("url", "")
if url.startswith("http"):
_, filename = os.path.split(url)
future = executor.submit(
oss.download_from_oss, url, filename, True
)
download_futures.append((future, url, filename))
# 等待所有下载完成,并记录失败的下载
failed_downloads = []
for future, url, filename in download_futures:
try:
result = future.result(timeout=30) # 30秒超时
if not result:
failed_downloads.append((url, filename))
except Exception as e:
logger.warning(f"Failed to download {url}: {e}")
failed_downloads.append((url, filename))
if failed_downloads:
logger.warning(
f"Failed to download {len(failed_downloads)} resources: {[f[1] for f in failed_downloads]}"
)
def _parse_video_source(
self,
source: str,
task_params: Dict[str, Any],
template_info: Dict[str, Any],
) -> tuple[Optional[str], Dict[str, Any]]:
"""解析视频源"""
if source.startswith("PLACEHOLDER_"):
placeholder_id = source.replace("PLACEHOLDER_", "")
new_sources = task_params.get(placeholder_id, [])
pick_source = {}
if isinstance(new_sources, list):
if len(new_sources) == 0:
logger.debug("No video found for placeholder: %s", placeholder_id)
return None, pick_source
else:
pick_source = new_sources.pop(0)
new_sources = pick_source.get("url", "")
if new_sources.startswith("http"):
_, source_name = os.path.split(new_sources)
oss.download_from_oss(new_sources, source_name, True)
return source_name, pick_source
return new_sources, pick_source
return os.path.join(template_info.get("local_path", ""), source), {}
def _check_placeholder_exist_with_count(
self,
placeholder_id: str,
task_params: Dict[str, Any],
required_count: int = 1,
) -> bool:
"""检查占位符是否存在足够数量的片段"""
if placeholder_id in task_params:
new_sources = task_params.get(placeholder_id, [])
if isinstance(new_sources, list):
return len(new_sources) >= required_count
return required_count <= 1
return False
def _create_sub_task(
self,
part: Dict[str, Any],
source: str,
ext_data: Dict[str, Any],
template_info: Dict[str, Any],
) -> RenderTask:
"""创建子任务"""
sub_task = RenderTask(
input_files=[source],
resolution=template_info.get("video_size", ""),
frame_rate=template_info.get("frame_rate", 25),
annexb=True,
center_cut=part.get("crop_mode"),
zoom_cut=part.get("zoom_cut"),
ext_data=ext_data,
)
# 应用部分模板设置
self._apply_template_settings(sub_task, part, template_info)
return sub_task
def _apply_template_settings(
self,
task: RenderTask,
template_part: Dict[str, Any],
template_info: Dict[str, Any],
):
"""应用模板设置到任务"""
# 添加效果
for effect in template_part.get("effects", []):
task.add_effect(effect)
# 添加LUT
for lut in template_part.get("luts", []):
full_path = os.path.join(template_info.get("local_path", ""), lut)
task.add_lut(full_path.replace("\\", "/"))
# 添加音频
for audio in template_part.get("audios", []):
full_path = os.path.join(template_info.get("local_path", ""), audio)
task.add_audios(full_path)
# 添加覆盖层
for overlay in template_part.get("overlays", []):
full_path = os.path.join(template_info.get("local_path", ""), overlay)
task.add_overlay(full_path)
def _cleanup_temp_files(self, task: RenderTask):
"""清理临时文件"""
try:
template_dir = os.getenv("TEMPLATE_DIR", "")
if template_dir and template_dir not in task.output_file:
if os.path.exists(task.output_file):
os.remove(task.output_file)
logger.info("Cleaned up temp file: %s", task.output_file)
else:
logger.info("Skipped cleanup of template file: %s", task.output_file)
except OSError as e:
logger.warning("Failed to cleanup temp file %s: %s", task.output_file, e)

View File

@@ -0,0 +1,287 @@
import json
import os
import logging
from abc import ABC, abstractmethod
from typing import Dict, Any, Optional
from opentelemetry.trace import Status, StatusCode
from util.exceptions import (
TemplateError,
TemplateNotFoundError,
TemplateValidationError,
)
from util import api, oss
from config.settings import get_storage_config
from telemetry import get_tracer
logger = logging.getLogger(__name__)
class TemplateService(ABC):
"""模板服务抽象接口"""
@abstractmethod
def get_template(self, template_id: str) -> Optional[Dict[str, Any]]:
"""
获取模板信息
Args:
template_id: 模板ID
Returns:
Dict[str, Any]: 模板信息,如果不存在则返回None
"""
pass
@abstractmethod
def load_local_templates(self):
"""加载本地模板"""
pass
@abstractmethod
def download_template(self, template_id: str) -> bool:
"""
下载模板
Args:
template_id: 模板ID
Returns:
bool: 下载是否成功
"""
pass
@abstractmethod
def validate_template(self, template_info: Dict[str, Any]) -> bool:
"""
验证模板
Args:
template_info: 模板信息
Returns:
bool: 验证是否通过
"""
pass
class DefaultTemplateService(TemplateService):
"""默认模板服务实现"""
def __init__(self):
self.templates: Dict[str, Dict[str, Any]] = {}
self.storage_config = get_storage_config()
def get_template(self, template_id: str) -> Optional[Dict[str, Any]]:
"""获取模板信息"""
if template_id not in self.templates:
# 尝试下载模板
if not self.download_template(template_id):
return None
return self.templates.get(template_id)
def load_local_templates(self):
"""加载本地模板"""
template_dir = self.storage_config.template_dir
if not os.path.exists(template_dir):
logger.warning("Template directory does not exist: %s", template_dir)
return
for template_name in os.listdir(template_dir):
if template_name.startswith("_") or template_name.startswith("."):
continue
target_path = os.path.join(template_dir, template_name)
if os.path.isdir(target_path):
try:
self._load_template(template_name, target_path)
except Exception as e:
logger.error("Failed to load template %s: %s", template_name, e)
def download_template(self, template_id: str) -> bool:
"""下载模板"""
tracer = get_tracer(__name__)
with tracer.start_as_current_span("download_template") as span:
try:
span.set_attribute("template.id", template_id)
# 获取远程模板信息
template_info = api.get_template_info(template_id)
if template_info is None:
logger.warning("Failed to get template info: %s", template_id)
return False
local_path = template_info.get("local_path")
if not local_path:
local_path = os.path.join(
self.storage_config.template_dir, str(template_id)
)
template_info["local_path"] = local_path
# 创建本地目录
if not os.path.isdir(local_path):
os.makedirs(local_path)
# 下载模板资源
overall_template = template_info.get("overall_template", {})
video_parts = template_info.get("video_parts", [])
self._download_template_assets(overall_template, template_info)
for video_part in video_parts:
self._download_template_assets(video_part, template_info)
# 保存模板定义文件
template_file = os.path.join(local_path, "template.json")
with open(template_file, "w", encoding="utf-8") as f:
json.dump(template_info, f, ensure_ascii=False, indent=2)
# 加载到内存
self._load_template(template_id, local_path)
span.set_status(Status(StatusCode.OK))
logger.info("Template downloaded successfully: %s", template_id)
return True
except Exception as e:
span.set_status(Status(StatusCode.ERROR))
logger.error("Failed to download template %s: %s", template_id, e)
return False
def validate_template(self, template_info: Dict[str, Any]) -> bool:
"""验证模板"""
try:
local_path = template_info.get("local_path")
if not local_path:
raise TemplateValidationError("Template missing local_path")
# 验证视频部分
for video_part in template_info.get("video_parts", []):
self._validate_template_part(video_part, local_path)
# 验证整体模板
overall_template = template_info.get("overall_template", {})
if overall_template:
self._validate_template_part(overall_template, local_path)
return True
except TemplateValidationError:
raise
except Exception as e:
raise TemplateValidationError(f"Template validation failed: {e}")
def _load_template(self, template_name: str, local_path: str):
"""加载单个模板"""
logger.info("Loading template: %s (%s)", template_name, local_path)
template_def_file = os.path.join(local_path, "template.json")
if not os.path.exists(template_def_file):
raise TemplateNotFoundError(
f"Template definition file not found: {template_def_file}"
)
try:
with open(template_def_file, "r", encoding="utf-8") as f:
template_info = json.load(f)
except json.JSONDecodeError as e:
raise TemplateError(f"Invalid template JSON: {e}")
template_info["local_path"] = local_path
try:
self.validate_template(template_info)
self.templates[template_name] = template_info
logger.info("Template loaded successfully: %s", template_name)
except TemplateValidationError as e:
logger.error(
"Template validation failed for %s: %s. Attempting to re-download.",
template_name,
e,
)
# 模板验证失败,尝试重新下载
if self.download_template(template_name):
logger.info("Template re-downloaded successfully: %s", template_name)
else:
logger.error("Failed to re-download template: %s", template_name)
raise
def _download_template_assets(
self, template_part: Dict[str, Any], template_info: Dict[str, Any]
):
"""下载模板资源"""
local_path = template_info["local_path"]
# 下载源文件
if "source" in template_part:
source = template_part["source"]
if isinstance(source, str) and source.startswith("http"):
_, filename = os.path.split(source)
new_file_path = os.path.join(local_path, filename)
oss.download_from_oss(source, new_file_path)
if filename.endswith(".mp4"):
from util.ffmpeg import re_encode_and_annexb
new_file_path = re_encode_and_annexb(new_file_path)
template_part["source"] = os.path.relpath(new_file_path, local_path)
# 下载覆盖层
if "overlays" in template_part:
for i, overlay in enumerate(template_part["overlays"]):
if isinstance(overlay, str) and overlay.startswith("http"):
_, filename = os.path.split(overlay)
oss.download_from_oss(overlay, os.path.join(local_path, filename))
template_part["overlays"][i] = filename
# 下载LUT
if "luts" in template_part:
for i, lut in enumerate(template_part["luts"]):
if isinstance(lut, str) and lut.startswith("http"):
_, filename = os.path.split(lut)
oss.download_from_oss(lut, os.path.join(local_path, filename))
template_part["luts"][i] = filename
# 下载音频
if "audios" in template_part:
for i, audio in enumerate(template_part["audios"]):
if isinstance(audio, str) and audio.startswith("http"):
_, filename = os.path.split(audio)
oss.download_from_oss(audio, os.path.join(local_path, filename))
template_part["audios"][i] = filename
def _validate_template_part(self, template_part: Dict[str, Any], base_dir: str):
"""验证模板部分"""
# 验证源文件
source_file = template_part.get("source", "")
if (
source_file
and not source_file.startswith("http")
and not source_file.startswith("PLACEHOLDER_")
):
if not os.path.isabs(source_file):
source_file = os.path.join(base_dir, source_file)
if not os.path.exists(source_file):
raise TemplateValidationError(f"Source file not found: {source_file}")
# 验证音频文件
for audio in template_part.get("audios", []):
if not os.path.isabs(audio):
audio = os.path.join(base_dir, audio)
if not os.path.exists(audio):
raise TemplateValidationError(f"Audio file not found: {audio}")
# 验证LUT文件
for lut in template_part.get("luts", []):
if not os.path.isabs(lut):
lut = os.path.join(base_dir, lut)
if not os.path.exists(lut):
raise TemplateValidationError(f"LUT file not found: {lut}")
# 验证覆盖层文件
for overlay in template_part.get("overlays", []):
if not os.path.isabs(overlay):
overlay = os.path.join(base_dir, overlay)
if not os.path.exists(overlay):
raise TemplateValidationError(f"Overlay file not found: {overlay}")

55
telemetry/__init__.py Normal file
View File

@@ -0,0 +1,55 @@
import os
from constant import SOFTWARE_VERSION
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
OTLPSpanExporter as OTLPSpanHttpExporter,
)
from opentelemetry.sdk.resources import (
DEPLOYMENT_ENVIRONMENT,
HOST_NAME,
Resource,
SERVICE_NAME,
SERVICE_VERSION,
)
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SimpleSpanProcessor
from opentelemetry.instrumentation.threading import ThreadingInstrumentor
ThreadingInstrumentor().instrument()
def get_tracer(name):
return trace.get_tracer(name)
# 初始化 OpenTelemetry
def init_opentelemetry(batch=True):
# 设置服务名、主机名
resource = Resource(
attributes={
SERVICE_NAME: "RENDER_WORKER",
SERVICE_VERSION: SOFTWARE_VERSION,
DEPLOYMENT_ENVIRONMENT: "Python",
HOST_NAME: os.getenv("ACCESS_KEY"),
}
)
# 使用HTTP协议上报
if batch:
span_processor = BatchSpanProcessor(
OTLPSpanHttpExporter(
endpoint="https://oltp.jerryyan.top/v1/traces",
)
)
else:
span_processor = SimpleSpanProcessor(
OTLPSpanHttpExporter(
endpoint="https://oltp.jerryyan.top/v1/traces",
)
)
trace_provider = TracerProvider(
resource=resource, active_span_processor=span_processor
)
trace.set_tracer_provider(trace_provider)

1
template/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
**/*

69
template/__init__.py Normal file
View File

@@ -0,0 +1,69 @@
import json
import os
import logging
from telemetry import get_tracer
from util import api, oss
from services.template_service import DefaultTemplateService
logger = logging.getLogger("template")
# 全局模板服务实例
_template_service = None
def _get_template_service():
"""获取模板服务实例"""
global _template_service
if _template_service is None:
_template_service = DefaultTemplateService()
return _template_service
# 向后兼容的全局变量和函数
TEMPLATES: dict = {}
def _update_templates_dict():
"""更新全局TEMPLATES字典以保持向后兼容"""
service = _get_template_service()
TEMPLATES.clear()
TEMPLATES.update(service.templates)
def check_local_template(local_name):
"""向后兼容函数"""
service = _get_template_service()
template_def = service.templates.get(local_name)
if template_def:
try:
service.validate_template(template_def)
except Exception as e:
logger.error(f"Template validation failed: {e}")
raise
def load_template(template_name, local_path):
"""向后兼容函数"""
service = _get_template_service()
service._load_template(template_name, local_path)
_update_templates_dict()
def load_local_template():
"""加载本地模板(向后兼容函数)"""
service = _get_template_service()
service.load_local_templates()
_update_templates_dict()
def get_template_def(template_id):
"""获取模板定义(向后兼容函数)"""
service = _get_template_service()
template = service.get_template(template_id)
_update_templates_dict()
return template
def download_template(template_id):
"""下载模板(向后兼容函数)"""
service = _get_template_service()
success = service.download_template(template_id)
_update_templates_dict()
return success
def analyze_template(template_id):
"""分析模板(占位符函数)"""
pass

313
tests/README.md Normal file
View File

@@ -0,0 +1,313 @@
# RenderWorker 测试文档
本目录包含 RenderWorker 项目的完整测试套件,用于验证特效参数生成和 FFmpeg 渲染功能。
## 目录结构
```
tests/
├── conftest.py # pytest 配置和公共 fixtures
├── test_data/ # 测试数据
│ ├── videos/ # 测试视频文件
│ ├── templates/ # 测试模板数据
│ └── expected_outputs/ # 预期输出结果
├── test_effects/ # 特效单元测试
│ ├── test_base.py # 基础类测试
│ ├── test_zoom_effect.py # 缩放特效测试
│ ├── test_speed_effect.py # 变速特效测试
│ └── ... # 其他特效测试
├── test_ffmpeg_builder/ # FFmpeg 命令构建测试
│ └── test_ffmpeg_command_builder.py
├── test_integration/ # 集成测试
│ └── test_ffmpeg_execution.py # FFmpeg 执行测试
└── utils/ # 测试工具
└── test_helpers.py # 测试辅助函数
```
## 快速开始
### 1. 安装测试依赖
```bash
pip install -r requirements-test.txt
```
### 2. 运行测试
使用测试运行器脚本:
```bash
# 运行所有测试
python run_tests.py all --coverage
# 只运行单元测试
python run_tests.py unit --coverage --html-report
# 只运行集成测试
python run_tests.py integration
# 运行特定特效测试
python run_tests.py effects --effect zoom
# 运行压力测试
python run_tests.py stress
```
或直接使用 pytest:
```bash
# 运行所有测试
pytest tests/ -v --cov=entity --cov=services
# 只运行单元测试
pytest tests/test_effects/ tests/test_ffmpeg_builder/ -v
# 只运行集成测试
pytest tests/test_integration/ -v -m integration
# 生成覆盖率报告
pytest tests/ --cov=entity --cov=services --cov-report=html
```
## 测试类型说明
### 单元测试 (Unit Tests)
位置:`tests/test_effects/`, `tests/test_ffmpeg_builder/`
测试内容:
- 各个特效处理器的参数验证
- FFmpeg 滤镜参数生成
- 特效注册表功能
- FFmpeg 命令构建逻辑
特点:
- 运行速度快
- 不依赖外部工具
- 测试覆盖率高
### 集成测试 (Integration Tests)
位置:`tests/test_integration/`
测试内容:
- 实际 FFmpeg 命令执行
- 视频文件处理验证
- 特效组合效果测试
- 错误处理验证
依赖:
- 需要系统安装 FFmpeg
- 需要测试视频文件
### 压力测试 (Stress Tests)
测试内容:
- 大量特效链处理
- 长时间运行稳定性
- 资源使用情况
运行条件:
- 设置环境变量 `RUN_STRESS_TESTS=1`
- 较长的超时时间
## 测试配置
### pytest 配置
`pytest.ini` 中配置:
- 测试路径和文件模式
- 覆盖率设置
- 标记定义
- 输出格式
### 环境变量
- `RUN_STRESS_TESTS`: 启用压力测试
- `FFMPEG_PATH`: 自定义 FFmpeg 路径
## 特效测试详解
### 缩放特效 (ZoomEffect)
测试用例:
- 有效参数验证:`"0,2.0,3.0"`(开始时间,缩放因子,持续时间)
- 无效参数处理:负值、非数字、参数不足
- 静态缩放:持续时间为 0
- 动态缩放:指定时间段内的缩放
- 位置 JSON 解析:自定义缩放中心点
生成滤镜格式:
```
# 静态缩放
[0:v]trim=start=0,zoompan=z=2.0:x=iw/2:y=ih/2:d=1[v_eff1]
# 动态缩放
[0:v]zoompan=z=if(between(t\,0\,3.0)\,2.0\,1):x=iw/2:y=ih/2:d=1[v_eff1]
```
### 变速特效 (SpeedEffect)
测试用例:
- 加速效果:`"2.0"`(2倍速)
- 减速效果:`"0.5"`(0.5倍速)
- 无效参数:零值、负值、非数字
- 默认处理:空参数或 1.0 倍速
生成滤镜格式:
```
[0:v]setpts=2.0*PTS[v_eff1]
```
## FFmpeg 命令构建测试
### 测试场景
1. **单文件复制**:直接复制无需编码
2. **多文件拼接**:使用 concat 滤镜
3. **特效处理**:复杂滤镜链构建
4. **错误处理**:缺失文件、无效参数
### 命令验证
使用 `FFmpegValidator` 验证:
- 命令结构完整性
- 输入输出文件存在
- 滤镜语法正确性
- 流标识符格式
## 集成测试详解
### 真实 FFmpeg 执行
测试流程:
1. 创建测试视频文件
2. 构建 FFmpeg 命令
3. 执行命令并检查返回码
4. 验证输出文件存在且有效
### 测试用例
- 简单复制操作
- 单特效处理
- 多特效组合
- 视频拼接
- 错误情况处理
### 性能测试
监控指标:
- 执行时间
- 内存使用
- 文件大小
- 处理速度
## 代码覆盖率
目标覆盖率:≥ 70%
覆盖范围:
- `entity/` 目录下的所有模块
- `services/` 目录下的所有模块
报告格式:
- XML 格式:用于 Jenkins CI/CD
- HTML 格式:用于本地查看
- 终端输出:实时查看缺失覆盖
## CI/CD 集成
### Jenkins Pipeline
配置文件:`Jenkinsfile`
阶段:
1. 环境准备
2. 代码质量检查
3. 单元测试
4. 集成测试
5. 完整测试套件
6. 性能测试
报告生成:
- JUnit XML 测试报告
- Cobertura 覆盖率报告
- HTML 测试和覆盖率报告
### 自动化触发
- 代码提交时运行单元测试
- PR 创建时运行完整测试
- 主分支更新时运行包含性能测试的完整套件
## 故障排除
### 常见问题
1. **FFmpeg 未找到**
```bash
# 安装 FFmpeg
sudo apt-get install ffmpeg # Ubuntu/Debian
brew install ffmpeg # macOS
```
2. **测试视频创建失败**
```bash
# 手动创建测试视频
python run_tests.py setup
```
3. **覆盖率过低**
- 检查是否有未测试的代码路径
- 添加边界条件测试
- 验证测试是否实际运行
4. **集成测试超时**
- 增加超时时间:`--timeout=600`
- 使用更小的测试文件
- 检查系统资源使用情况
### 调试技巧
1. **查看详细输出**:
```bash
pytest tests/ -v -s
```
2. **只运行失败的测试**:
```bash
pytest tests/ --lf
```
3. **停在第一个失败**:
```bash
pytest tests/ -x
```
4. **查看覆盖率详情**:
```bash
pytest tests/ --cov=entity --cov-report=term-missing
```
## 贡献指南
### 添加新测试
1. 在相应目录创建测试文件
2. 使用描述性的测试函数名
3. 添加适当的测试标记
4. 更新文档说明
### 测试最佳实践
1. **独立性**:每个测试应该独立运行
2. **可重复性**:测试结果应该一致
3. **清晰性**:测试意图应该明确
4. **完整性**:覆盖正常和异常情况
### 代码质量
- 遵循 PEP 8 代码风格
- 使用类型注解
- 添加适当的文档字符串
- 使用有意义的变量名

108
tests/conftest.py Normal file
View File

@@ -0,0 +1,108 @@
"""pytest配置文件"""
import pytest
import os
import tempfile
import shutil
from pathlib import Path
from typing import Dict, Any
from entity.render_task import RenderTask
from config.settings import FFmpegConfig, APIConfig, StorageConfig
@pytest.fixture
def temp_dir():
"""创建临时目录"""
temp_path = tempfile.mkdtemp()
yield temp_path
shutil.rmtree(temp_path, ignore_errors=True)
@pytest.fixture
def test_ffmpeg_config():
"""测试用FFmpeg配置"""
return FFmpegConfig(
encoder_args=["-c:v", "h264"],
video_args=["-profile:v", "high"],
audio_args=["-c:a", "aac"],
default_args=["-shortest"],
)
@pytest.fixture
def test_api_config():
"""测试用API配置"""
return APIConfig(
endpoint="http://test.local",
access_key="test_key",
)
@pytest.fixture
def test_storage_config():
"""测试用存储配置"""
return StorageConfig(
template_dir="tests/test_data/templates",
)
@pytest.fixture
def sample_render_task():
"""示例渲染任务"""
return RenderTask(
input_files=["test_input.mp4"],
output_file="test_output.mp4",
frame_rate=25,
effects=["zoom:0,2.0,3.0", "ospeed:1.5"],
ext_data={
"posJson": '{"ltX": 100, "ltY": 100, "rbX": 200, "rbY": 200, "imgWidth": 300, "imgHeight": 300}'
},
)
@pytest.fixture
def sample_video_file(temp_dir):
"""创建测试用的样本视频文件路径"""
video_path = os.path.join(temp_dir, "sample.mp4")
# 这里只返回路径,实际测试中可能需要真实视频文件
return video_path
@pytest.fixture(scope="session")
def ffmpeg_available():
"""检查FFmpeg是否可用"""
import subprocess
try:
subprocess.run(["ffmpeg", "-version"], capture_output=True, check=True)
return True
except (subprocess.CalledProcessError, FileNotFoundError):
return False
@pytest.fixture
def mock_ext_data():
"""模拟扩展数据"""
return {
"posJson": '{"ltX": 50, "ltY": 50, "rbX": 150, "rbY": 150, "imgWidth": 200, "imgHeight": 200}',
"templateData": {"width": 1920, "height": 1080},
}
# 标记:只在FFmpeg可用时运行集成测试
def pytest_collection_modifyitems(config, items):
"""根据FFmpeg可用性修改测试收集"""
try:
import subprocess
subprocess.run(["ffmpeg", "-version"], capture_output=True, check=True)
ffmpeg_available = True
except (subprocess.CalledProcessError, FileNotFoundError):
ffmpeg_available = False
if not ffmpeg_available:
skip_ffmpeg = pytest.mark.skip(reason="FFmpeg not available")
for item in items:
if "integration" in item.keywords:
item.add_marker(skip_ffmpeg)

15
tests/test_data/README.md Normal file
View File

@@ -0,0 +1,15 @@
# 测试数据说明
这个目录包含测试所需的样本数据:
## 目录结构
- `videos/` - 测试用视频文件
- `templates/` - 测试用模板数据
- `expected_outputs/` - 预期输出结果
## 使用说明
- 小尺寸视频文件用于快速测试
- 模板文件包含各种特效配置
- 预期输出用于验证测试结果
注意:实际的视频文件可能较大,建议使用小尺寸测试文件。

View File

@@ -0,0 +1,187 @@
"""测试特效基础类和注册表"""
import pytest
from unittest.mock import Mock
from entity.effects.base import EffectProcessor, EffectRegistry
class TestEffectProcessor:
"""测试特效处理器基类"""
def test_init(self):
"""测试初始化"""
processor = Mock(spec=EffectProcessor)
processor.__init__("test_params", {"key": "value"})
assert processor.params == "test_params"
assert processor.ext_data == {"key": "value"}
assert processor.frame_rate == 25
def test_init_default_ext_data(self):
"""测试默认扩展数据"""
processor = Mock(spec=EffectProcessor)
processor.__init__("test_params")
assert processor.ext_data == {}
def test_parse_params_empty(self):
"""测试解析空参数"""
processor = EffectProcessor("", {})
result = processor.parse_params()
assert result == []
def test_parse_params_single(self):
"""测试解析单个参数"""
processor = EffectProcessor("123", {})
result = processor.parse_params()
assert result == ["123"]
def test_parse_params_multiple(self):
"""测试解析多个参数"""
processor = EffectProcessor("1,2.5,test", {})
result = processor.parse_params()
assert result == ["1", "2.5", "test"]
def test_get_pos_json_empty(self):
"""测试获取空位置JSON"""
processor = EffectProcessor("", {})
result = processor.get_pos_json()
assert result == {}
def test_get_pos_json_valid(self):
"""测试获取有效位置JSON"""
ext_data = {"posJson": '{"ltX": 100, "ltY": 200, "rbX": 300, "rbY": 400}'}
processor = EffectProcessor("", ext_data)
result = processor.get_pos_json()
expected = {"ltX": 100, "ltY": 200, "rbX": 300, "rbY": 400}
assert result == expected
def test_get_pos_json_invalid(self):
"""测试获取无效位置JSON"""
ext_data = {"posJson": "invalid_json"}
processor = EffectProcessor("", ext_data)
result = processor.get_pos_json()
assert result == {}
def test_get_pos_json_default_value(self):
"""测试默认位置JSON值"""
ext_data = {"posJson": "{}"}
processor = EffectProcessor("", ext_data)
result = processor.get_pos_json()
assert result == {}
class TestEffectRegistry:
"""测试特效注册表"""
def test_init(self):
"""测试初始化"""
registry = EffectRegistry()
assert registry._processors == {}
def test_register_valid_processor(self):
"""测试注册有效处理器"""
registry = EffectRegistry()
class TestEffect(EffectProcessor):
def validate_params(self):
return True
def generate_filter_args(self, video_input, effect_index):
return [], video_input
def get_effect_name(self):
return "test"
registry.register("test_effect", TestEffect)
assert "test_effect" in registry._processors
assert registry._processors["test_effect"] == TestEffect
def test_register_invalid_processor(self):
"""测试注册无效处理器"""
registry = EffectRegistry()
class InvalidEffect:
pass
with pytest.raises(ValueError, match="must be a subclass of EffectProcessor"):
registry.register("invalid", InvalidEffect)
def test_get_processor_exists(self):
"""测试获取存在的处理器"""
registry = EffectRegistry()
class TestEffect(EffectProcessor):
def validate_params(self):
return True
def generate_filter_args(self, video_input, effect_index):
return [], video_input
def get_effect_name(self):
return "test"
registry.register("test_effect", TestEffect)
processor = registry.get_processor("test_effect", "params", {"data": "value"})
assert isinstance(processor, TestEffect)
assert processor.params == "params"
assert processor.ext_data == {"data": "value"}
def test_get_processor_not_exists(self):
"""测试获取不存在的处理器"""
registry = EffectRegistry()
processor = registry.get_processor("nonexistent")
assert processor is None
def test_list_effects_empty(self):
"""测试列出空特效"""
registry = EffectRegistry()
effects = registry.list_effects()
assert effects == []
def test_list_effects_with_processors(self):
"""测试列出已注册特效"""
registry = EffectRegistry()
class TestEffect(EffectProcessor):
def validate_params(self):
return True
def generate_filter_args(self, video_input, effect_index):
return [], video_input
def get_effect_name(self):
return "test"
registry.register("effect1", TestEffect)
registry.register("effect2", TestEffect)
effects = registry.list_effects()
assert set(effects) == {"effect1", "effect2"}
def test_parse_effect_string_with_params(self):
"""测试解析带参数的特效字符串"""
registry = EffectRegistry()
name, params = registry.parse_effect_string("zoom:0,2.0,3.0")
assert name == "zoom"
assert params == "0,2.0,3.0"
def test_parse_effect_string_without_params(self):
"""测试解析无参数的特效字符串"""
registry = EffectRegistry()
name, params = registry.parse_effect_string("zoom")
assert name == "zoom"
assert params == ""
def test_parse_effect_string_multiple_colons(self):
"""测试解析多个冒号的特效字符串"""
registry = EffectRegistry()
name, params = registry.parse_effect_string("effect:param1:param2")
assert name == "effect"
assert params == "param1:param2"

View File

@@ -0,0 +1,138 @@
"""测试变速特效"""
import pytest
from entity.effects.speed import SpeedEffect
from tests.utils.test_helpers import EffectTestHelper
class TestSpeedEffect:
"""测试变速特效处理器"""
def test_validate_params_valid_cases(self):
"""测试有效参数验证"""
test_cases = [
{"params": "2.0", "expected": True, "description": "2倍速"},
{"params": "0.5", "expected": True, "description": "0.5倍速(慢速)"},
{"params": "1.0", "expected": True, "description": "正常速度"},
{"params": "10.0", "expected": True, "description": "10倍速"},
{"params": "", "expected": True, "description": "空参数(默认不变速)"},
]
effect = SpeedEffect()
results = EffectTestHelper.test_effect_params_validation(effect, test_cases)
for result in results:
assert result["passed"], f"Failed case: {result['description']} - {result}"
def test_validate_params_invalid_cases(self):
"""测试无效参数验证"""
test_cases = [
{"params": "0", "expected": False, "description": "零速度"},
{"params": "-1.0", "expected": False, "description": "负速度"},
{"params": "abc", "expected": False, "description": "非数字参数"},
{"params": "1.0,2.0", "expected": False, "description": "多余参数"},
]
effect = SpeedEffect()
results = EffectTestHelper.test_effect_params_validation(effect, test_cases)
for result in results:
assert result["passed"], f"Failed case: {result['description']} - {result}"
def test_generate_filter_args_speed_change(self):
"""测试变速滤镜生成"""
effect = SpeedEffect("2.0") # 2倍速
result = EffectTestHelper.test_filter_generation(effect, "[0:v]", 1)
assert result["success"], f"Filter generation failed: {result.get('error')}"
assert result["filter_count"] == 1
assert result["valid_syntax"]
assert result["output_stream"] == "[v_eff1]"
# 检查滤镜内容
filter_str = result["filters"][0]
assert "setpts=2.0*PTS" in filter_str
assert "[0:v]" in filter_str
assert "[v_eff1]" in filter_str
def test_generate_filter_args_slow_motion(self):
"""测试慢动作滤镜生成"""
effect = SpeedEffect("0.5") # 0.5倍速(慢动作)
result = EffectTestHelper.test_filter_generation(effect, "[input]", 3)
assert result["success"]
assert result["filter_count"] == 1
assert result["valid_syntax"]
assert result["output_stream"] == "[v_eff3]"
filter_str = result["filters"][0]
assert "setpts=0.5*PTS" in filter_str
assert "[input]" in filter_str
assert "[v_eff3]" in filter_str
def test_generate_filter_args_no_change(self):
"""测试无变速效果"""
test_cases = [
{"params": "", "description": "空参数"},
{"params": "1", "description": "1倍速"},
{"params": "1.0", "description": "1.0倍速"},
]
for case in test_cases:
effect = SpeedEffect(case["params"])
result = EffectTestHelper.test_filter_generation(effect, "[0:v]", 1)
assert result["success"], f"Failed for {case['description']}"
assert (
result["filter_count"] == 0
), f"Should not generate filter for {case['description']}"
assert (
result["output_stream"] == "[0:v]"
), f"Output should equal input for {case['description']}"
def test_generate_filter_args_invalid_params(self):
"""测试无效参数的滤镜生成"""
effect = SpeedEffect("invalid")
result = EffectTestHelper.test_filter_generation(effect, "[0:v]", 1)
assert result["success"]
assert result["filter_count"] == 0
assert result["output_stream"] == "[0:v]"
def test_get_effect_name(self):
"""测试获取特效名称"""
effect = SpeedEffect()
assert effect.get_effect_name() == "ospeed"
def test_various_speed_factors(self):
"""测试各种速度因子"""
speed_factors = ["0.1", "0.25", "0.75", "1.5", "3.0", "5.0"]
for speed in speed_factors:
effect = SpeedEffect(speed)
result = EffectTestHelper.test_filter_generation(effect, "[test]", 10)
if speed == "1.0":
# 1倍速不应该生成滤镜
assert result["filter_count"] == 0
assert result["output_stream"] == "[test]"
else:
# 其他速度应该生成滤镜
assert result["success"], f"Failed for speed {speed}"
assert result["filter_count"] == 1
assert result["valid_syntax"]
assert f"setpts={speed}*PTS" in result["filters"][0]
def test_effect_chaining(self):
"""测试特效链式处理"""
# 模拟在特效链中的使用
effect = SpeedEffect("2.0")
# 第一个特效
result1 = EffectTestHelper.test_filter_generation(effect, "[0:v]", 1)
assert result1["output_stream"] == "[v_eff1]"
# 作为链中的第二个特效
result2 = EffectTestHelper.test_filter_generation(effect, "[v_eff1]", 2)
assert result2["output_stream"] == "[v_eff2]"
assert "[v_eff1]" in result2["filters"][0]

View File

@@ -0,0 +1,157 @@
"""测试缩放特效"""
import pytest
from entity.effects.zoom import ZoomEffect
from tests.utils.test_helpers import EffectTestHelper, FFmpegValidator
class TestZoomEffect:
"""测试缩放特效处理器"""
def test_validate_params_valid_cases(self):
"""测试有效参数验证"""
test_cases = [
{"params": "0,2.0,3.0", "expected": True, "description": "标准缩放参数"},
{
"params": "1.5,1.5,0",
"expected": True,
"description": "静态缩放(duration=0)",
},
{
"params": "0,1.0,5.0",
"expected": True,
"description": "无缩放效果(factor=1.0)",
},
{"params": "10,0.5,2.0", "expected": True, "description": "缩小效果"},
]
effect = ZoomEffect()
results = EffectTestHelper.test_effect_params_validation(effect, test_cases)
for result in results:
assert result["passed"], f"Failed case: {result['description']} - {result}"
def test_validate_params_invalid_cases(self):
"""测试无效参数验证"""
test_cases = [
{"params": "", "expected": False, "description": "空参数"},
{"params": "1,2", "expected": False, "description": "参数不足"},
{"params": "-1,2.0,3.0", "expected": False, "description": "负开始时间"},
{"params": "0,0,3.0", "expected": False, "description": "零缩放因子"},
{"params": "0,-2.0,3.0", "expected": False, "description": "负缩放因子"},
{"params": "0,2.0,-1.0", "expected": False, "description": "负持续时间"},
{"params": "abc,2.0,3.0", "expected": False, "description": "非数字参数"},
]
effect = ZoomEffect()
results = EffectTestHelper.test_effect_params_validation(effect, test_cases)
for result in results:
assert result["passed"], f"Failed case: {result['description']} - {result}"
def test_generate_filter_args_static_zoom(self):
"""测试静态缩放滤镜生成"""
effect = ZoomEffect("0,2.0,0") # duration=0表示静态缩放
result = EffectTestHelper.test_filter_generation(effect, "[0:v]", 1)
assert result["success"], f"Filter generation failed: {result.get('error')}"
assert result["filter_count"] == 1
assert result["valid_syntax"]
assert result["output_stream"] == "[v_eff1]"
# 检查滤镜内容
filter_str = result["filters"][0]
assert "trim=start=0" in filter_str
assert "zoompan=z=2.0" in filter_str
assert "[v_eff1]" in filter_str
def test_generate_filter_args_dynamic_zoom(self):
"""测试动态缩放滤镜生成"""
effect = ZoomEffect("1.0,1.5,2.0") # 从1秒开始,持续2秒的1.5倍缩放
result = EffectTestHelper.test_filter_generation(effect, "[0:v]", 2)
assert result["success"], f"Filter generation failed: {result.get('error')}"
assert result["filter_count"] == 1
assert result["valid_syntax"]
assert result["output_stream"] == "[v_eff2]"
# 检查滤镜内容
filter_str = result["filters"][0]
assert "zoompan=z=" in filter_str
assert "between(t\\\\,1.0\\\\,3.0)" in filter_str # 检查时间范围
assert "[v_eff2]" in filter_str
def test_generate_filter_args_no_zoom(self):
"""测试无缩放效果(factor=1.0)"""
effect = ZoomEffect("0,1.0,3.0") # 缩放因子为1.0,应该不生成滤镜
result = EffectTestHelper.test_filter_generation(effect, "[0:v]", 1)
assert result["success"]
assert result["filter_count"] == 0
assert result["output_stream"] == "[0:v]" # 输出应该等于输入
def test_generate_filter_args_invalid_params(self):
"""测试无效参数的滤镜生成"""
effect = ZoomEffect("invalid,params")
result = EffectTestHelper.test_filter_generation(effect, "[0:v]", 1)
assert result["success"]
assert result["filter_count"] == 0
assert result["output_stream"] == "[0:v]" # 无效参数时应该返回原输入
def test_get_zoom_center_default(self):
"""测试默认缩放中心点"""
effect = ZoomEffect("0,2.0,3.0")
center_x, center_y = effect._get_zoom_center()
assert center_x == "iw/2"
assert center_y == "ih/2"
def test_get_zoom_center_with_pos_json(self):
"""测试基于posJson的缩放中心点"""
ext_data = {
"posJson": '{"ltX": 100, "ltY": 100, "rbX": 200, "rbY": 200, "imgWidth": 400, "imgHeight": 300}'
}
effect = ZoomEffect("0,2.0,3.0", ext_data)
center_x, center_y = effect._get_zoom_center()
# 中心点应该是矩形的中心
# center_x_ratio = (100 + 200) / (2 * 400) = 0.375
# center_y_ratio = (100 + 200) / (2 * 300) = 0.5
assert "iw*0.375" in center_x
assert "ih*0.5" in center_y
def test_get_zoom_center_invalid_pos_json(self):
"""测试无效posJson时的缩放中心点"""
ext_data = {"posJson": '{"imgWidth": 0, "imgHeight": 0}'} # 无效尺寸
effect = ZoomEffect("0,2.0,3.0", ext_data)
center_x, center_y = effect._get_zoom_center()
# 应该回退到默认中心点
assert center_x == "iw/2"
assert center_y == "ih/2"
def test_get_effect_name(self):
"""测试获取特效名称"""
effect = ZoomEffect()
assert effect.get_effect_name() == "zoom"
def test_complex_zoom_scenario(self):
"""测试复杂缩放场景"""
# 带有复杂posJson数据的动态缩放
ext_data = {
"posJson": '{"ltX": 50, "ltY": 75, "rbX": 350, "rbY": 225, "imgWidth": 400, "imgHeight": 300}'
}
effect = ZoomEffect("2.5,3.0,1.5", ext_data)
result = EffectTestHelper.test_filter_generation(effect, "[input]", 5)
assert result["success"]
assert result["filter_count"] == 1
assert result["valid_syntax"]
assert result["output_stream"] == "[v_eff5]"
filter_str = result["filters"][0]
assert "zoompan=z=" in filter_str
assert "between(t\\\\,2.5\\\\,4.0)" in filter_str # 2.5 + 1.5 = 4.0
assert "[input]" in filter_str
assert "[v_eff5]" in filter_str

View File

@@ -0,0 +1,234 @@
"""测试FFmpeg命令构建器"""
import pytest
from unittest.mock import Mock, patch
from entity.ffmpeg_command_builder import FFmpegCommandBuilder
from entity.render_task import RenderTask
from config.settings import FFmpegConfig, APIConfig, StorageConfig
from tests.utils.test_helpers import MockRenderTask, FFmpegValidator
class TestFFmpegCommandBuilder:
"""测试FFmpeg命令构建器"""
@pytest.fixture
def simple_task(self):
"""简单的渲染任务"""
task = MockRenderTask()
task.input_files = ["input1.mp4", "input2.mp4"]
task.output_path = "output.mp4"
task.effects = []
return task
@pytest.fixture
def task_with_effects(self):
"""带特效的渲染任务"""
task = MockRenderTask()
task.input_files = ["input.mp4"]
task.output_path = "output.mp4"
task.effects = ["zoom:0,2.0,3.0", "ospeed:1.5"]
task.ext_data = {
"posJson": '{"ltX": 100, "ltY": 100, "rbX": 200, "rbY": 200, "imgWidth": 300, "imgHeight": 300}'
}
return task
def test_init(self, simple_task):
"""测试初始化"""
builder = FFmpegCommandBuilder(simple_task)
assert builder.task == simple_task
assert builder.config is not None
def test_build_copy_command(self, simple_task):
"""测试构建复制命令"""
builder = FFmpegCommandBuilder(simple_task)
command = builder._build_copy_command()
# 验证命令结构
validation = FFmpegValidator.validate_ffmpeg_command(command)
assert validation["valid"], f"Invalid command: {validation['errors']}"
assert validation["has_input"]
assert validation["has_output"]
# 验证具体内容
assert command[0] == "ffmpeg"
assert "-i" in command
assert "input1.mp4" in command
assert "output.mp4" in command
assert "-c" in command and "copy" in command
def test_build_concat_command_multiple_files(self):
"""测试构建多文件拼接命令"""
task = MockRenderTask()
task.input_files = ["file1.mp4", "file2.mp4", "file3.mp4"]
task.output_path = "concat_output.mp4"
builder = FFmpegCommandBuilder(task)
command = builder._build_concat_command()
validation = FFmpegValidator.validate_ffmpeg_command(command)
assert validation["valid"]
assert validation["has_filter"]
# 验证拼接相关参数
assert "concat=n=3:v=1:a=1" in " ".join(command)
assert all(f"file{i}.mp4" in command for i in range(1, 4))
def test_build_encode_command_with_effects(self, task_with_effects):
"""测试构建带特效的编码命令"""
builder = FFmpegCommandBuilder(task_with_effects)
command = builder._build_encode_command()
validation = FFmpegValidator.validate_ffmpeg_command(command)
assert validation["valid"]
assert validation["has_filter"]
command_str = " ".join(command)
# 应该包含特效相关的滤镜
assert "-filter_complex" in command
# 应该包含编码参数
assert "-c:v" in command and "h264" in command
def test_add_effects_single_effect(self):
"""测试添加单个特效"""
task = MockRenderTask()
task.effects = ["zoom:0,2.0,3.0"]
task.ext_data = {"posJson": "{}"}
builder = FFmpegCommandBuilder(task)
filter_args: list[str] = []
result_input, result_index = builder._add_effects(filter_args, "[0:v]", 1)
# 验证结果
assert len(filter_args) > 0 # 应该有滤镜被添加
assert result_input == "[v_eff1]" # 输出流应该更新
assert result_index == 2 # 索引应该递增
def test_add_effects_multiple_effects(self):
"""测试添加多个特效"""
task = MockRenderTask()
task.effects = ["zoom:0,2.0,3.0", "ospeed:1.5"]
task.ext_data = {"posJson": "{}"}
builder = FFmpegCommandBuilder(task)
filter_args: list[str] = []
result_input, result_index = builder._add_effects(filter_args, "[0:v]", 1)
# 验证特效链
assert len(filter_args) >= 2 # 应该有两个特效的滤镜
assert result_input == "[v_eff2]" # 最终输出流
assert result_index == 3 # 索引应该递增两次
def test_add_effects_invalid_effect(self):
"""测试添加无效特效"""
task = MockRenderTask()
task.effects = ["invalid_effect:params"]
builder = FFmpegCommandBuilder(task)
filter_args: list[str] = []
result_input, result_index = builder._add_effects(filter_args, "[0:v]", 1)
# 无效特效应该被忽略
assert result_input == "[0:v]" # 输入流不变
assert result_index == 1 # 索引不变
def test_add_effects_no_effects(self):
"""测试无特效情况"""
task = MockRenderTask()
task.effects = []
builder = FFmpegCommandBuilder(task)
filter_args: list[str] = []
result_input, result_index = builder._add_effects(filter_args, "[0:v]", 1)
assert result_input == "[0:v]" # 输入流不变
assert result_index == 1 # 索引不变
assert len(filter_args) == 0 # 无滤镜添加
def test_build_command_copy_mode(self, simple_task):
"""测试构建复制模式命令"""
simple_task.input_files = ["single_file.mp4"]
builder = FFmpegCommandBuilder(simple_task)
command = builder.build_command()
validation = FFmpegValidator.validate_ffmpeg_command(command)
assert validation["valid"]
# 应该是复制模式
command_str = " ".join(command)
assert "-c copy" in command_str
def test_build_command_concat_mode(self):
"""测试构建拼接模式命令"""
task = MockRenderTask()
task.input_files = ["file1.mp4", "file2.mp4"]
task.effects = []
builder = FFmpegCommandBuilder(task)
command = builder.build_command()
validation = FFmpegValidator.validate_ffmpeg_command(command)
assert validation["valid"]
assert validation["has_filter"]
# 应该包含拼接滤镜
command_str = " ".join(command)
assert "concat=" in command_str
def test_build_command_encode_mode(self, task_with_effects):
"""测试构建编码模式命令"""
builder = FFmpegCommandBuilder(task_with_effects)
command = builder.build_command()
validation = FFmpegValidator.validate_ffmpeg_command(command)
assert validation["valid"]
assert validation["has_filter"]
# 应该包含编码参数和特效滤镜
command_str = " ".join(command)
assert "-c:v h264" in command_str
@patch("entity.effects.registry.get_processor")
def test_effect_processor_integration(self, mock_get_processor):
"""测试与特效处理器的集成"""
# 模拟特效处理器
mock_processor = Mock()
mock_processor.frame_rate = 25
mock_processor.generate_filter_args.return_value = (
["[0:v]zoompan=z=2.0:x=iw/2:y=ih/2:d=1[v_eff1]"],
"[v_eff1]",
)
mock_get_processor.return_value = mock_processor
task = MockRenderTask()
task.effects = ["zoom:0,2.0,3.0"]
task.frame_rate = 30
builder = FFmpegCommandBuilder(task)
filter_args: list[str] = []
builder._add_effects(filter_args, "[0:v]", 1)
# 验证处理器被正确调用
mock_processor.generate_filter_args.assert_called_once_with("[0:v]", 1)
assert mock_processor.frame_rate == 30 # 帧率应该被设置
def test_error_handling_missing_input(self):
"""测试缺少输入文件的错误处理"""
task = MockRenderTask()
task.input_files = []
builder = FFmpegCommandBuilder(task)
# 构建命令时应该处理错误情况
# 具体的错误处理依赖于实现
command = builder.build_command()
# 验证至少返回了基本的ffmpeg命令结构
assert isinstance(command, list)
assert len(command) > 0

View File

@@ -0,0 +1,275 @@
"""FFmpeg执行集成测试"""
import pytest
import subprocess
import tempfile
import os
from pathlib import Path
from entity.ffmpeg_command_builder import FFmpegCommandBuilder
from entity.render_task import RenderTask
from config.settings import FFmpegConfig
from services.render_service import DefaultRenderService
from tests.utils.test_helpers import MockRenderTask, create_test_video_file
@pytest.mark.integration
class TestFFmpegExecution:
"""FFmpeg执行集成测试"""
@pytest.fixture
def sample_video(self, temp_dir):
"""创建测试视频文件"""
video_path = os.path.join(temp_dir, "test_input.mp4")
success = create_test_video_file(video_path, duration=3, resolution="320x240")
if not success:
pytest.skip("Cannot create test video file")
return video_path
@pytest.fixture
def render_service(self):
"""渲染服务实例"""
return DefaultRenderService()
def test_simple_copy_execution(self, sample_video, temp_dir):
"""测试简单复制执行"""
output_path = os.path.join(temp_dir, "copy_output.mp4")
task = MockRenderTask()
task.input_files = [sample_video]
task.output_path = output_path
task.effects = []
builder = FFmpegCommandBuilder(task)
command = builder.build_command()
# 执行命令
try:
result = subprocess.run(command, capture_output=True, text=True, timeout=30)
assert result.returncode == 0, f"FFmpeg failed: {result.stderr}"
assert os.path.exists(output_path), "Output file was not created"
assert os.path.getsize(output_path) > 0, "Output file is empty"
except subprocess.TimeoutExpired:
pytest.fail("FFmpeg execution timed out")
def test_zoom_effect_execution(self, sample_video, temp_dir):
"""测试缩放特效执行"""
output_path = os.path.join(temp_dir, "zoom_output.mp4")
task = MockRenderTask()
task.input_files = [sample_video]
task.output_path = output_path
task.effects = ["zoom:0,2.0,2.0"] # 2倍缩放,持续2秒
task.ext_data = {"posJson": "{}"}
builder = FFmpegCommandBuilder(task)
command = builder.build_command()
try:
result = subprocess.run(command, capture_output=True, text=True, timeout=60)
assert result.returncode == 0, f"FFmpeg failed: {result.stderr}"
assert os.path.exists(output_path), "Output file was not created"
assert os.path.getsize(output_path) > 0, "Output file is empty"
except subprocess.TimeoutExpired:
pytest.fail("FFmpeg execution timed out")
def test_speed_effect_execution(self, sample_video, temp_dir):
"""测试变速特效执行"""
output_path = os.path.join(temp_dir, "speed_output.mp4")
task = MockRenderTask()
task.input_files = [sample_video]
task.output_path = output_path
task.effects = ["ospeed:2.0"] # 2倍速
task.ext_data = {}
builder = FFmpegCommandBuilder(task)
command = builder.build_command()
try:
result = subprocess.run(command, capture_output=True, text=True, timeout=60)
assert result.returncode == 0, f"FFmpeg failed: {result.stderr}"
assert os.path.exists(output_path), "Output file was not created"
assert os.path.getsize(output_path) > 0, "Output file is empty"
except subprocess.TimeoutExpired:
pytest.fail("FFmpeg execution timed out")
def test_multiple_effects_execution(self, sample_video, temp_dir):
"""测试多特效组合执行"""
output_path = os.path.join(temp_dir, "multi_effects_output.mp4")
task = MockRenderTask()
task.input_files = [sample_video]
task.output_path = output_path
task.effects = ["zoom:0,1.5,1.0", "ospeed:1.5"] # 缩放+变速
task.ext_data = {"posJson": "{}"}
builder = FFmpegCommandBuilder(task)
command = builder.build_command()
try:
result = subprocess.run(command, capture_output=True, text=True, timeout=60)
assert result.returncode == 0, f"FFmpeg failed: {result.stderr}"
assert os.path.exists(output_path), "Output file was not created"
assert os.path.getsize(output_path) > 0, "Output file is empty"
except subprocess.TimeoutExpired:
pytest.fail("FFmpeg execution timed out")
def test_concat_execution(self, temp_dir):
"""测试视频拼接执行"""
# 创建两个测试视频
video1_path = os.path.join(temp_dir, "video1.mp4")
video2_path = os.path.join(temp_dir, "video2.mp4")
output_path = os.path.join(temp_dir, "concat_output.mp4")
success1 = create_test_video_file(video1_path, duration=2, resolution="320x240")
success2 = create_test_video_file(video2_path, duration=2, resolution="320x240")
if not (success1 and success2):
pytest.skip("Cannot create test video files")
task = MockRenderTask()
task.input_files = [video1_path, video2_path]
task.output_path = output_path
task.effects = []
builder = FFmpegCommandBuilder(task)
command = builder.build_command()
try:
result = subprocess.run(command, capture_output=True, text=True, timeout=60)
assert result.returncode == 0, f"FFmpeg failed: {result.stderr}"
assert os.path.exists(output_path), "Output file was not created"
assert os.path.getsize(output_path) > 0, "Output file is empty"
except subprocess.TimeoutExpired:
pytest.fail("FFmpeg execution timed out")
def test_invalid_effect_execution(self, sample_video, temp_dir):
"""测试无效特效的执行处理"""
output_path = os.path.join(temp_dir, "invalid_effect_output.mp4")
task = MockRenderTask()
task.input_files = [sample_video]
task.output_path = output_path
task.effects = ["invalid_effect:params", "zoom:0,2.0,1.0"] # 混合有效和无效特效
task.ext_data = {"posJson": "{}"}
builder = FFmpegCommandBuilder(task)
command = builder.build_command()
# 应该忽略无效特效,继续处理有效特效
try:
result = subprocess.run(command, capture_output=True, text=True, timeout=60)
assert result.returncode == 0, f"FFmpeg failed: {result.stderr}"
assert os.path.exists(output_path), "Output file was not created"
except subprocess.TimeoutExpired:
pytest.fail("FFmpeg execution timed out")
def test_render_service_integration(self, sample_video, temp_dir, render_service):
"""测试渲染服务集成"""
output_path = os.path.join(temp_dir, "service_output.mp4")
# 创建真实的RenderTask(不是Mock)
task_data = {
"task_id": "integration_test",
"template_id": "test_template",
"input_files": [sample_video],
"output_path": output_path,
"effects": ["zoom:0,1.8,2.0"],
"ext_data": {"posJson": "{}"},
"frame_rate": 25,
}
# 这里需要根据实际的RenderTask构造方法调整
task = MockRenderTask(**task_data)
# 使用渲染服务执行
try:
# 这里的方法调用需要根据实际的渲染服务接口调整
# success = render_service.render(task)
# assert success, "Render service failed"
# 临时直接使用FFmpegCommandBuilder测试
builder = FFmpegCommandBuilder(task)
command = builder.build_command()
result = subprocess.run(command, capture_output=True, text=True, timeout=60)
assert result.returncode == 0, f"Render failed: {result.stderr}"
assert os.path.exists(output_path), "Output file was not created"
except Exception as e:
pytest.fail(f"Render service integration failed: {e}")
def test_error_handling_missing_input(self, temp_dir):
"""测试缺失输入文件的错误处理"""
missing_file = os.path.join(temp_dir, "nonexistent.mp4")
output_path = os.path.join(temp_dir, "error_output.mp4")
task = MockRenderTask()
task.input_files = [missing_file]
task.output_path = output_path
task.effects = []
builder = FFmpegCommandBuilder(task)
command = builder.build_command()
# 应该失败,因为输入文件不存在
result = subprocess.run(command, capture_output=True, text=True, timeout=30)
assert result.returncode != 0, "FFmpeg should fail with missing input file"
def test_performance_multiple_effects(self, sample_video, temp_dir):
"""测试多特效性能"""
output_path = os.path.join(temp_dir, "performance_output.mp4")
task = MockRenderTask()
task.input_files = [sample_video]
task.output_path = output_path
# 多个特效组合
task.effects = ["zoom:0,1.5,1.0", "ospeed:1.2", "zoom:1.5,2.0,1.0"]
task.ext_data = {"posJson": "{}"}
builder = FFmpegCommandBuilder(task)
command = builder.build_command()
import time
start_time = time.time()
try:
result = subprocess.run(
command, capture_output=True, text=True, timeout=120
)
execution_time = time.time() - start_time
assert result.returncode == 0, f"FFmpeg failed: {result.stderr}"
assert os.path.exists(output_path), "Output file was not created"
assert execution_time < 60, f"Execution took too long: {execution_time}s"
except subprocess.TimeoutExpired:
pytest.fail("FFmpeg execution timed out")
@pytest.mark.skipif(
not os.environ.get("RUN_STRESS_TESTS"), reason="Stress tests disabled"
)
def test_stress_test_large_effects_chain(self, sample_video, temp_dir):
"""压力测试:大量特效链"""
output_path = os.path.join(temp_dir, "stress_output.mp4")
task = MockRenderTask()
task.input_files = [sample_video]
task.output_path = output_path
# 创建大量特效
task.effects = [f"zoom:{i*0.5},1.{i+5},{i*0.2+0.5}" for i in range(10)]
task.ext_data = {"posJson": "{}"}
builder = FFmpegCommandBuilder(task)
command = builder.build_command()
try:
result = subprocess.run(
command, capture_output=True, text=True, timeout=300
)
assert result.returncode == 0, f"Stress test failed: {result.stderr}"
assert os.path.exists(output_path), "Output file was not created"
except subprocess.TimeoutExpired:
pytest.fail("Stress test timed out")

View File

@@ -1,15 +0,0 @@
# -*- coding: utf-8 -*-
import os
from services.cache import MaterialCache, _extract_cache_key
def test_cache_lock_acquire_release(tmp_path):
cache = MaterialCache(cache_dir=str(tmp_path), enabled=True, max_size_gb=0)
cache_key = _extract_cache_key("https://example.com/path/file.mp4?token=abc")
lock_path = cache._acquire_lock(cache_key)
assert lock_path
assert os.path.exists(lock_path)
cache._release_lock(lock_path)
assert not os.path.exists(lock_path)

241
tests/utils/test_helpers.py Normal file
View File

@@ -0,0 +1,241 @@
"""测试辅助工具"""
import json
import tempfile
import subprocess
from typing import Dict, Any, List, Optional
from pathlib import Path
from entity.render_task import RenderTask
from entity.effects.base import EffectProcessor
from config.settings import FFmpegConfig
class MockRenderTask:
"""模拟渲染任务,用于测试"""
def __init__(
self,
input_files: Optional[List[str]] = None,
output_file: str = "test_output.mp4",
effects: Optional[List[str]] = None,
ext_data: Optional[Dict[str, Any]] = None,
frame_rate: int = 25,
):
# RenderTask required fields
self.input_files = input_files or []
self.output_file = output_file
self.task_type = "copy" # TaskType.COPY equivalent
# Optional fields that match RenderTask
self.resolution = None
self.frame_rate = frame_rate
self.speed = 1.0
self.mute = True
self.annexb = False
# Cut parameters
self.zoom_cut = None
self.center_cut = None
# Resource lists
self.subtitles: List[str] = []
self.luts: List[str] = []
self.audios: List[str] = []
self.overlays: List[str] = []
self.effects = effects or []
# Extension data
self.ext_data = ext_data or {}
# Legacy compatibility
self.task_id = "test_task"
self.template_id = "test_template"
self.use_center_cut = False
self.use_zoom_cut = False
self.audio_file = None
class FFmpegValidator:
"""FFmpeg命令验证器"""
@staticmethod
def validate_filter_syntax(filter_str: str) -> bool:
"""验证滤镜语法是否正确"""
try:
# 基本语法检查
if not filter_str:
return False
# 检查是否包含基本的滤镜结构
if "[" in filter_str and "]" in filter_str:
return True
# 检查常见的滤镜格式
common_filters = ["zoompan", "setpts", "trim", "scale", "crop"]
return any(f in filter_str for f in common_filters)
except Exception:
return False
@staticmethod
def validate_stream_identifier(stream_id: str) -> bool:
"""验证流标识符格式"""
if not stream_id:
return False
return stream_id.startswith("[") and stream_id.endswith("]")
@staticmethod
def validate_ffmpeg_command(command: List[str]) -> Dict[str, Any]:
"""验证完整的FFmpeg命令"""
result = {
"valid": False,
"has_input": False,
"has_output": False,
"has_filter": False,
"errors": [],
}
if not command or command[0] != "ffmpeg":
result["errors"].append("Command must start with 'ffmpeg'")
return result
# 检查输入文件
if "-i" in command:
result["has_input"] = True
else:
result["errors"].append("No input file specified")
# 检查输出文件
if len(command) > 1 and not command[-1].startswith("-"):
result["has_output"] = True
else:
result["errors"].append("No output file specified")
# 检查滤镜
if "-filter_complex" in command or "-vf" in command:
result["has_filter"] = True
result["valid"] = (
result["has_input"] and result["has_output"] and len(result["errors"]) == 0
)
return result
class EffectTestHelper:
"""特效测试辅助类"""
@staticmethod
def create_test_effect(
effect_class, params: str = "", ext_data: Dict[str, Any] = None
):
"""创建测试用特效实例"""
return effect_class(params, ext_data)
@staticmethod
def test_effect_params_validation(
effect: EffectProcessor, test_cases: List[Dict[str, Any]]
):
"""批量测试特效参数验证"""
results = []
for case in test_cases:
effect.params = case.get("params", "")
effect.ext_data = case.get("ext_data", {})
is_valid = effect.validate_params()
expected = case.get("expected", True)
results.append(
{
"params": effect.params,
"expected": expected,
"actual": is_valid,
"passed": is_valid == expected,
"description": case.get("description", ""),
}
)
return results
@staticmethod
def test_filter_generation(
effect: EffectProcessor, video_input: str = "[0:v]", effect_index: int = 1
):
"""测试滤镜生成"""
try:
filters, output_stream = effect.generate_filter_args(
video_input, effect_index
)
result = {
"success": True,
"filters": filters,
"output_stream": output_stream,
"filter_count": len(filters),
"valid_syntax": all(
FFmpegValidator.validate_filter_syntax(f) for f in filters
),
"valid_output": (
FFmpegValidator.validate_stream_identifier(output_stream)
if output_stream != video_input
else True
),
}
except Exception as e:
result = {
"success": False,
"error": str(e),
"filters": [],
"output_stream": "",
"filter_count": 0,
"valid_syntax": False,
"valid_output": False,
}
return result
def create_test_video_file(
output_path: str, duration: int = 5, resolution: str = "640x480"
) -> bool:
"""创建测试用视频文件"""
try:
cmd = [
"ffmpeg",
"-y", # 覆盖输出文件
"-f",
"lavfi", # 使用libavfilter输入
"-i",
f"testsrc=duration={duration}:size={resolution}:rate=25",
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-crf",
"23",
output_path,
]
result = subprocess.run(cmd, capture_output=True, text=True)
return result.returncode == 0
except Exception:
return False
def create_sample_template_data() -> Dict[str, Any]:
"""创建示例模板数据"""
return {
"templateId": "test_template_001",
"name": "测试模板",
"parts": [
{
"id": "part1",
"type": "video",
"duration": 10.0,
"effects": ["zoom:0,2.0,3.0", "ospeed:1.5"],
}
],
"settings": {"width": 1920, "height": 1080, "frameRate": 25},
}

View File

@@ -1,15 +0,0 @@
# -*- coding: utf-8 -*-
"""
工具模块
提供系统信息采集等工具函数。
"""
from util.system import get_sys_info, get_capabilities, get_gpu_info, get_ffmpeg_version
__all__ = [
'get_sys_info',
'get_capabilities',
'get_gpu_info',
'get_ffmpeg_version',
]

344
util/api.py Normal file
View File

@@ -0,0 +1,344 @@
import json
import logging
import os
import threading
import time
from urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
import requests
from opentelemetry.trace import Status, StatusCode
import util.system
from telemetry import get_tracer
from util import oss
# 创建带有连接池和重试策略的会话
session = requests.Session()
# 配置重试策略
retry_strategy = Retry(
total=3,
status_forcelist=[429, 500, 502, 503, 504],
backoff_factor=1,
respect_retry_after_header=True,
)
# 配置HTTP适配器(连接池)
adapter = HTTPAdapter(pool_connections=10, pool_maxsize=20, max_retries=retry_strategy)
session.mount("http://", adapter)
session.mount("https://", adapter)
# 设置默认超时
session.timeout = 30
logger = logging.getLogger(__name__)
def normalize_task(task_info):
...
return task_info
def sync_center():
"""
通过接口获取任务
:return: 任务列表
"""
from services import DefaultTemplateService
template_service = DefaultTemplateService()
try:
response = session.post(
os.getenv("API_ENDPOINT") + "/sync",
json={
"accessKey": os.getenv("ACCESS_KEY"),
"clientStatus": util.system.get_sys_info(),
"templateList": [
{"id": t.get("id", ""), "updateTime": t.get("updateTime", "")}
for t in template_service.templates.values()
],
},
timeout=10,
)
response.raise_for_status()
except requests.RequestException as e:
logger.error("请求失败!", e)
return []
data = response.json()
logger.debug("获取任务结果:【%s", data)
if data.get("code", 0) == 200:
templates = data.get("data", {}).get("templates", [])
tasks = data.get("data", {}).get("tasks", [])
else:
tasks = []
templates = []
logger.warning("获取任务失败")
if os.getenv("REDIRECT_TO_URL", False) != False:
for task in tasks:
_sess = requests.Session()
logger.info(
"重定向任务【%s】至配置的地址:%s",
task.get("id"),
os.getenv("REDIRECT_TO_URL"),
)
url = f"{os.getenv('REDIRECT_TO_URL')}{task.get('id')}"
threading.Thread(target=requests.post, args=(url,)).start()
return []
for template in templates:
template_id = template.get("id", "")
if template_id:
logger.info("更新模板:【%s", template_id)
template_service.download_template(template_id)
return tasks
def get_template_info(template_id):
"""
通过接口获取模板信息
:rtype: Template
:param template_id: 模板id
:type template_id: str
:return: 模板信息
"""
tracer = get_tracer(__name__)
with tracer.start_as_current_span("get_template_info"):
with tracer.start_as_current_span("get_template_info.request") as req_span:
try:
req_span.set_attribute("http.method", "POST")
req_span.set_attribute(
"http.url",
"{0}/template/{1}".format(os.getenv("API_ENDPOINT"), template_id),
)
response = session.post(
"{0}/template/{1}".format(os.getenv("API_ENDPOINT"), template_id),
json={
"accessKey": os.getenv("ACCESS_KEY"),
},
timeout=10,
)
req_span.set_attribute("http.status_code", response.status_code)
req_span.set_attribute("http.response", response.text)
response.raise_for_status()
except requests.RequestException as e:
req_span.set_attribute("api.error", str(e))
logger.error("请求失败!", e)
return None
data = response.json()
logger.debug("获取模板信息结果:【%s", data)
remote_template_info = data.get("data", {})
if not remote_template_info:
logger.warning("获取模板信息结果为空", data)
return None
template = {
"id": template_id,
"updateTime": remote_template_info.get("updateTime", template_id),
"scenic_name": remote_template_info.get("scenicName", "景区"),
"name": remote_template_info.get("name", "模版"),
"video_size": remote_template_info.get("resolution", "1920x1080"),
"frame_rate": 25,
"overall_duration": 30,
"video_parts": [],
}
def _template_normalizer(template_info):
_template = {}
_placeholder_type = template_info.get("isPlaceholder", -1)
if _placeholder_type == 0:
# 固定视频
_template["source"] = template_info.get("sourceUrl", "")
elif _placeholder_type == 1:
# 占位符
_template["source"] = "PLACEHOLDER_" + template_info.get(
"sourceUrl", ""
)
_template["mute"] = template_info.get("mute", True)
_template["crop_mode"] = template_info.get("cropEnable", None)
_template["zoom_cut"] = template_info.get("zoomCut", None)
else:
_template["source"] = None
_overlays = template_info.get("overlays", "")
if _overlays:
_template["overlays"] = _overlays.split(",")
_audios = template_info.get("audios", "")
if _audios:
_template["audios"] = _audios.split(",")
_luts = template_info.get("luts", "")
if _luts:
_template["luts"] = _luts.split(",")
_only_if = template_info.get("onlyIf", "")
if _only_if:
_template["only_if"] = _only_if
_effects = template_info.get("effects", "")
if _effects:
_template["effects"] = _effects.split("|")
return _template
# outer template definition
overall_template = _template_normalizer(remote_template_info)
template["overall_template"] = overall_template
# inter template definition
inter_template_list = remote_template_info.get("children", [])
for children_template in inter_template_list:
parts = _template_normalizer(children_template)
template["video_parts"].append(parts)
template["local_path"] = os.path.join(
os.getenv("TEMPLATE_DIR"), str(template_id)
)
with get_tracer("api").start_as_current_span(
"get_template_info.template"
) as res_span:
res_span.set_attribute("normalized.response", json.dumps(template))
return template
def report_task_success(task_info, **kwargs):
tracer = get_tracer(__name__)
with tracer.start_as_current_span("report_task_success"):
with tracer.start_as_current_span("report_task_success.request") as req_span:
try:
req_span.set_attribute("http.method", "POST")
req_span.set_attribute(
"http.url",
"{0}/{1}/success".format(
os.getenv("API_ENDPOINT"), task_info.get("id")
),
)
response = session.post(
"{0}/{1}/success".format(
os.getenv("API_ENDPOINT"), task_info.get("id")
),
json={"accessKey": os.getenv("ACCESS_KEY"), **kwargs},
timeout=10,
)
req_span.set_attribute("http.status_code", response.status_code)
req_span.set_attribute("http.response", response.text)
response.raise_for_status()
req_span.set_status(Status(StatusCode.OK))
except requests.RequestException as e:
req_span.set_attribute("api.error", str(e))
logger.error("请求失败!", e)
return None
def report_task_start(task_info):
tracer = get_tracer(__name__)
with tracer.start_as_current_span("report_task_start"):
with tracer.start_as_current_span("report_task_start.request") as req_span:
try:
req_span.set_attribute("http.method", "POST")
req_span.set_attribute(
"http.url",
"{0}/{1}/start".format(
os.getenv("API_ENDPOINT"), task_info.get("id")
),
)
response = session.post(
"{0}/{1}/start".format(
os.getenv("API_ENDPOINT"), task_info.get("id")
),
json={
"accessKey": os.getenv("ACCESS_KEY"),
},
timeout=10,
)
req_span.set_attribute("http.status_code", response.status_code)
req_span.set_attribute("http.response", response.text)
response.raise_for_status()
req_span.set_status(Status(StatusCode.OK))
except requests.RequestException as e:
req_span.set_attribute("api.error", str(e))
logger.error("请求失败!", e)
return None
def report_task_failed(task_info, reason=""):
tracer = get_tracer(__name__)
with tracer.start_as_current_span("report_task_failed") as span:
span.set_attribute("task_id", task_info.get("id"))
span.set_attribute("reason", reason)
with tracer.start_as_current_span("report_task_failed.request") as req_span:
try:
req_span.set_attribute("http.method", "POST")
req_span.set_attribute(
"http.url",
"{0}/{1}/fail".format(
os.getenv("API_ENDPOINT"), task_info.get("id")
),
)
response = session.post(
"{0}/{1}/fail".format(
os.getenv("API_ENDPOINT"), task_info.get("id")
),
json={"accessKey": os.getenv("ACCESS_KEY"), "reason": reason},
timeout=10,
)
req_span.set_attribute("http.status_code", response.status_code)
req_span.set_attribute("http.response", response.text)
response.raise_for_status()
req_span.set_status(Status(StatusCode.OK))
except requests.RequestException as e:
req_span.set_attribute("api.error", str(e))
req_span.set_status(Status(StatusCode.ERROR))
logger.error("请求失败!", e)
return None
def upload_task_file(task_info, ffmpeg_task):
tracer = get_tracer(__name__)
with get_tracer("api").start_as_current_span("upload_task_file") as span:
logger.info("开始上传文件: %s", task_info.get("id"))
span.set_attribute("file.id", task_info.get("id"))
with tracer.start_as_current_span(
"upload_task_file.request_upload_url"
) as req_span:
try:
req_span.set_attribute("http.method", "POST")
req_span.set_attribute(
"http.url",
"{0}/{1}/uploadUrl".format(
os.getenv("API_ENDPOINT"), task_info.get("id")
),
)
response = session.post(
"{0}/{1}/uploadUrl".format(
os.getenv("API_ENDPOINT"), task_info.get("id")
),
json={
"accessKey": os.getenv("ACCESS_KEY"),
},
timeout=10,
)
req_span.set_attribute("http.status_code", response.status_code)
req_span.set_attribute("http.response", response.text)
response.raise_for_status()
req_span.set_status(Status(StatusCode.OK))
except requests.RequestException as e:
span.set_attribute("api.error", str(e))
req_span.set_status(Status(StatusCode.ERROR))
logger.error("请求失败!", e)
return False
data = response.json()
url = data.get("data", "")
logger.info("开始上传文件: %s%s", task_info.get("id"), url)
return oss.upload_to_oss(url, ffmpeg_task.get_output_file())
def get_task_info(id):
try:
response = session.get(
os.getenv("API_ENDPOINT") + "/" + id + "/info",
params={
"accessKey": os.getenv("ACCESS_KEY"),
},
timeout=10,
)
response.raise_for_status()
except requests.RequestException as e:
logger.error("请求失败!", e)
return []
data = response.json()
logger.debug("获取任务结果:【%s", data)
if data.get("code", 0) == 200:
return data.get("data", {})

113
util/exceptions.py Normal file
View File

@@ -0,0 +1,113 @@
from typing import Optional
class RenderWorkerError(Exception):
"""RenderWorker基础异常类"""
def __init__(self, message: str, error_code: Optional[str] = None):
super().__init__(message)
self.message = message
self.error_code = error_code or self.__class__.__name__
class ConfigurationError(RenderWorkerError):
"""配置错误"""
pass
class TemplateError(RenderWorkerError):
"""模板相关错误"""
pass
class TemplateNotFoundError(TemplateError):
"""模板未找到错误"""
pass
class TemplateValidationError(TemplateError):
"""模板验证错误"""
pass
class TaskError(RenderWorkerError):
"""任务处理错误"""
pass
class TaskValidationError(TaskError):
"""任务参数验证错误"""
pass
class RenderError(RenderWorkerError):
"""渲染处理错误"""
pass
class FFmpegError(RenderError):
"""FFmpeg执行错误"""
def __init__(
self,
message: str,
command: Optional[list] = None,
return_code: Optional[int] = None,
stderr: Optional[str] = None,
):
super().__init__(message)
self.command = command
self.return_code = return_code
self.stderr = stderr
class EffectError(RenderError):
"""效果处理错误"""
def __init__(
self, message: str, effect_name: Optional[str] = None, effect_params: Optional[str] = None
):
super().__init__(message)
self.effect_name = effect_name
self.effect_params = effect_params
class StorageError(RenderWorkerError):
"""存储相关错误"""
pass
class APIError(RenderWorkerError):
"""API调用错误"""
def __init__(
self, message: str, status_code: Optional[int] = None, response_body: Optional[str] = None
):
super().__init__(message)
self.status_code = status_code
self.response_body = response_body
class ResourceError(RenderWorkerError):
"""资源相关错误"""
pass
class ResourceNotFoundError(ResourceError):
"""资源未找到错误"""
pass
class DownloadError(ResourceError):
"""下载错误"""
pass

298
util/ffmpeg.py Normal file
View File

@@ -0,0 +1,298 @@
import json
import logging
import os
import subprocess
from datetime import datetime
from typing import Optional, IO
from opentelemetry.trace import Status, StatusCode
from entity.ffmpeg import (
FfmpegTask,
ENCODER_ARGS,
VIDEO_ARGS,
AUDIO_ARGS,
MUTE_AUDIO_INPUT,
get_mp4toannexb_filter,
)
from telemetry import get_tracer
logger = logging.getLogger(__name__)
def re_encode_and_annexb(file):
with get_tracer("ffmpeg").start_as_current_span("re_encode_and_annexb") as span:
span.set_attribute("file.path", file)
if not os.path.exists(file):
span.set_status(Status(StatusCode.ERROR))
return file
logger.info("ReEncodeAndAnnexb: %s", file)
has_audio = not not probe_video_audio(file)
# 优先使用RE_ENCODE_VIDEO_ARGS环境变量,其次使用默认的VIDEO_ARGS
if os.getenv("RE_ENCODE_VIDEO_ARGS", False):
_video_args = tuple(os.getenv("RE_ENCODE_VIDEO_ARGS", "").split(" "))
else:
_video_args = VIDEO_ARGS
# 优先使用RE_ENCODE_ENCODER_ARGS环境变量,其次使用默认的ENCODER_ARGS
if os.getenv("RE_ENCODE_ENCODER_ARGS", False):
_encoder_args = tuple(os.getenv("RE_ENCODE_ENCODER_ARGS", "").split(" "))
else:
_encoder_args = ENCODER_ARGS
ffmpeg_process = subprocess.run(
[
"ffmpeg",
"-y",
"-hide_banner",
"-i",
file,
*(set() if has_audio else MUTE_AUDIO_INPUT),
"-fps_mode",
"cfr",
"-map",
"0:v",
"-map",
"0:a" if has_audio else "1:a",
*_video_args,
"-bsf:v",
get_mp4toannexb_filter(),
*AUDIO_ARGS,
"-bsf:a",
"setts=pts=DTS",
*_encoder_args,
"-shortest",
"-fflags",
"+genpts",
"-f",
"mpegts",
file + ".ts",
]
)
logger.info(" ".join(ffmpeg_process.args))
span.set_attribute("ffmpeg.args", json.dumps(ffmpeg_process.args))
logger.info(
"ReEncodeAndAnnexb: %s, returned: %s", file, ffmpeg_process.returncode
)
span.set_attribute("ffmpeg.code", ffmpeg_process.returncode)
if ffmpeg_process.returncode == 0:
span.set_status(Status(StatusCode.OK))
span.set_attribute("file.size", os.path.getsize(file + ".ts"))
# os.remove(file)
return file + ".ts"
else:
span.set_status(Status(StatusCode.ERROR))
return file
# start_render函数已迁移到services/render_service.py中的DefaultRenderService
# 保留原有签名用于向后兼容,但建议使用新的服务架构
def start_render(ffmpeg_task):
"""
已迁移到新架构,建议使用 DefaultRenderService.render()
保留用于向后兼容
"""
logger.warning(
"start_render is deprecated, use DefaultRenderService.render() instead"
)
from services import DefaultRenderService
render_service = DefaultRenderService()
return render_service.render(ffmpeg_task)
def handle_ffmpeg_output(stdout: Optional[bytes]) -> str:
out_time = "0:0:0.0"
if stdout is None:
print("[!]STDOUT is null")
return out_time
speed = "0"
for line in stdout.split(b"\n"):
if line == b"":
break
if line.strip() == b"progress=end":
# 处理完毕
break
if line.startswith(b"out_time="):
out_time = line.replace(b"out_time=", b"").decode().strip()
if line.startswith(b"speed="):
speed = line.replace(b"speed=", b"").decode().strip()
print("[ ]Speed:", out_time, "@", speed)
return out_time + "@" + speed
def duration_str_to_float(duration_str: str) -> float:
_duration = datetime.strptime(duration_str, "%H:%M:%S.%f") - datetime(1900, 1, 1)
return _duration.total_seconds()
def probe_video_info(video_file):
tracer = get_tracer(__name__)
with tracer.start_as_current_span("probe_video_info") as span:
span.set_attribute("video.file", video_file)
# 获取宽度和高度
result = subprocess.run(
[
"ffprobe",
"-v",
"error",
"-select_streams",
"v:0",
"-show_entries",
"stream=width,height:format=duration",
"-of",
"csv=s=x:p=0",
video_file,
],
stderr=subprocess.STDOUT,
**subprocess_args(True)
)
span.set_attribute("ffprobe.args", json.dumps(result.args))
span.set_attribute("ffprobe.code", result.returncode)
if result.returncode != 0:
span.set_status(Status(StatusCode.ERROR))
return 0, 0, 0
all_result = result.stdout.decode("utf-8").strip()
span.set_attribute("ffprobe.out", all_result)
if all_result == "":
span.set_status(Status(StatusCode.ERROR))
return 0, 0, 0
span.set_status(Status(StatusCode.OK))
wh, duration = all_result.split("\n")
width, height = wh.strip().split("x")
return int(width), int(height), float(duration)
def probe_video_audio(video_file, type=None):
tracer = get_tracer(__name__)
with tracer.start_as_current_span("probe_video_audio") as span:
span.set_attribute("video.file", video_file)
args = [
"ffprobe",
"-hide_banner",
"-v",
"error",
"-select_streams",
"a",
"-show_entries",
"stream=index",
"-of",
"csv=p=0",
]
if type == "concat":
args.append("-safe")
args.append("0")
args.append("-f")
args.append("concat")
args.append(video_file)
logger.info(" ".join(args))
result = subprocess.run(args, stderr=subprocess.STDOUT, **subprocess_args(True))
span.set_attribute("ffprobe.args", json.dumps(result.args))
span.set_attribute("ffprobe.code", result.returncode)
logger.info("probe_video_audio: %s", result.stdout.decode("utf-8").strip())
if result.returncode != 0:
return False
if result.stdout.decode("utf-8").strip() == "":
return False
return True
# 音频淡出2秒
def fade_out_audio(file, duration, fade_out_sec=2):
if type(duration) == str:
try:
duration = float(duration)
except Exception as e:
logger.error("duration is not float: %s", e)
return file
tracer = get_tracer(__name__)
with tracer.start_as_current_span("fade_out_audio") as span:
span.set_attribute("audio.file", file)
if duration <= fade_out_sec:
return file
else:
new_fn = file + "_.mp4"
if os.path.exists(new_fn):
os.remove(new_fn)
logger.info("delete tmp file: " + new_fn)
try:
process = subprocess.run(
[
"ffmpeg",
"-i",
file,
"-c:v",
"copy",
"-c:a",
"aac",
"-af",
"afade=t=out:st="
+ str(duration - fade_out_sec)
+ ":d="
+ str(fade_out_sec),
"-y",
new_fn,
],
**subprocess_args(True)
)
span.set_attribute("ffmpeg.args", json.dumps(process.args))
logger.info(" ".join(process.args))
if process.returncode != 0:
span.set_status(Status(StatusCode.ERROR))
logger.error("FFMPEG ERROR: %s", process.stderr)
return file
else:
span.set_status(Status(StatusCode.OK))
return new_fn
except Exception as e:
span.set_status(Status(StatusCode.ERROR))
logger.error("FFMPEG ERROR: %s", e)
return file
# Create a set of arguments which make a ``subprocess.Popen`` (and
# variants) call work with or without Pyinstaller, ``--noconsole`` or
# not, on Windows and Linux. Typical use::
#
# subprocess.call(['program_to_run', 'arg_1'], **subprocess_args())
#
# When calling ``check_output``::
#
# subprocess.check_output(['program_to_run', 'arg_1'],
# **subprocess_args(False))
def subprocess_args(include_stdout=True):
# The following is true only on Windows.
if hasattr(subprocess, "STARTUPINFO"):
# On Windows, subprocess calls will pop up a command window by default
# when run from Pyinstaller with the ``--noconsole`` option. Avoid this
# distraction.
si = subprocess.STARTUPINFO()
si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
# Windows doesn't search the path by default. Pass it an environment so
# it will.
env = os.environ
else:
si = None
env = None
# ``subprocess.check_output`` doesn't allow specifying ``stdout``::
#
# Traceback (most recent call last):
# File "test_subprocess.py", line 58, in <module>
# **subprocess_args(stdout=None))
# File "C:\Python27\lib\subprocess.py", line 567, in check_output
# raise ValueError('stdout argument not allowed, it will be overridden.')
# ValueError: stdout argument not allowed, it will be overridden.
#
# So, add it only if it's needed.
if include_stdout:
ret = {"stdout": subprocess.PIPE}
else:
ret = {}
# On Windows, running this from the binary produced by Pyinstaller
# with the ``--noconsole`` option requires redirecting everything
# (stdin, stdout, stderr) to avoid an OSError exception
# "[Error 6] the handle is invalid."
ret.update({"stdin": subprocess.PIPE, "startupinfo": si, "env": env})
return ret

161
util/ffmpeg_utils.py Normal file
View File

@@ -0,0 +1,161 @@
"""
FFmpeg工具模块 - 提供FFmpeg命令构建和处理的公共函数
"""
import logging
from typing import List, Tuple, Optional
from config.settings import get_ffmpeg_config
logger = logging.getLogger(__name__)
def build_base_ffmpeg_args() -> List[str]:
"""
构建基础FFmpeg参数
Returns:
基础参数列表
"""
config = get_ffmpeg_config()
args = ["ffmpeg", "-y", "-hide_banner"]
args.extend(config.progress_args)
args.extend(config.loglevel_args)
return args
def build_null_audio_input() -> List[str]:
"""
构建空音频输入参数
Returns:
空音频输入参数列表
"""
config = get_ffmpeg_config()
return config.null_audio_args
def build_amix_filter(input1: str, input2: str, output: str) -> str:
"""
构建音频混合滤镜
Args:
input1: 第一个音频输入
input2: 第二个音频输入
output: 输出流名称
Returns:
混合滤镜字符串
"""
config = get_ffmpeg_config()
return f"{input1}[{input2}]{config.amix_args[0]}[{output}]"
def build_overlay_scale_filter(
video_input: str, overlay_input: str, output: str
) -> str:
"""
构建覆盖层缩放滤镜
Args:
video_input: 视频输入流
overlay_input: 覆盖层输入流
output: 输出流名称
Returns:
缩放滤镜字符串
"""
config = get_ffmpeg_config()
if config.overlay_scale_mode == "scale":
return f"{video_input}[{overlay_input}]scale=iw:ih[{output}]"
else:
return (
f"{video_input}[{overlay_input}]{config.overlay_scale_mode}=iw:ih[{output}]"
)
def get_annexb_filter() -> str:
"""
获取annexb转换滤镜
Returns:
annexb滤镜名称
"""
config = get_ffmpeg_config()
encoder_args_str = " ".join(config.encoder_args).lower()
if "hevc" in encoder_args_str:
return "hevc_mp4toannexb"
return "h264_mp4toannexb"
def build_standard_output_args() -> List[str]:
"""
构建标准输出参数
Returns:
输出参数列表
"""
config = get_ffmpeg_config()
return [
*config.video_args,
*config.audio_args,
*config.encoder_args,
*config.default_args,
]
def validate_ffmpeg_file_extensions(file_path: str) -> bool:
"""
验证文件扩展名是否为FFmpeg支持的格式
Args:
file_path: 文件路径
Returns:
是否为支持的格式
"""
supported_extensions = {
".mp4",
".avi",
".mov",
".mkv",
".flv",
".wmv",
".webm",
".ts",
".m2ts",
".mts",
".m4v",
".3gp",
".asf",
".rm",
".mp3",
".wav",
".aac",
".flac",
".ogg",
".m4a",
".wma",
}
import os
_, ext = os.path.splitext(file_path.lower())
return ext in supported_extensions
def estimate_processing_time(
input_duration: float, complexity_factor: float = 1.0
) -> float:
"""
估算处理时间
Args:
input_duration: 输入文件时长(秒)
complexity_factor: 复杂度因子(1.0为普通处理)
Returns:
预估处理时间(秒)
"""
# 基础处理速度假设为实时的0.5倍(即处理1秒视频需要2秒)
base_processing_ratio = 2.0
return input_duration * base_processing_ratio * complexity_factor

99
util/json_utils.py Normal file
View File

@@ -0,0 +1,99 @@
"""
JSON处理工具模块 - 提供安全的JSON解析和处理功能
"""
import json
import logging
from typing import Dict, Any, Optional, Union
logger = logging.getLogger(__name__)
def safe_json_loads(json_str: Union[str, bytes], default: Any = None) -> Any:
"""
安全解析JSON字符串
Args:
json_str: JSON字符串
default: 解析失败时返回的默认值
Returns:
解析后的对象,或默认值
"""
if not json_str or json_str == "{}":
return default or {}
try:
return json.loads(json_str)
except (json.JSONDecodeError, TypeError) as e:
logger.warning(f"Failed to parse JSON: {e}, input: {json_str!r}")
return default or {}
def safe_json_dumps(
obj: Any, indent: Optional[int] = None, ensure_ascii: bool = False
) -> str:
"""
安全序列化对象为JSON字符串
Args:
obj: 要序列化的对象
indent: 缩进空格数
ensure_ascii: 是否确保ASCII编码
Returns:
JSON字符串
"""
try:
return json.dumps(obj, indent=indent, ensure_ascii=ensure_ascii)
except (TypeError, ValueError) as e:
logger.error(f"Failed to serialize to JSON: {e}")
return "{}"
def get_nested_value(data: Dict[str, Any], key_path: str, default: Any = None) -> Any:
"""
从嵌套字典中安全获取值
Args:
data: 字典数据
key_path: 键路径,用点分隔(如 "user.profile.name"
default: 默认值
Returns:
找到的值或默认值
"""
if not isinstance(data, dict):
return default
try:
keys = key_path.split(".")
current = data
for key in keys:
if isinstance(current, dict) and key in current:
current = current[key]
else:
return default
return current
except Exception as e:
logger.warning(f"Failed to get nested value for path '{key_path}': {e}")
return default
def merge_dicts(*dicts: Dict[str, Any]) -> Dict[str, Any]:
"""
合并多个字典,后面的字典会覆盖前面的字典中相同的键
Args:
*dicts: 要合并的字典
Returns:
合并后的字典
"""
result = {}
for d in dicts:
if isinstance(d, dict):
result.update(d)
return result

143
util/oss.py Normal file
View File

@@ -0,0 +1,143 @@
import logging
import os
import sys
import requests
from opentelemetry.trace import Status, StatusCode
from telemetry import get_tracer
logger = logging.getLogger(__name__)
def upload_to_oss(url, file_path):
"""
使用签名URL上传文件到OSS
:param str url: 签名URL
:param str file_path: 文件路径
:return bool: 是否成功
"""
tracer = get_tracer(__name__)
with tracer.start_as_current_span("upload_to_oss") as span:
span.set_attribute("file.url", url)
span.set_attribute("file.path", file_path)
span.set_attribute("file.size", os.path.getsize(file_path))
max_retries = 5
retries = 0
if os.getenv("UPLOAD_METHOD") == "rclone":
with tracer.start_as_current_span("rclone_to_oss") as r_span:
replace_map = os.getenv("RCLONE_REPLACE_MAP")
r_span.set_attribute("rclone.replace_map", replace_map)
if replace_map != "":
replace_list = [i.split("|", 1) for i in replace_map.split(",")]
new_url = url
for _src, _dst in replace_list:
new_url = new_url.replace(_src, _dst)
new_url = new_url.split("?", 1)[0]
r_span.set_attribute("rclone.target_dir", new_url)
if new_url != url:
result = os.system(
f"rclone copyto --no-check-dest --ignore-existing --multi-thread-chunk-size 8M --multi-thread-streams 8 {file_path} {new_url}"
)
r_span.set_attribute("rclone.result", result)
if result == 0:
span.set_status(Status(StatusCode.OK))
return True
else:
span.set_status(Status(StatusCode.ERROR))
while retries < max_retries:
with tracer.start_as_current_span("upload_to_oss.request") as req_span:
req_span.set_attribute("http.retry_count", retries)
try:
req_span.set_attribute("http.method", "PUT")
req_span.set_attribute("http.url", url)
with open(file_path, "rb") as f:
response = requests.put(
url,
data=f,
stream=True,
timeout=60,
headers={"Content-Type": "video/mp4"},
)
req_span.set_attribute("http.status_code", response.status_code)
req_span.set_attribute("http.response", response.text)
response.raise_for_status()
req_span.set_status(Status(StatusCode.OK))
span.set_status(Status(StatusCode.OK))
return True
except requests.exceptions.Timeout:
req_span.set_attribute("http.error", "Timeout")
req_span.set_status(Status(StatusCode.ERROR))
retries += 1
logger.warning(
f"Upload timed out. Retrying {retries}/{max_retries}..."
)
except Exception as e:
req_span.set_attribute("http.error", str(e))
req_span.set_status(Status(StatusCode.ERROR))
retries += 1
logger.warning(
f"Upload failed. Retrying {retries}/{max_retries}..."
)
span.set_status(Status(StatusCode.ERROR))
return False
def download_from_oss(url, file_path, skip_if_exist=None):
"""
使用签名URL下载文件到OSS
:param skip_if_exist: 如果存在就不下载了
:param str url: 签名URL
:param Union[LiteralString, str, bytes] file_path: 文件路径
:return bool: 是否成功
"""
tracer = get_tracer(__name__)
with tracer.start_as_current_span("download_from_oss") as span:
span.set_attribute("file.url", url)
span.set_attribute("file.path", file_path)
# 如果skip_if_exist为None,则从启动参数中读取
if skip_if_exist is None:
skip_if_exist = "skip_if_exist" in sys.argv
if skip_if_exist and os.path.exists(file_path):
span.set_attribute("file.exist", True)
span.set_attribute("file.size", os.path.getsize(file_path))
return True
logging.info("download_from_oss: %s", url)
file_dir, file_name = os.path.split(file_path)
if file_dir:
if not os.path.exists(file_dir):
os.makedirs(file_dir)
max_retries = 5
retries = 0
while retries < max_retries:
with tracer.start_as_current_span("download_from_oss.request") as req_span:
req_span.set_attribute("http.retry_count", retries)
try:
req_span.set_attribute("http.method", "GET")
req_span.set_attribute("http.url", url)
response = requests.get(url, timeout=15) # 设置超时时间
req_span.set_attribute("http.status_code", response.status_code)
with open(file_path, "wb") as f:
f.write(response.content)
req_span.set_attribute("file.size", os.path.getsize(file_path))
req_span.set_status(Status(StatusCode.OK))
span.set_status(Status(StatusCode.OK))
return True
except requests.exceptions.Timeout:
req_span.set_attribute("http.error", "Timeout")
req_span.set_status(Status(StatusCode.ERROR))
retries += 1
logger.warning(
f"Download timed out. Retrying {retries}/{max_retries}..."
)
except Exception as e:
req_span.set_attribute("http.error", str(e))
req_span.set_status(Status(StatusCode.ERROR))
retries += 1
logger.warning(
f"Download failed. Retrying {retries}/{max_retries}..."
)
span.set_status(Status(StatusCode.ERROR))
return False

View File

@@ -1,345 +1,24 @@
# -*- coding: utf-8 -*-
"""
系统信息工具
提供系统信息采集功能。
"""
import logging
import os import os
import platform import platform
import subprocess from datetime import datetime
from typing import Optional, Dict, Any, List
import psutil import psutil
from constant import SOFTWARE_VERSION, DEFAULT_CAPABILITIES, HW_ACCEL_NONE, HW_ACCEL_QSV, HW_ACCEL_CUDA from constant import SUPPORT_FEATURE, SOFTWARE_VERSION
from domain.gpu import GPUDevice
logger = logging.getLogger(__name__)
def get_sys_info(): def get_sys_info():
""" """
获取系统信息 Returns a dictionary with system information.
Returns:
dict: 系统信息字典
""" """
mem = psutil.virtual_memory()
info = { info = {
'os': platform.system(), "version": SOFTWARE_VERSION,
'cpu': f"{os.cpu_count()} cores", "client_datetime": datetime.now().isoformat(),
'memory': f"{mem.total // (1024**3)}GB", "platform": platform.system(),
'cpuUsage': f"{psutil.cpu_percent()}%", "runtime_version": "Python " + platform.python_version(),
'memoryAvailable': f"{mem.available // (1024**3)}GB", "cpu_count": os.cpu_count(),
'platform': platform.system(), "cpu_usage": psutil.cpu_percent(),
'pythonVersion': platform.python_version(), "memory_total": psutil.virtual_memory().total,
'version': SOFTWARE_VERSION, "memory_available": psutil.virtual_memory().available,
"support_feature": SUPPORT_FEATURE,
} }
# 尝试获取 GPU 信息
gpu_info = get_gpu_info()
if gpu_info:
info['gpu'] = gpu_info
return info return info
def get_capabilities():
"""
获取 Worker 支持的能力列表
Returns:
list: 能力列表
"""
return DEFAULT_CAPABILITIES.copy()
def get_gpu_info() -> Optional[str]:
"""
尝试获取 GPU 信息
Returns:
str: GPU 信息,失败返回 None
"""
try:
# 尝试使用 nvidia-smi
result = subprocess.run(
['nvidia-smi', '--query-gpu=name', '--format=csv,noheader'],
capture_output=True,
text=True,
timeout=5
)
if result.returncode == 0:
gpu_name = result.stdout.strip().split('\n')[0]
return gpu_name
except Exception:
pass
return None
def get_ffmpeg_version() -> str:
"""
获取 FFmpeg 版本
Returns:
str: FFmpeg 版本号
"""
try:
result = subprocess.run(
['ffmpeg', '-version'],
capture_output=True,
text=True,
timeout=5
)
if result.returncode == 0:
first_line = result.stdout.split('\n')[0]
# 解析版本号,例如 "ffmpeg version 6.0 ..."
parts = first_line.split()
for i, part in enumerate(parts):
if part == 'version' and i + 1 < len(parts):
return parts[i + 1]
except Exception:
pass
return 'unknown'
def check_ffmpeg_encoder(encoder: str) -> bool:
"""
检查 FFmpeg 是否支持指定的编码器
Args:
encoder: 编码器名称,如 'h264_nvenc', 'h264_qsv'
Returns:
bool: 是否支持该编码器
"""
try:
result = subprocess.run(
['ffmpeg', '-hide_banner', '-encoders'],
capture_output=True,
text=True,
timeout=5
)
if result.returncode == 0:
return encoder in result.stdout
except Exception:
pass
return False
def check_ffmpeg_decoder(decoder: str) -> bool:
"""
检查 FFmpeg 是否支持指定的解码器
Args:
decoder: 解码器名称,如 'h264_cuvid', 'h264_qsv'
Returns:
bool: 是否支持该解码器
"""
try:
result = subprocess.run(
['ffmpeg', '-hide_banner', '-decoders'],
capture_output=True,
text=True,
timeout=5
)
if result.returncode == 0:
return decoder in result.stdout
except Exception:
pass
return False
def check_ffmpeg_hwaccel(hwaccel: str) -> bool:
"""
检查 FFmpeg 是否支持指定的硬件加速方法
Args:
hwaccel: 硬件加速方法,如 'cuda', 'qsv', 'dxva2', 'd3d11va'
Returns:
bool: 是否支持该硬件加速方法
"""
try:
result = subprocess.run(
['ffmpeg', '-hide_banner', '-hwaccels'],
capture_output=True,
text=True,
timeout=5
)
if result.returncode == 0:
return hwaccel in result.stdout
except Exception:
pass
return False
def detect_hw_accel_support() -> Dict[str, Any]:
"""
检测系统的硬件加速支持情况
Returns:
dict: 硬件加速支持信息
{
'cuda': {
'available': bool,
'gpu': str or None,
'encoder': bool, # h264_nvenc
'decoder': bool, # h264_cuvid
},
'qsv': {
'available': bool,
'encoder': bool, # h264_qsv
'decoder': bool, # h264_qsv
},
'recommended': str # 推荐的加速方式: 'cuda', 'qsv', 'none'
}
"""
result = {
'cuda': {
'available': False,
'gpu': None,
'encoder': False,
'decoder': False,
},
'qsv': {
'available': False,
'encoder': False,
'decoder': False,
},
'recommended': HW_ACCEL_NONE
}
# 检测 CUDA/NVENC 支持
gpu_info = get_gpu_info()
if gpu_info:
result['cuda']['gpu'] = gpu_info
result['cuda']['available'] = check_ffmpeg_hwaccel('cuda')
result['cuda']['encoder'] = check_ffmpeg_encoder('h264_nvenc')
result['cuda']['decoder'] = check_ffmpeg_decoder('h264_cuvid')
# 检测 QSV 支持
result['qsv']['available'] = check_ffmpeg_hwaccel('qsv')
result['qsv']['encoder'] = check_ffmpeg_encoder('h264_qsv')
result['qsv']['decoder'] = check_ffmpeg_decoder('h264_qsv')
# 推荐硬件加速方式(优先 CUDA,其次 QSV)
if result['cuda']['available'] and result['cuda']['encoder']:
result['recommended'] = HW_ACCEL_CUDA
elif result['qsv']['available'] and result['qsv']['encoder']:
result['recommended'] = HW_ACCEL_QSV
return result
def get_hw_accel_info_str() -> str:
"""
获取硬件加速支持信息的可读字符串
Returns:
str: 硬件加速支持信息描述
"""
support = detect_hw_accel_support()
parts = []
if support['cuda']['available']:
gpu = support['cuda']['gpu'] or 'Unknown GPU'
status = 'encoder+decoder' if support['cuda']['encoder'] and support['cuda']['decoder'] else (
'encoder only' if support['cuda']['encoder'] else 'decoder only' if support['cuda']['decoder'] else 'hwaccel only'
)
parts.append(f"CUDA({gpu}, {status})")
if support['qsv']['available']:
status = 'encoder+decoder' if support['qsv']['encoder'] and support['qsv']['decoder'] else (
'encoder only' if support['qsv']['encoder'] else 'decoder only' if support['qsv']['decoder'] else 'hwaccel only'
)
parts.append(f"QSV({status})")
if not parts:
return "No hardware acceleration available"
return ', '.join(parts) + f" [recommended: {support['recommended']}]"
def get_all_gpu_info() -> List[GPUDevice]:
"""
获取所有 NVIDIA GPU 信息
使用 nvidia-smi 查询所有 GPU 设备。
Returns:
GPU 设备列表,失败返回空列表
"""
try:
result = subprocess.run(
[
'nvidia-smi',
'--query-gpu=index,name,memory.total',
'--format=csv,noheader,nounits'
],
capture_output=True,
text=True,
timeout=10
)
if result.returncode != 0:
return []
devices = []
for line in result.stdout.strip().split('\n'):
if not line.strip():
continue
parts = [p.strip() for p in line.split(',')]
if len(parts) >= 2:
index = int(parts[0])
name = parts[1]
memory = int(parts[2]) if len(parts) >= 3 else None
devices.append(GPUDevice(
index=index,
name=name,
memory_total=memory,
available=True
))
return devices
except Exception as e:
logger.warning(f"Failed to detect GPUs: {e}")
return []
def validate_gpu_device(index: int) -> bool:
"""
验证指定索引的 GPU 设备是否可用
Args:
index: GPU 设备索引
Returns:
设备是否可用
"""
try:
result = subprocess.run(
[
'nvidia-smi',
'-i', str(index),
'--query-gpu=name',
'--format=csv,noheader'
],
capture_output=True,
text=True,
timeout=5
)
return result.returncode == 0 and bool(result.stdout.strip())
except Exception:
return False