You've already forked FrameTour-RenderWorker
feat(base): 添加单任务内文件传输并发功能
- 引入 ThreadPoolExecutor 实现并行下载和上传 - 新增 download_files_parallel 和 upload_files_parallel 方法 - 添加任务传输并发数配置选项 TASK_DOWNLOAD_CONCURRENCY 和 TASK_UPLOAD_CONCURRENCY - 实现并发数配置的环境变量解析和验证逻辑 - 在多个处理器中应用并行下载优化文件获取性能 - 更新 .env.example 配置文件模板 - 移除 FFmpeg 命令日志长度限制
This commit is contained in:
@@ -85,13 +85,63 @@ class RenderSegmentVideoHandler(BaseHandler):
|
||||
else:
|
||||
input_file = os.path.join(work_dir, 'input.mp4')
|
||||
|
||||
# 2. 下载素材
|
||||
if not self.download_file(material_url, input_file):
|
||||
# 2. 构建并行下载任务（主素材 + 可选 LUT + 可选叠加层）
|
||||
lut_file = os.path.join(work_dir, 'lut.cube') if render_spec.lut_url else None
|
||||
overlay_file = None
|
||||
if render_spec.overlay_url:
|
||||
# 根据 URL 后缀确定文件扩展名
|
||||
overlay_url_lower = render_spec.overlay_url.lower()
|
||||
if overlay_url_lower.endswith('.jpg') or overlay_url_lower.endswith('.jpeg'):
|
||||
overlay_ext = '.jpg'
|
||||
elif overlay_url_lower.endswith('.mov'):
|
||||
overlay_ext = '.mov'
|
||||
else:
|
||||
overlay_ext = '.png'
|
||||
overlay_file = os.path.join(work_dir, f'overlay{overlay_ext}')
|
||||
|
||||
download_jobs = [
|
||||
{
|
||||
'key': 'material',
|
||||
'url': material_url,
|
||||
'dest': input_file,
|
||||
'required': True
|
||||
}
|
||||
]
|
||||
if render_spec.lut_url and lut_file:
|
||||
download_jobs.append({
|
||||
'key': 'lut',
|
||||
'url': render_spec.lut_url,
|
||||
'dest': lut_file,
|
||||
'required': False
|
||||
})
|
||||
if render_spec.overlay_url and overlay_file:
|
||||
download_jobs.append({
|
||||
'key': 'overlay',
|
||||
'url': render_spec.overlay_url,
|
||||
'dest': overlay_file,
|
||||
'required': False
|
||||
})
|
||||
download_results = self.download_files_parallel(download_jobs)
|
||||
|
||||
material_result = download_results.get('material')
|
||||
if not material_result or not material_result['success']:
|
||||
return TaskResult.fail(
|
||||
ErrorCode.E_INPUT_UNAVAILABLE,
|
||||
f"Failed to download material: {material_url}"
|
||||
)
|
||||
|
||||
if render_spec.lut_url:
|
||||
lut_result = download_results.get('lut')
|
||||
if not lut_result or not lut_result['success']:
|
||||
logger.warning(f"[task:{task.task_id}] Failed to download LUT, continuing without it")
|
||||
lut_file = None
|
||||
|
||||
if render_spec.overlay_url:
|
||||
overlay_result = download_results.get('overlay')
|
||||
if not overlay_result or not overlay_result['success']:
|
||||
logger.warning(f"[task:{task.task_id}] Failed to download overlay, continuing without it")
|
||||
overlay_file = None
|
||||
|
||||
# 3. 图片素材转换为视频
|
||||
if is_image:
|
||||
video_input_file = os.path.join(work_dir, 'input_video.mp4')
|
||||
@@ -111,31 +161,7 @@ class RenderSegmentVideoHandler(BaseHandler):
|
||||
input_file = video_input_file
|
||||
logger.info(f"[task:{task.task_id}] Image converted to video successfully")
|
||||
|
||||
# 4. 下载 LUT（如有）
|
||||
lut_file = None
|
||||
if render_spec.lut_url:
|
||||
lut_file = os.path.join(work_dir, 'lut.cube')
|
||||
if not self.download_file(render_spec.lut_url, lut_file):
|
||||
logger.warning(f"[task:{task.task_id}] Failed to download LUT, continuing without it")
|
||||
lut_file = None
|
||||
|
||||
# 5. 下载叠加层（如有）
|
||||
overlay_file = None
|
||||
if render_spec.overlay_url:
|
||||
# 根据 URL 后缀确定文件扩展名
|
||||
url_lower = render_spec.overlay_url.lower()
|
||||
if url_lower.endswith('.jpg') or url_lower.endswith('.jpeg'):
|
||||
ext = '.jpg'
|
||||
elif url_lower.endswith('.mov'):
|
||||
ext = '.mov'
|
||||
else:
|
||||
ext = '.png' # 默认
|
||||
overlay_file = os.path.join(work_dir, f'overlay{ext}')
|
||||
if not self.download_file(render_spec.overlay_url, overlay_file):
|
||||
logger.warning(f"[task:{task.task_id}] Failed to download overlay, continuing without it")
|
||||
overlay_file = None
|
||||
|
||||
# 6. 探测源视频时长（仅对视频素材）
|
||||
# 4. 探测源视频时长（仅对视频素材）
|
||||
# 用于检测时长不足并通过冻结最后一帧补足
|
||||
source_duration_sec = None
|
||||
if not is_image:
|
||||
@@ -158,13 +184,13 @@ class RenderSegmentVideoHandler(BaseHandler):
|
||||
f"will freeze last frame for {shortage_sec:.2f}s"
|
||||
)
|
||||
|
||||
# 7. 计算 overlap 时长（用于转场帧冻结）
|
||||
# 5. 计算 overlap 时长（用于转场帧冻结）
|
||||
# 头部 overlap: 来自前一片段的出场转场
|
||||
overlap_head_ms = task.get_overlap_head_ms()
|
||||
# 尾部 overlap: 当前片段的出场转场
|
||||
overlap_tail_ms = task.get_overlap_tail_ms_v2()
|
||||
|
||||
# 8. 构建 FFmpeg 命令
|
||||
# 6. 构建 FFmpeg 命令
|
||||
output_file = os.path.join(work_dir, 'output.mp4')
|
||||
cmd = self._build_command(
|
||||
input_file=input_file,
|
||||
@@ -179,25 +205,25 @@ class RenderSegmentVideoHandler(BaseHandler):
|
||||
source_duration_sec=source_duration_sec
|
||||
)
|
||||
|
||||
# 9. 执行 FFmpeg
|
||||
# 7. 执行 FFmpeg
|
||||
if not self.run_ffmpeg(cmd, task.task_id):
|
||||
return TaskResult.fail(
|
||||
ErrorCode.E_FFMPEG_FAILED,
|
||||
"FFmpeg rendering failed"
|
||||
)
|
||||
|
||||
# 10. 验证输出文件
|
||||
# 8. 验证输出文件
|
||||
if not self.ensure_file_exists(output_file, min_size=4096):
|
||||
return TaskResult.fail(
|
||||
ErrorCode.E_FFMPEG_FAILED,
|
||||
"Output file is missing or too small"
|
||||
)
|
||||
|
||||
# 11. 获取实际时长
|
||||
# 9. 获取实际时长
|
||||
actual_duration = self.probe_duration(output_file)
|
||||
actual_duration_ms = int(actual_duration * 1000) if actual_duration else duration_ms
|
||||
|
||||
# 12. 上传产物
|
||||
# 10. 上传产物
|
||||
video_url = self.upload_file(task.task_id, 'video', output_file)
|
||||
if not video_url:
|
||||
return TaskResult.fail(
|
||||
@@ -205,7 +231,7 @@ class RenderSegmentVideoHandler(BaseHandler):
|
||||
"Failed to upload video"
|
||||
)
|
||||
|
||||
# 13. 构建结果（包含 overlap 信息）
|
||||
# 11. 构建结果（包含 overlap 信息）
|
||||
result_data = {
|
||||
'videoUrl': video_url,
|
||||
'actualDurationMs': actual_duration_ms,
|
||||
|
||||
Reference in New Issue
Block a user