feat(base): 添加单任务内文件传输并发功能

- 引入 ThreadPoolExecutor 实现并行下载和上传
- 新增 download_files_parallel 和 upload_files_parallel 方法
- 添加任务传输并发数配置选项 TASK_DOWNLOAD_CONCURRENCY 和 TASK_UPLOAD_CONCURRENCY
- 实现并发数配置的环境变量解析和验证逻辑
- 在多个处理器中应用并行下载优化文件获取性能
- 更新 .env.example 配置文件模板
- 移除 FFmpeg 命令日志长度限制
This commit is contained in:
2026-02-07 00:38:43 +08:00
parent d955def63c
commit 88aa3adca1
7 changed files with 435 additions and 88 deletions

View File

@@ -91,23 +91,37 @@ class ComposeTransitionHandler(BaseHandler):
f"overlap_tail={overlap_tail_ms}ms, overlap_head={overlap_head_ms}ms"
)
# 1. 下载前一个片段视频
# 1. 并行下载前片段视频
prev_video_file = os.path.join(work_dir, 'prev_segment.mp4')
if not self.download_file(prev_segment['videoUrl'], prev_video_file):
next_video_file = os.path.join(work_dir, 'next_segment.mp4')
download_results = self.download_files_parallel([
{
'key': 'prev_video',
'url': prev_segment['videoUrl'],
'dest': prev_video_file,
'required': True
},
{
'key': 'next_video',
'url': next_segment['videoUrl'],
'dest': next_video_file,
'required': True
}
])
prev_result = download_results.get('prev_video')
if not prev_result or not prev_result['success']:
return TaskResult.fail(
ErrorCode.E_INPUT_UNAVAILABLE,
f"Failed to download prev segment video: {prev_segment['videoUrl']}"
)
# 2. 下载后一个片段视频
next_video_file = os.path.join(work_dir, 'next_segment.mp4')
if not self.download_file(next_segment['videoUrl'], next_video_file):
next_result = download_results.get('next_video')
if not next_result or not next_result['success']:
return TaskResult.fail(
ErrorCode.E_INPUT_UNAVAILABLE,
f"Failed to download next segment video: {next_segment['videoUrl']}"
)
# 3. 获取前一个片段的实际时长
# 2. 获取前一个片段的实际时长
prev_duration = self.probe_duration(prev_video_file)
if not prev_duration:
return TaskResult.fail(
@@ -115,7 +129,7 @@ class ComposeTransitionHandler(BaseHandler):
"Failed to probe prev segment duration"
)
# 4. 构建转场合成命令
# 3. 构建转场合成命令
output_file = os.path.join(work_dir, 'transition.mp4')
cmd = self._build_command(
prev_video_file=prev_video_file,
@@ -128,25 +142,25 @@ class ComposeTransitionHandler(BaseHandler):
output_spec=output_spec
)
# 5. 执行 FFmpeg
# 4. 执行 FFmpeg
if not self.run_ffmpeg(cmd, task.task_id):
return TaskResult.fail(
ErrorCode.E_FFMPEG_FAILED,
"FFmpeg transition composition failed"
)
# 6. 验证输出文件
# 5. 验证输出文件
if not self.ensure_file_exists(output_file, min_size=1024):
return TaskResult.fail(
ErrorCode.E_FFMPEG_FAILED,
"Transition output file is missing or too small"
)
# 7. 获取实际时长
# 6. 获取实际时长
actual_duration = self.probe_duration(output_file)
actual_duration_ms = int(actual_duration * 1000) if actual_duration else transition_duration_ms
# 8. 上传产物
# 7. 上传产物
transition_video_url = self.upload_file(task.task_id, 'video', output_file)
if not transition_video_url:
return TaskResult.fail(