feat(base): 添加单任务内文件传输并发功能

- 引入 ThreadPoolExecutor 实现并行下载和上传
- 新增 download_files_parallel 和 upload_files_parallel 方法
- 添加任务传输并发数配置选项 TASK_DOWNLOAD_CONCURRENCY 和 TASK_UPLOAD_CONCURRENCY
- 实现并发数配置的环境变量解析和验证逻辑
- 在多个处理器中应用并行下载优化文件获取性能
- 更新 .env.example 配置文件模板
- 移除 FFmpeg 命令日志长度限制
This commit is contained in:
2026-02-07 00:38:43 +08:00
parent d955def63c
commit 88aa3adca1
7 changed files with 435 additions and 88 deletions

View File

@@ -81,29 +81,43 @@ class PackageSegmentTsHandler(BaseHandler):
start_sec = start_time_ms / 1000.0
duration_sec = duration_ms / 1000.0
# 1. 下载视频片段
# 1. 并行下载视频片段与全局音频
video_file = os.path.join(work_dir, 'video.mp4')
if not self.download_file(video_url, video_file):
audio_file = os.path.join(work_dir, 'audio.aac')
download_results = self.download_files_parallel([
{
'key': 'video',
'url': video_url,
'dest': video_file,
'required': True
},
{
'key': 'audio',
'url': audio_url,
'dest': audio_file,
'required': True
}
])
video_result = download_results.get('video')
if not video_result or not video_result['success']:
return TaskResult.fail(
ErrorCode.E_INPUT_UNAVAILABLE,
f"Failed to download video: {video_url}"
)
# 2. 下载全局音频
audio_file = os.path.join(work_dir, 'audio.aac')
if not self.download_file(audio_url, audio_file):
audio_result = download_results.get('audio')
if not audio_result or not audio_result['success']:
return TaskResult.fail(
ErrorCode.E_INPUT_UNAVAILABLE,
f"Failed to download audio: {audio_url}"
)
# 3. 判断是否需要精确裁剪视频
# 2. 判断是否需要精确裁剪视频
needs_video_trim = not is_transition_segment and (
(trim_head and trim_head_ms > 0) or
(trim_tail and trim_tail_ms > 0)
)
# 4. 如果需要裁剪,先重编码裁剪视频
# 3. 如果需要裁剪,先重编码裁剪视频
processed_video_file = video_file
if needs_video_trim:
processed_video_file = os.path.join(work_dir, 'trimmed_video.mp4')
@@ -129,7 +143,7 @@ class PackageSegmentTsHandler(BaseHandler):
"Trimmed video file is missing or too small"
)
# 5. 构建 TS 封装命令
# 4. 构建 TS 封装命令
output_file = os.path.join(work_dir, 'segment.ts')
cmd = self._build_package_command(
video_file=processed_video_file,
@@ -139,14 +153,14 @@ class PackageSegmentTsHandler(BaseHandler):
duration_sec=duration_sec
)
# 6. 执行 FFmpeg
# 5. 执行 FFmpeg
if not self.run_ffmpeg(cmd, task.task_id):
return TaskResult.fail(
ErrorCode.E_FFMPEG_FAILED,
"TS packaging failed"
)
# 7. 验证输出文件
# 6. 验证输出文件
if not self.ensure_file_exists(output_file, min_size=1024):
return TaskResult.fail(
ErrorCode.E_FFMPEG_FAILED,