feat(base): 添加单任务内文件传输并发功能

- 引入 ThreadPoolExecutor 实现并行下载和上传
- 新增 download_files_parallel 和 upload_files_parallel 方法
- 添加任务传输并发数配置选项 TASK_DOWNLOAD_CONCURRENCY 和 TASK_UPLOAD_CONCURRENCY
- 实现并发数配置的环境变量解析和验证逻辑
- 在多个处理器中应用并行下载优化文件获取性能
- 更新 .env.example 配置文件模板
- 移除 FFmpeg 命令日志长度限制
This commit is contained in:
2026-02-07 00:38:43 +08:00
parent d955def63c
commit 88aa3adca1
7 changed files with 435 additions and 88 deletions

View File

@@ -110,16 +110,26 @@ class FinalizeMp4Handler(BaseHandler):
Returns:
TaskResult
"""
# 1. 下载所有 TS 分片
# 1. 并行下载所有 TS 分片
download_jobs = []
for i, ts_url in enumerate(ts_list):
download_jobs.append({
'key': str(i),
'url': ts_url,
'dest': os.path.join(work_dir, f'seg_{i}.ts'),
'required': True
})
download_results = self.download_files_parallel(download_jobs)
ts_files = []
for i, ts_url in enumerate(ts_list):
ts_file = os.path.join(work_dir, f'seg_{i}.ts')
if not self.download_file(ts_url, ts_file):
result = download_results.get(str(i))
if not result or not result['success']:
return TaskResult.fail(
ErrorCode.E_INPUT_UNAVAILABLE,
f"Failed to download TS segment {i}: {ts_url}"
)
ts_files.append(ts_file)
ts_files.append(result['dest'])
logger.info(f"[task:{task.task_id}] Downloaded {len(ts_files)} TS segments")