You've already forked FrameTour-RenderWorker
refactor(core): 移除旧版 FFmpeg 业务逻辑并重构常量配置
- 删除 biz/ffmpeg.py 和 biz/task.py 旧版业务模块 - 删除 entity/ffmpeg.py FFmpeg 任务实体类 - 删除 config/__init__.py 旧版配置初始化 - 更新 constant/__init__.py 常量定义,从 v1/v2 版本改为统一版本 - 修改 handlers/base.py 基础处理器,替换 OSS 相关导入为存储服务 - 添加 subprocess_args 工具函数支持跨平台进程参数配置 - 新增 probe_video_info 函数用于视频信息探测 - 新增 probe_duration_json 函数用于媒体时长探测
This commit is contained in:
@@ -2,15 +2,17 @@
|
||||
"""
|
||||
服务层
|
||||
|
||||
包含 API 客户端、任务执行器、租约服务等服务组件。
|
||||
包含 API 客户端、任务执行器、租约服务、存储服务等组件。
|
||||
"""
|
||||
|
||||
from services.api_client import APIClientV2
|
||||
from services.lease_service import LeaseService
|
||||
from services.task_executor import TaskExecutor
|
||||
from services import storage
|
||||
|
||||
__all__ = [
|
||||
'APIClientV2',
|
||||
'LeaseService',
|
||||
'TaskExecutor',
|
||||
'storage',
|
||||
]
|
||||
|
||||
200
services/storage.py
Normal file
200
services/storage.py
Normal file
@@ -0,0 +1,200 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
存储服务
|
||||
|
||||
提供文件上传/下载功能,支持 OSS 签名 URL 和 HTTP_REPLACE_MAP 环境变量。
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _apply_http_replace_map(url: str) -> str:
|
||||
"""
|
||||
应用 HTTP_REPLACE_MAP 环境变量替换 URL
|
||||
|
||||
Args:
|
||||
url: 原始 URL
|
||||
|
||||
Returns:
|
||||
替换后的 URL
|
||||
"""
|
||||
replace_map = os.getenv("HTTP_REPLACE_MAP", "")
|
||||
if not replace_map:
|
||||
return url
|
||||
|
||||
new_url = url
|
||||
replace_list = [i.split("|", 1) for i in replace_map.split(",") if "|" in i]
|
||||
for src, dst in replace_list:
|
||||
new_url = new_url.replace(src, dst)
|
||||
|
||||
if new_url != url:
|
||||
logger.debug(f"HTTP_REPLACE_MAP: {url} -> {new_url}")
|
||||
|
||||
return new_url
|
||||
|
||||
|
||||
def upload_file(url: str, file_path: str, max_retries: int = 5, timeout: int = 60) -> bool:
|
||||
"""
|
||||
使用签名 URL 上传文件到 OSS
|
||||
|
||||
Args:
|
||||
url: 签名 URL
|
||||
file_path: 本地文件路径
|
||||
max_retries: 最大重试次数
|
||||
timeout: 超时时间(秒)
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
if not os.path.exists(file_path):
|
||||
logger.error(f"File not found: {file_path}")
|
||||
return False
|
||||
|
||||
file_size = os.path.getsize(file_path)
|
||||
logger.info(f"Uploading: {file_path} ({file_size} bytes)")
|
||||
|
||||
# 检查是否使用 rclone 上传
|
||||
if os.getenv("UPLOAD_METHOD") == "rclone":
|
||||
result = _upload_with_rclone(url, file_path)
|
||||
if result:
|
||||
return True
|
||||
# rclone 失败时回退到 HTTP
|
||||
|
||||
# 应用 HTTP_REPLACE_MAP 替换 URL
|
||||
http_url = _apply_http_replace_map(url)
|
||||
|
||||
retries = 0
|
||||
while retries < max_retries:
|
||||
try:
|
||||
with open(file_path, 'rb') as f:
|
||||
response = requests.put(
|
||||
http_url,
|
||||
data=f,
|
||||
stream=True,
|
||||
timeout=timeout,
|
||||
headers={"Content-Type": "application/octet-stream"}
|
||||
)
|
||||
response.raise_for_status()
|
||||
logger.info(f"Upload succeeded: {file_path}")
|
||||
return True
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
retries += 1
|
||||
logger.warning(f"Upload timed out. Retrying {retries}/{max_retries}...")
|
||||
except requests.exceptions.RequestException as e:
|
||||
retries += 1
|
||||
logger.warning(f"Upload failed ({e}). Retrying {retries}/{max_retries}...")
|
||||
|
||||
logger.error(f"Upload failed after {max_retries} retries: {file_path}")
|
||||
return False
|
||||
|
||||
|
||||
def _upload_with_rclone(url: str, file_path: str) -> bool:
|
||||
"""
|
||||
使用 rclone 上传文件
|
||||
|
||||
Args:
|
||||
url: 目标 URL
|
||||
file_path: 本地文件路径
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
replace_map = os.getenv("RCLONE_REPLACE_MAP", "")
|
||||
if not replace_map:
|
||||
return False
|
||||
|
||||
config_file = os.getenv("RCLONE_CONFIG_FILE", "")
|
||||
rclone_config = f"--config {config_file}" if config_file else ""
|
||||
|
||||
# 替换 URL
|
||||
new_url = url
|
||||
replace_list = [i.split("|", 1) for i in replace_map.split(",") if "|" in i]
|
||||
for src, dst in replace_list:
|
||||
new_url = new_url.replace(src, dst)
|
||||
new_url = new_url.split("?", 1)[0] # 移除查询参数
|
||||
|
||||
if new_url == url:
|
||||
return False
|
||||
|
||||
cmd = (
|
||||
f"rclone copyto --no-check-dest --ignore-existing "
|
||||
f"--multi-thread-chunk-size 8M --multi-thread-streams 8 "
|
||||
f"{rclone_config} {file_path} {new_url}"
|
||||
)
|
||||
logger.debug(f"rclone command: {cmd}")
|
||||
|
||||
result = os.system(cmd)
|
||||
if result == 0:
|
||||
logger.info(f"rclone upload succeeded: {file_path}")
|
||||
return True
|
||||
|
||||
logger.warning(f"rclone upload failed (code={result}): {file_path}")
|
||||
return False
|
||||
|
||||
|
||||
def download_file(
|
||||
url: str,
|
||||
file_path: str,
|
||||
max_retries: int = 5,
|
||||
timeout: int = 30,
|
||||
skip_if_exist: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
使用签名 URL 下载文件
|
||||
|
||||
Args:
|
||||
url: 签名 URL
|
||||
file_path: 本地文件路径
|
||||
max_retries: 最大重试次数
|
||||
timeout: 超时时间(秒)
|
||||
skip_if_exist: 如果文件存在则跳过
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
# 如果文件已存在且跳过
|
||||
if skip_if_exist and os.path.exists(file_path):
|
||||
logger.debug(f"File exists, skipping download: {file_path}")
|
||||
return True
|
||||
|
||||
logger.info(f"Downloading: {url}")
|
||||
|
||||
# 确保目标目录存在
|
||||
file_dir = os.path.dirname(file_path)
|
||||
if file_dir:
|
||||
os.makedirs(file_dir, exist_ok=True)
|
||||
|
||||
# 应用 HTTP_REPLACE_MAP 替换 URL
|
||||
http_url = _apply_http_replace_map(url)
|
||||
|
||||
retries = 0
|
||||
while retries < max_retries:
|
||||
try:
|
||||
response = requests.get(http_url, timeout=timeout, stream=True)
|
||||
response.raise_for_status()
|
||||
|
||||
with open(file_path, 'wb') as f:
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
|
||||
file_size = os.path.getsize(file_path)
|
||||
logger.info(f"Download succeeded: {file_path} ({file_size} bytes)")
|
||||
return True
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
retries += 1
|
||||
logger.warning(f"Download timed out. Retrying {retries}/{max_retries}...")
|
||||
except requests.exceptions.RequestException as e:
|
||||
retries += 1
|
||||
logger.warning(f"Download failed ({e}). Retrying {retries}/{max_retries}...")
|
||||
|
||||
logger.error(f"Download failed after {max_retries} retries: {url}")
|
||||
return False
|
||||
Reference in New Issue
Block a user