You've already forked FrameTour-RenderWorker
refactor(core): 移除旧版 FFmpeg 业务逻辑并重构常量配置
- 删除 biz/ffmpeg.py 和 biz/task.py 旧版业务模块 - 删除 entity/ffmpeg.py FFmpeg 任务实体类 - 删除 config/__init__.py 旧版配置初始化 - 更新 constant/__init__.py 常量定义,从 v1/v2 版本改为统一版本 - 修改 handlers/base.py 基础处理器,替换 OSS 相关导入为存储服务 - 添加 subprocess_args 工具函数支持跨平台进程参数配置 - 新增 probe_video_info 函数用于视频信息探测 - 新增 probe_duration_json 函数用于媒体时长探测
This commit is contained in:
15
util/__init__.py
Normal file
15
util/__init__.py
Normal file
@@ -0,0 +1,15 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
工具模块
|
||||
|
||||
提供系统信息采集等工具函数。
|
||||
"""
|
||||
|
||||
from util.system import get_sys_info, get_capabilities, get_gpu_info, get_ffmpeg_version
|
||||
|
||||
__all__ = [
|
||||
'get_sys_info',
|
||||
'get_capabilities',
|
||||
'get_gpu_info',
|
||||
'get_ffmpeg_version',
|
||||
]
|
||||
260
util/api.py
260
util/api.py
@@ -1,260 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
|
||||
import requests
|
||||
from opentelemetry.trace import Status, StatusCode
|
||||
|
||||
import util.system
|
||||
from telemetry import get_tracer
|
||||
from util import oss
|
||||
|
||||
session = requests.Session()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def normalize_task(task_info):
|
||||
...
|
||||
return task_info
|
||||
|
||||
|
||||
def sync_center():
|
||||
"""
|
||||
通过接口获取任务
|
||||
:return: 任务列表
|
||||
"""
|
||||
from template import TEMPLATES, download_template
|
||||
try:
|
||||
response = session.post(os.getenv('API_ENDPOINT') + "/sync", json={
|
||||
'accessKey': os.getenv('ACCESS_KEY'),
|
||||
'clientStatus': util.system.get_sys_info(),
|
||||
'templateList': [{'id': t.get('id', ''), 'updateTime': t.get('updateTime', '')} for t in
|
||||
TEMPLATES.values()]
|
||||
}, timeout=10)
|
||||
response.raise_for_status()
|
||||
except requests.RequestException as e:
|
||||
logger.error("请求失败!", e)
|
||||
return []
|
||||
data = response.json()
|
||||
logger.debug("获取任务结果:【%s】", data)
|
||||
if data.get('code', 0) == 200:
|
||||
templates = data.get('data', {}).get('templates', [])
|
||||
tasks = data.get('data', {}).get('tasks', [])
|
||||
else:
|
||||
tasks = []
|
||||
templates = []
|
||||
logger.warning("获取任务失败")
|
||||
if os.getenv("REDIRECT_TO_URL", False) != False:
|
||||
for task in tasks:
|
||||
_sess = requests.Session()
|
||||
logger.info("重定向任务【%s】至配置的地址:%s", task.get("id"), os.getenv("REDIRECT_TO_URL"))
|
||||
url = f"{os.getenv('REDIRECT_TO_URL')}{task.get('id')}"
|
||||
threading.Thread(target=requests.post, args=(url,)).start()
|
||||
return []
|
||||
for template in templates:
|
||||
template_id = template.get('id', '')
|
||||
if template_id:
|
||||
logger.info("更新模板:【%s】", template_id)
|
||||
download_template(template_id)
|
||||
return tasks
|
||||
|
||||
|
||||
def get_template_info(template_id):
|
||||
"""
|
||||
通过接口获取模板信息
|
||||
:rtype: Template
|
||||
:param template_id: 模板id
|
||||
:type template_id: str
|
||||
:return: 模板信息
|
||||
"""
|
||||
tracer = get_tracer(__name__)
|
||||
with tracer.start_as_current_span("get_template_info"):
|
||||
with tracer.start_as_current_span("get_template_info.request") as req_span:
|
||||
try:
|
||||
req_span.set_attribute("http.method", "POST")
|
||||
req_span.set_attribute("http.url", '{0}/template/{1}'.format(os.getenv('API_ENDPOINT'), template_id))
|
||||
response = session.post('{0}/template/{1}'.format(os.getenv('API_ENDPOINT'), template_id), json={
|
||||
'accessKey': os.getenv('ACCESS_KEY'),
|
||||
}, timeout=10)
|
||||
req_span.set_attribute("http.status_code", response.status_code)
|
||||
req_span.set_attribute("http.response", response.text)
|
||||
response.raise_for_status()
|
||||
except requests.RequestException as e:
|
||||
req_span.set_attribute("api.error", str(e))
|
||||
logger.error("请求失败!", e)
|
||||
return None
|
||||
data = response.json()
|
||||
logger.debug("获取模板信息结果:【%s】", data)
|
||||
remote_template_info = data.get('data', {})
|
||||
if not remote_template_info:
|
||||
logger.warning("获取模板信息结果为空", data)
|
||||
return None
|
||||
template = {
|
||||
'id': template_id,
|
||||
'updateTime': remote_template_info.get('updateTime', template_id),
|
||||
'scenic_name': remote_template_info.get('scenicName', '景区'),
|
||||
'name': remote_template_info.get('name', '模版'),
|
||||
'video_size': remote_template_info.get('resolution', '1920x1080'),
|
||||
'frame_rate': 25,
|
||||
'overall_duration': 30,
|
||||
'video_parts': [
|
||||
|
||||
]
|
||||
}
|
||||
|
||||
def _template_normalizer(template_info):
|
||||
_template = {}
|
||||
_placeholder_type = template_info.get('isPlaceholder', -1)
|
||||
if _placeholder_type == 0:
|
||||
# 固定视频
|
||||
_template['source'] = template_info.get('sourceUrl', '')
|
||||
elif _placeholder_type == 1:
|
||||
# 占位符
|
||||
_template['source'] = "PLACEHOLDER_" + template_info.get('sourceUrl', '')
|
||||
_template['mute'] = template_info.get('mute', True)
|
||||
_template['crop_mode'] = template_info.get('cropEnable', None)
|
||||
_template['zoom_cut'] = template_info.get('zoomCut', None)
|
||||
else:
|
||||
_template['source'] = None
|
||||
_overlays = template_info.get('overlays', '')
|
||||
if _overlays:
|
||||
_template['overlays'] = _overlays.split(",")
|
||||
_audios = template_info.get('audios', '')
|
||||
if _audios:
|
||||
_template['audios'] = _audios.split(",")
|
||||
_luts = template_info.get('luts', '')
|
||||
if _luts:
|
||||
_template['luts'] = _luts.split(",")
|
||||
_only_if = template_info.get('onlyIf', '')
|
||||
if _only_if:
|
||||
_template['only_if'] = _only_if
|
||||
_effects = template_info.get('effects', '')
|
||||
if _effects:
|
||||
_template['effects'] = _effects.split("|")
|
||||
return _template
|
||||
|
||||
# outer template definition
|
||||
overall_template = _template_normalizer(remote_template_info)
|
||||
template['overall_template'] = overall_template
|
||||
# inter template definition
|
||||
inter_template_list = remote_template_info.get('children', [])
|
||||
for children_template in inter_template_list:
|
||||
parts = _template_normalizer(children_template)
|
||||
template['video_parts'].append(parts)
|
||||
template['local_path'] = os.path.join(os.getenv('TEMPLATE_DIR'), str(template_id))
|
||||
with get_tracer("api").start_as_current_span("get_template_info.template") as res_span:
|
||||
res_span.set_attribute("normalized.response", json.dumps(template))
|
||||
return template
|
||||
|
||||
|
||||
def report_task_success(task_info, **kwargs):
|
||||
tracer = get_tracer(__name__)
|
||||
with tracer.start_as_current_span("report_task_success"):
|
||||
with tracer.start_as_current_span("report_task_success.request") as req_span:
|
||||
try:
|
||||
req_span.set_attribute("http.method", "POST")
|
||||
req_span.set_attribute("http.url",
|
||||
'{0}/{1}/success'.format(os.getenv('API_ENDPOINT'), task_info.get("id")))
|
||||
response = session.post('{0}/{1}/success'.format(os.getenv('API_ENDPOINT'), task_info.get("id")), json={
|
||||
'accessKey': os.getenv('ACCESS_KEY'),
|
||||
**kwargs
|
||||
}, timeout=10)
|
||||
req_span.set_attribute("http.status_code", response.status_code)
|
||||
req_span.set_attribute("http.response", response.text)
|
||||
response.raise_for_status()
|
||||
req_span.set_status(Status(StatusCode.OK))
|
||||
except requests.RequestException as e:
|
||||
req_span.set_attribute("api.error", str(e))
|
||||
logger.error("请求失败!", e)
|
||||
return None
|
||||
|
||||
|
||||
def report_task_start(task_info):
|
||||
tracer = get_tracer(__name__)
|
||||
with tracer.start_as_current_span("report_task_start"):
|
||||
with tracer.start_as_current_span("report_task_start.request") as req_span:
|
||||
try:
|
||||
req_span.set_attribute("http.method", "POST")
|
||||
req_span.set_attribute("http.url",
|
||||
'{0}/{1}/start'.format(os.getenv('API_ENDPOINT'), task_info.get("id")))
|
||||
response = session.post('{0}/{1}/start'.format(os.getenv('API_ENDPOINT'), task_info.get("id")), json={
|
||||
'accessKey': os.getenv('ACCESS_KEY'),
|
||||
}, timeout=10)
|
||||
req_span.set_attribute("http.status_code", response.status_code)
|
||||
req_span.set_attribute("http.response", response.text)
|
||||
response.raise_for_status()
|
||||
req_span.set_status(Status(StatusCode.OK))
|
||||
except requests.RequestException as e:
|
||||
req_span.set_attribute("api.error", str(e))
|
||||
logger.error("请求失败!", e)
|
||||
return None
|
||||
|
||||
|
||||
def report_task_failed(task_info, reason=''):
|
||||
tracer = get_tracer(__name__)
|
||||
with tracer.start_as_current_span("report_task_failed") as span:
|
||||
span.set_attribute("task_id", task_info.get("id"))
|
||||
span.set_attribute("reason", reason)
|
||||
with tracer.start_as_current_span("report_task_failed.request") as req_span:
|
||||
try:
|
||||
req_span.set_attribute("http.method", "POST")
|
||||
req_span.set_attribute("http.url",
|
||||
'{0}/{1}/fail'.format(os.getenv('API_ENDPOINT'), task_info.get("id")))
|
||||
response = session.post('{0}/{1}/fail'.format(os.getenv('API_ENDPOINT'), task_info.get("id")), json={
|
||||
'accessKey': os.getenv('ACCESS_KEY'),
|
||||
'reason': reason
|
||||
}, timeout=10)
|
||||
req_span.set_attribute("http.status_code", response.status_code)
|
||||
req_span.set_attribute("http.response", response.text)
|
||||
response.raise_for_status()
|
||||
req_span.set_status(Status(StatusCode.OK))
|
||||
except requests.RequestException as e:
|
||||
req_span.set_attribute("api.error", str(e))
|
||||
req_span.set_status(Status(StatusCode.ERROR))
|
||||
logger.error("请求失败!", e)
|
||||
return None
|
||||
|
||||
|
||||
def upload_task_file(task_info, ffmpeg_task):
|
||||
tracer = get_tracer(__name__)
|
||||
with get_tracer("api").start_as_current_span("upload_task_file") as span:
|
||||
logger.info("开始上传文件: %s", task_info.get("id"))
|
||||
span.set_attribute("file.id", task_info.get("id"))
|
||||
with tracer.start_as_current_span("upload_task_file.request_upload_url") as req_span:
|
||||
try:
|
||||
req_span.set_attribute("http.method", "POST")
|
||||
req_span.set_attribute("http.url",
|
||||
'{0}/{1}/uploadUrl'.format(os.getenv('API_ENDPOINT'), task_info.get("id")))
|
||||
response = session.post('{0}/{1}/uploadUrl'.format(os.getenv('API_ENDPOINT'), task_info.get("id")),
|
||||
json={
|
||||
'accessKey': os.getenv('ACCESS_KEY'),
|
||||
}, timeout=10)
|
||||
req_span.set_attribute("http.status_code", response.status_code)
|
||||
req_span.set_attribute("http.response", response.text)
|
||||
response.raise_for_status()
|
||||
req_span.set_status(Status(StatusCode.OK))
|
||||
except requests.RequestException as e:
|
||||
span.set_attribute("api.error", str(e))
|
||||
req_span.set_status(Status(StatusCode.ERROR))
|
||||
logger.error("请求失败!", e)
|
||||
return False
|
||||
data = response.json()
|
||||
url = data.get('data', "")
|
||||
logger.info("开始上传文件: %s 至 %s", task_info.get("id"), url)
|
||||
return oss.upload_to_oss(url, ffmpeg_task.get_output_file())
|
||||
|
||||
|
||||
def get_task_info(id):
|
||||
try:
|
||||
response = session.get(os.getenv('API_ENDPOINT') + "/" + id + "/info", params={
|
||||
'accessKey': os.getenv('ACCESS_KEY'),
|
||||
}, timeout=10)
|
||||
response.raise_for_status()
|
||||
except requests.RequestException as e:
|
||||
logger.error("请求失败!", e)
|
||||
return []
|
||||
data = response.json()
|
||||
logger.debug("获取任务结果:【%s】", data)
|
||||
if data.get('code', 0) == 200:
|
||||
return data.get('data', {})
|
||||
257
util/ffmpeg.py
257
util/ffmpeg.py
@@ -1,257 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from typing import Optional, IO
|
||||
|
||||
from opentelemetry.trace import Status, StatusCode
|
||||
|
||||
from entity.ffmpeg import FfmpegTask, ENCODER_ARGS, VIDEO_ARGS, AUDIO_ARGS, MUTE_AUDIO_INPUT, get_mp4toannexb_filter
|
||||
from telemetry import get_tracer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def re_encode_and_annexb(file):
|
||||
with get_tracer("ffmpeg").start_as_current_span("re_encode_and_annexb") as span:
|
||||
span.set_attribute("file.path", file)
|
||||
if not os.path.exists(file):
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
return file
|
||||
logger.info("ReEncodeAndAnnexb: %s", file)
|
||||
has_audio = not not probe_video_audio(file)
|
||||
# 优先使用RE_ENCODE_VIDEO_ARGS环境变量,其次使用默认的VIDEO_ARGS
|
||||
if os.getenv("RE_ENCODE_VIDEO_ARGS", False):
|
||||
_video_args = tuple(os.getenv("RE_ENCODE_VIDEO_ARGS", "").split(" "))
|
||||
else:
|
||||
_video_args = VIDEO_ARGS
|
||||
# 优先使用RE_ENCODE_ENCODER_ARGS环境变量,其次使用默认的ENCODER_ARGS
|
||||
if os.getenv("RE_ENCODE_ENCODER_ARGS", False):
|
||||
_encoder_args = tuple(os.getenv("RE_ENCODE_ENCODER_ARGS", "").split(" "))
|
||||
else:
|
||||
_encoder_args = ENCODER_ARGS
|
||||
ffmpeg_process = subprocess.run(["ffmpeg", "-y", "-hide_banner", "-i", file,
|
||||
*(set() if has_audio else MUTE_AUDIO_INPUT),
|
||||
"-fps_mode", "cfr",
|
||||
"-map", "0:v", "-map", "0:a" if has_audio else "1:a",
|
||||
*_video_args, "-bsf:v", get_mp4toannexb_filter(),
|
||||
*AUDIO_ARGS, "-bsf:a", "setts=pts=DTS",
|
||||
*_encoder_args, "-shortest", "-fflags", "+genpts",
|
||||
"-f", "mpegts", file + ".ts"])
|
||||
logger.info(" ".join(ffmpeg_process.args))
|
||||
span.set_attribute("ffmpeg.args", json.dumps(ffmpeg_process.args))
|
||||
logger.info("ReEncodeAndAnnexb: %s, returned: %s", file, ffmpeg_process.returncode)
|
||||
span.set_attribute("ffmpeg.code", ffmpeg_process.returncode)
|
||||
if ffmpeg_process.returncode == 0:
|
||||
span.set_status(Status(StatusCode.OK))
|
||||
span.set_attribute("file.size", os.path.getsize(file+".ts"))
|
||||
# os.remove(file)
|
||||
return file+".ts"
|
||||
else:
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
return file
|
||||
|
||||
def start_render(ffmpeg_task: FfmpegTask):
|
||||
tracer = get_tracer(__name__)
|
||||
with tracer.start_as_current_span("start_render") as span:
|
||||
span.set_attribute("ffmpeg.task", str(ffmpeg_task))
|
||||
if not ffmpeg_task.need_run():
|
||||
ffmpeg_task.set_output_file(ffmpeg_task.input_file[0])
|
||||
span.set_status(Status(StatusCode.OK))
|
||||
return True
|
||||
ffmpeg_args = ffmpeg_task.get_ffmpeg_args()
|
||||
if len(ffmpeg_args) == 0:
|
||||
ffmpeg_task.set_output_file(ffmpeg_task.input_file[0])
|
||||
span.set_status(Status(StatusCode.OK))
|
||||
return True
|
||||
# 通过环境变量传入通用FFmpeg参数
|
||||
common_args = os.getenv("FFMPEG_COMMON_ARGS", "").split() if os.getenv("FFMPEG_COMMON_ARGS") else []
|
||||
ffmpeg_process = subprocess.run(["ffmpeg", "-progress", "-", "-loglevel", "error", *common_args, *ffmpeg_args], stderr=subprocess.PIPE, **subprocess_args(True))
|
||||
span.set_attribute("ffmpeg.args", json.dumps(ffmpeg_process.args))
|
||||
logger.info(" ".join(ffmpeg_process.args))
|
||||
ffmpeg_final_out = handle_ffmpeg_output(ffmpeg_process.stdout)
|
||||
span.set_attribute("ffmpeg.out", ffmpeg_final_out)
|
||||
logger.info("FINISH TASK, OUTPUT IS %s", ffmpeg_final_out)
|
||||
code = ffmpeg_process.returncode
|
||||
span.set_attribute("ffmpeg.code", code)
|
||||
if code != 0:
|
||||
span.set_attribute("ffmpeg.err", str(ffmpeg_process.stderr))
|
||||
span.set_status(Status(StatusCode.ERROR, "FFMPEG异常退出"))
|
||||
logger.error("FFMPEG ERROR: %s", ffmpeg_process.stderr)
|
||||
return False
|
||||
span.set_attribute("ffmpeg.out_file", ffmpeg_task.output_file)
|
||||
try:
|
||||
file_size = os.path.getsize(ffmpeg_task.output_file)
|
||||
span.set_attribute("file.size", file_size)
|
||||
if file_size < 4096:
|
||||
span.set_status(Status(StatusCode.ERROR, "输出文件过小"))
|
||||
logger.error("FFMPEG ERROR: OUTPUT FILE IS TOO SMALL")
|
||||
return False
|
||||
except OSError as e:
|
||||
span.set_attribute("file.size", 0)
|
||||
span.set_attribute("file.error", e.strerror)
|
||||
span.set_status(Status(StatusCode.ERROR, "输出文件不存在"))
|
||||
logger.error("FFMPEG ERROR: OUTPUT FILE NOT FOUND")
|
||||
return False
|
||||
span.set_status(Status(StatusCode.OK))
|
||||
return True
|
||||
|
||||
def handle_ffmpeg_output(stdout: Optional[bytes]) -> str:
|
||||
out_time = "0:0:0.0"
|
||||
if stdout is None:
|
||||
print("[!]STDOUT is null")
|
||||
return out_time
|
||||
speed = "0"
|
||||
for line in stdout.split(b"\n"):
|
||||
if line == b"":
|
||||
break
|
||||
if line.strip() == b"progress=end":
|
||||
# 处理完毕
|
||||
break
|
||||
if line.startswith(b"out_time="):
|
||||
out_time = line.replace(b"out_time=", b"").decode().strip()
|
||||
if line.startswith(b"speed="):
|
||||
speed = line.replace(b"speed=", b"").decode().strip()
|
||||
print("[ ]Speed:", out_time, "@", speed)
|
||||
return out_time+"@"+speed
|
||||
|
||||
def duration_str_to_float(duration_str: str) -> float:
|
||||
_duration = datetime.strptime(duration_str, "%H:%M:%S.%f") - datetime(1900, 1, 1)
|
||||
return _duration.total_seconds()
|
||||
|
||||
|
||||
def probe_video_info(video_file):
|
||||
tracer = get_tracer(__name__)
|
||||
with tracer.start_as_current_span("probe_video_info") as span:
|
||||
span.set_attribute("video.file", video_file)
|
||||
# 获取宽度和高度
|
||||
result = subprocess.run(
|
||||
["ffprobe", '-v', 'error', '-select_streams', 'v:0', '-show_entries', 'stream=width,height:format=duration', '-of',
|
||||
'csv=s=x:p=0', video_file],
|
||||
stderr=subprocess.STDOUT,
|
||||
**subprocess_args(True)
|
||||
)
|
||||
span.set_attribute("ffprobe.args", json.dumps(result.args))
|
||||
span.set_attribute("ffprobe.code", result.returncode)
|
||||
if result.returncode != 0:
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
return 0, 0, 0
|
||||
all_result = result.stdout.decode('utf-8').strip()
|
||||
span.set_attribute("ffprobe.out", all_result)
|
||||
if all_result == '':
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
return 0, 0, 0
|
||||
span.set_status(Status(StatusCode.OK))
|
||||
wh, duration = all_result.split('\n')
|
||||
width, height = wh.strip().split('x')
|
||||
return int(width), int(height), float(duration)
|
||||
|
||||
|
||||
def probe_video_audio(video_file, type=None):
|
||||
tracer = get_tracer(__name__)
|
||||
with tracer.start_as_current_span("probe_video_audio") as span:
|
||||
span.set_attribute("video.file", video_file)
|
||||
args = ["ffprobe", "-hide_banner", "-v", "error", "-select_streams", "a", "-show_entries", "stream=index", "-of", "csv=p=0"]
|
||||
if type == 'concat':
|
||||
args.append("-safe")
|
||||
args.append("0")
|
||||
args.append("-f")
|
||||
args.append("concat")
|
||||
args.append(video_file)
|
||||
logger.info(" ".join(args))
|
||||
result = subprocess.run(args, stderr=subprocess.STDOUT, **subprocess_args(True))
|
||||
span.set_attribute("ffprobe.args", json.dumps(result.args))
|
||||
span.set_attribute("ffprobe.code", result.returncode)
|
||||
logger.info("probe_video_audio: %s", result.stdout.decode('utf-8').strip())
|
||||
if result.returncode != 0:
|
||||
return False
|
||||
if result.stdout.decode('utf-8').strip() == '':
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# 音频淡出2秒
|
||||
def fade_out_audio(file, duration, fade_out_sec = 2):
|
||||
if type(duration) == str:
|
||||
try:
|
||||
duration = float(duration)
|
||||
except Exception as e:
|
||||
logger.error("duration is not float: %s", e)
|
||||
return file
|
||||
tracer = get_tracer(__name__)
|
||||
with tracer.start_as_current_span("fade_out_audio") as span:
|
||||
span.set_attribute("audio.file", file)
|
||||
if duration <= fade_out_sec:
|
||||
return file
|
||||
else:
|
||||
new_fn = file + "_.mp4"
|
||||
if os.path.exists(new_fn):
|
||||
os.remove(new_fn)
|
||||
logger.info("delete tmp file: " + new_fn)
|
||||
try:
|
||||
process = subprocess.run(["ffmpeg", "-i", file, "-c:v", "copy", "-c:a", "aac", "-af", "afade=t=out:st=" + str(duration - fade_out_sec) + ":d=" + str(fade_out_sec), "-y", new_fn], **subprocess_args(True))
|
||||
span.set_attribute("ffmpeg.args", json.dumps(process.args))
|
||||
logger.info(" ".join(process.args))
|
||||
if process.returncode != 0:
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
logger.error("FFMPEG ERROR: %s", process.stderr)
|
||||
return file
|
||||
else:
|
||||
span.set_status(Status(StatusCode.OK))
|
||||
return new_fn
|
||||
except Exception as e:
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
logger.error("FFMPEG ERROR: %s", e)
|
||||
return file
|
||||
|
||||
|
||||
|
||||
# Create a set of arguments which make a ``subprocess.Popen`` (and
|
||||
# variants) call work with or without Pyinstaller, ``--noconsole`` or
|
||||
# not, on Windows and Linux. Typical use::
|
||||
#
|
||||
# subprocess.call(['program_to_run', 'arg_1'], **subprocess_args())
|
||||
#
|
||||
# When calling ``check_output``::
|
||||
#
|
||||
# subprocess.check_output(['program_to_run', 'arg_1'],
|
||||
# **subprocess_args(False))
|
||||
def subprocess_args(include_stdout=True):
|
||||
# The following is true only on Windows.
|
||||
if hasattr(subprocess, 'STARTUPINFO'):
|
||||
# On Windows, subprocess calls will pop up a command window by default
|
||||
# when run from Pyinstaller with the ``--noconsole`` option. Avoid this
|
||||
# distraction.
|
||||
si = subprocess.STARTUPINFO()
|
||||
si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
|
||||
# Windows doesn't search the path by default. Pass it an environment so
|
||||
# it will.
|
||||
env = os.environ
|
||||
else:
|
||||
si = None
|
||||
env = None
|
||||
|
||||
# ``subprocess.check_output`` doesn't allow specifying ``stdout``::
|
||||
#
|
||||
# Traceback (most recent call last):
|
||||
# File "test_subprocess.py", line 58, in <module>
|
||||
# **subprocess_args(stdout=None))
|
||||
# File "C:\Python27\lib\subprocess.py", line 567, in check_output
|
||||
# raise ValueError('stdout argument not allowed, it will be overridden.')
|
||||
# ValueError: stdout argument not allowed, it will be overridden.
|
||||
#
|
||||
# So, add it only if it's needed.
|
||||
if include_stdout:
|
||||
ret = {'stdout': subprocess.PIPE}
|
||||
else:
|
||||
ret = {}
|
||||
|
||||
# On Windows, running this from the binary produced by Pyinstaller
|
||||
# with the ``--noconsole`` option requires redirecting everything
|
||||
# (stdin, stdout, stderr) to avoid an OSError exception
|
||||
# "[Error 6] the handle is invalid."
|
||||
ret.update({'stdin': subprocess.PIPE,
|
||||
'startupinfo': si,
|
||||
'env': env})
|
||||
return ret
|
||||
|
||||
155
util/oss.py
155
util/oss.py
@@ -1,155 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
import requests
|
||||
from opentelemetry.trace import Status, StatusCode
|
||||
|
||||
from telemetry import get_tracer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _apply_http_replace_map(url):
|
||||
"""
|
||||
应用 HTTP_REPLACE_MAP 环境变量替换 URL
|
||||
:param str url: 原始 URL
|
||||
:return str: 替换后的 URL
|
||||
"""
|
||||
replace_map = os.getenv("HTTP_REPLACE_MAP", "")
|
||||
if not replace_map:
|
||||
return url
|
||||
replace_list = [i.split("|", 1) for i in replace_map.split(",")]
|
||||
new_url = url
|
||||
for (_src, _dst) in replace_list:
|
||||
new_url = new_url.replace(_src, _dst)
|
||||
if new_url != url:
|
||||
logger.debug(f"HTTP_REPLACE_MAP: {url} -> {new_url}")
|
||||
return new_url
|
||||
|
||||
|
||||
def upload_to_oss(url, file_path):
|
||||
"""
|
||||
使用签名URL上传文件到OSS
|
||||
:param str url: 签名URL
|
||||
:param str file_path: 文件路径
|
||||
:return bool: 是否成功
|
||||
"""
|
||||
tracer = get_tracer(__name__)
|
||||
with tracer.start_as_current_span("upload_to_oss") as span:
|
||||
span.set_attribute("file.url", url)
|
||||
span.set_attribute("file.path", file_path)
|
||||
span.set_attribute("file.size", os.path.getsize(file_path))
|
||||
max_retries = 5
|
||||
retries = 0
|
||||
if os.getenv("UPLOAD_METHOD") == "rclone":
|
||||
with tracer.start_as_current_span("rclone_to_oss") as r_span:
|
||||
replace_map = os.getenv("RCLONE_REPLACE_MAP")
|
||||
config_file = os.getenv("RCLONE_CONFIG_FILE")
|
||||
rclone_config = ""
|
||||
if config_file != "":
|
||||
rclone_config = f"--config {config_file}"
|
||||
r_span.set_attribute("rclone.replace_map", replace_map)
|
||||
if replace_map != "":
|
||||
replace_list = [i.split("|", 1) for i in replace_map.split(",")]
|
||||
new_url = url
|
||||
for (_src, _dst) in replace_list:
|
||||
new_url = new_url.replace(_src, _dst)
|
||||
new_url = new_url.split("?", 1)[0]
|
||||
r_span.set_attribute("rclone.target_dir", new_url)
|
||||
if new_url != url:
|
||||
result = os.system(f"rclone copyto --no-check-dest --ignore-existing --multi-thread-chunk-size 8M --multi-thread-streams 8 {rclone_config} {file_path} {new_url}")
|
||||
r_span.set_attribute("rclone.result", result)
|
||||
if result == 0:
|
||||
span.set_status(Status(StatusCode.OK))
|
||||
return True
|
||||
else:
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
# 应用 HTTP_REPLACE_MAP 替换 URL
|
||||
http_url = _apply_http_replace_map(url)
|
||||
span.set_attribute("file.http_url", http_url)
|
||||
while retries < max_retries:
|
||||
with tracer.start_as_current_span("upload_to_oss.request") as req_span:
|
||||
req_span.set_attribute("http.retry_count", retries)
|
||||
try:
|
||||
req_span.set_attribute("http.method", "PUT")
|
||||
req_span.set_attribute("http.url", http_url)
|
||||
with open(file_path, 'rb') as f:
|
||||
response = requests.put(http_url, data=f, stream=True, timeout=60, headers={"Content-Type": "video/mp4"})
|
||||
req_span.set_attribute("http.status_code", response.status_code)
|
||||
req_span.set_attribute("http.response", response.text)
|
||||
response.raise_for_status()
|
||||
req_span.set_status(Status(StatusCode.OK))
|
||||
span.set_status(Status(StatusCode.OK))
|
||||
return True
|
||||
except requests.exceptions.Timeout:
|
||||
req_span.set_attribute("http.error", "Timeout")
|
||||
req_span.set_status(Status(StatusCode.ERROR))
|
||||
retries += 1
|
||||
logger.warning(f"Upload timed out. Retrying {retries}/{max_retries}...")
|
||||
except Exception as e:
|
||||
req_span.set_attribute("http.error", str(e))
|
||||
req_span.set_status(Status(StatusCode.ERROR))
|
||||
retries += 1
|
||||
logger.warning(f"Upload failed. Retrying {retries}/{max_retries}...")
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
return False
|
||||
|
||||
|
||||
def download_from_oss(url, file_path, skip_if_exist=None):
|
||||
"""
|
||||
使用签名URL下载文件到OSS
|
||||
:param skip_if_exist: 如果存在就不下载了
|
||||
:param str url: 签名URL
|
||||
:param Union[LiteralString, str, bytes] file_path: 文件路径
|
||||
:return bool: 是否成功
|
||||
"""
|
||||
tracer = get_tracer(__name__)
|
||||
with tracer.start_as_current_span("download_from_oss") as span:
|
||||
span.set_attribute("file.url", url)
|
||||
span.set_attribute("file.path", file_path)
|
||||
|
||||
# 如果skip_if_exist为None,则从启动参数中读取
|
||||
if skip_if_exist is None:
|
||||
skip_if_exist = 'skip_if_exist' in sys.argv
|
||||
|
||||
if skip_if_exist and os.path.exists(file_path):
|
||||
span.set_attribute("file.exist", True)
|
||||
span.set_attribute("file.size", os.path.getsize(file_path))
|
||||
return True
|
||||
logging.info("download_from_oss: %s", url)
|
||||
file_dir, file_name = os.path.split(file_path)
|
||||
if file_dir:
|
||||
if not os.path.exists(file_dir):
|
||||
os.makedirs(file_dir)
|
||||
# 应用 HTTP_REPLACE_MAP 替换 URL
|
||||
http_url = _apply_http_replace_map(url)
|
||||
span.set_attribute("file.http_url", http_url)
|
||||
max_retries = 5
|
||||
retries = 0
|
||||
while retries < max_retries:
|
||||
with tracer.start_as_current_span("download_from_oss.request") as req_span:
|
||||
req_span.set_attribute("http.retry_count", retries)
|
||||
try:
|
||||
req_span.set_attribute("http.method", "GET")
|
||||
req_span.set_attribute("http.url", http_url)
|
||||
response = requests.get(http_url, timeout=15) # 设置超时时间
|
||||
req_span.set_attribute("http.status_code", response.status_code)
|
||||
with open(file_path, 'wb') as f:
|
||||
f.write(response.content)
|
||||
req_span.set_attribute("file.size", os.path.getsize(file_path))
|
||||
req_span.set_status(Status(StatusCode.OK))
|
||||
span.set_status(Status(StatusCode.OK))
|
||||
return True
|
||||
except requests.exceptions.Timeout:
|
||||
req_span.set_attribute("http.error", "Timeout")
|
||||
req_span.set_status(Status(StatusCode.ERROR))
|
||||
retries += 1
|
||||
logger.warning(f"Download timed out. Retrying {retries}/{max_retries}...")
|
||||
except Exception as e:
|
||||
req_span.set_attribute("http.error", str(e))
|
||||
req_span.set_status(Status(StatusCode.ERROR))
|
||||
retries += 1
|
||||
logger.warning(f"Download failed. Retrying {retries}/{max_retries}...")
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
return False
|
||||
@@ -7,40 +7,20 @@
|
||||
|
||||
import os
|
||||
import platform
|
||||
from datetime import datetime
|
||||
import subprocess
|
||||
from typing import Optional
|
||||
|
||||
import psutil
|
||||
from constant import SUPPORT_FEATURE, SOFTWARE_VERSION, V2_DEFAULT_CAPABILITIES
|
||||
from constant import SOFTWARE_VERSION, DEFAULT_CAPABILITIES
|
||||
|
||||
|
||||
def get_sys_info():
|
||||
"""
|
||||
获取系统信息(v1 格式)
|
||||
获取系统信息
|
||||
|
||||
Returns:
|
||||
dict: 系统信息字典
|
||||
"""
|
||||
info = {
|
||||
'version': SOFTWARE_VERSION,
|
||||
'client_datetime': datetime.now().isoformat(),
|
||||
'platform': platform.system(),
|
||||
'runtime_version': 'Python ' + platform.python_version(),
|
||||
'cpu_count': os.cpu_count(),
|
||||
'cpu_usage': psutil.cpu_percent(),
|
||||
'memory_total': psutil.virtual_memory().total,
|
||||
'memory_available': psutil.virtual_memory().available,
|
||||
'support_feature': SUPPORT_FEATURE
|
||||
}
|
||||
return info
|
||||
|
||||
|
||||
def get_sys_info_v2():
|
||||
"""
|
||||
获取系统信息(v2 格式)
|
||||
|
||||
Returns:
|
||||
dict: v2 API 所需的系统信息字典
|
||||
"""
|
||||
mem = psutil.virtual_memory()
|
||||
|
||||
info = {
|
||||
@@ -51,10 +31,11 @@ def get_sys_info_v2():
|
||||
'memoryAvailable': f"{mem.available // (1024**3)}GB",
|
||||
'platform': platform.system(),
|
||||
'pythonVersion': platform.python_version(),
|
||||
'version': SOFTWARE_VERSION,
|
||||
}
|
||||
|
||||
# 尝试获取 GPU 信息
|
||||
gpu_info = _get_gpu_info()
|
||||
gpu_info = get_gpu_info()
|
||||
if gpu_info:
|
||||
info['gpu'] = gpu_info
|
||||
|
||||
@@ -68,10 +49,10 @@ def get_capabilities():
|
||||
Returns:
|
||||
list: 能力列表
|
||||
"""
|
||||
return V2_DEFAULT_CAPABILITIES.copy()
|
||||
return DEFAULT_CAPABILITIES.copy()
|
||||
|
||||
|
||||
def _get_gpu_info():
|
||||
def get_gpu_info() -> Optional[str]:
|
||||
"""
|
||||
尝试获取 GPU 信息
|
||||
|
||||
@@ -79,7 +60,6 @@ def _get_gpu_info():
|
||||
str: GPU 信息,失败返回 None
|
||||
"""
|
||||
try:
|
||||
import subprocess
|
||||
# 尝试使用 nvidia-smi
|
||||
result = subprocess.run(
|
||||
['nvidia-smi', '--query-gpu=name', '--format=csv,noheader'],
|
||||
@@ -94,3 +74,30 @@ def _get_gpu_info():
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_ffmpeg_version() -> str:
|
||||
"""
|
||||
获取 FFmpeg 版本
|
||||
|
||||
Returns:
|
||||
str: FFmpeg 版本号
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
['ffmpeg', '-version'],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5
|
||||
)
|
||||
if result.returncode == 0:
|
||||
first_line = result.stdout.split('\n')[0]
|
||||
# 解析版本号,例如 "ffmpeg version 6.0 ..."
|
||||
parts = first_line.split()
|
||||
for i, part in enumerate(parts):
|
||||
if part == 'version' and i + 1 < len(parts):
|
||||
return parts[i + 1]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return 'unknown'
|
||||
|
||||
Reference in New Issue
Block a user