q792602257/FrameTour-RenderWorker
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

埋点

Browse Source
This commit is contained in:
Jerry Yan 2025-03-04 11:42:31 +08:00
parent 1f9632761f
commit 9d178a3d34
7 changed files with 443 additions and 295 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
biz/ffmpeg.py
View File

@ -6,54 +6,57 @@ from entity.ffmpeg import FfmpegTask
import logging import logging
from util import ffmpeg, oss from util import ffmpeg, oss
from telemetry import get_tracer
logger = logging.getLogger('biz/ffmpeg') logger = logging.getLogger('biz/ffmpeg')
def parse_ffmpeg_task(task_info, template_info): def parse_ffmpeg_task(task_info, template_info):
tasks = [] tracer = get_tracer(__name__)
# 中间片段 with tracer.start_as_current_span("parse_ffmpeg_task"):
task_params_str = task_info.get("taskParams", "{}") tasks = []
task_params = json.loads(task_params_str) # 中间片段
for part in template_info.get("video_parts"): task_params_str = task_info.get("taskParams", "{}")
source = parse_video(part.get('source'), task_params, template_info) task_params = json.loads(task_params_str)
if not source: for part in template_info.get("video_parts"):
logger.warning("no video found for part: " + str(part)) source = parse_video(part.get('source'), task_params, template_info)
continue if not source:
only_if = part.get('only_if', '') logger.warning("no video found for part: " + str(part))
if only_if:
if not check_placeholder_exist(only_if, task_params):
logger.info("because only_if exist, placeholder: %s not exist, skip part: %s", only_if, part)
continue continue
sub_ffmpeg_task = FfmpegTask(source) only_if = part.get('only_if', '')
sub_ffmpeg_task.annexb = True if only_if:
sub_ffmpeg_task.frame_rate = template_info.get("frame_rate", 25) if not check_placeholder_exist(only_if, task_params):
for effect in part.get('effects', []): logger.info("because only_if exist, placeholder: %s not exist, skip part: %s", only_if, part)
sub_ffmpeg_task.add_effect(effect) continue
for lut in part.get('filters', []): sub_ffmpeg_task = FfmpegTask(source)
sub_ffmpeg_task.add_lut(os.path.join(template_info.get("local_path"), lut)) sub_ffmpeg_task.annexb = True
for audio in part.get('audios', []): sub_ffmpeg_task.frame_rate = template_info.get("frame_rate", 25)
sub_ffmpeg_task.add_audios(os.path.join(template_info.get("local_path"), audio)) for effect in part.get('effects', []):
for overlay in part.get('overlays', []): sub_ffmpeg_task.add_effect(effect)
sub_ffmpeg_task.add_overlay(os.path.join(template_info.get("local_path"), overlay)) for lut in part.get('filters', []):
tasks.append(sub_ffmpeg_task) sub_ffmpeg_task.add_lut(os.path.join(template_info.get("local_path"), lut))
output_file = "out_" + str(time.time()) + ".mp4" for audio in part.get('audios', []):
task = FfmpegTask(tasks, output_file=output_file) sub_ffmpeg_task.add_audios(os.path.join(template_info.get("local_path"), audio))
overall = template_info.get("overall_template") for overlay in part.get('overlays', []):
task.center_cut = template_info.get("crop_mode", None) sub_ffmpeg_task.add_overlay(os.path.join(template_info.get("local_path"), overlay))
task.frame_rate = template_info.get("frame_rate", 25) tasks.append(sub_ffmpeg_task)
if overall.get('source', ''): output_file = "out_" + str(time.time()) + ".mp4"
source = parse_video(overall.get('source'), task_params, template_info) task = FfmpegTask(tasks, output_file=output_file)
task.add_inputs(source) overall = template_info.get("overall_template")
for effect in overall.get('effects', []): task.center_cut = template_info.get("crop_mode", None)
task.add_effect(effect) task.frame_rate = template_info.get("frame_rate", 25)
for lut in overall.get('filters', []): if overall.get('source', ''):
task.add_lut(os.path.join(template_info.get("local_path"), lut)) source = parse_video(overall.get('source'), task_params, template_info)
for audio in overall.get('audios', []): task.add_inputs(source)
task.add_audios(os.path.join(template_info.get("local_path"), audio)) for effect in overall.get('effects', []):
for overlay in overall.get('overlays', []): task.add_effect(effect)
task.add_overlay(os.path.join(template_info.get("local_path"), overlay)) for lut in overall.get('filters', []):
return task task.add_lut(os.path.join(template_info.get("local_path"), lut))
for audio in overall.get('audios', []):
task.add_audios(os.path.join(template_info.get("local_path"), audio))
for overlay in overall.get('overlays', []):
task.add_overlay(os.path.join(template_info.get("local_path"), overlay))
return task
def parse_video(source, task_params, template_info): def parse_video(source, task_params, template_info):
@ -87,13 +90,16 @@ def check_placeholder_exist(placeholder_id, task_params):
return True return True
return False return False
def start_ffmpeg_task(ffmpeg_task): def start_ffmpeg_task(ffmpeg_task):
for task in ffmpeg_task.analyze_input_render_tasks(): tracer = get_tracer(__name__)
result = start_ffmpeg_task(task) with tracer.start_as_current_span("start_ffmpeg_task"):
if not result: for task in ffmpeg_task.analyze_input_render_tasks():
return False result = start_ffmpeg_task(task)
ffmpeg_task.correct_task_type() if not result:
return ffmpeg.start_render(ffmpeg_task) return False
ffmpeg_task.correct_task_type()
return ffmpeg.start_render(ffmpeg_task)
def clear_task_tmp_file(ffmpeg_task): def clear_task_tmp_file(ffmpeg_task):

41
biz/task.py
View File

@ -1,24 +1,27 @@
from biz.ffmpeg import parse_ffmpeg_task, start_ffmpeg_task, clear_task_tmp_file, probe_video_info
from telemetry import get_tracer
from template import get_template_def from template import get_template_def
from util import api from util import api
def start_task(task_info): def start_task(task_info):
from biz.ffmpeg import parse_ffmpeg_task, start_ffmpeg_task, clear_task_tmp_file, probe_video_info tracer = get_tracer(__name__)
task_info = api.normalize_task(task_info) with tracer.start_as_current_span("start_task"):
template_info = get_template_def(task_info.get("templateId")) task_info = api.normalize_task(task_info)
api.report_task_start(task_info) template_info = get_template_def(task_info.get("templateId"))
ffmpeg_task = parse_ffmpeg_task(task_info, template_info) api.report_task_start(task_info)
result = start_ffmpeg_task(ffmpeg_task) ffmpeg_task = parse_ffmpeg_task(task_info, template_info)
if not result: result = start_ffmpeg_task(ffmpeg_task)
return api.report_task_failed(task_info) if not result:
oss_result = api.upload_task_file(task_info, ffmpeg_task) return api.report_task_failed(task_info)
if not oss_result: oss_result = api.upload_task_file(task_info, ffmpeg_task)
return api.report_task_failed(task_info) if not oss_result:
# 获取视频长度宽度和时长 return api.report_task_failed(task_info)
width, height, duration = probe_video_info(ffmpeg_task) # 获取视频长度宽度和时长
clear_task_tmp_file(ffmpeg_task) width, height, duration = probe_video_info(ffmpeg_task)
api.report_task_success(task_info, videoInfo={ clear_task_tmp_file(ffmpeg_task)
"width": width, api.report_task_success(task_info, videoInfo={
"height": height, "width": width,
"duration": duration "height": height,
}) "duration": duration
})

2
index.py
View File

@ -2,6 +2,7 @@ from time import sleep
import biz.task import biz.task
import config import config
from telemetry import init_opentelemetry
from template import load_local_template from template import load_local_template
from util import api from util import api
@ -12,6 +13,7 @@ load_local_template()
import logging import logging
LOGGER = logging.getLogger(__name__) LOGGER = logging.getLogger(__name__)
init_opentelemetry()
while True: while True:
# print(get_sys_info()) # print(get_sys_info())

30
telemetry/__init__.py Normal file
View File

@ -0,0 +1,30 @@
import os
from constant import SOFTWARE_VERSION
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter as OTLPSpanHttpExporter
from opentelemetry.sdk.resources import DEPLOYMENT_ENVIRONMENT, HOST_NAME, Resource, SERVICE_NAME, SERVICE_VERSION
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
def get_tracer(name):
return trace.get_tracer(name)
# 初始化 OpenTelemetry
def init_opentelemetry():
# 设置服务名、主机名
resource = Resource(attributes={
SERVICE_NAME: "RENDER_WORKER",
SERVICE_VERSION: SOFTWARE_VERSION,
DEPLOYMENT_ENVIRONMENT: "Python",
HOST_NAME: os.getenv("ACCESS_KEY"),
})
# 使用HTTP协议上报
span_processor = BatchSpanProcessor(OTLPSpanHttpExporter(
endpoint="http://tracing-analysis-dc-sh.aliyuncs.com/adapt_e7qojqi4e0@aa79b4d367fb6b7_e7qojqi4e0@53df7ad2afe8301/api/otlp/traces",
))
trace_provider = TracerProvider(resource=resource, active_span_processor=span_processor)
trace.set_tracer_provider(trace_provider)

331
util/api.py
View File

@ -1,9 +1,11 @@
import json
import logging import logging
import os import os
import requests import requests
import util.system import util.system
from telemetry import get_tracer
session = requests.Session() session = requests.Session()
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -19,33 +21,41 @@ def sync_center():
通过接口获取任务 通过接口获取任务
:return: 任务列表 :return: 任务列表
""" """
try: from template import TEMPLATES, download_template
from template import TEMPLATES, download_template tracer = get_tracer(__name__)
with tracer.start_as_current_span("sync_center"):
response = session.post(os.getenv('API_ENDPOINT') + "/sync", json={ with get_tracer("api").start_as_current_span("sync_center.request") as req_span:
'accessKey': os.getenv('ACCESS_KEY'), try:
'clientStatus': util.system.get_sys_info(), req_span.set_attribute("http.method", "POST")
'templateList': [{'id': t.get('id', ''), 'updateTime': t.get('updateTime', '')} for t in TEMPLATES.values()] req_span.set_attribute("http.url", os.getenv('API_ENDPOINT') + "/sync")
}, timeout=10) response = session.post(os.getenv('API_ENDPOINT') + "/sync", json={
response.raise_for_status() 'accessKey': os.getenv('ACCESS_KEY'),
except requests.RequestException as e: 'clientStatus': util.system.get_sys_info(),
logger.error("请求失败！", e) 'templateList': [{'id': t.get('id', ''), 'updateTime': t.get('updateTime', '')} for t in
return [] TEMPLATES.values()]
data = response.json() }, timeout=10)
logger.debug("获取任务结果：【%s】", data) req_span.set_attribute("http.status_code", response.status_code)
if data.get('code', 0) == 200: response.raise_for_status()
templates = data.get('data', {}).get('templates', []) req_span.set_attribute("api.response", response.text)
tasks = data.get('data', {}).get('tasks', []) except requests.RequestException as e:
else: req_span.set_attribute("api.error", str(e))
tasks = [] logger.error("请求失败！", e)
templates = [] return []
logger.warning("获取任务失败") data = response.json()
for template in templates: logger.debug("获取任务结果：【%s】", data)
template_id = template.get('id', '') if data.get('code', 0) == 200:
if template_id: templates = data.get('data', {}).get('templates', [])
logger.info("更新模板：【%s】", template_id) tasks = data.get('data', {}).get('tasks', [])
download_template(template_id) else:
return tasks tasks = []
templates = []
logger.warning("获取任务失败")
for template in templates:
template_id = template.get('id', '')
if template_id:
logger.info("更新模板：【%s】", template_id)
download_template(template_id)
return tasks
def get_template_info(template_id): def get_template_info(template_id):
@ -56,126 +66,177 @@ def get_template_info(template_id):
:type template_id: str :type template_id: str
:return: 模板信息 :return: 模板信息
""" """
try: tracer = get_tracer(__name__)
response = session.post('{0}/template/{1}'.format(os.getenv('API_ENDPOINT'), template_id), json={ with tracer.start_as_current_span("get_template_info"):
'accessKey': os.getenv('ACCESS_KEY'), with get_tracer("api").start_as_current_span("get_template_info.request") as req_span:
}, timeout=10) try:
response.raise_for_status() req_span.set_attribute("http.method", "POST")
except requests.RequestException as e: req_span.set_attribute("http.url", '{0}/template/{1}'.format(os.getenv('API_ENDPOINT'), template_id))
logger.error("请求失败！", e) response = session.post('{0}/template/{1}'.format(os.getenv('API_ENDPOINT'), template_id), json={
return None 'accessKey': os.getenv('ACCESS_KEY'),
data = response.json() }, timeout=10)
remote_template_info = data.get('data', {}) req_span.set_attribute("http.status_code", response.status_code)
logger.debug("获取模板信息结果：【%s】", remote_template_info) response.raise_for_status()
template = { req_span.set_attribute("api.response", response.text)
'id': template_id, except requests.RequestException as e:
'updateTime': remote_template_info.get('updateTime', template_id), req_span.set_attribute("api.error", str(e))
'scenic_name': remote_template_info.get('scenicName', '景区'), logger.error("请求失败！", e)
'name': remote_template_info.get('name', '模版'), return None
'video_size': '1920x1080', data = response.json()
'frame_rate': 25, logger.debug("获取模板信息结果：【%s】", data)
'overall_duration': 30, remote_template_info = data.get('data', {})
'video_parts': [ template = {
'id': template_id,
'updateTime': remote_template_info.get('updateTime', template_id),
'scenic_name': remote_template_info.get('scenicName', '景区'),
'name': remote_template_info.get('name', '模版'),
'video_size': '1920x1080',
'frame_rate': 25,
'overall_duration': 30,
'video_parts': [
] ]
} }
def _template_normalizer(template_info): def _template_normalizer(template_info):
_template = {} _template = {}
_placeholder_type = template_info.get('isPlaceholder', -1) _placeholder_type = template_info.get('isPlaceholder', -1)
if _placeholder_type == 0: if _placeholder_type == 0:
# 固定视频 # 固定视频
_template['source'] = template_info.get('sourceUrl', '') _template['source'] = template_info.get('sourceUrl', '')
elif _placeholder_type == 1: elif _placeholder_type == 1:
# 占位符 # 占位符
_template['source'] = "PLACEHOLDER_" + template_info.get('sourceUrl', '') _template['source'] = "PLACEHOLDER_" + template_info.get('sourceUrl', '')
_template['mute'] = template_info.get('mute', True) _template['mute'] = template_info.get('mute', True)
_template['crop_mode'] = template_info.get('cropEnable', None) _template['crop_mode'] = template_info.get('cropEnable', None)
else: else:
_template['source'] = None _template['source'] = None
_overlays = template_info.get('overlays', '') _overlays = template_info.get('overlays', '')
if _overlays: if _overlays:
_template['overlays'] = _overlays.split(",") _template['overlays'] = _overlays.split(",")
_audios = template_info.get('audios', '') _audios = template_info.get('audios', '')
if _audios: if _audios:
_template['audios'] = _audios.split(",") _template['audios'] = _audios.split(",")
_luts = template_info.get('luts', '') _luts = template_info.get('luts', '')
if _luts: if _luts:
_template['luts'] = _luts.split(",") _template['luts'] = _luts.split(",")
_only_if = template_info.get('onlyIf', '') _only_if = template_info.get('onlyIf', '')
if _only_if: if _only_if:
_template['only_if'] = _only_if _template['only_if'] = _only_if
_effects = template_info.get('effects', '') _effects = template_info.get('effects', '')
if _effects: if _effects:
_template['effects'] = _effects.split("|") _template['effects'] = _effects.split("|")
return _template return _template
# outer template definition # outer template definition
overall_template = _template_normalizer(remote_template_info) overall_template = _template_normalizer(remote_template_info)
template['overall_template'] = overall_template template['overall_template'] = overall_template
# inter template definition # inter template definition
inter_template_list = remote_template_info.get('children', []) inter_template_list = remote_template_info.get('children', [])
for children_template in inter_template_list: for children_template in inter_template_list:
parts = _template_normalizer(children_template) parts = _template_normalizer(children_template)
template['video_parts'].append(parts) template['video_parts'].append(parts)
template['local_path'] = os.path.join(os.getenv('TEMPLATE_DIR'), str(template_id)) template['local_path'] = os.path.join(os.getenv('TEMPLATE_DIR'), str(template_id))
return template with get_tracer("api").start_as_current_span("get_template_info.template") as res_span:
res_span.set_attribute("normalized.response", json.dumps(template))
return template
def report_task_success(task_info, **kwargs): def report_task_success(task_info, **kwargs):
try: tracer = get_tracer(__name__)
response = session.post('{0}/{1}/success'.format(os.getenv('API_ENDPOINT'), task_info.get("id")), json={ with tracer.start_as_current_span("report_task_success"):
'accessKey': os.getenv('ACCESS_KEY'), with get_tracer("api").start_as_current_span("report_task_success.request") as req_span:
**kwargs try:
}, timeout=10) req_span.set_attribute("http.method", "POST")
response.raise_for_status() req_span.set_attribute("http.url",
except requests.RequestException as e: '{0}/{1}/success'.format(os.getenv('API_ENDPOINT'), task_info.get("id")))
logger.error("请求失败！", e) response = session.post('{0}/{1}/success'.format(os.getenv('API_ENDPOINT'), task_info.get("id")), json={
return None 'accessKey': os.getenv('ACCESS_KEY'),
**kwargs
}, timeout=10)
req_span.set_attribute("http.status_code", response.status_code)
response.raise_for_status()
req_span.set_attribute("api.response", response.text)
except requests.RequestException as e:
req_span.set_attribute("api.error", str(e))
logger.error("请求失败！", e)
return None
def report_task_start(task_info): def report_task_start(task_info):
try: tracer = get_tracer(__name__)
response = session.post('{0}/{1}/start'.format(os.getenv('API_ENDPOINT'), task_info.get("id")), json={ with tracer.start_as_current_span("report_task_start"):
'accessKey': os.getenv('ACCESS_KEY'), with get_tracer("api").start_as_current_span("report_task_start.request") as req_span:
}, timeout=10) try:
response.raise_for_status() req_span.set_attribute("http.method", "POST")
except requests.RequestException as e: req_span.set_attribute("http.url",
logger.error("请求失败！", e) '{0}/{1}/start'.format(os.getenv('API_ENDPOINT'), task_info.get("id")))
return None response = session.post('{0}/{1}/start'.format(os.getenv('API_ENDPOINT'), task_info.get("id")), json={
'accessKey': os.getenv('ACCESS_KEY'),
}, timeout=10)
req_span.set_attribute("http.status_code", response.status_code)
response.raise_for_status()
req_span.set_attribute("api.response", response.text)
except requests.RequestException as e:
req_span.set_attribute("api.error", str(e))
logger.error("请求失败！", e)
return None
def report_task_failed(task_info, reason=''): def report_task_failed(task_info, reason=''):
try: tracer = get_tracer(__name__)
response = session.post('{0}/{1}/fail'.format(os.getenv('API_ENDPOINT'), task_info.get("id")), json={ with tracer.start_as_current_span("report_task_failed"):
'accessKey': os.getenv('ACCESS_KEY'), with tracer.start_as_current_span("report_task_failed.request") as req_span:
'reason': reason try:
}, timeout=10) req_span.set_attribute("http.method", "POST")
response.raise_for_status() req_span.set_attribute("http.url",
except requests.RequestException as e: '{0}/{1}/fail'.format(os.getenv('API_ENDPOINT'), task_info.get("id")))
logger.error("请求失败！", e) response = session.post('{0}/{1}/fail'.format(os.getenv('API_ENDPOINT'), task_info.get("id")), json={
return None 'accessKey': os.getenv('ACCESS_KEY'),
'reason': reason
}, timeout=10)
req_span.set_attribute("http.status_code", response.status_code)
response.raise_for_status()
req_span.set_attribute("api.response", response.text)
except requests.RequestException as e:
req_span.set_attribute("api.error", str(e))
logger.error("请求失败！", e)
return None
def upload_task_file(task_info, ffmpeg_task): def upload_task_file(task_info, ffmpeg_task):
logger.info("开始上传文件: %s", task_info.get("id")) tracer = get_tracer(__name__)
try: with tracer.start_as_current_span("upload_task_file") as span:
response = session.post('{0}/{1}/uploadUrl'.format(os.getenv('API_ENDPOINT'), task_info.get("id")), json={ logger.info("开始上传文件: %s", task_info.get("id"))
'accessKey': os.getenv('ACCESS_KEY'), span.set_attribute("file.id", task_info.get("id"))
}, timeout=10) try:
response.raise_for_status() with tracer.start_as_current_span("upload_task_file.request_upload_url") as req_span:
except requests.RequestException as e: req_span.set_attribute("http.method", "POST")
logger.error("请求失败！", e) req_span.set_attribute("http.url",
return False '{0}/{1}/uploadUrl'.format(os.getenv('API_ENDPOINT'), task_info.get("id")))
data = response.json() response = session.post('{0}/{1}/uploadUrl'.format(os.getenv('API_ENDPOINT'), task_info.get("id")),
url = data.get('data', "") json={
logger.info("开始上传文件: %s 至 %s", task_info.get("id"), url) 'accessKey': os.getenv('ACCESS_KEY'),
try: }, timeout=10)
with open(ffmpeg_task.get_output_file(), 'rb') as f: response.raise_for_status()
requests.put(url, data=f, headers={"Content-Type": "video/mp4"}) req_span.set_attribute("http.status_code", response.status_code)
except requests.RequestException as e: except requests.RequestException as e:
logger.error("上传失败！", e) span.set_attribute("api.error", str(e))
return False logger.error("请求失败！", e)
finally: return False
logger.info("上传文件结束: %s", task_info.get("id")) data = response.json()
return True url = data.get('data', "")
logger.info("开始上传文件: %s 至 %s", task_info.get("id"), url)
try:
with tracer.start_as_current_span("upload_task_file.start_upload_file") as upload_span:
upload_span.set_attribute("http.method", "PUT")
upload_span.set_attribute("http.url", url)
with open(ffmpeg_task.get_output_file(), 'rb') as f:
requests.put(url, data=f, headers={"Content-Type": "video/mp4"})
except requests.RequestException as e:
span.set_attribute("api.error", str(e))
logger.error("上传失败！", e)
return False
finally:
logger.info("上传文件结束: %s", task_info.get("id"))
return True

146
util/ffmpeg.py
View File

@ -1,3 +1,4 @@
import json
import logging import logging
import os import os
import subprocess import subprocess
@ -5,60 +6,81 @@ from datetime import datetime
from typing import Optional, IO from typing import Optional, IO
from entity.ffmpeg import FfmpegTask, ENCODER_ARGS, PROFILE_LEVEL_ARGS from entity.ffmpeg import FfmpegTask, ENCODER_ARGS, PROFILE_LEVEL_ARGS
from telemetry import get_tracer
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def to_annexb(file): def to_annexb(file):
if not os.path.exists(file): with get_tracer("ffmpeg").start_as_current_span("to_annexb") as span:
return file span.set_attribute("file.path", file)
logger.info("ToAnnexb: %s", file) if not os.path.exists(file):
ffmpeg_process = subprocess.run(["ffmpeg", "-y", "-hide_banner", "-i", file, "-c", "copy", "-bsf:v", "h264_mp4toannexb", return file
"-f", "mpegts", file+".ts"]) logger.info("ToAnnexb: %s", file)
logger.info("ToAnnexb: %s, returned: %s", file, ffmpeg_process.returncode) ffmpeg_process = subprocess.run(["ffmpeg", "-y", "-hide_banner", "-i", file, "-c", "copy", "-bsf:v", "h264_mp4toannexb",
if ffmpeg_process.returncode == 0: "-f", "mpegts", file+".ts"])
os.remove(file) span.set_attribute("ffmpeg.args", json.dumps(ffmpeg_process.args))
return file+".ts" logger.info("ToAnnexb: %s, returned: %s", file, ffmpeg_process.returncode)
else: span.set_attribute("ffmpeg.code", ffmpeg_process.returncode)
return file if ffmpeg_process.returncode == 0:
span.set_attribute("file.size", os.path.getsize(file+".ts"))
os.remove(file)
return file+".ts"
else:
return file
def re_encode_and_annexb(file): def re_encode_and_annexb(file):
if not os.path.exists(file): with get_tracer("ffmpeg").start_as_current_span("re_encode_and_annexb") as span:
return file span.set_attribute("file.path", file)
logger.info("ReEncodeAndAnnexb: %s", file) if not os.path.exists(file):
ffmpeg_process = subprocess.run(["ffmpeg", "-y", "-hide_banner", "-i", file, *PROFILE_LEVEL_ARGS, *ENCODER_ARGS, "-bsf:v", "h264_mp4toannexb", return file
"-f", "mpegts", file +".ts"]) logger.info("ReEncodeAndAnnexb: %s", file)
logger.info("ReEncodeAndAnnexb: %s, returned: %s", file, ffmpeg_process.returncode) ffmpeg_process = subprocess.run(["ffmpeg", "-y", "-hide_banner", "-i", file, *PROFILE_LEVEL_ARGS, *ENCODER_ARGS, "-bsf:v", "h264_mp4toannexb",
if ffmpeg_process.returncode == 0: "-f", "mpegts", file +".ts"])
os.remove(file) span.set_attribute("ffmpeg.args", json.dumps(ffmpeg_process.args))
return file+".ts" logger.info("ReEncodeAndAnnexb: %s, returned: %s", file, ffmpeg_process.returncode)
else: span.set_attribute("ffmpeg.code", ffmpeg_process.returncode)
return file if ffmpeg_process.returncode == 0:
span.set_attribute("file.size", os.path.getsize(file+".ts"))
os.remove(file)
return file+".ts"
else:
return file
def start_render(ffmpeg_task: FfmpegTask): def start_render(ffmpeg_task: FfmpegTask):
logger.info(ffmpeg_task) tracer = get_tracer(__name__)
if not ffmpeg_task.need_run(): with tracer.start_as_current_span("start_render") as span:
ffmpeg_task.set_output_file(ffmpeg_task.input_file[0]) span.set_attribute("ffmpeg.task", str(ffmpeg_task))
return True logger.info(ffmpeg_task)
ffmpeg_args = ffmpeg_task.get_ffmpeg_args() if not ffmpeg_task.need_run():
logger.info(ffmpeg_args) ffmpeg_task.set_output_file(ffmpeg_task.input_file[0])
if len(ffmpeg_args) == 0: return True
ffmpeg_task.set_output_file(ffmpeg_task.input_file[0]) ffmpeg_args = ffmpeg_task.get_ffmpeg_args()
return True logger.info(ffmpeg_args)
ffmpeg_process = subprocess.run(["ffmpeg", "-progress", "-", "-loglevel", "error", *ffmpeg_args], **subprocess_args(True)) if len(ffmpeg_args) == 0:
logger.info("FINISH TASK, OUTPUT IS %s", handle_ffmpeg_output(ffmpeg_process.stdout)) ffmpeg_task.set_output_file(ffmpeg_task.input_file[0])
code = ffmpeg_process.returncode return True
if code != 0: ffmpeg_process = subprocess.run(["ffmpeg", "-progress", "-", "-loglevel", "error", *ffmpeg_args], **subprocess_args(True))
logger.error("FFMPEG ERROR: %s", ffmpeg_process.stderr) span.set_attribute("ffmpeg.args", json.dumps(ffmpeg_process.args))
return False ffmpeg_final_out = handle_ffmpeg_output(ffmpeg_process.stdout)
try: span.set_attribute("ffmpeg.out", ffmpeg_final_out)
out_file_stat = os.stat(ffmpeg_task.output_file) logger.info("FINISH TASK, OUTPUT IS %s", ffmpeg_final_out)
if out_file_stat.st_size < 4096: code = ffmpeg_process.returncode
logger.error("FFMPEG ERROR: OUTPUT FILE IS TOO SMALL") span.set_attribute("ffmpeg.code", code)
if code != 0:
logger.error("FFMPEG ERROR: %s", ffmpeg_process.stderr)
return False return False
except OSError: span.set_attribute("ffmpeg.out_file", ffmpeg_task.output_file)
logger.error("FFMPEG ERROR: OUTPUT FILE NOT FOUND") try:
return False file_size = os.path.getsize(ffmpeg_task.output_file)
return True span.set_attribute("file.size", file_size)
if file_size < 4096:
logger.error("FFMPEG ERROR: OUTPUT FILE IS TOO SMALL")
return False
except OSError:
span.set_attribute("file.size", 0)
logger.error("FFMPEG ERROR: OUTPUT FILE NOT FOUND")
return False
return True
def handle_ffmpeg_output(stdout: Optional[bytes]) -> str: def handle_ffmpeg_output(stdout: Optional[bytes]) -> str:
out_time = "0:0:0.0" out_time = "0:0:0.0"
@ -79,27 +101,31 @@ def handle_ffmpeg_output(stdout: Optional[bytes]) -> str:
print("[ ]Speed:", out_time, "@", speed) print("[ ]Speed:", out_time, "@", speed)
return out_time+"@"+speed return out_time+"@"+speed
def duration_str_to_float(duration_str: str) -> float: def duration_str_to_float(duration_str: str) -> float:
_duration = datetime.strptime(duration_str, "%H:%M:%S.%f") - datetime(1900, 1, 1) _duration = datetime.strptime(duration_str, "%H:%M:%S.%f") - datetime(1900, 1, 1)
return _duration.total_seconds() return _duration.total_seconds()
def probe_video_info(video_file): def probe_video_info(video_file):
# 获取宽度和高度 tracer = get_tracer(__name__)
result = subprocess.run( with tracer.start_as_current_span("probe_video_info") as span:
["ffprobe", '-v', 'error', '-select_streams', 'v:0', '-show_entries', 'stream=width,height:format=duration', '-of', span.set_attribute("video.file", video_file)
'csv=s=x:p=0', video_file], # 获取宽度和高度
stderr=subprocess.STDOUT, result = subprocess.run(
**subprocess_args(True) ["ffprobe", '-v', 'error', '-select_streams', 'v:0', '-show_entries', 'stream=width,height:format=duration', '-of',
) 'csv=s=x:p=0', video_file],
if result.returncode != 0: stderr=subprocess.STDOUT,
return 0, 0, 0 **subprocess_args(True)
all_result = result.stdout.decode('utf-8').strip() )
wh, duration = all_result.split('\n') span.set_attribute("ffprobe.args", json.dumps(result.args))
width, height = wh.strip().split('x') span.set_attribute("ffprobe.code", result.returncode)
if result.returncode != 0:
return int(width), int(height), float(duration) return 0, 0, 0
all_result = result.stdout.decode('utf-8').strip()
span.set_attribute("ffprobe.out", all_result)
wh, duration = all_result.split('\n')
width, height = wh.strip().split('x')
return int(width), int(height), float(duration)
# Create a set of arguments which make a ``subprocess.Popen`` (and # Create a set of arguments which make a ``subprocess.Popen`` (and

86
util/oss.py
View File

@ -3,6 +3,8 @@ import os
import requests import requests
from telemetry import get_tracer
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -13,20 +15,29 @@ def upload_to_oss(url, file_path):
:param str file_path: 文件路径 :param str file_path: 文件路径
:return bool: 是否成功 :return bool: 是否成功
""" """
max_retries = 5 tracer = get_tracer(__name__)
retries = 0 with tracer.start_as_current_span("upload_to_oss") as span:
while retries < max_retries: span.set_attribute("file.url", url)
try: span.set_attribute("file.path", file_path)
with open(file_path, 'rb') as f: max_retries = 5
response = requests.put(url, data=f, timeout=60) # 设置超时时间为1分钟 retries = 0
if response.status_code == 200: while retries < max_retries:
return True try:
except requests.exceptions.Timeout: with tracer.start_as_current_span("upload_to_oss.request") as req_span:
retries += 1 req_span.set_attribute("http.method", "PUT")
logger.warning(f"Upload timed out. Retrying {retries}/{max_retries}...") req_span.set_attribute("http.url", url)
except Exception as e: with open(file_path, 'rb') as f:
logger.warning(f"Upload failed. Retrying {retries}/{max_retries}...") response = requests.put(url, data=f, timeout=60) # 设置超时时间为1分钟
retries += 1 req_span.set_attribute("http.status_code", response.status_code)
if response.status_code == 200:
return True
except requests.exceptions.Timeout:
retries += 1
logger.warning(f"Upload timed out. Retrying {retries}/{max_retries}...")
except Exception as e:
retries += 1
span.set_attribute("oss.error", str(e))
logger.warning(f"Upload failed. Retrying {retries}/{max_retries}...")
return False return False
@ -37,23 +48,32 @@ def download_from_oss(url, file_path):
:param Union[LiteralString, str, bytes] file_path: 文件路径 :param Union[LiteralString, str, bytes] file_path: 文件路径
:return bool: 是否成功 :return bool: 是否成功
""" """
logging.info("download_from_oss: %s", url) tracer = get_tracer(__name__)
file_dir, file_name = os.path.split(file_path) with tracer.start_as_current_span("download_from_oss") as span:
if file_dir: span.set_attribute("file.url", url)
if not os.path.exists(file_dir): span.set_attribute("file.path", file_path)
os.makedirs(file_dir) logging.info("download_from_oss: %s", url)
max_retries = 5 file_dir, file_name = os.path.split(file_path)
retries = 0 if file_dir:
while retries < max_retries: if not os.path.exists(file_dir):
try: os.makedirs(file_dir)
response = requests.get(url, timeout=15) # 设置超时时间 max_retries = 5
with open(file_path, 'wb') as f: retries = 0
f.write(response.content) while retries < max_retries:
return True try:
except requests.exceptions.Timeout: with tracer.start_as_current_span("download_from_oss.request") as req_span:
retries += 1 req_span.set_attribute("http.method", "GET")
logger.warning(f"Download timed out. Retrying {retries}/{max_retries}...") req_span.set_attribute("http.url", url)
except Exception as e: response = requests.get(url, timeout=15) # 设置超时时间
logger.warning(f"Download failed. Retrying {retries}/{max_retries}...") req_span.set_attribute("http.status_code", response.status_code)
retries += 1 with open(file_path, 'wb') as f:
f.write(response.content)
return True
except requests.exceptions.Timeout:
retries += 1
logger.warning(f"Download timed out. Retrying {retries}/{max_retries}...")
except Exception as e:
retries += 1
span.set_attribute("oss.error", str(e))
logger.warning(f"Download failed. Retrying {retries}/{max_retries}...")
return False return False