q792602257/FrameTour-RenderWorker
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mypy

Browse Source
This commit is contained in:
Jerry Yan
2025-09-24 10:17:11 +08:00
parent dfb07d679f
commit c055a68592
31 changed files with 1357 additions and 811 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
util/oss.py
View File

@@ -31,12 +31,14 @@ def upload_to_oss(url, file_path):
if replace_map != "":
replace_list = [i.split("|", 1) for i in replace_map.split(",")]
new_url = url
for (_src, _dst) in replace_list:
for _src, _dst in replace_list:
new_url = new_url.replace(_src, _dst)
new_url = new_url.split("?", 1)[0]
r_span.set_attribute("rclone.target_dir", new_url)
if new_url != url:
result = os.system(f"rclone copyto --no-check-dest --ignore-existing --multi-thread-chunk-size 8M --multi-thread-streams 8 {file_path} {new_url}")
result = os.system(
f"rclone copyto --no-check-dest --ignore-existing --multi-thread-chunk-size 8M --multi-thread-streams 8 {file_path} {new_url}"
)
r_span.set_attribute("rclone.result", result)
if result == 0:
span.set_status(Status(StatusCode.OK))
@@ -49,8 +51,14 @@ def upload_to_oss(url, file_path):
try:
req_span.set_attribute("http.method", "PUT")
req_span.set_attribute("http.url", url)
with open(file_path, 'rb') as f:
response = requests.put(url, data=f, stream=True, timeout=60, headers={"Content-Type": "video/mp4"})
with open(file_path, "rb") as f:
response = requests.put(
url,
data=f,
stream=True,
timeout=60,
headers={"Content-Type": "video/mp4"},
)
req_span.set_attribute("http.status_code", response.status_code)
req_span.set_attribute("http.response", response.text)
response.raise_for_status()
@@ -61,12 +69,16 @@ def upload_to_oss(url, file_path):
req_span.set_attribute("http.error", "Timeout")
req_span.set_status(Status(StatusCode.ERROR))
retries += 1
logger.warning(f"Upload timed out. Retrying {retries}/{max_retries}...")
logger.warning(
f"Upload timed out. Retrying {retries}/{max_retries}..."
)
except Exception as e:
req_span.set_attribute("http.error", str(e))
req_span.set_status(Status(StatusCode.ERROR))
retries += 1
logger.warning(f"Upload failed. Retrying {retries}/{max_retries}...")
logger.warning(
f"Upload failed. Retrying {retries}/{max_retries}..."
)
span.set_status(Status(StatusCode.ERROR))
return False
@@ -83,11 +95,11 @@ def download_from_oss(url, file_path, skip_if_exist=None):
with tracer.start_as_current_span("download_from_oss") as span:
span.set_attribute("file.url", url)
span.set_attribute("file.path", file_path)
# 如果skip_if_exist为None，则从启动参数中读取
if skip_if_exist is None:
skip_if_exist = 'skip_if_exist' in sys.argv
skip_if_exist = "skip_if_exist" in sys.argv
if skip_if_exist and os.path.exists(file_path):
span.set_attribute("file.exist", True)
span.set_attribute("file.size", os.path.getsize(file_path))
@@ -107,7 +119,7 @@ def download_from_oss(url, file_path, skip_if_exist=None):
req_span.set_attribute("http.url", url)
response = requests.get(url, timeout=15) # 设置超时时间
req_span.set_attribute("http.status_code", response.status_code)
with open(file_path, 'wb') as f:
with open(file_path, "wb") as f:
f.write(response.content)
req_span.set_attribute("file.size", os.path.getsize(file_path))
req_span.set_status(Status(StatusCode.OK))
@@ -117,11 +129,15 @@ def download_from_oss(url, file_path, skip_if_exist=None):
req_span.set_attribute("http.error", "Timeout")
req_span.set_status(Status(StatusCode.ERROR))
retries += 1
logger.warning(f"Download timed out. Retrying {retries}/{max_retries}...")
logger.warning(
f"Download timed out. Retrying {retries}/{max_retries}..."
)
except Exception as e:
req_span.set_attribute("http.error", str(e))
req_span.set_status(Status(StatusCode.ERROR))
retries += 1
logger.warning(f"Download failed. Retrying {retries}/{max_retries}...")
logger.warning(
f"Download failed. Retrying {retries}/{max_retries}..."
)
span.set_status(Status(StatusCode.ERROR))
return False