From c74de5cbf9f30461586042d27d1411851e40bb85 Mon Sep 17 00:00:00 2001 From: JerryYan_at_Remote <792602257@qq.com> Date: Mon, 15 May 2023 06:34:48 +0800 Subject: [PATCH] =?UTF-8?q?=E9=81=BF=E5=85=8D=E5=BC=82=E5=B8=B8=EF=BC=8C?= =?UTF-8?q?=E6=9B=B4=E6=8D=A2=E9=80=BB=E8=BE=91=EF=BC=8C=E5=88=A0=E9=99=A4?= =?UTF-8?q?B=E7=AB=99=E6=8A=95=E7=A8=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 +- app.py | 2 + config.py | 4 +- controller/api/bilirecorder_blueprint.py | 47 +- controller/api/biliuploader_blueprint.py | 20 - workflow/__init__.py | 2 +- workflow/bilibili.py | 161 ++--- workflow/bilibiliupload/README.md | 4 - workflow/bilibiliupload/__init__.py | 4 - workflow/bilibiliupload/bilibiliuploader.py | 91 --- workflow/bilibiliupload/core.py | 641 -------------------- workflow/bilibiliupload/util/__init__.py | 1 - workflow/bilibiliupload/util/cipher.py | 119 ---- workflow/bilibiliupload/util/retry.py | 21 - workflow/video.py | 3 +- workflow/worker.py | 2 +- 16 files changed, 52 insertions(+), 1073 deletions(-) delete mode 100644 workflow/bilibiliupload/README.md delete mode 100644 workflow/bilibiliupload/__init__.py delete mode 100644 workflow/bilibiliupload/bilibiliuploader.py delete mode 100644 workflow/bilibiliupload/core.py delete mode 100644 workflow/bilibiliupload/util/__init__.py delete mode 100644 workflow/bilibiliupload/util/cipher.py delete mode 100644 workflow/bilibiliupload/util/retry.py diff --git a/.gitignore b/.gitignore index 19a5799..9b0e97b 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ build/ dist/ access_token winsw.* -*.json \ No newline at end of file +*.json +*.log \ No newline at end of file diff --git a/app.py b/app.py index 392e15a..fc6e58e 100644 --- a/app.py +++ b/app.py @@ -11,6 +11,7 @@ from controller.api.posting_blueprint import blueprint as api_posting_blueprint from controller.api.video_part_blueprint import blueprint as api_video_part_blueprint from controller.api.biliuploader_blueprint import blueprint as api_biliuploader_blueprint from model import db +from workflow.bilibili import Bilibili app = Flask(__name__) app.config['JSON_AS_ASCII'] = False @@ -35,4 +36,5 @@ with app.app_context(): db.create_all(app=app) if __name__ == '__main__': + Bilibili().start() app.run() diff --git a/config.py b/config.py index c513791..2a015c3 100644 --- a/config.py +++ b/config.py @@ -90,8 +90,8 @@ def load_config(): if config.has_section("clip"): section = config['clip'] global VIDEO_CLIP_EACH_SEC, VIDEO_CLIP_OVERFLOW_SEC - VIDEO_CLIP_EACH_SEC = section.getfloat('each_sec', VIDEO_CLIP_EACH_SEC) - VIDEO_CLIP_OVERFLOW_SEC = section.getfloat('overflow_sec', VIDEO_CLIP_OVERFLOW_SEC) + VIDEO_CLIP_EACH_SEC = section.getint('each_sec', VIDEO_CLIP_EACH_SEC) + VIDEO_CLIP_OVERFLOW_SEC = section.getint('overflow_sec', VIDEO_CLIP_OVERFLOW_SEC) if config.has_section("ffmpeg"): section = config['ffmpeg'] global FFMPEG_EXEC diff --git a/controller/api/bilirecorder_blueprint.py b/controller/api/bilirecorder_blueprint.py index cafe29c..2f086ab 100644 --- a/controller/api/bilirecorder_blueprint.py +++ b/controller/api/bilirecorder_blueprint.py @@ -6,18 +6,15 @@ from typing import Optional from flask import Blueprint, jsonify, request, current_app -from config import BILILIVE_RECORDER_DIRECTORY, VIDEO_TITLE, XIGUALIVE_RECORDER_DIRECTORY, VIDEO_DESC, \ - VIDEO_TAGS, VIDEO_TID, VIDEO_ENABLED +from config import BILILIVE_RECORDER_DIRECTORY, VIDEO_TITLE, XIGUALIVE_RECORDER_DIRECTORY from exception.danmaku import DanmakuException from model import db from model.DanmakuClip import DanmakuClip from model.VideoClip import VideoClip from model.Workflow import Workflow -from workflow.bilibili import IS_LIVING, IS_UPLOADING, INSTANCE as bilibili_instance, IS_ENCODING -from workflow.bilibili import VideoPart +from workflow.bilibili import ENCODING_QUEUE, IS_LIVING from workflow.danmaku import get_file_start from workflow.video import get_video_real_duration, duration_str_to_float -from workflow.worker import do_workflow blueprint = Blueprint("api_bilirecorder", __name__, url_prefix="/api/bilirecorder") @@ -35,50 +32,13 @@ def auto_submit_task(): if len(bili_record_workflow_item.video_clips) == 0: print("[!]Auto Submit Fail: No Video Clips") return - if VIDEO_ENABLED: - bilibili_instance.login() - video_title = bili_record_workflow_item.name - _future = None for video_clip in bili_record_workflow_item.video_clips: if len(video_clip.danmaku_clips) > 0: print("[+]Workflow:", bili_record_workflow_item.id, "; Video:", video_clip.full_path) - _started = True - IS_ENCODING.set() - _future = pool.submit( - do_workflow, - video_clip.full_path, - video_clip.danmaku_clips[0].full_path, - *[clip.full_path for clip in video_clip.danmaku_clips[1:]] - ) + ENCODING_QUEUE.put(bili_record_workflow_item) clear_item() - def _clear_encode_flag_callback(_f: "Future"): - IS_ENCODING.clear() - _future.add_done_callback(_clear_encode_flag_callback) - if VIDEO_ENABLED: - def _encode_finish_callback(_f: "Future"): - _result = _f.result() - if _result: - # start uploading - bilibili_instance.pre_upload( - parts=[VideoPart(os.path.join(_item['base_path'], _item['file']), _item['file']) - for _item in _result], - max_retry=10 - ) - - _future.add_done_callback(_encode_finish_callback) else: print("[-]Workflow:", bili_record_workflow_item.id, "; Video:", video_clip.full_path, "; No Danmaku") - if VIDEO_ENABLED and _future is not None: - def _on_upload_finish(_f: "Future"): - if IS_UPLOADING.is_set() or IS_LIVING.is_set() or IS_ENCODING.is_set(): - return - bilibili_instance.finish_upload( - title=video_title, - desc=VIDEO_DESC, - tid=VIDEO_TID, - tag=VIDEO_TAGS, - no_reprint=0) - _future.add_done_callback(_on_upload_finish) def clear_item(): @@ -109,6 +69,7 @@ def safe_create_item(): commit_item() auto_submit_task() bili_record_workflow_item = Workflow() + bili_record_workflow_item.name = VIDEO_TITLE.format(datetime.utcnow().strftime("%Y%m%d")) else: bili_record_workflow_item.name = VIDEO_TITLE.format(datetime.utcnow().strftime("%Y%m%d")) bili_record_workflow_item.automatic = True diff --git a/controller/api/biliuploader_blueprint.py b/controller/api/biliuploader_blueprint.py index 4584f64..fafe2a1 100644 --- a/controller/api/biliuploader_blueprint.py +++ b/controller/api/biliuploader_blueprint.py @@ -2,23 +2,3 @@ from flask import Blueprint, jsonify from workflow.bilibili import INSTANCE as BILIBILI_INSTANCE blueprint = Blueprint("api_biliuploader", __name__, url_prefix="/api/biliuploader") - - -@blueprint.get("/") -def get_login_info(): - return jsonify({ - "mid": BILIBILI_INSTANCE.user_id, - "expires": BILIBILI_INSTANCE.expires, - "login_at": BILIBILI_INSTANCE.login_time, - }) - - -@blueprint.post("/") -def do_login(): - BILIBILI_INSTANCE.login() - return get_login_info() - - -@blueprint.post("/finish") -def finish_uploading(): - BILIBILI_INSTANCE.finish_upload() diff --git a/workflow/__init__.py b/workflow/__init__.py index 0dc593a..885c793 100644 --- a/workflow/__init__.py +++ b/workflow/__init__.py @@ -1,7 +1,7 @@ import logging LOGGER = logging.getLogger("WORKFLOW") -_ch = logging.StreamHandler() +_ch = logging.FileHandler("workflow.log", "w", encoding="UTF-8", delay=False) _ch.setLevel(logging.DEBUG) LOGGER.setLevel(logging.DEBUG) LOGGER.addHandler(_ch) diff --git a/workflow/bilibili.py b/workflow/bilibili.py index 65f4780..0e94a7b 100644 --- a/workflow/bilibili.py +++ b/workflow/bilibili.py @@ -1,143 +1,58 @@ import threading -from datetime import datetime +from queue import SimpleQueue as Queue +from time import sleep + +from model.Workflow import Workflow +from workflow.video import quick_split_video +from workflow.worker import do_workflow from . import LOGGER -from .bilibiliupload import core, VideoPart IS_LIVING = threading.Event() IS_ENCODING = threading.Event() IS_UPLOADING = threading.Event() +ENCODING_QUEUE: "Queue[Workflow]" = Queue() -class Bilibili: - def __init__(self): - self.access_token = "" - self.session_id = "" - self.user_id = "" - self.expires = 0 - self.login_time = None +class Bilibili(threading.Thread): + + def __init__(self) -> None: + super().__init__() self.parts = [] - def login(self): - with open("access_token", "r") as f: - self.access_token = f.read(64).strip() - self.session_id, self.user_id, self.expires = core.login_by_access_token(self.access_token) - self.login_time = datetime.now() - LOGGER.info("B站登录，UID【{}】，过期时间【{}】".format(self.user_id, self.expires)) - - def upload(self, - parts, - title, - tid, - tag, - desc, - source='', - cover='', - no_reprint=1, - ): - """ - - :param parts: e.g. VideoPart('part path', 'part title', 'part desc'), or [VideoPart(...), VideoPart(...)] - :type parts: VideoPart or list - :param title: video's title - :type title: str - :param tid: video type, see: https://member.bilibili.com/x/web/archive/pre - or https://github.com/uupers/BiliSpider/wiki/%E8%A7%86%E9%A2%91%E5%88%86%E5%8C%BA%E5%AF%B9%E5%BA%94%E8%A1%A8 - :type tid: int - :param tag: video's tag - :type tag: list - :param desc: video's description - :type desc: str - :param source: (optional) 转载地址 - :type source: str - :param cover: (optional) cover's URL, use method *cover_up* to get - :type cover: str - :param no_reprint: (optional) 0=可以转载, 1=禁止转载(default) - :type no_reprint: int - """ - self.pre_upload(parts) - self.finish_upload(title, tid, tag, desc, source, cover, no_reprint) - self.clear() - - def pre_upload(self, parts: "VideoPart", max_retry=5): - """ - :param max_retry: - :param parts: e.g. VideoPart('part path', 'part title', 'part desc'), or [VideoPart(...), VideoPart(...)] - :type parts: VideoPart or list - """ - if not isinstance(parts, list): - parts = [parts] - - IS_UPLOADING.set() - for part in parts: - if isinstance(part, str): - part = VideoPart(part) - LOGGER.info("Start Uploading >{}<".format(part.path)) - status = core.upload_video_part(self.access_token, self.session_id, self.user_id, part, max_retry) - if status: - # 上传完毕 - LOGGER.info("Upload >{}< Finished；【{}】".format(part.path, part.server_file_name)) - self.parts.append(part) + def run(self) -> None: + while True: + if ENCODING_QUEUE.empty(): + sleep(5) + if len(self.parts) > 0 and not IS_UPLOADING.is_set(): + self.do_upload() else: - LOGGER.warn("Upload >{}< Failed".format(part.path)) - IS_UPLOADING.clear() + workflow_item = ENCODING_QUEUE.get() + LOGGER.info("收到工作流请求：ID:【{}】".format(workflow_item.id)) + for video_clip in workflow_item.video_clips: + IS_ENCODING.set() + try: + LOGGER.info("工作流视频：ID：【{}】，路径：【{}】".format(video_clip.id, video_clip.full_path)) + if len(video_clip.danmaku_clips) < 1: + _parts = quick_split_video(video_clip.full_path) + else: + _parts = do_workflow(video_clip.full_path, video_clip.danmaku_clips[0].full_path) + LOGGER.info("工作流视频压制完成：结果：【{}】".format(_parts)) + for _part in _parts: + self.parts.append(_part) + except: + LOGGER.error("压制异常！工作流视频：ID：【{}】，路径：【{}】".format(video_clip.id, video_clip.full_path)) + finally: + IS_ENCODING.clear() - def finish_upload(self, - title, - tid, - tag, - desc, - source='', - cover='', - no_reprint=1, - ): - """ - :param title: video's title - :type title: str - :param tid: video type, see: https://member.bilibili.com/x/web/archive/pre - or https://github.com/uupers/BiliSpider/wiki/%E8%A7%86%E9%A2%91%E5%88%86%E5%8C%BA%E5%AF%B9%E5%BA%94%E8%A1%A8 - :type tid: int - :param tag: video's tag - :type tag: str - :param desc: video's description - :type desc: str - :param source: (optional) 转载地址 - :type source: str - :param cover: (optional) cover's URL, use method *cover_up* to get - :type cover: str - :param no_reprint: (optional) 0=可以转载, 1=禁止转载(default) - :type no_reprint: int - :param copyright: (optional) 0=转载的, 1=自制的(default) - :type copyright: int - """ - if len(self.parts) == 0: - return - if IS_ENCODING.is_set(): - LOGGER.info("[{}]仍在压制，取消发布".format(title)) - return - if IS_LIVING.is_set(): - LOGGER.info("[{}]仍在直播，取消发布".format(title)) - return - if IS_UPLOADING.is_set(): - LOGGER.info("[{}]仍在上传，取消发布".format(title)) - return - LOGGER.info("[{}]投稿中，请稍后".format(title)) - copyright = 2 if source else 1 - try: - avid, bvid = core.upload(self.access_token, self.session_id, self.user_id, self.parts, copyright, - title=title, tid=tid, tag=tag, desc=desc, source=source, cover=cover, no_reprint=no_reprint) - LOGGER.info("[{}]投稿成功；AVID【{}】，BVID【{}】".format(title, avid, bvid)) - self.clear() - except Exception as e: - LOGGER.error("[{}]投稿失败".format(title), exc_info=e) - - - def reloadFromPrevious(self): - ... + def do_upload(self): + LOGGER.info("尝试投稿：内容【{}】".format(self.parts)) + self.clear() def clear(self): self.parts = [] + INSTANCE = Bilibili() diff --git a/workflow/bilibiliupload/README.md b/workflow/bilibiliupload/README.md deleted file mode 100644 index 68c4d09..0000000 --- a/workflow/bilibiliupload/README.md +++ /dev/null @@ -1,4 +0,0 @@ -修改自 -[BilibiliUploader](https://github.com/FortuneDayssss/BilibiliUploader/) - -LICENSE：GPL \ No newline at end of file diff --git a/workflow/bilibiliupload/__init__.py b/workflow/bilibiliupload/__init__.py deleted file mode 100644 index 49b4a54..0000000 --- a/workflow/bilibiliupload/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .bilibiliuploader import BilibiliUploader -from .core import VideoPart - -__version__ = '0.0.6' diff --git a/workflow/bilibiliupload/bilibiliuploader.py b/workflow/bilibiliupload/bilibiliuploader.py deleted file mode 100644 index da6152a..0000000 --- a/workflow/bilibiliupload/bilibiliuploader.py +++ /dev/null @@ -1,91 +0,0 @@ -from .core import login_by_access_token, upload, edit_videos -from .util import cipher - - -class BilibiliUploader(): - def __init__(self): - self.access_token = None - self.refresh_token = None - self.sid = None - self.mid = None - - def login_by_access_token(self, access_token, refresh_token=None): - self.access_token = access_token - self.refresh_token = refresh_token - self.sid, self.mid, _ = login_by_access_token(access_token) - - def upload(self, - parts, - copyright: int, - title: str, - tid: int, - tag: str, - desc: str, - source: str = '', - cover: str = '', - no_reprint: int = 0, - open_elec: int = 1, - max_retry: int = 5, - thread_pool_workers: int = 1): - return upload(self.access_token, - self.sid, - self.mid, - parts, - copyright, - title, - tid, - tag, - desc, - source, - cover, - no_reprint, - open_elec, - max_retry, - thread_pool_workers) - - def edit(self, - avid=None, - bvid=None, - parts=None, - insert_index=None, - copyright=None, - title=None, - tid=None, - tag=None, - desc=None, - source=None, - cover=None, - no_reprint=None, - open_elec=None, - max_retry: int = 5, - thread_pool_workers: int = 1): - - if not avid and not bvid: - print("please provide avid or bvid") - return None, None - if not avid: - avid = cipher.bv2av(bvid) - if not isinstance(parts, list): - parts = [parts] - if type(avid) is str: - avid = int(avid) - edit_videos( - self.access_token, - self.sid, - self.mid, - avid, - bvid, - parts, - insert_index, - copyright, - title, - tid, - tag, - desc, - source, - cover, - no_reprint, - open_elec, - max_retry, - thread_pool_workers - ) diff --git a/workflow/bilibiliupload/core.py b/workflow/bilibiliupload/core.py deleted file mode 100644 index f0d45c0..0000000 --- a/workflow/bilibiliupload/core.py +++ /dev/null @@ -1,641 +0,0 @@ -import requests -from datetime import datetime -from .util import cipher as cipher -import os -import math -import hashlib -from .util.retry import Retry -from concurrent.futures import ThreadPoolExecutor, as_completed - -# From PC ugc_assisstant -# APPKEY = 'aae92bc66f3edfab' -# APPSECRET = 'af125a0d5279fd576c1b4418a3e8276d' -APPKEY = '1d8b6e7d45233436' -APPSECRET = '560c52ccd288fed045859ed18bffd973' -LOGIN_APPKEY = '783bbb7264451d82' - -# upload chunk size = 2MB -CHUNK_SIZE = 2 * 1024 * 1024 - - -class VideoPart: - """ - Video Part of a post. - 每个对象代表一个分P - - Attributes: - path: file path in local file system. - title: title of the video part. - desc: description of the video part. - server_file_name: file name in bilibili server. generated by pre-upload API. - """ - - def __init__(self, path, title='', desc='', server_file_name=None): - self.path = path - self.title = title - self.desc = desc - self.server_file_name = server_file_name - - def __repr__(self): - return '<{clazz}, path: {path}, title: {title}, desc: {desc}, server_file_name:{server_file_name}>' \ - .format(clazz=self.__class__.__name__, - path=self.path, - title=self.title, - desc=self.desc, - server_file_name=self.server_file_name) - - -def get_key_old(sid=None, jsessionid=None): - """ - get public key, hash and session id for login. - Args: - sid: session id. only for captcha login. - jsessionid: j-session id. only for captcha login. - Returns: - hash: salt for password encryption. - pubkey: rsa public key for password encryption. - sid: session id. - """ - headers = { - 'Content-Type': 'application/x-www-form-urlencoded', - 'Accept': "application/json, text/javascript, */*; q=0.01" - } - post_data = { - 'appkey': APPKEY, - 'platform': "pc", - 'ts': str(int(datetime.now().timestamp())) - } - post_data['sign'] = cipher.sign_dict(post_data, APPSECRET) - cookie = {} - if sid: - cookie['sid'] = sid - if jsessionid: - cookie['JSESSIONID'] = jsessionid - r = requests.post( - # "https://passport.bilibili.com/api/oauth2/getKey", - "https://passport.bilibili.com/x/passport-login/web/key", - headers=headers, - data=post_data, - cookies=cookie - ) - print(r.content.decode()) - r_data = r.json()['data'] - if sid: - return r_data['hash'], r_data['key'], sid - return r_data['hash'], r_data['key'], r.cookies['sid'] - - -def get_key(): - headers = { - 'Content-Type': 'application/x-www-form-urlencoded', - 'Accept': "application/json, text/javascript, */*; q=0.01" - } - params_data = { - 'appkey': LOGIN_APPKEY, - # 'ts': str(int(datetime.now().timestamp())) - } - params_data['sign'] = cipher.login_sign_dict_bin(params_data) - r = requests.get( - "https://passport.bilibili.com/x/passport-login/web/key", - headers=headers, - params=params_data - ) - r_data = r.json()['data'] - return r_data['hash'], r_data['key'], '' - - -def get_capcha(sid): - headers = { - 'User-Agent': '', - 'Accept-Encoding': 'gzip,deflate', - } - - params = { - 'appkey': APPKEY, - 'platform': 'pc', - 'ts': str(int(datetime.now().timestamp())) - } - params['sign'] = cipher.sign_dict(params, APPSECRET) - - r = requests.get( - "https://passport.bilibili.com/captcha", - headers=headers, - params=params, - cookies={ - 'sid': sid - } - ) - - print(r.status_code) - - capcha_data = r.content - - return r.cookies['JSESSIONID'], capcha_data - - -def login_by_access_token(access_token): - """ - bilibili access token login. - Args: - access_token: Bilibili access token got by previous username/password login. - - Returns: - sid: session id. - mid: member id. - expires_in: access token expire time - """ - headers = { - 'Connection': 'keep-alive', - 'Accept-Encoding': 'gzip,deflate', - 'Host': 'passport.bilibili.com', - 'User-Agent': '', - } - - login_params = { - 'appkey': APPKEY, - 'access_token': access_token, - 'platform': "pc", - 'ts': str(int(datetime.now().timestamp())), - } - login_params['sign'] = cipher.sign_dict(login_params, APPSECRET) - - r = requests.get( - url="https://passport.bilibili.com/api/oauth2/info", - headers=headers, - params=login_params - ) - - login_data = r.json()['data'] - - return r.cookies['sid'], login_data['mid'], login_data["expires_in"] - - -def upload_cover(access_token, sid, cover_file_path): - with open(cover_file_path, "rb") as f: - cover_pic = f.read() - - headers = { - 'Connection': 'keep-alive', - 'Host': 'member.bilibili.com', - 'Accept-Encoding': 'gzip,deflate', - 'User-Agent': '', - } - - params = { - "access_key": access_token, - } - - params["sign"] = cipher.sign_dict(params, APPSECRET) - - files = { - 'file': ("cover.png", cover_pic, "Content-Type: image/png"), - } - - r = requests.post( - "http://member.bilibili.com/x/vu/client/cover/up", - headers=headers, - params=params, - files=files, - cookies={ - 'sid': sid - }, - verify=False, - ) - - return r.json()["data"]["url"] - - -def upload_chunk(upload_url, server_file_name, local_file_name, chunk_data, chunk_size, chunk_id, chunk_total_num): - """ - upload video chunk. - Args: - upload_url: upload url by pre_upload api. - server_file_name: file name on server by pre_upload api. - local_file_name: video file name in local fs. - chunk_data: binary data of video chunk. - chunk_size: default of ugc_assisstant is 2M. - chunk_id: chunk number. - chunk_total_num: total chunk number. - - Returns: - True: upload chunk success. - False: upload chunk fail. - """ - print("filename: {}".format(local_file_name), "chunk{}/{}".format(chunk_id, chunk_total_num)) - files = { - 'version': (None, '2.0.0.1054'), - 'filesize': (None, chunk_size), - 'chunk': (None, chunk_id), - 'chunks': (None, chunk_total_num), - 'md5': (None, cipher.md5_bytes(chunk_data)), - 'file': (local_file_name, chunk_data, 'application/octet-stream') - } - - r = requests.post( - url=upload_url, - files=files, - cookies={ - 'PHPSESSID': server_file_name - }, - ) - r.raise_for_status() - if r.status_code == 200 and r.json().get("OK", 0) == 1: - return True - else: - print(r.status_code) - print(r.content) - return False - - -def upload_video_part(access_token, sid, mid, video_part: VideoPart, max_retry=5): - """ - upload a video file. - Args: - access_token: access token generated by login api. - sid: session id. - mid: member id. - video_part: local video file data. - max_retry: max retry number for each chunk. - - Returns: - status: success or fail. - server_file_name: server file name by pre_upload api. - """ - if not isinstance(video_part, VideoPart): - return False - if video_part.server_file_name is not None: - return True - headers = { - 'Connection': 'keep-alive', - 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', - 'User-Agent': '', - 'Accept-Encoding': 'gzip,deflate', - } - - r = requests.get( - "http://member.bilibili.com/preupload?access_key={}&mid={}&profile=ugcfr%2Fpc3".format(access_token, mid), - headers=headers, - cookies={ - 'sid': sid - }, - verify=False, - ) - - pre_upload_data = r.json() - upload_url = pre_upload_data['url'] - complete_upload_url = pre_upload_data['complete'] - server_file_name = pre_upload_data['filename'] - local_file_name = video_part.path - - file_size = os.path.getsize(local_file_name) - chunk_total_num = int(math.ceil(file_size / CHUNK_SIZE)) - file_hash = hashlib.md5() - with open(local_file_name, 'rb') as f: - for chunk_id in range(0, chunk_total_num): - chunk_data = f.read(CHUNK_SIZE) - status = Retry(max_retry=max_retry, success_return_value=True).run( - upload_chunk, - upload_url, - server_file_name, - os.path.basename(local_file_name), - chunk_data, - CHUNK_SIZE, - chunk_id, - chunk_total_num - ) - - if not status: - return False - file_hash.update(chunk_data) - print(file_hash.hexdigest()) - - # complete upload - post_data = { - 'chunks': chunk_total_num, - 'filesize': file_size, - 'md5': file_hash.hexdigest(), - 'name': os.path.basename(local_file_name), - 'version': '2.0.0.1054', - } - - r = requests.post( - url=complete_upload_url, - data=post_data, - headers=headers, - ) - print(r.status_code) - print(r.content) - - video_part.server_file_name = server_file_name - - return True - - -def upload(access_token, - sid, - mid, - parts, - copyright: int, - title: str, - tid: int, - tag: str, - desc: str, - source: str = '', - cover: str = '', - no_reprint: int = 0, - open_elec: int = 1, - max_retry: int = 5, - thread_pool_workers: int = 1): - """ - upload video. - - Args: - access_token: oauth2 access token. - sid: session id. - mid: member id. - parts: VideoPart list. - copyright: 原创/转载. - title: 投稿标题. - tid: 分区id. - tag: 标签. - desc: 投稿简介. - source: 转载地址. - cover: 封面图片文件路径. - no_reprint: 可否转载. - open_elec: 充电. - max_retry: max retry time for each chunk. - thread_pool_workers: max upload threads. - - Returns: - (aid, bvid) - aid: av号 - bvid: bv号 - """ - if not isinstance(parts, list): - parts = [parts] - - status = True - with ThreadPoolExecutor(max_workers=thread_pool_workers) as tpe: - t_list = [] - for video_part in parts: - print("upload {} added in pool".format(video_part.title)) - t_obj = tpe.submit(upload_video_part, access_token, sid, mid, video_part, max_retry) - t_obj.video_part = video_part - t_list.append(t_obj) - - for t_obj in as_completed(t_list): - status = status and t_obj.result() - print("video part {} finished, status: {}".format(t_obj.video_part.title, t_obj.result())) - if not status: - print("upload failed") - return None, None - - # cover - if os.path.isfile(cover): - try: - cover = upload_cover(access_token, sid, cover) - except: - cover = '' - else: - cover = '' - - # submit - headers = { - 'Connection': 'keep-alive', - 'Content-Type': 'application/json', - 'User-Agent': '', - } - post_data = { - 'build': 1054, - 'copyright': copyright, - 'cover': cover, - 'desc': desc, - 'no_reprint': no_reprint, - 'open_elec': open_elec, - 'source': source, - 'tag': tag, - 'tid': tid, - 'title': title, - 'videos': [] - } - for video_part in parts: - post_data['videos'].append({ - "desc": video_part.desc, - "filename": video_part.server_file_name, - "title": video_part.title - }) - - params = { - 'access_key': access_token, - } - params['sign'] = cipher.sign_dict(params, APPSECRET) - r = requests.post( - url="http://member.bilibili.com/x/vu/client/add", - params=params, - headers=headers, - verify=False, - cookies={ - 'sid': sid - }, - json=post_data, - ) - - print("submit") - print(r.status_code) - print(r.content.decode()) - - data = r.json()["data"] - return data["aid"], data["bvid"] - - -def get_post_data(access_token, sid, avid): - headers = { - 'Connection': 'keep-alive', - 'Host': 'member.bilibili.com', - 'Accept-Encoding': 'gzip,deflate', - 'User-Agent': '', - } - - params = { - "access_key": access_token, - "aid": avid, - "build": "1054" - } - - params["sign"] = cipher.sign_dict(params, APPSECRET) - - r = requests.get( - url="http://member.bilibili.com/x/client/archive/view", - headers=headers, - params=params, - cookies={ - 'sid': sid - } - ) - - return r.json()["data"] - - -def edit_videos( - access_token, - sid, - mid, - avid=None, - bvid=None, - parts=None, - insert_index=None, - copyright=None, - title=None, - tid=None, - tag=None, - desc=None, - source=None, - cover=None, - no_reprint=None, - open_elec=None, - max_retry: int = 5, - thread_pool_workers: int = 1): - """ - insert videos into existed post. - - Args: - access_token: oauth2 access token. - sid: session id. - mid: member id. - avid: av number, - bvid: bv string, - parts: VideoPart list. - insert_index: new video index. - copyright: 原创/转载. - title: 投稿标题. - tid: 分区id. - tag: 标签. - desc: 投稿简介. - source: 转载地址. - cover: cover url. - no_reprint: 可否转载. - open_elec: 充电. - max_retry: max retry time for each chunk. - thread_pool_workers: max upload threads. - - Returns: - (aid, bvid) - aid: av号 - bvid: bv号 - """ - if not avid and not bvid: - print("please provide avid or bvid") - return None, None - if not avid: - avid = cipher.bv2av(bvid) - if not isinstance(parts, list): - parts = [parts] - if type(avid) is str: - avid = int(avid) - - post_video_data = get_post_data(access_token, sid, avid) - - status = True - with ThreadPoolExecutor(max_workers=thread_pool_workers) as tpe: - t_list = [] - for video_part in parts: - print("upload {} added in pool".format(video_part.title)) - t_obj = tpe.submit(upload_video_part, access_token, sid, mid, video_part, max_retry) - t_obj.video_part = video_part - t_list.append(t_obj) - - for t_obj in as_completed(t_list): - status = status and t_obj.result() - print("video part {} finished, status: {}".format(t_obj.video_part.title, t_obj.result())) - if not status: - print("upload failed") - return None, None - - headers = { - 'Connection': 'keep-alive', - 'Content-Type': 'application/json', - 'User-Agent': '', - } - submit_data = { - 'aid': avid, - 'build': 1054, - 'copyright': post_video_data["archive"]["copyright"], - 'cover': post_video_data["archive"]["cover"], - 'desc': post_video_data["archive"]["desc"], - 'no_reprint': post_video_data["archive"]["no_reprint"], - 'open_elec': post_video_data["archive_elec"]["state"], # open_elec not tested - 'source': post_video_data["archive"]["source"], - 'tag': post_video_data["archive"]["tag"], - 'tid': post_video_data["archive"]["tid"], - 'title': post_video_data["archive"]["title"], - 'videos': post_video_data["videos"] - } - - # cover - if os.path.isfile(cover): - try: - cover = upload_cover(access_token, sid, cover) - except: - cover = '' - else: - cover = '' - - # edit archive data - if copyright: - submit_data["copyright"] = copyright - if title: - submit_data["title"] = title - if tid: - submit_data["tid"] = tid - if tag: - submit_data["tag"] = tag - if desc: - submit_data["desc"] = desc - if source: - submit_data["source"] = source - if cover: - submit_data["cover"] = cover - if no_reprint: - submit_data["no_reprint"] = no_reprint - if open_elec: - submit_data["open_elec"] = open_elec - - if type(insert_index) is int: - for i, video_part in enumerate(parts): - submit_data['videos'].insert(insert_index + i, { - "desc": video_part.desc, - "filename": video_part.server_file_name, - "title": video_part.title - }) - elif insert_index is None: - for video_part in parts: - submit_data['videos'].append({ - "desc": video_part.desc, - "filename": video_part.server_file_name, - "title": video_part.title - }) - else: - print("wrong insert index") - return None, None - - params = { - 'access_key': access_token, - } - params['sign'] = cipher.sign_dict(params, APPSECRET) - r = requests.post( - url="http://member.bilibili.com/x/vu/client/edit", - params=params, - headers=headers, - verify=False, - cookies={ - 'sid': sid - }, - json=submit_data, - ) - - print("edit submit") - print(r.status_code) - print(r.content.decode()) - - data = r.json()["data"] - return data["aid"], data["bvid"] diff --git a/workflow/bilibiliupload/util/__init__.py b/workflow/bilibiliupload/util/__init__.py deleted file mode 100644 index fe276f7..0000000 --- a/workflow/bilibiliupload/util/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .cipher import * \ No newline at end of file diff --git a/workflow/bilibiliupload/util/cipher.py b/workflow/bilibiliupload/util/cipher.py deleted file mode 100644 index a88c948..0000000 --- a/workflow/bilibiliupload/util/cipher.py +++ /dev/null @@ -1,119 +0,0 @@ -import hashlib -import rsa -import base64 -import subprocess -import platform -import os.path - - -def md5(data: str): - """ - generate md5 hash of utf-8 encoded string. - """ - return hashlib.md5(data.encode("utf-8")).hexdigest() - - -def md5_bytes(data: bytes): - """ - generate md5 hash of binary. - """ - return hashlib.md5(data).hexdigest() - - -def sign_str(data: str, app_secret: str): - """ - sign a string of request parameters - Args: - data: string of request parameters, must be sorted by key before input. - app_secret: a secret string coupled with app_key. - - Returns: - A hash string. len=32 - """ - return md5(data + app_secret) - - -def sign_dict(data: dict, app_secret: str): - """ - sign a dictionary of request parameters - Args: - data: dictionary of request parameters. - app_secret: a secret string coupled with app_key. - - Returns: - A hash string. len=32 - """ - data_str = [] - keys = list(data.keys()) - keys.sort() - for key in keys: - data_str.append("{}={}".format(key, data[key])) - data_str = "&".join(data_str) - data_str = data_str + app_secret - return md5(data_str) - - -def login_sign_dict_bin(data: dict): - data_str = [] - keys = list(data.keys()) - keys.sort() - for key in keys: - data_str.append("{}={}".format(key, data[key])) - data_str = "&".join(data_str) - package_directory = os.path.dirname(os.path.abspath(__file__)) - if platform.system().lower() == 'windows': - print(data_str) - print(subprocess.Popen([os.path.join(package_directory, "sign.exe"), data_str], stdout=subprocess.PIPE).communicate()[0].decode().strip()) - - return subprocess.Popen([os.path.join(package_directory, "sign.exe"), data_str], stdout=subprocess.PIPE).communicate()[0].decode().strip() - if platform.system().lower() == 'linux': - return subprocess.Popen([os.path.join(package_directory, "sign.out"), data_str], stdout=subprocess.PIPE).communicate()[0].decode().strip() - raise Exception("Operating System is not supported.") - - -def encrypt_login_password(password, hash, pubkey): - """ - encrypt password for login api. - Args: - password: plain text of user password. - hash: hash provided by /api/oauth2/getKey. - pubkey: public key provided by /api/oauth2/getKey. - - Returns: - An encrypted cipher of password. - """ - return base64.b64encode(rsa.encrypt( - (hash + password).encode('utf-8'), - rsa.PublicKey.load_pkcs1_openssl_pem(pubkey.encode()), - )) - - -def av2bv(av: int): - table = 'fZodR9XQDSUm21yCkr6zBqiveYah8bt4xsWpHnJE7jL5VG3guMTKNPAwcF' - tr = {} - for i in range(58): - tr[table[i]] = i - s = [11, 10, 3, 8, 4, 6] - xor = 177451812 - add = 8728348608 - - av = (av ^ xor) + add - r = list('BV1 4 1 7 ') - for i in range(6): - r[s[i]] = table[av // 58 ** i % 58] - return ''.join(r) - - -def bv2av(bv: str): - table = 'fZodR9XQDSUm21yCkr6zBqiveYah8bt4xsWpHnJE7jL5VG3guMTKNPAwcF' - tr = {} - for i in range(58): - tr[table[i]] = i - s = [11, 10, 3, 8, 4, 6] - xor = 177451812 - add = 8728348608 - - r = 0 - for i in range(6): - r += tr[bv[s[i]]] * 58 ** i - return (r - add) ^ xor diff --git a/workflow/bilibiliupload/util/retry.py b/workflow/bilibiliupload/util/retry.py deleted file mode 100644 index 96e5f40..0000000 --- a/workflow/bilibiliupload/util/retry.py +++ /dev/null @@ -1,21 +0,0 @@ -from time import sleep - - -class Retry: - def __init__(self, max_retry, success_return_value, sleep_sec = 60): - self.max_retry = max_retry - self.success_return_value = success_return_value - self.sleep_sec = 60 - - def run(self, func, *args, **kwargs): - status = False - for i in range(0, self.max_retry): - try: - return_value = func(*args, **kwargs) - except Exception: - sleep(self.sleep_sec) - continue - if return_value == self.success_return_value: - status = True - break - return status diff --git a/workflow/video.py b/workflow/video.py index 6661ce2..ef549a7 100644 --- a/workflow/video.py +++ b/workflow/video.py @@ -48,6 +48,7 @@ def encode_video_with_subtitles(orig_filename: str, subtitles: list[str], base_t "file": "{}.mp4".format(current_dt), }) current_sec += VIDEO_CLIP_EACH_SEC + return _video_parts def get_encode_process_use_handbrake(orig_filename: str, subtitles: list[str], new_filename: str, start_time: int, stop_time: int): @@ -136,7 +137,7 @@ def quick_split_video(file): raise FileNotFoundError(file) file_name = os.path.split(file)[-1] _create_dt = os.path.splitext(file_name)[0] - create_dt = datetime.strptime(_create_dt, "%Y%m%d_%H%M") + create_dt = datetime.strptime(_create_dt[:13], "%Y%m%d_%H%M") _duration_str = get_video_real_duration(file) duration = duration_str_to_float(_duration_str) current_sec = 0 diff --git a/workflow/worker.py b/workflow/worker.py index b125085..9b7a323 100644 --- a/workflow/worker.py +++ b/workflow/worker.py @@ -13,7 +13,7 @@ def do_workflow(video_file, danmaku_base_file, *danmaku_files): start_ts = get_file_start(danmaku_base_file) except DanmakuException: print("基准弹幕文件异常，跳过") - return + return [] result.append(danmaku_to_subtitle(danmaku_base_file, 0)) for danmaku_file in danmaku_files: try: