From ef10d0bf0f92f76a9ca1c0fcdc27b3f9f2582f81 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Mon, 25 Jul 2022 16:24:11 +0800 Subject: [PATCH] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=8A=95=E7=A8=BFalpha?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 +- config.py | 19 +- controller/api/bilirecorder_blueprint.py | 30 +- controller/api/collector_blueprint.py | 5 +- requirements.txt | 4 +- templates/index.html | 14 +- workflow/bilibili.py | 132 ++++ workflow/bilibiliupload/README.md | 4 + workflow/bilibiliupload/__init__.py | 4 + workflow/bilibiliupload/bilibiliuploader.py | 91 +++ workflow/bilibiliupload/core.py | 646 ++++++++++++++++++++ workflow/bilibiliupload/util/__init__.py | 1 + workflow/bilibiliupload/util/cipher.py | 119 ++++ workflow/bilibiliupload/util/retry.py | 18 + workflow/video.py | 6 + workflow/worker.py | 5 +- 16 files changed, 1086 insertions(+), 15 deletions(-) create mode 100644 workflow/bilibili.py create mode 100644 workflow/bilibiliupload/README.md create mode 100644 workflow/bilibiliupload/__init__.py create mode 100644 workflow/bilibiliupload/bilibiliuploader.py create mode 100644 workflow/bilibiliupload/core.py create mode 100644 workflow/bilibiliupload/util/__init__.py create mode 100644 workflow/bilibiliupload/util/cipher.py create mode 100644 workflow/bilibiliupload/util/retry.py diff --git a/.gitignore b/.gitignore index 9d533f5..294b402 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ __pycache__ *.py[cod] venv build/ -dist/ \ No newline at end of file +dist/ +access_token \ No newline at end of file diff --git a/config.py b/config.py index 5b02614..e382429 100644 --- a/config.py +++ b/config.py @@ -29,6 +29,17 @@ VIDEO_CRF = 28 # [video] # title VIDEO_TITLE = "【永恒de草薙直播录播】直播于 {}" +# desc +VIDEO_DESC = "弹幕来源：B站直播\r\n\r\n" + \ + "原主播：永恒de草薙\r\n往期节目单查询：https://comment.sc.jerryyan.top\r\n\r\n" + \ + "好多人这时候就开始问了，在哪直播呀，哎。对吧…咱来啦啊。在哔哩哔哩啊，" \ + "无论你是用网页百度搜索哔哩哔哩官网或者是用手机的APP下载一个哔哩哔哩，" \ + "啊，都能找到原主播。大概每天晚七点半左右吧…一般都是往左。然后的话呢搜索永恒de草薙就能找到他。" \ + "那么今天的话呢今天的录播也就发完了…他是本期的主播永恒。咱明天同一时间不见不散…拜拜！！" +# tid +VIDEO_TID = 17 +# tags +VIDEO_TAGS = "永恒de草薙,三国,三国战记,直播录像,录播,怀旧,街机" # [clip] # each_sec VIDEO_CLIP_EACH_SEC = 6000 @@ -59,8 +70,11 @@ def load_config(): VIDEO_RESOLUTION = section.get('resolution', VIDEO_RESOLUTION) if config.has_section("video"): section = config['video'] - global VIDEO_TITLE + global VIDEO_TITLE, VIDEO_DESC, VIDEO_TID, VIDEO_TAGS VIDEO_TITLE = section.get('title', VIDEO_TITLE) + VIDEO_DESC = section.get('desc', VIDEO_DESC) + VIDEO_TID = section.getint('tid', VIDEO_TID) + VIDEO_TAGS = section.getint('tags', VIDEO_TAGS) if config.has_section("clip"): section = config['clip'] global VIDEO_CLIP_EACH_SEC, VIDEO_CLIP_OVERFLOW_SEC @@ -95,6 +109,9 @@ def get_config(): }, 'video': { 'title': VIDEO_TITLE, + 'desc': VIDEO_DESC, + 'tid': VIDEO_TID, + 'tags': VIDEO_TAGS, }, 'clip': { 'each_sec': VIDEO_CLIP_EACH_SEC, diff --git a/controller/api/bilirecorder_blueprint.py b/controller/api/bilirecorder_blueprint.py index fb3efce..ba18709 100644 --- a/controller/api/bilirecorder_blueprint.py +++ b/controller/api/bilirecorder_blueprint.py @@ -1,25 +1,28 @@ import os.path -import threading +from concurrent.futures import ProcessPoolExecutor, Future from datetime import datetime from glob import glob from typing import Optional from flask import Blueprint, jsonify, request, current_app -from config import BILILIVE_RECORDER_DIRECTORY, VIDEO_TITLE, XIGUALIVE_RECORDER_DIRECTORY +from config import BILILIVE_RECORDER_DIRECTORY, VIDEO_TITLE, XIGUALIVE_RECORDER_DIRECTORY, VIDEO_OUTPUT_DIR, VIDEO_DESC, \ + VIDEO_TAGS, VIDEO_TID from exception.danmaku import DanmakuException from model import db from model.DanmakuClip import DanmakuClip from model.VideoClip import VideoClip +from model.VideoPart import VideoPart from model.Workflow import Workflow from workflow.danmaku import get_file_start from workflow.video import get_video_real_duration, duration_str_to_float from workflow.worker import do_workflow +from workflow.bilibili import IS_LIVING, INSTANCE as bilibili_instance blueprint = Blueprint("api_bilirecorder", __name__, url_prefix="/api/bilirecorder") bili_record_workflow_item: Optional[Workflow] = None - +pool = ProcessPoolExecutor(max_workers=4) def auto_submit_task(): global bili_record_workflow_item @@ -32,20 +35,29 @@ def auto_submit_task(): if len(bili_record_workflow_item.video_clips) == 0: print("[!]Auto Submit Fail: No Video Clips") return - _started = False for video_clip in bili_record_workflow_item.video_clips: if len(video_clip.danmaku_clips) > 0: print("[+]Workflow:", bili_record_workflow_item.id, "; Video:", video_clip.full_path) _started = True - threading.Thread(target=do_workflow, args=( + _future = pool.submit( + do_workflow, video_clip.full_path, video_clip.danmaku_clips[0].full_path, *[clip.full_path for clip in video_clip.danmaku_clips[1:]] - )).start() + ) + clear_item() + def _encode_finish_callback(_f: "Future"): + _result = _f.result() + if _result: + # start uploading + bilibili_instance.upload(parts=_result, + title=bili_record_workflow_item.name, + desc=VIDEO_DESC, + tid=VIDEO_TID, + tag=VIDEO_TAGS) + _future.add_done_callback(_encode_finish_callback) else: print("[-]Workflow:", bili_record_workflow_item.id, "; Video:", video_clip.full_path, "; No Danmaku") - if _started: - clear_item() def clear_item(): @@ -156,9 +168,11 @@ def bilirecorder_event(): return response if payload['EventType'] == "SessionStarted": + IS_LIVING.set() # 录制开始 safe_create_item() elif payload['EventType'] == "SessionEnded": + IS_LIVING.clear() # 录制结束 item = safe_get_item() item.editing = False diff --git a/controller/api/collector_blueprint.py b/controller/api/collector_blueprint.py index 5759c2e..98aee72 100644 --- a/controller/api/collector_blueprint.py +++ b/controller/api/collector_blueprint.py @@ -6,6 +6,7 @@ from flask import Blueprint, jsonify from config import DANMAKU_FACTORY_EXEC, FFMPEG_EXEC, BILILIVE_RECORDER_DIRECTORY, XIGUALIVE_RECORDER_DIRECTORY, VIDEO_OUTPUT_DIR from util.system import check_exec +from workflow.bilibili import IS_LIVING, IS_UPLOADING blueprint = Blueprint("api_collector", __name__, url_prefix="/api/collector") @@ -63,5 +64,7 @@ def collect_basic_status(): }, 'system': { 'os': platform.system(), - } + }, + 'living': IS_LIVING.is_set(), + 'uploading': IS_UPLOADING.is_set(), }) diff --git a/requirements.txt b/requirements.txt index 1adfbf0..e084ff6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,6 @@ bs4~=0.0.1 Flask~=2.1.2 psutil~=5.9.0 Flask-SQLAlchemy~=2.5.1 -lxml~=4.8 \ No newline at end of file +lxml~=4.8 +requests~=2.28.1 +rsa~=4.8 \ No newline at end of file diff --git a/templates/index.html b/templates/index.html index c56490e..984da12 100644 --- a/templates/index.html +++ b/templates/index.html @@ -49,6 +49,14 @@ 弹幕工具状态 + + 当前录制状态 + + + + 当前上传状态 + +

配置状态

@@ -214,7 +222,8 @@ free: "", total: "" } - } + }, + living: false }, }, config: { @@ -227,6 +236,9 @@ }, video: { title: "", + desc: "", + tid: 0, + tags: "", }, clip: { each_sec: 0, diff --git a/workflow/bilibili.py b/workflow/bilibili.py new file mode 100644 index 0000000..85ccc20 --- /dev/null +++ b/workflow/bilibili.py @@ -0,0 +1,132 @@ +import threading + +from . import LOGGER +from .bilibiliupload import core, VideoPart + + +IS_LIVING = threading.Event() +IS_UPLOADING = threading.Event() + + +class Bilibili: + def __init__(self): + self.access_token = "" + self.session_id = "" + self.user_id = "" + self.parts = [] + + def login(self): + with open("access_token", "r") as f: + self.access_token = f.read(64).strip() + self.session_id, self.user_id, expires = core.login_by_access_token(self.access_token) + LOGGER.info("B站登录，UID【{}】，过期时间【{}】".format(self.user_id, expires)) + + def upload(self, + parts, + title, + tid, + tag, + desc, + source='', + cover='', + no_reprint=1, + ): + """ + + :param parts: e.g. VideoPart('part path', 'part title', 'part desc'), or [VideoPart(...), VideoPart(...)] + :type parts: VideoPart or list + :param title: video's title + :type title: str + :param tid: video type, see: https://member.bilibili.com/x/web/archive/pre + or https://github.com/uupers/BiliSpider/wiki/%E8%A7%86%E9%A2%91%E5%88%86%E5%8C%BA%E5%AF%B9%E5%BA%94%E8%A1%A8 + :type tid: int + :param tag: video's tag + :type tag: list + :param desc: video's description + :type desc: str + :param source: (optional) 转载地址 + :type source: str + :param cover: (optional) cover's URL, use method *cover_up* to get + :type cover: str + :param no_reprint: (optional) 0=可以转载, 1=禁止转载(default) + :type no_reprint: int + """ + self.pre_upload(parts) + self.finish_upload(title, tid, tag, desc, source, cover, no_reprint) + self.clear() + + def pre_upload(self, parts: "VideoPart", max_retry=5): + """ + :param max_retry: + :param parts: e.g. VideoPart('part path', 'part title', 'part desc'), or [VideoPart(...), VideoPart(...)] + :type parts: VideoPart or list + """ + if not isinstance(parts, list): + parts = [parts] + + def log_status(video_part, chunks_index: int, chunks_num: int): + LOGGER.debug("Uploading >{}< @ {:.2f}%".format(video_part.path, 100.0 * chunks_index / chunks_num)) + + IS_UPLOADING.set() + for part in parts: + if isinstance(part, str): + part = VideoPart(part) + LOGGER.info("Start Uploading >{}<".format(part.path)) + status = core.upload_video_part(self.access_token, self.session_id, self.user_id, part, max_retry, cb=log_status) + if status: + # 上传完毕 + LOGGER.info("Upload >{}< Finished；【{}】".format(part.path, part.server_file_name)) + self.parts.append(part) + else: + LOGGER.warn("Upload >{}< Failed".format(part.path)) + IS_UPLOADING.clear() + + def finish_upload(self, + title, + tid, + tag, + desc, + source='', + cover='', + no_reprint=1, + ): + """ + :param title: video's title + :type title: str + :param tid: video type, see: https://member.bilibili.com/x/web/archive/pre + or https://github.com/uupers/BiliSpider/wiki/%E8%A7%86%E9%A2%91%E5%88%86%E5%8C%BA%E5%AF%B9%E5%BA%94%E8%A1%A8 + :type tid: int + :param tag: video's tag + :type tag: str + :param desc: video's description + :type desc: str + :param source: (optional) 转载地址 + :type source: str + :param cover: (optional) cover's URL, use method *cover_up* to get + :type cover: str + :param no_reprint: (optional) 0=可以转载, 1=禁止转载(default) + :type no_reprint: int + :param copyright: (optional) 0=转载的, 1=自制的(default) + :type copyright: int + """ + if len(self.parts) == 0: + return + LOGGER.info("[{}]投稿中，请稍后".format(title)) + copyright = 2 if source else 1 + try: + avid, bvid = core.upload(self.access_token, self.session_id, self.user_id, self.parts, copyright, + title=title, tid=tid, tag=tag, desc=desc, source=source, cover=cover, no_reprint=no_reprint) + LOGGER.info("[{}]投稿成功；AVID【{}】，BVID【{}】".format(title, avid, bvid)) + self.clear() + except Exception as e: + LOGGER.error("[{}]投稿失败".format(title), exc_info=e) + + + def reloadFromPrevious(self): + ... + + def clear(self): + self.parts = [] + + +INSTANCE = Bilibili() diff --git a/workflow/bilibiliupload/README.md b/workflow/bilibiliupload/README.md new file mode 100644 index 0000000..68c4d09 --- /dev/null +++ b/workflow/bilibiliupload/README.md @@ -0,0 +1,4 @@ +修改自 +[BilibiliUploader](https://github.com/FortuneDayssss/BilibiliUploader/) + +LICENSE：GPL \ No newline at end of file diff --git a/workflow/bilibiliupload/__init__.py b/workflow/bilibiliupload/__init__.py new file mode 100644 index 0000000..49b4a54 --- /dev/null +++ b/workflow/bilibiliupload/__init__.py @@ -0,0 +1,4 @@ +from .bilibiliuploader import BilibiliUploader +from .core import VideoPart + +__version__ = '0.0.6' diff --git a/workflow/bilibiliupload/bilibiliuploader.py b/workflow/bilibiliupload/bilibiliuploader.py new file mode 100644 index 0000000..da6152a --- /dev/null +++ b/workflow/bilibiliupload/bilibiliuploader.py @@ -0,0 +1,91 @@ +from .core import login_by_access_token, upload, edit_videos +from .util import cipher + + +class BilibiliUploader(): + def __init__(self): + self.access_token = None + self.refresh_token = None + self.sid = None + self.mid = None + + def login_by_access_token(self, access_token, refresh_token=None): + self.access_token = access_token + self.refresh_token = refresh_token + self.sid, self.mid, _ = login_by_access_token(access_token) + + def upload(self, + parts, + copyright: int, + title: str, + tid: int, + tag: str, + desc: str, + source: str = '', + cover: str = '', + no_reprint: int = 0, + open_elec: int = 1, + max_retry: int = 5, + thread_pool_workers: int = 1): + return upload(self.access_token, + self.sid, + self.mid, + parts, + copyright, + title, + tid, + tag, + desc, + source, + cover, + no_reprint, + open_elec, + max_retry, + thread_pool_workers) + + def edit(self, + avid=None, + bvid=None, + parts=None, + insert_index=None, + copyright=None, + title=None, + tid=None, + tag=None, + desc=None, + source=None, + cover=None, + no_reprint=None, + open_elec=None, + max_retry: int = 5, + thread_pool_workers: int = 1): + + if not avid and not bvid: + print("please provide avid or bvid") + return None, None + if not avid: + avid = cipher.bv2av(bvid) + if not isinstance(parts, list): + parts = [parts] + if type(avid) is str: + avid = int(avid) + edit_videos( + self.access_token, + self.sid, + self.mid, + avid, + bvid, + parts, + insert_index, + copyright, + title, + tid, + tag, + desc, + source, + cover, + no_reprint, + open_elec, + max_retry, + thread_pool_workers + ) diff --git a/workflow/bilibiliupload/core.py b/workflow/bilibiliupload/core.py new file mode 100644 index 0000000..3e78f53 --- /dev/null +++ b/workflow/bilibiliupload/core.py @@ -0,0 +1,646 @@ +import requests +from datetime import datetime +from .util import cipher as cipher +import os +import math +import hashlib +from .util.retry import Retry +from concurrent.futures import ThreadPoolExecutor, as_completed + +# From PC ugc_assisstant +# APPKEY = 'aae92bc66f3edfab' +# APPSECRET = 'af125a0d5279fd576c1b4418a3e8276d' +APPKEY = '1d8b6e7d45233436' +APPSECRET = '560c52ccd288fed045859ed18bffd973' +LOGIN_APPKEY = '783bbb7264451d82' + +# upload chunk size = 2MB +CHUNK_SIZE = 2 * 1024 * 1024 + + +class VideoPart: + """ + Video Part of a post. + 每个对象代表一个分P + + Attributes: + path: file path in local file system. + title: title of the video part. + desc: description of the video part. + server_file_name: file name in bilibili server. generated by pre-upload API. + """ + + def __init__(self, path, title='', desc='', server_file_name=None): + self.path = path + self.title = title + self.desc = desc + self.server_file_name = server_file_name + + def __repr__(self): + return '<{clazz}, path: {path}, title: {title}, desc: {desc}, server_file_name:{server_file_name}>' \ + .format(clazz=self.__class__.__name__, + path=self.path, + title=self.title, + desc=self.desc, + server_file_name=self.server_file_name) + + +def get_key_old(sid=None, jsessionid=None): + """ + get public key, hash and session id for login. + Args: + sid: session id. only for captcha login. + jsessionid: j-session id. only for captcha login. + Returns: + hash: salt for password encryption. + pubkey: rsa public key for password encryption. + sid: session id. + """ + headers = { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Accept': "application/json, text/javascript, */*; q=0.01" + } + post_data = { + 'appkey': APPKEY, + 'platform': "pc", + 'ts': str(int(datetime.now().timestamp())) + } + post_data['sign'] = cipher.sign_dict(post_data, APPSECRET) + cookie = {} + if sid: + cookie['sid'] = sid + if jsessionid: + cookie['JSESSIONID'] = jsessionid + r = requests.post( + # "https://passport.bilibili.com/api/oauth2/getKey", + "https://passport.bilibili.com/x/passport-login/web/key", + headers=headers, + data=post_data, + cookies=cookie + ) + print(r.content.decode()) + r_data = r.json()['data'] + if sid: + return r_data['hash'], r_data['key'], sid + return r_data['hash'], r_data['key'], r.cookies['sid'] + + +def get_key(): + headers = { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Accept': "application/json, text/javascript, */*; q=0.01" + } + params_data = { + 'appkey': LOGIN_APPKEY, + # 'ts': str(int(datetime.now().timestamp())) + } + params_data['sign'] = cipher.login_sign_dict_bin(params_data) + r = requests.get( + "https://passport.bilibili.com/x/passport-login/web/key", + headers=headers, + params=params_data + ) + r_data = r.json()['data'] + return r_data['hash'], r_data['key'], '' + + +def get_capcha(sid): + headers = { + 'User-Agent': '', + 'Accept-Encoding': 'gzip,deflate', + } + + params = { + 'appkey': APPKEY, + 'platform': 'pc', + 'ts': str(int(datetime.now().timestamp())) + } + params['sign'] = cipher.sign_dict(params, APPSECRET) + + r = requests.get( + "https://passport.bilibili.com/captcha", + headers=headers, + params=params, + cookies={ + 'sid': sid + } + ) + + print(r.status_code) + + capcha_data = r.content + + return r.cookies['JSESSIONID'], capcha_data + + +def login_by_access_token(access_token): + """ + bilibili access token login. + Args: + access_token: Bilibili access token got by previous username/password login. + + Returns: + sid: session id. + mid: member id. + expires_in: access token expire time + """ + headers = { + 'Connection': 'keep-alive', + 'Accept-Encoding': 'gzip,deflate', + 'Host': 'passport.bilibili.com', + 'User-Agent': '', + } + + login_params = { + 'appkey': APPKEY, + 'access_token': access_token, + 'platform': "pc", + 'ts': str(int(datetime.now().timestamp())), + } + login_params['sign'] = cipher.sign_dict(login_params, APPSECRET) + + r = requests.get( + url="https://passport.bilibili.com/api/oauth2/info", + headers=headers, + params=login_params + ) + + login_data = r.json()['data'] + + return r.cookies['sid'], login_data['mid'], login_data["expires_in"] + + +def upload_cover(access_token, sid, cover_file_path): + with open(cover_file_path, "rb") as f: + cover_pic = f.read() + + headers = { + 'Connection': 'keep-alive', + 'Host': 'member.bilibili.com', + 'Accept-Encoding': 'gzip,deflate', + 'User-Agent': '', + } + + params = { + "access_key": access_token, + } + + params["sign"] = cipher.sign_dict(params, APPSECRET) + + files = { + 'file': ("cover.png", cover_pic, "Content-Type: image/png"), + } + + r = requests.post( + "http://member.bilibili.com/x/vu/client/cover/up", + headers=headers, + params=params, + files=files, + cookies={ + 'sid': sid + }, + verify=False, + ) + + return r.json()["data"]["url"] + + +def upload_chunk(upload_url, server_file_name, local_file_name, chunk_data, chunk_size, chunk_id, chunk_total_num): + """ + upload video chunk. + Args: + upload_url: upload url by pre_upload api. + server_file_name: file name on server by pre_upload api. + local_file_name: video file name in local fs. + chunk_data: binary data of video chunk. + chunk_size: default of ugc_assisstant is 2M. + chunk_id: chunk number. + chunk_total_num: total chunk number. + + Returns: + True: upload chunk success. + False: upload chunk fail. + """ + print("chunk{}/{}".format(chunk_id, chunk_total_num)) + print("filename: {}".format(local_file_name)) + files = { + 'version': (None, '2.0.0.1054'), + 'filesize': (None, chunk_size), + 'chunk': (None, chunk_id), + 'chunks': (None, chunk_total_num), + 'md5': (None, cipher.md5_bytes(chunk_data)), + 'file': (local_file_name, chunk_data, 'application/octet-stream') + } + + r = requests.post( + url=upload_url, + files=files, + cookies={ + 'PHPSESSID': server_file_name + }, + ) + print(r.status_code) + print(r.content) + + if r.status_code == 200 and r.json()['OK'] == 1: + return True + else: + return False + + +def upload_video_part(access_token, sid, mid, video_part: VideoPart, max_retry=5, cb=None): + """ + upload a video file. + Args: + access_token: access token generated by login api. + sid: session id. + mid: member id. + video_part: local video file data. + max_retry: max retry number for each chunk. + cb: 回调 + + Returns: + status: success or fail. + server_file_name: server file name by pre_upload api. + """ + if cb is None: + cb = lambda f, c, t: None + if not isinstance(video_part, VideoPart): + return False + if video_part.server_file_name is not None: + return True + headers = { + 'Connection': 'keep-alive', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', + 'User-Agent': '', + 'Accept-Encoding': 'gzip,deflate', + } + + r = requests.get( + "http://member.bilibili.com/preupload?access_key={}&mid={}&profile=ugcfr%2Fpc3".format(access_token, mid), + headers=headers, + cookies={ + 'sid': sid + }, + verify=False, + ) + + pre_upload_data = r.json() + upload_url = pre_upload_data['url'] + complete_upload_url = pre_upload_data['complete'] + server_file_name = pre_upload_data['filename'] + local_file_name = video_part.path + + file_size = os.path.getsize(local_file_name) + chunk_total_num = int(math.ceil(file_size / CHUNK_SIZE)) + file_hash = hashlib.md5() + with open(local_file_name, 'rb') as f: + for chunk_id in range(0, chunk_total_num): + chunk_data = f.read(CHUNK_SIZE) + cb(video_part, chunk_id, chunk_total_num) + status = Retry(max_retry=max_retry, success_return_value=True).run( + upload_chunk, + upload_url, + server_file_name, + os.path.basename(local_file_name), + chunk_data, + CHUNK_SIZE, + chunk_id, + chunk_total_num + ) + + if not status: + return False + file_hash.update(chunk_data) + print(file_hash.hexdigest()) + + # complete upload + post_data = { + 'chunks': chunk_total_num, + 'filesize': file_size, + 'md5': file_hash.hexdigest(), + 'name': os.path.basename(local_file_name), + 'version': '2.0.0.1054', + } + + r = requests.post( + url=complete_upload_url, + data=post_data, + headers=headers, + ) + print(r.status_code) + print(r.content) + + video_part.server_file_name = server_file_name + + return True + + +def upload(access_token, + sid, + mid, + parts, + copyright: int, + title: str, + tid: int, + tag: str, + desc: str, + source: str = '', + cover: str = '', + no_reprint: int = 0, + open_elec: int = 1, + max_retry: int = 5, + thread_pool_workers: int = 1): + """ + upload video. + + Args: + access_token: oauth2 access token. + sid: session id. + mid: member id. + parts: VideoPart list. + copyright: 原创/转载. + title: 投稿标题. + tid: 分区id. + tag: 标签. + desc: 投稿简介. + source: 转载地址. + cover: 封面图片文件路径. + no_reprint: 可否转载. + open_elec: 充电. + max_retry: max retry time for each chunk. + thread_pool_workers: max upload threads. + + Returns: + (aid, bvid) + aid: av号 + bvid: bv号 + """ + if not isinstance(parts, list): + parts = [parts] + + status = True + with ThreadPoolExecutor(max_workers=thread_pool_workers) as tpe: + t_list = [] + for video_part in parts: + print("upload {} added in pool".format(video_part.title)) + t_obj = tpe.submit(upload_video_part, access_token, sid, mid, video_part, max_retry) + t_obj.video_part = video_part + t_list.append(t_obj) + + for t_obj in as_completed(t_list): + status = status and t_obj.result() + print("video part {} finished, status: {}".format(t_obj.video_part.title, t_obj.result())) + if not status: + print("upload failed") + return None, None + + # cover + if os.path.isfile(cover): + try: + cover = upload_cover(access_token, sid, cover) + except: + cover = '' + else: + cover = '' + + # submit + headers = { + 'Connection': 'keep-alive', + 'Content-Type': 'application/json', + 'User-Agent': '', + } + post_data = { + 'build': 1054, + 'copyright': copyright, + 'cover': cover, + 'desc': desc, + 'no_reprint': no_reprint, + 'open_elec': open_elec, + 'source': source, + 'tag': tag, + 'tid': tid, + 'title': title, + 'videos': [] + } + for video_part in parts: + post_data['videos'].append({ + "desc": video_part.desc, + "filename": video_part.server_file_name, + "title": video_part.title + }) + + params = { + 'access_key': access_token, + } + params['sign'] = cipher.sign_dict(params, APPSECRET) + r = requests.post( + url="http://member.bilibili.com/x/vu/client/add", + params=params, + headers=headers, + verify=False, + cookies={ + 'sid': sid + }, + json=post_data, + ) + + print("submit") + print(r.status_code) + print(r.content.decode()) + + data = r.json()["data"] + return data["aid"], data["bvid"] + + +def get_post_data(access_token, sid, avid): + headers = { + 'Connection': 'keep-alive', + 'Host': 'member.bilibili.com', + 'Accept-Encoding': 'gzip,deflate', + 'User-Agent': '', + } + + params = { + "access_key": access_token, + "aid": avid, + "build": "1054" + } + + params["sign"] = cipher.sign_dict(params, APPSECRET) + + r = requests.get( + url="http://member.bilibili.com/x/client/archive/view", + headers=headers, + params=params, + cookies={ + 'sid': sid + } + ) + + return r.json()["data"] + + +def edit_videos( + access_token, + sid, + mid, + avid=None, + bvid=None, + parts=None, + insert_index=None, + copyright=None, + title=None, + tid=None, + tag=None, + desc=None, + source=None, + cover=None, + no_reprint=None, + open_elec=None, + max_retry: int = 5, + thread_pool_workers: int = 1): + """ + insert videos into existed post. + + Args: + access_token: oauth2 access token. + sid: session id. + mid: member id. + avid: av number, + bvid: bv string, + parts: VideoPart list. + insert_index: new video index. + copyright: 原创/转载. + title: 投稿标题. + tid: 分区id. + tag: 标签. + desc: 投稿简介. + source: 转载地址. + cover: cover url. + no_reprint: 可否转载. + open_elec: 充电. + max_retry: max retry time for each chunk. + thread_pool_workers: max upload threads. + + Returns: + (aid, bvid) + aid: av号 + bvid: bv号 + """ + if not avid and not bvid: + print("please provide avid or bvid") + return None, None + if not avid: + avid = cipher.bv2av(bvid) + if not isinstance(parts, list): + parts = [parts] + if type(avid) is str: + avid = int(avid) + + post_video_data = get_post_data(access_token, sid, avid) + + status = True + with ThreadPoolExecutor(max_workers=thread_pool_workers) as tpe: + t_list = [] + for video_part in parts: + print("upload {} added in pool".format(video_part.title)) + t_obj = tpe.submit(upload_video_part, access_token, sid, mid, video_part, max_retry) + t_obj.video_part = video_part + t_list.append(t_obj) + + for t_obj in as_completed(t_list): + status = status and t_obj.result() + print("video part {} finished, status: {}".format(t_obj.video_part.title, t_obj.result())) + if not status: + print("upload failed") + return None, None + + headers = { + 'Connection': 'keep-alive', + 'Content-Type': 'application/json', + 'User-Agent': '', + } + submit_data = { + 'aid': avid, + 'build': 1054, + 'copyright': post_video_data["archive"]["copyright"], + 'cover': post_video_data["archive"]["cover"], + 'desc': post_video_data["archive"]["desc"], + 'no_reprint': post_video_data["archive"]["no_reprint"], + 'open_elec': post_video_data["archive_elec"]["state"], # open_elec not tested + 'source': post_video_data["archive"]["source"], + 'tag': post_video_data["archive"]["tag"], + 'tid': post_video_data["archive"]["tid"], + 'title': post_video_data["archive"]["title"], + 'videos': post_video_data["videos"] + } + + # cover + if os.path.isfile(cover): + try: + cover = upload_cover(access_token, sid, cover) + except: + cover = '' + else: + cover = '' + + # edit archive data + if copyright: + submit_data["copyright"] = copyright + if title: + submit_data["title"] = title + if tid: + submit_data["tid"] = tid + if tag: + submit_data["tag"] = tag + if desc: + submit_data["desc"] = desc + if source: + submit_data["source"] = source + if cover: + submit_data["cover"] = cover + if no_reprint: + submit_data["no_reprint"] = no_reprint + if open_elec: + submit_data["open_elec"] = open_elec + + if type(insert_index) is int: + for i, video_part in enumerate(parts): + submit_data['videos'].insert(insert_index + i, { + "desc": video_part.desc, + "filename": video_part.server_file_name, + "title": video_part.title + }) + elif insert_index is None: + for video_part in parts: + submit_data['videos'].append({ + "desc": video_part.desc, + "filename": video_part.server_file_name, + "title": video_part.title + }) + else: + print("wrong insert index") + return None, None + + params = { + 'access_key': access_token, + } + params['sign'] = cipher.sign_dict(params, APPSECRET) + r = requests.post( + url="http://member.bilibili.com/x/vu/client/edit", + params=params, + headers=headers, + verify=False, + cookies={ + 'sid': sid + }, + json=submit_data, + ) + + print("edit submit") + print(r.status_code) + print(r.content.decode()) + + data = r.json()["data"] + return data["aid"], data["bvid"] diff --git a/workflow/bilibiliupload/util/__init__.py b/workflow/bilibiliupload/util/__init__.py new file mode 100644 index 0000000..fe276f7 --- /dev/null +++ b/workflow/bilibiliupload/util/__init__.py @@ -0,0 +1 @@ +from .cipher import * \ No newline at end of file diff --git a/workflow/bilibiliupload/util/cipher.py b/workflow/bilibiliupload/util/cipher.py new file mode 100644 index 0000000..a88c948 --- /dev/null +++ b/workflow/bilibiliupload/util/cipher.py @@ -0,0 +1,119 @@ +import hashlib +import rsa +import base64 +import subprocess +import platform +import os.path + + +def md5(data: str): + """ + generate md5 hash of utf-8 encoded string. + """ + return hashlib.md5(data.encode("utf-8")).hexdigest() + + +def md5_bytes(data: bytes): + """ + generate md5 hash of binary. + """ + return hashlib.md5(data).hexdigest() + + +def sign_str(data: str, app_secret: str): + """ + sign a string of request parameters + Args: + data: string of request parameters, must be sorted by key before input. + app_secret: a secret string coupled with app_key. + + Returns: + A hash string. len=32 + """ + return md5(data + app_secret) + + +def sign_dict(data: dict, app_secret: str): + """ + sign a dictionary of request parameters + Args: + data: dictionary of request parameters. + app_secret: a secret string coupled with app_key. + + Returns: + A hash string. len=32 + """ + data_str = [] + keys = list(data.keys()) + keys.sort() + for key in keys: + data_str.append("{}={}".format(key, data[key])) + data_str = "&".join(data_str) + data_str = data_str + app_secret + return md5(data_str) + + +def login_sign_dict_bin(data: dict): + data_str = [] + keys = list(data.keys()) + keys.sort() + for key in keys: + data_str.append("{}={}".format(key, data[key])) + data_str = "&".join(data_str) + package_directory = os.path.dirname(os.path.abspath(__file__)) + if platform.system().lower() == 'windows': + print(data_str) + print(subprocess.Popen([os.path.join(package_directory, "sign.exe"), data_str], stdout=subprocess.PIPE).communicate()[0].decode().strip()) + + return subprocess.Popen([os.path.join(package_directory, "sign.exe"), data_str], stdout=subprocess.PIPE).communicate()[0].decode().strip() + if platform.system().lower() == 'linux': + return subprocess.Popen([os.path.join(package_directory, "sign.out"), data_str], stdout=subprocess.PIPE).communicate()[0].decode().strip() + raise Exception("Operating System is not supported.") + + +def encrypt_login_password(password, hash, pubkey): + """ + encrypt password for login api. + Args: + password: plain text of user password. + hash: hash provided by /api/oauth2/getKey. + pubkey: public key provided by /api/oauth2/getKey. + + Returns: + An encrypted cipher of password. + """ + return base64.b64encode(rsa.encrypt( + (hash + password).encode('utf-8'), + rsa.PublicKey.load_pkcs1_openssl_pem(pubkey.encode()), + )) + + +def av2bv(av: int): + table = 'fZodR9XQDSUm21yCkr6zBqiveYah8bt4xsWpHnJE7jL5VG3guMTKNPAwcF' + tr = {} + for i in range(58): + tr[table[i]] = i + s = [11, 10, 3, 8, 4, 6] + xor = 177451812 + add = 8728348608 + + av = (av ^ xor) + add + r = list('BV1 4 1 7 ') + for i in range(6): + r[s[i]] = table[av // 58 ** i % 58] + return ''.join(r) + + +def bv2av(bv: str): + table = 'fZodR9XQDSUm21yCkr6zBqiveYah8bt4xsWpHnJE7jL5VG3guMTKNPAwcF' + tr = {} + for i in range(58): + tr[table[i]] = i + s = [11, 10, 3, 8, 4, 6] + xor = 177451812 + add = 8728348608 + + r = 0 + for i in range(6): + r += tr[bv[s[i]]] * 58 ** i + return (r - add) ^ xor diff --git a/workflow/bilibiliupload/util/retry.py b/workflow/bilibiliupload/util/retry.py new file mode 100644 index 0000000..d9f3151 --- /dev/null +++ b/workflow/bilibiliupload/util/retry.py @@ -0,0 +1,18 @@ + + +class Retry: + def __init__(self, max_retry, success_return_value): + self.max_retry = max_retry + self.success_return_value = success_return_value + + def run(self, func, *args, **kwargs): + status = False + for i in range(0, self.max_retry): + try: + return_value = func(*args, **kwargs) + except Exception: + return_value = not self.success_return_value + if return_value == self.success_return_value: + status = True + break + return status diff --git a/workflow/video.py b/workflow/video.py index 45e095e..603ef74 100644 --- a/workflow/video.py +++ b/workflow/video.py @@ -168,6 +168,7 @@ def quick_split_video(file): _duration_str = get_video_real_duration(file) duration = duration_str_to_float(_duration_str) current_sec = 0 + _video_parts = [] while current_sec < duration: if (current_sec + VIDEO_CLIP_OVERFLOW_SEC * 2) > duration: print("[-]Less than 2 overflow sec, skip") @@ -185,7 +186,12 @@ def quick_split_video(file): ], stdout=subprocess.PIPE) handle_ffmpeg_output(split_process.stdout) split_process.wait() + _video_parts.append({ + "base_path": VIDEO_OUTPUT_DIR, + "file": "{}.mp4".format(current_dt), + }) current_sec += VIDEO_CLIP_EACH_SEC + return _video_parts def _common_ffmpeg_setting(): diff --git a/workflow/worker.py b/workflow/worker.py index 48fd76d..f66e75c 100644 --- a/workflow/worker.py +++ b/workflow/worker.py @@ -27,8 +27,9 @@ def do_workflow(video_file, danmaku_base_file, *danmaku_files): continue print(result) file_need_split = encode_video_with_subtitles(video_file, result, start_ts) + _video_parts = [] for file in file_need_split: - quick_split_video(file) + _video_parts += quick_split_video(file) # clean files for file in result: if os.path.isfile(file): @@ -36,4 +37,4 @@ def do_workflow(video_file, danmaku_base_file, *danmaku_files): for file in file_need_split: if os.path.isfile(file): os.remove(file) - + return _video_parts