diff --git a/Common.py b/Common.py index 164f060..7c42f59 100644 --- a/Common.py +++ b/Common.py @@ -15,6 +15,7 @@ def reloadConfig(): config = json.load(_config_fp) _config_fp.close() + dt_format="%Y/%m/%d %H:%M:%S" broadcaster = "" @@ -23,6 +24,7 @@ isBroadcasting = False updateTime = "" forceStopDownload = False +forceNotBroadcasting = False uploadQueue = queue.Queue() encodeQueue = queue.Queue() @@ -33,6 +35,18 @@ encodeStatus = [] errors = [] +def parseSize(size): + K = size/1024.0 + if K > 1000: + M = K/1024.0 + if M > 1000: + return "{:.2f}GB".format(M / 1024.0) + else: + return "{:.2f}MB".format(M) + else: + return "{:.2f}MB".format(K) + + def appendUploadStatus(obj): global uploadStatus if isinstance(obj, dict): @@ -44,7 +58,7 @@ def appendUploadStatus(obj): "datetime": datetime.strftime(datetime.now(), dt_format), "message": str(obj) }) - uploadStatus = uploadStatus[-10:] + uploadStatus = uploadStatus[-config["l_c"]:] def modifyLastUploadStatus(obj): @@ -69,7 +83,7 @@ def appendEncodeStatus(obj): "datetime": datetime.strftime(datetime.now(), dt_format), "message": str(obj) }) - encodeStatus = encodeStatus[-10:] + encodeStatus = encodeStatus[-config["l_c"]:] def modifyLastEncodeStatus(obj): @@ -94,7 +108,7 @@ def appendDownloadStatus(obj): "datetime": datetime.strftime(datetime.now(), dt_format), "message": str(obj) }) - downloadStatus = downloadStatus[-10:] + downloadStatus = downloadStatus[-config["l_c"]:] def modifyLastDownloadStatus(obj): @@ -119,7 +133,7 @@ def appendError(obj): "datetime": datetime.strftime(datetime.now(), dt_format), "message": str(obj) }) - errors = errors[-10:] + errors = errors[-config["elc"]:] class downloader(XiGuaLiveApi): @@ -127,7 +141,7 @@ class downloader(XiGuaLiveApi): playlist = None def updRoomInfo(self): - global broadcaster, isBroadcasting, updateTime + global broadcaster, isBroadcasting, updateTime, forceNotBroadcasting, forceStopDownload super(downloader, self).updRoomInfo() updateTime = datetime.strftime(datetime.now(), dt_format) broadcaster = self.roomLiver @@ -135,6 +149,9 @@ class downloader(XiGuaLiveApi): if self.isLive: self.updPlayList() else: + forceStopDownload = False + forceNotBroadcasting = False + self.playlist = False self.files = [] def updPlayList(self): diff --git a/WebMain.py b/WebMain.py index 713f394..22a9204 100644 --- a/WebMain.py +++ b/WebMain.py @@ -1,6 +1,8 @@ +import os +from glob import glob from time import sleep from flask_cors import CORS -from flask import Flask, jsonify, request, redirect, url_for +from flask import Flask, jsonify, request, redirect, render_template, Response import Common import threading from liveDownloader import run as RUN @@ -58,19 +60,25 @@ def getAllStats(): return jsonify({"message":"ok","code":200,"status":0,"data":{ "download":Common.downloadStatus, "encode": Common.encodeStatus, + "encodeQueueSize": Common.encodeQueue.qsize(), "upload": Common.uploadStatus, + "uploadQueueSize": Common.uploadQueue.qsize(), "error": Common.errors, "broadcast": { "broadcaster": Common.broadcaster.__str__(), "isBroadcasting": Common.isBroadcasting, "streamUrl": Common.streamUrl, "updateTime": Common.updateTime + }, + "config": { + "forceNotBroadcasting": Common.forceNotBroadcasting, + "forceStopDownload": Common.forceStopDownload, } }}) @app.route("/stats/broadcast", methods=["GET"]) -def geBroadcastStats(): +def getBroadcastStats(): return jsonify({"message":"ok","code":200,"status":0,"data":{ "broadcast": { "broadcaster": Common.broadcaster.__str__(), @@ -81,8 +89,18 @@ def geBroadcastStats(): }}) +@app.route("/stats/config", methods=["GET"]) +def getConfigStats(): + return jsonify({"message":"ok","code":200,"status":0,"data":{ + "config": { + "forceNotBroadcasting": Common.forceNotBroadcasting, + "forceStopDownload": Common.forceStopDownload, + } + }}) + + @app.route("/stats/download", methods=["GET"]) -def geDownloadStats(): +def getDownloadStats(): return jsonify({"message":"ok","code":200,"status":0,"data":{ "download":Common.downloadStatus, }}) @@ -92,6 +110,7 @@ def geDownloadStats(): def getEncodeStats(): return jsonify({"message":"ok","code":200,"status":0,"data":{ "encode": Common.encodeStatus, + "encodeQueueSize": Common.encodeQueue.qsize(), }}) @@ -99,9 +118,30 @@ def getEncodeStats(): def getUploadStats(): return jsonify({"message":"ok","code":200,"status":0,"data":{ "upload": Common.uploadStatus, + "uploadQueueSize": Common.uploadQueue.qsize(), }}) +@app.route("/files/", methods=["GET"]) +def fileIndex(): + a = [] + for i in (glob("*.mp4") + glob("*.flv")): + a.append({ + "name": i, + "size": Common.parseSize(os.path.getsize(i)) + }) + return render_template("files.html",files=a) + + +@app.route("/files/download/", methods=["GET"]) +def fileDownload(path): + def generate(file): + with open(file, "rb") as f: + for row in f: + yield row + return Response(generate(path), mimetype='application/octet-stream') + + def SubThread(): t = threading.Thread(target=RUN, args=(Common.config['l_u'],)) t.setDaemon(True) diff --git a/bilibili.py b/bilibili.py index 22c9dd7..001b698 100644 --- a/bilibili.py +++ b/bilibili.py @@ -4,6 +4,7 @@ import os import re import json as JSON from datetime import datetime +from time import sleep from Common import appendUploadStatus, modifyLastUploadStatus import rsa import math @@ -11,7 +12,7 @@ import base64 import hashlib import requests from urllib import parse -requests.adapters.DEFAULT_RETRIES = 10 + class VideoPart: def __init__(self, path, title='', desc=''): @@ -25,6 +26,7 @@ class Bilibili: self.files = [] self.videos = [] self.session = requests.session() + self.session.keep_alive = False if cookie: self.session.headers["cookie"] = cookie self.csrf = re.search('bili_jct=(.*?);', cookie).group(1) @@ -203,6 +205,7 @@ class Bilibili: filepath = part.path filename = os.path.basename(filepath) filesize = os.path.getsize(filepath) + appendUploadStatus("Upload >{}< Started".format(filepath)) self.files.append(part) r = self.session.get('https://member.bilibili.com/preupload?' 'os=upos&upcdn=ws&name={name}&size={size}&r=upos&profile=ugcupos%2Fyb&ssl=0' @@ -237,12 +240,13 @@ class Bilibili: # {"upload_id":"72eb747b9650b8c7995fdb0efbdc2bb6","key":"\/i181012ws2wg1tb7tjzswk2voxrwlk1u.mp4","OK":1,"bucket":"ugc"} json = r.json() upload_id = json['upload_id'] - appendUploadStatus("Upload >{}< Started".format(filepath)) with open(filepath, 'rb') as f: chunks_num = math.ceil(filesize / chunk_size) chunks_index = 0 chunks_data = f.read(chunk_size) + modifyLastUploadStatus("Uploading >{}< @ {:.2f}%".format(filepath, 100.0 * chunks_index / chunks_num)) while True: + _d = datetime.now() if not chunks_data: break r = self.session.put('https:{endpoint}/{upos_uri}?' @@ -265,6 +269,8 @@ class Bilibili: chunks_data = f.read(chunk_size) chunks_index += 1 # start with 0 modifyLastUploadStatus("Uploading >{}< @ {:.2f}%".format(filepath, 100.0*chunks_index/chunks_num)) + if (datetime.now()-_d).seconds < 2: + sleep(1) # NOT DELETE! Refer to https://github.com/comwrg/bilibiliupload/issues/15#issuecomment-424379769 self.session.post('https:{endpoint}/{upos_uri}?' @@ -330,7 +336,7 @@ class Bilibili: "order_id": 0, "videos": self.videos} ) - appendUploadStatus(">{}< Published | Result : {}".format(title, r.text)) + appendUploadStatus("[{}] Published | Result : {}".format(title, r.text)) def reloadFromPrevious(self): if os.path.exists("uploaded.json"): diff --git a/liveDownloader.py b/liveDownloader.py index 5829dcf..cb284d0 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -17,20 +17,20 @@ def download(url): path = datetime.strftime(datetime.now(), "%Y%m%d_%H%M.flv") p = requests.get(url, stream=True) if p.status_code != 200: - appendDownloadStatus("Download [{}] Response 404 ,will stop looping".format(url)) + appendDownloadStatus("Download with Response 404, maybe broadcaster is not broadcasting") return True isDownload = True - appendDownloadStatus("Starting Download >{}<".format(path)) + appendDownloadStatus("Download >{}< Start".format(path)) f = open(path, "wb") try: for t in p.iter_content(chunk_size=64 * 1024): if t: f.write(t) _size = os.path.getsize(path) - modifyLastDownloadStatus("Download >{}< @ {:.2f}%".format(path, 100.0 * _size/config["p_s"])) + modifyLastDownloadStatus("Downloading >{}< @ {:.2f}%".format(path, 100.0 * _size/config["p_s"])) if _size > config["p_s"] or forceStopDownload: break - modifyLastDownloadStatus("Finished Download >{}<".format(path)) + modifyLastDownloadStatus("Download >{}< Finished".format(path)) except Exception as e: appendError("Download >{}< With Exception {}".format(path, datetime.strftime(datetime.now(), "%y%m%d %H%M"), e.__str__())) @@ -53,10 +53,10 @@ def encode(): i = encodeQueue.get() if os.path.exists(i): isEncode = True - appendEncodeStatus("Start Encoding >{}<".format(i)) + appendEncodeStatus("Encoding >{}< Start".format(i)) os.system("ffmpeg -i {} -c:v copy -c:a copy -f mp4 {} -y".format(i, i[:13] + ".mp4")) uploadQueue.put(i[:13] + ".mp4") - modifyLastEncodeStatus("Finished Encoding >{}<".format(i)) + modifyLastEncodeStatus("Encode >{}< Finished".format(i)) if config["mv"]: shutil.move(i, config["mtd"]) elif config["del"]: @@ -107,10 +107,10 @@ def run(name): _count = 0 _count_error = 0 while True: - if api.isLive: + if api.isLive and not forceNotBroadcasting: if d is None: d = datetime.strftime(datetime.now(), "%Y_%m_%d") - if not t.is_alive(): + if not t.is_alive() and not forceStopDownload: _count_error += 1 _preT = api.playlist t = threading.Thread(target=download, args=(_preT,)) diff --git a/static/index.html b/static/index.html new file mode 100644 index 0000000..a457651 --- /dev/null +++ b/static/index.html @@ -0,0 +1,73 @@ + + + + + 录播 + + + + +
+

基本信息

+ + + + + + + + + + + + + + + + + +
主播名
是否正在直播
直播视频流地址
信息更新时间
+

特殊设置

+ + + + + + + + + +
是否设置强制认为不直播
是否设置强制不下载
+
+

当前状态

+ + + + + + + + + + + + + + + + + + + + + +
下载日志
转码日志
上传日志
错误日志
操作日志
+
+
+ + + \ No newline at end of file diff --git a/static/index.js b/static/index.js new file mode 100644 index 0000000..86f19b6 --- /dev/null +++ b/static/index.js @@ -0,0 +1,45 @@ +function update(){ + $.ajax( + "/stats", + { + success: function (res){ + $("#broadcaster").text(res.data.broadcast.broadcaster) + $("#isBroadcasting").text(res.data.broadcast.isBroadcasting) + $("#streamUrl").text(res.data.broadcast.streamUrl) + $("#forceNotBroadcasting").text(res.data.config.forceNotBroadcasting) + $("#forceStopDownload").text(res.data.config.forceStopDownload) + $("#updateTime").text(res.data.broadcast.updateTime) + $("#download").html(function(){ + var ret = "" + res.data.download.reverse().forEach(function(obj){ + ret += "" + obj.datetime + "" + obj.message + "" + }) + return "" + ret + "
" + }) + $("#encode").html(function(){ + var ret = "" + res.data.encode.reverse().forEach(function(obj){ + ret += "" + obj.datetime + "" + obj.message + "" + }) + return "" + ret + "
" + }) + $("#upload").html(function(){ + var ret = "" + res.data.upload.reverse().forEach(function(obj){ + ret += "" + obj.datetime + "" + obj.message + "" + }) + return "" + ret + "
" + }) + $("#error").html(function(){ + var ret = "" + res.data.error.reverse().forEach(function(obj){ + ret += "" + obj.datetime + "" + obj.message + "" + }) + return "" + ret + "
" + }) + } + } + ) +} +update() +setInterval(update,10000) \ No newline at end of file diff --git a/templates/files.html b/templates/files.html new file mode 100644 index 0000000..891e537 --- /dev/null +++ b/templates/files.html @@ -0,0 +1,25 @@ + + + + + 文件 + + + +

所有录像文件

+ + + + + {%for i in files %} + + + + {% endfor %} +
文件名文件大小链接
{{i.name}}{{i.size}}下载文件
+ + \ No newline at end of file