From b9b0994f4c56a96ec17e3ac06c02efef3b689459 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Sun, 7 Apr 2019 15:30:57 +0800 Subject: [PATCH 01/58] =?UTF-8?q?=E6=94=AF=E6=8C=81api=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E7=8A=B6=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Common.py | 170 ++++++++++++++++++++++++++++++++++++++++++++++ Model/DataBase.py | 8 +++ Model/Files.py | 10 +++ WebMain.py | 68 +++++++++++++++++++ bilibili.py | 17 +++-- liveDownloader.py | 133 ++++++++++-------------------------- 6 files changed, 300 insertions(+), 106 deletions(-) create mode 100644 Common.py create mode 100644 Model/DataBase.py create mode 100644 Model/Files.py create mode 100644 WebMain.py diff --git a/Common.py b/Common.py new file mode 100644 index 0000000..953a411 --- /dev/null +++ b/Common.py @@ -0,0 +1,170 @@ +import queue +from datetime import datetime + +from api import XiGuaLiveApi +import json + +_config_fp = open("config.json","r",encoding="utf8") +config = json.load(_config_fp) + +dt_format="%Y/%m/%d %H:%M:%S" + +broadcaster = "" +streamUrl = "" +isBroadcasting = False +updateTime = "" + +uploadQueue = queue.Queue() +encodeQueue = queue.Queue() + +uploadStatus = [] +downloadStatus = [] +encodeStatus = [] +errors = [] + + +def appendUploadStatus(obj): + global uploadStatus + if isinstance(obj, dict): + if "datetime" not in obj: + obj["datetime"] = datetime.strftime(datetime.now(), dt_format) + uploadStatus.append(obj) + else: + uploadStatus.append({ + "datetime": datetime.strftime(datetime.now(), dt_format), + "message": str(obj) + }) + uploadStatus = uploadStatus[-10:] + + +def modifyLastUploadStatus(obj): + global uploadStatus + if isinstance(obj, dict): + if "datetime" not in obj: + obj["datetime"] = datetime.strftime(datetime.now(), dt_format) + uploadStatus[-1] = obj + else: + uploadStatus[-1]["message"] = str(obj) + uploadStatus[-1]["datetime"] = datetime.strftime(datetime.now(), dt_format) + + +def appendEncodeStatus(obj): + global encodeStatus + if isinstance(obj, dict): + if "datetime" not in obj: + obj["datetime"] = datetime.strftime(datetime.now(), dt_format) + encodeStatus.append(obj) + else: + encodeStatus.append({ + "datetime": datetime.strftime(datetime.now(), dt_format), + "message": str(obj) + }) + encodeStatus = encodeStatus[-10:] + + +def modifyLastEncodeStatus(obj): + global encodeStatus + if isinstance(obj, dict): + if "datetime" not in obj: + obj["datetime"] = datetime.strftime(datetime.now(), dt_format) + encodeStatus[-1] = obj + else: + encodeStatus[-1]["message"] = str(obj) + encodeStatus[-1]["datetime"] = datetime.strftime(datetime.now(), dt_format) + + +def appendDownloadStatus(obj): + global downloadStatus + if isinstance(obj, dict): + if "datetime" not in obj: + obj["datetime"] = datetime.strftime(datetime.now(), dt_format) + downloadStatus.append(obj) + else: + downloadStatus.append({ + "datetime": datetime.strftime(datetime.now(), dt_format), + "message": str(obj) + }) + downloadStatus = downloadStatus[-10:] + + +def modifyLastDownloadStatus(obj): + global downloadStatus + if isinstance(obj, dict): + if "datetime" not in obj: + obj["datetime"] = datetime.strftime(datetime.now(), dt_format) + downloadStatus[-1] = obj + else: + downloadStatus[-1]["message"] = str(obj) + downloadStatus[-1]["datetime"] = datetime.strftime(datetime.now(), dt_format) + + +def appendError(obj): + global errors + if isinstance(obj, dict): + if "datetime" not in obj: + obj["datetime"] = datetime.strftime(datetime.now(), dt_format) + errors.append(obj) + else: + errors.append({ + "datetime": datetime.strftime(datetime.now(), dt_format), + "message": str(obj) + }) + errors = errors[-10:] + + +class downloader(XiGuaLiveApi): + files = [] + playlist: str = None + + def updRoomInfo(self): + global broadcaster, isBroadcasting, updateTime + super(downloader, self).updRoomInfo() + updateTime = datetime.strftime(datetime.now(), dt_format) + broadcaster = self.roomLiver + isBroadcasting = self.isLive + if self.isLive: + self.updPlayList() + else: + self.files = [] + + def updPlayList(self): + global streamUrl + if self.isLive: + if "stream_url" in self._rawRoomInfo: + if self.playlist is None: + self.playlist = False + else: + self.playlist = self._rawRoomInfo["stream_url"]["flv_pull_url"] + self.playlist = self.playlist.replace("_uhd", "").replace("_sd", "").replace("_ld", "") + streamUrl = self.playlist + + def onLike(self, user): + pass + + def onAd(self, i): + pass + + def onChat(self, chat): + pass + + def onEnter(self, msg): + pass + + def onJoin(self, user): + pass + + def onLeave(self, json): + self.updRoomInfo() + + def onMessage(self, msg): + pass + + def onPresent(self, gift): + pass + + def onPresentEnd(self, gift): + pass + + def onSubscribe(self, user): + pass + diff --git a/Model/DataBase.py b/Model/DataBase.py new file mode 100644 index 0000000..de475fd --- /dev/null +++ b/Model/DataBase.py @@ -0,0 +1,8 @@ +from flask import Flask +from flask_sqlalchemy import SQLAlchemy + +app = Flask(__name__) +app.config["SQLALCHEMY_DATABASE_URL"] = "sqlite://data.db" +app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True +app.config['SQLALCHEMY_COMMIT_ON_TEARDOWN'] = True +db = SQLAlchemy(app) \ No newline at end of file diff --git a/Model/Files.py b/Model/Files.py new file mode 100644 index 0000000..3b6d595 --- /dev/null +++ b/Model/Files.py @@ -0,0 +1,10 @@ +from datetime import datetime +from sqlalchemy import func +from .DataBase import db + + +class Files(db.Model): + id = db.Column(db.Integer, primary_key=True, autoincrement=True) + filename = db.Column(db.String(50)) + is_upload = db.Column(db.Integer(1), server_default=0, default=0) + create_time = db.Column(db.TIMESTAMP, server_default=func.now(), default=datetime.now()) diff --git a/WebMain.py b/WebMain.py new file mode 100644 index 0000000..15db91c --- /dev/null +++ b/WebMain.py @@ -0,0 +1,68 @@ +from flask import Flask, jsonify, request +import Common +import threading +from liveDownloader import run as RUN + +app = Flask("liveStatus") +app.config['JSON_AS_ASCII'] = False + + +@app.route("/config", methods=["GET"]) +def readConfig(): + config = Common.config.copy() + config.pop("b_p") + config.pop("mtd") + config.pop("del") + config.pop("mv") + return jsonify(config) + + +@app.route("/config", methods=["POST"]) +def writeConfig(): + # TODO : 完善 + return jsonify({"message":"ok","code":200,"status":0,"data":request.form}) + + +@app.route("/stats", methods=["GET"]) +def getAllStats(): + return jsonify({"message":"ok","code":200,"status":0,"data":{ + "download":Common.downloadStatus, + "encode": Common.encodeStatus, + "upload": Common.uploadStatus, + "error": Common.errors, + "broadcast": { + "broadcaster": Common.broadcaster.__str__(), + "isBroadcasting": Common.isBroadcasting, + "streamUrl": Common.streamUrl, + "updateTime": Common.updateTime + } + }}) + + +@app.route("/stats/download", methods=["GET"]) +def geDownloadStats(): + return jsonify({"message":"ok","code":200,"status":0,"data":{ + "download":Common.downloadStatus, + }}) + + +@app.route("/stats/encode", methods=["GET"]) +def getEncodeStats(): + return jsonify({"message":"ok","code":200,"status":0,"data":{ + "encode": Common.encodeStatus, + }}) + + +@app.route("/stats/upload", methods=["GET"]) +def getUploadStats(): + return jsonify({"message":"ok","code":200,"status":0,"data":{ + "upload": Common.uploadStatus, + }}) + + +t = threading.Thread(target=RUN, args=(Common.config['l_u'],)) +t.setDaemon(True) + +if __name__ == "__main__": + t.start() + app.run() diff --git a/bilibili.py b/bilibili.py index 76a55b0..a7ffa03 100644 --- a/bilibili.py +++ b/bilibili.py @@ -4,7 +4,7 @@ import os import re import json as JSON from datetime import datetime - +from Common import appendUploadStatus, modifyLastUploadStatus import rsa import math import base64 @@ -195,7 +195,6 @@ class Bilibili: :param parts: e.g. VideoPart('part path', 'part title', 'part desc'), or [VideoPart(...), VideoPart(...)] :type parts: VideoPart or list """ - self.session.headers['Content-Type'] = 'application/json; charset=utf-8' if not isinstance(parts, list): parts = [parts] @@ -238,7 +237,7 @@ class Bilibili: # {"upload_id":"72eb747b9650b8c7995fdb0efbdc2bb6","key":"\/i181012ws2wg1tb7tjzswk2voxrwlk1u.mp4","OK":1,"bucket":"ugc"} json = r.json() upload_id = json['upload_id'] - + appendUploadStatus("Upload >{}< Started".format(filepath)) with open(filepath, 'rb') as f: chunks_num = math.ceil(filesize / chunk_size) chunks_index = 0 @@ -263,10 +262,9 @@ class Bilibili: ) if r.status_code != 200: continue - print('{} : UPLOAD {}/{}'.format(datetime.strftime(datetime.now(), "%y%m%d %H%M"), chunks_index, - chunks_num), r.text) chunks_data = f.read(chunk_size) chunks_index += 1 # start with 0 + modifyLastUploadStatus("Uploading >{}< @ {:.2f}%".format(filepath, 100.0*chunks_index/chunks_num)) # NOT DELETE! Refer to https://github.com/comwrg/bilibiliupload/issues/15#issuecomment-424379769 self.session.post('https:{endpoint}/{upos_uri}?' @@ -282,6 +280,7 @@ class Bilibili: self.videos.append({'filename': upos_uri.replace('upos://ugc/', '').split('.')[0], 'title': part.title, 'desc': part.desc}) + modifyLastUploadStatus("Upload >{}< Finished".format(filepath)) __f = open("uploaded.json","w") JSON.dump(self.videos, __f) @@ -330,21 +329,21 @@ class Bilibili: "order_id": 0, "videos": self.videos} ) - print(r.text) + appendUploadStatus(">{}< Published | Result : {}".format(title, r.text)) def reloadFromPrevious(self): if os.path.exists("uploaded.json"): __f = open("uploaded.json", "r") try: self.videos = JSON.load(__f) - print("RELOAD Success") + appendUploadStatus("RELOAD SUCCESS") except: - print("RELOAD Failed") + appendUploadStatus("RELOAD Failed") self.videos = [] __f.close() os.remove("uploaded.json") else: - print("RELOAD Failed") + appendUploadStatus("RELOAD Failed") self.videos = [] def clear(self): diff --git a/liveDownloader.py b/liveDownloader.py index 4c03283..9f324b7 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -2,71 +2,14 @@ import shutil import sys import time from datetime import datetime -import queue import threading -from config import config -from api import XiGuaLiveApi from bilibili import * +from Common import * +import os +import requests -q = queue.Queue() -base_uri = "" isEncode = False isDownload = False -uq = queue.Queue() -eq = queue.Queue() - - -class downloader(XiGuaLiveApi): - files = [] - playlist: str = None - - def updRoomInfo(self): - super(downloader, self).updRoomInfo() - if self.isLive: - self.updPlayList() - else: - print("未开播，等待开播") - self.files = [] - - def updPlayList(self): - if self.isLive: - if "stream_url" in self._rawRoomInfo: - if self.playlist is None: - self.apiChangedError("无法获取直播链接") - self.playlist = False - else: - self.playlist = self._rawRoomInfo["stream_url"]["flv_pull_url"] - self.playlist = self.playlist.replace("_uhd", "").replace("_sd", "").replace("_ld", "") - - def onLike(self, user): - pass - - def onAd(self, i): - pass - - def onChat(self, chat): - pass - - def onEnter(self, msg): - pass - - def onJoin(self, user): - pass - - def onLeave(self, json): - self.updRoomInfo() - - def onMessage(self, msg): - pass - - def onPresent(self, gift): - pass - - def onPresentEnd(self, gift): - pass - - def onSubscribe(self, user): - pass def download(url): @@ -74,38 +17,42 @@ def download(url): path = datetime.strftime(datetime.now(), "%Y%m%d_%H%M.flv") p = requests.get(url, stream=True) if p.status_code != 200: - print("{} : Download Response 404 ,will stop looping".format(datetime.strftime(datetime.now(), "%y%m%d %H%M"))) + appendDownloadStatus("Download [{}] Response 404 ,will stop looping".format(url)) return True isDownload = True - print("{} : Download {}".format(datetime.strftime(datetime.now(), "%y%m%d %H%M"), path)) + appendDownloadStatus("Starting Download >{}<".format(path)) f = open(path, "wb") try: for t in p.iter_content(chunk_size=64 * 1024): if t: f.write(t) - if os.path.getsize(path) > 1024 * 1024 * 1024 * 1.5: + _size = os.path.getsize(path) + modifyLastDownloadStatus("Download >{}< @ {:.2f}%".format(path, 100.0 * _size/config["size"])) + if _size > config["size"]: break - print("{} : Download Quiting".format(datetime.strftime(datetime.now(), "%y%m%d %H%M"))) + modifyLastDownloadStatus("Finished Download >{}<".format(path)) except Exception as e: - print("{} : Download Quiting With Exception {}".format(datetime.strftime(datetime.now(), "%y%m%d %H%M"), + appendError("Download >{}< With Exception {}".format(path, datetime.strftime(datetime.now(), "%y%m%d %H%M"), e.__str__())) f.close() isDownload = False if os.path.getsize(path) == 0: os.remove(path) return False - eq.put(path) + encodeQueue.put(path) download(url) def encode(): global isEncode while True: - i = eq.get() + i = encodeQueue.get() if os.path.exists(i): isEncode = True - os.system("ffmpeg -i {} -c:v copy -c:a copy -f mp4 {}".format(i, i[:13] + ".mp4")) - uq.put(i[:13] + ".mp4") + appendEncodeStatus("Start Encoding >{}<".format(i)) + os.system("ffmpeg -i {} -c:v copy -c:a copy -f mp4 {} -y".format(i, i[:13] + ".mp4")) + uploadQueue.put(i[:13] + ".mp4") + modifyLastEncodeStatus("Finished Encoding >{}<".format(i)) if config["mv"]: shutil.move(i, config["mtd"]) elif config["del"]: @@ -114,51 +61,45 @@ def encode(): def upload(date=datetime.strftime(datetime.now(), "%Y_%m_%d")): - print("{} : Upload Daemon Starting".format(datetime.strftime(datetime.now(), "%y%m%d %H%M"))) - i = uq.get() + i = uploadQueue.get() while True: if isinstance(i, bool): - print("{} : Upload Daemon Receive Command {}" - .format(datetime.strftime(datetime.now(), "%y%m%d %H%M"), i)) if i is True: - print("自动投稿中，请稍后") + appendUploadStatus("[{}]自动投稿中，请稍后".format(config["t_t"].format(date))) b.finishUpload(config["t_t"].format(date), 17, config["tag"], config["des"], source=config["src"], no_reprint=0) b.clear() break - print("{} : Upload {}".format(datetime.strftime(datetime.now(), "%y%m%d %H%M"), i)) if not os.path.exists(i): - print("{} : Upload File Not Exist {}".format(datetime.strftime(datetime.now(), "%y%m%d %H%M"), i)) - i = uq.get() + appendError("Upload File Not Exist {}".format(i)) + i = uploadQueue.get() continue try: b.preUpload(VideoPart(i, os.path.basename(i))) - except: - continue + except Exception as e: + appendError(e.__str__()) os.remove(i) - i = uq.get() + i = uploadQueue.get() - print("{} : Upload Daemon Quiting".format(datetime.strftime(datetime.now(), "%y%m%d %H%M"))) + appendUploadStatus("Upload Daemon Quiting") b = Bilibili() b.login(config["b_u"], config["b_p"]) -if __name__ == "__main__": - name = config["l_u"] - print("西瓜直播录播助手 by JerryYan") + +def run(name): + global isEncode, isDownload api = downloader(name) - print("进入", api.roomLiver, "的直播间") - if not api.isValidRoom: - input("房间不存在") - sys.exit() - print("=" * 30) - d = datetime.strftime(datetime.now(), "%Y_%m_%d") - t = threading.Thread(target=download) - ut = threading.Thread(target=upload, args=(d,)) et = threading.Thread(target=encode, args=()) et.setDaemon(True) et.start() + if not api.isValidRoom: + appendError("[{}]房间不存在".format(name)) + return + d = None + t = threading.Thread(target=download) + ut = threading.Thread(target=upload, args=(d,)) _count = 0 _count_error = 0 while True: @@ -185,7 +126,7 @@ if __name__ == "__main__": _count = 0 _count_error = 0 except Exception as e: - print(e.__str__()) + appendError(e.__str__()) time.sleep(20) _count_error += 1 continue @@ -197,15 +138,13 @@ if __name__ == "__main__": if d is not None: d = None if not isEncode and not isDownload: - uq.put(True) + uploadQueue.put(True) isEncode = True isDownload = True - del config - from config import config # print("主播未开播，等待1分钟后重试") time.sleep(60) try: api.updRoomInfo() _count_error = 0 except Exception as e: - print(e.__str__()) + appendError(e.__str__()) From 00190468a8afc65c3fd4e0706d8aba0f39eb3a7e Mon Sep 17 00:00:00 2001 From: JerryYan <792602257@qq.com> Date: Sun, 7 Apr 2019 15:34:17 +0800 Subject: [PATCH 02/58] =?UTF-8?q?=E6=94=AF=E6=8C=81api=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E7=8A=B6=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- liveDownloader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/liveDownloader.py b/liveDownloader.py index 9f324b7..a27679b 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -27,8 +27,8 @@ def download(url): if t: f.write(t) _size = os.path.getsize(path) - modifyLastDownloadStatus("Download >{}< @ {:.2f}%".format(path, 100.0 * _size/config["size"])) - if _size > config["size"]: + modifyLastDownloadStatus("Download >{}< @ {:.2f}%".format(path, 100.0 * _size/config["p_s"])) + if _size > config["p_s"]: break modifyLastDownloadStatus("Finished Download >{}<".format(path)) except Exception as e: From 5fa4ee929e3249e05b851a1ea054c1833f33de29 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Sun, 7 Apr 2019 16:01:27 +0800 Subject: [PATCH 03/58] =?UTF-8?q?=E6=94=B9=E5=96=84=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E5=85=BC=E5=AE=B9=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Common.py | 2 +- Struct/Chat.py | 6 +++--- Struct/Gift.py | 12 ++++++------ Struct/Lottery.py | 4 ++-- Struct/MemberMsg.py | 6 +++--- Struct/User.py | 17 ++++++++--------- WebMain.py | 26 ++++++++++++++++++++++---- api.py | 22 +++++++++++----------- 8 files changed, 56 insertions(+), 39 deletions(-) diff --git a/Common.py b/Common.py index 953a411..5f7790c 100644 --- a/Common.py +++ b/Common.py @@ -114,7 +114,7 @@ def appendError(obj): class downloader(XiGuaLiveApi): files = [] - playlist: str = None + playlist = None def updRoomInfo(self): global broadcaster, isBroadcasting, updateTime diff --git a/Struct/Chat.py b/Struct/Chat.py index 729ee78..31269c6 100644 --- a/Struct/Chat.py +++ b/Struct/Chat.py @@ -4,9 +4,9 @@ from .Lottery import Lottery class Chat: - content: str ="" - user: User=None - filterString:list = ["",] + content ="" + user=None + filterString = ["",] isFiltered = False def __init__(self, json=None, lottery:Lottery = None): diff --git a/Struct/Gift.py b/Struct/Gift.py index c856b28..64d11bb 100644 --- a/Struct/Gift.py +++ b/Struct/Gift.py @@ -3,12 +3,12 @@ from .User import User class Gift: - ID:int = 0 - count:int = 0 - roomID:int = 0 - giftList:dict = {} - amount:int = 0 - user:User = None + ID = 0 + count = 0 + roomID = 0 + giftList = {} + amount = 0 + user = None def __init__(self, json=None): if json: diff --git a/Struct/Lottery.py b/Struct/Lottery.py index d2b5ae1..e2d9eb8 100644 --- a/Struct/Lottery.py +++ b/Struct/Lottery.py @@ -5,14 +5,14 @@ from .LuckyUser import LuckyUser class Lottery: - ID: int = 0 + ID = 0 isActive = False content = "" isFinished = False luckyUsers = [] joinedUserCount = 0 prizeName = "" - finish:int = 0 + finish = 0 def __init__(self, json=None): if json: diff --git a/Struct/MemberMsg.py b/Struct/MemberMsg.py index 404a522..2dcdeea 100644 --- a/Struct/MemberMsg.py +++ b/Struct/MemberMsg.py @@ -2,9 +2,9 @@ from .User import User class MemberMsg: - type:int = 0 - content:str = "" - user:User = None + type = 0 + content = "" + user = None def __init__(self, json=None): if json: diff --git a/Struct/User.py b/Struct/User.py index f2253f3..628a234 100644 --- a/Struct/User.py +++ b/Struct/User.py @@ -1,11 +1,11 @@ class User: - ID: int = 0 - name: str = "" - brand: str = "" - level: int = 0 - type: int = 0 - block: bool = False - mute: bool = False + ID = 0 + name = "" + brand = "" + level = 0 + type = 0 + block = False + mute = False def __init__(self, json=None): if json: @@ -47,8 +47,7 @@ class User: else: if self.type != 0: return "[{}{}]{}".format(self.brand, self.level, self.name) - return "<{}{}>{}".format(self.brand,self.level,self.name) + return "<{}{}>{}".format(self.brand, self.level, self.name) def __unicode__(self): return self.__str__() - diff --git a/WebMain.py b/WebMain.py index 15db91c..bd2e5da 100644 --- a/WebMain.py +++ b/WebMain.py @@ -1,3 +1,5 @@ +from time import sleep + from flask import Flask, jsonify, request import Common import threading @@ -39,6 +41,18 @@ def getAllStats(): }}) +@app.route("/stats/broadcast", methods=["GET"]) +def geBroadcastStats(): + return jsonify({"message":"ok","code":200,"status":0,"data":{ + "broadcast": { + "broadcaster": Common.broadcaster.__str__(), + "isBroadcasting": Common.isBroadcasting, + "streamUrl": Common.streamUrl, + "updateTime": Common.updateTime + } + }}) + + @app.route("/stats/download", methods=["GET"]) def geDownloadStats(): return jsonify({"message":"ok","code":200,"status":0,"data":{ @@ -62,7 +76,11 @@ def getUploadStats(): t = threading.Thread(target=RUN, args=(Common.config['l_u'],)) t.setDaemon(True) - -if __name__ == "__main__": - t.start() - app.run() +t.start() +while True: + if t.is_alive(): + sleep(240) + else: + t = threading.Thread(target=RUN, args=(Common.config['l_u'],)) + t.setDaemon(True) + t.start() diff --git a/api.py b/api.py index fd272bb..b905e68 100644 --- a/api.py +++ b/api.py @@ -12,21 +12,21 @@ import time s = requests.Session() -DEBUG: bool = False +DEBUG = False class XiGuaLiveApi: - isLive: bool = False - isValidRoom: bool = False + isLive = False + isValidRoom = False _rawRoomInfo = {} - name: str = "" - roomID: int = 0 - roomTitle: str = "" - roomLiver: User = None - roomPopularity: int = 0 - _cursor:str = "0" - _updRoomCount:int = 0 - lottery:Lottery = None + name = "" + roomID = 0 + roomTitle = "" + roomLiver = None + roomPopularity = 0 + _cursor = "0" + _updRoomCount = 0 + lottery = None def __init__(self, name: str = "永恒de草薙"): self.name = name From 16c68e5ceb898837443dc3070f65cafafd9f7044 Mon Sep 17 00:00:00 2001 From: JerryYan <792602257@qq.com> Date: Sun, 7 Apr 2019 19:12:03 +0800 Subject: [PATCH 04/58] wc --- WebMain.py | 37 +++++++++++++++++++++++++------------ bilibili.py | 2 +- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/WebMain.py b/WebMain.py index bd2e5da..283acc6 100644 --- a/WebMain.py +++ b/WebMain.py @@ -1,14 +1,21 @@ from time import sleep - -from flask import Flask, jsonify, request +from flask_cors import CORS +from flask import Flask, jsonify, request, redirect, url_for import Common import threading from liveDownloader import run as RUN app = Flask("liveStatus") app.config['JSON_AS_ASCII'] = False +CORS(app, supports_credentials=True) +url_for('static', filename='index.html') +url_for('static', filename='index.js') +@app.route("/") +def index(): + return redirect("/static/index.html") + @app.route("/config", methods=["GET"]) def readConfig(): config = Common.config.copy() @@ -74,13 +81,19 @@ def getUploadStats(): }}) -t = threading.Thread(target=RUN, args=(Common.config['l_u'],)) -t.setDaemon(True) -t.start() -while True: - if t.is_alive(): - sleep(240) - else: - t = threading.Thread(target=RUN, args=(Common.config['l_u'],)) - t.setDaemon(True) - t.start() +def SubThread(): + t = threading.Thread(target=RUN, args=(Common.config['l_u'],)) + t.setDaemon(True) + t.start() + while True: + if t.is_alive(): + sleep(240) + else: + t = threading.Thread(target=RUN, args=(Common.config['l_u'],)) + t.setDaemon(True) + t.start() + + +p = threading.Thread(target = SubThread) +p.setDaemon(True) +p.start() diff --git a/bilibili.py b/bilibili.py index a7ffa03..6290f8a 100644 --- a/bilibili.py +++ b/bilibili.py @@ -188,7 +188,7 @@ class Bilibili: """ self.preUpload(parts) self.finishUpload(title, tid, tag, desc, source, cover, no_reprint) - self.clean() + self.clear() def preUpload(self, parts): """ From ce819185eb06debf5467f34ff8472ebf63682e63 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Sun, 7 Apr 2019 22:26:21 +0800 Subject: [PATCH 05/58] support change while running --- Common.py | 10 ++++++++++ WebMain.py | 25 +++++++++++++++++++++++-- bilibili.py | 1 + liveDownloader.py | 9 +++++---- 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/Common.py b/Common.py index 5f7790c..164f060 100644 --- a/Common.py +++ b/Common.py @@ -6,6 +6,14 @@ import json _config_fp = open("config.json","r",encoding="utf8") config = json.load(_config_fp) +_config_fp.close() + + +def reloadConfig(): + global config, _config_fp + _config_fp = open("config.json", "r", encoding="utf8") + config = json.load(_config_fp) + _config_fp.close() dt_format="%Y/%m/%d %H:%M:%S" @@ -14,6 +22,8 @@ streamUrl = "" isBroadcasting = False updateTime = "" +forceStopDownload = False + uploadQueue = queue.Queue() encodeQueue = queue.Queue() diff --git a/WebMain.py b/WebMain.py index 283acc6..713f394 100644 --- a/WebMain.py +++ b/WebMain.py @@ -8,8 +8,8 @@ from liveDownloader import run as RUN app = Flask("liveStatus") app.config['JSON_AS_ASCII'] = False CORS(app, supports_credentials=True) -url_for('static', filename='index.html') -url_for('static', filename='index.js') +# url_for('static', filename='index.html') +# url_for('static', filename='index.js') @app.route("/") @@ -29,9 +29,30 @@ def readConfig(): @app.route("/config", methods=["POST"]) def writeConfig(): # TODO : 完善 + Common.reloadConfig() return jsonify({"message":"ok","code":200,"status":0,"data":request.form}) +@app.route("/encode/insert", methods=["POST"]) +def insertEncode(): + if "filename" in request.form: + Common.encodeQueue.put(request.form["filename"]) + return jsonify({"message":"ok","code":200,"status":0}) + + +@app.route("/upload/insert", methods=["POST"]) +def insertUpload(): + if "filename" in request.form: + Common.uploadQueue.put(request.form["filename"]) + return jsonify({"message":"ok","code":200,"status":0}) + + +@app.route("/upload/finish", methods=["POST"]) +def finishUpload(): + Common.uploadQueue.put(True) + return jsonify({"message":"ok","code":200,"status":0}) + + @app.route("/stats", methods=["GET"]) def getAllStats(): return jsonify({"message":"ok","code":200,"status":0,"data":{ diff --git a/bilibili.py b/bilibili.py index 6290f8a..4368f2e 100644 --- a/bilibili.py +++ b/bilibili.py @@ -313,6 +313,7 @@ class Bilibili: """ if len(self.videos) == 0: return + appendUploadStatus("[{}]投稿中，请稍后".format(title)) self.session.headers['Content-Type'] = 'application/json; charset=utf-8' copyright = 2 if source else 1 r = self.session.post('https://member.bilibili.com/x/vu/web/add?csrf=' + self.csrf, diff --git a/liveDownloader.py b/liveDownloader.py index a27679b..8c7b883 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -13,7 +13,7 @@ isDownload = False def download(url): - global isDownload + global isDownload, forceStopDownload path = datetime.strftime(datetime.now(), "%Y%m%d_%H%M.flv") p = requests.get(url, stream=True) if p.status_code != 200: @@ -28,7 +28,7 @@ def download(url): f.write(t) _size = os.path.getsize(path) modifyLastDownloadStatus("Download >{}< @ {:.2f}%".format(path, 100.0 * _size/config["p_s"])) - if _size > config["p_s"]: + if _size > config["p_s"] or forceStopDownload: break modifyLastDownloadStatus("Finished Download >{}<".format(path)) except Exception as e: @@ -40,7 +40,9 @@ def download(url): os.remove(path) return False encodeQueue.put(path) - download(url) + if forceStopDownload: + download(url) + forceStopDownload = False def encode(): @@ -65,7 +67,6 @@ def upload(date=datetime.strftime(datetime.now(), "%Y_%m_%d")): while True: if isinstance(i, bool): if i is True: - appendUploadStatus("[{}]自动投稿中，请稍后".format(config["t_t"].format(date))) b.finishUpload(config["t_t"].format(date), 17, config["tag"], config["des"], source=config["src"], no_reprint=0) b.clear() From be1a1df00b8d75fdd26298d05bf63b253d39b65c Mon Sep 17 00:00:00 2001 From: JerryYan <792602257@qq.com> Date: Sun, 7 Apr 2019 22:27:17 +0800 Subject: [PATCH 06/58] support change while running --- liveDownloader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/liveDownloader.py b/liveDownloader.py index 8c7b883..5ba6f10 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -41,8 +41,10 @@ def download(url): return False encodeQueue.put(path) if forceStopDownload: - download(url) forceStopDownload = False + return + else: + download(url) def encode(): From 99d148666eb2a2f704ec5b390f1b859d64cd9094 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Sun, 7 Apr 2019 23:35:47 +0800 Subject: [PATCH 07/58] Fix: Max Retry --- bilibili.py | 2 +- liveDownloader.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bilibili.py b/bilibili.py index 4368f2e..22c9dd7 100644 --- a/bilibili.py +++ b/bilibili.py @@ -11,7 +11,7 @@ import base64 import hashlib import requests from urllib import parse - +requests.adapters.DEFAULT_RETRIES = 10 class VideoPart: def __init__(self, path, title='', desc=''): diff --git a/liveDownloader.py b/liveDownloader.py index 5ba6f10..5829dcf 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -81,6 +81,7 @@ def upload(date=datetime.strftime(datetime.now(), "%Y_%m_%d")): b.preUpload(VideoPart(i, os.path.basename(i))) except Exception as e: appendError(e.__str__()) + continue os.remove(i) i = uploadQueue.get() From c3a965f28489464d8fb8d08bc90b5194699c0de9 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Mon, 8 Apr 2019 11:32:05 +0800 Subject: [PATCH 08/58] =?UTF-8?q?=E9=A1=B5=E9=9D=A2=E5=8F=8A=E9=80=BB?= =?UTF-8?q?=E8=BE=91=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Common.py | 27 +++++++++++++--- WebMain.py | 46 ++++++++++++++++++++++++++-- bilibili.py | 12 ++++++-- liveDownloader.py | 16 +++++----- static/index.html | 73 ++++++++++++++++++++++++++++++++++++++++++++ static/index.js | 45 +++++++++++++++++++++++++++ templates/files.html | 25 +++++++++++++++ 7 files changed, 225 insertions(+), 19 deletions(-) create mode 100644 static/index.html create mode 100644 static/index.js create mode 100644 templates/files.html diff --git a/Common.py b/Common.py index 164f060..7c42f59 100644 --- a/Common.py +++ b/Common.py @@ -15,6 +15,7 @@ def reloadConfig(): config = json.load(_config_fp) _config_fp.close() + dt_format="%Y/%m/%d %H:%M:%S" broadcaster = "" @@ -23,6 +24,7 @@ isBroadcasting = False updateTime = "" forceStopDownload = False +forceNotBroadcasting = False uploadQueue = queue.Queue() encodeQueue = queue.Queue() @@ -33,6 +35,18 @@ encodeStatus = [] errors = [] +def parseSize(size): + K = size/1024.0 + if K > 1000: + M = K/1024.0 + if M > 1000: + return "{:.2f}GB".format(M / 1024.0) + else: + return "{:.2f}MB".format(M) + else: + return "{:.2f}MB".format(K) + + def appendUploadStatus(obj): global uploadStatus if isinstance(obj, dict): @@ -44,7 +58,7 @@ def appendUploadStatus(obj): "datetime": datetime.strftime(datetime.now(), dt_format), "message": str(obj) }) - uploadStatus = uploadStatus[-10:] + uploadStatus = uploadStatus[-config["l_c"]:] def modifyLastUploadStatus(obj): @@ -69,7 +83,7 @@ def appendEncodeStatus(obj): "datetime": datetime.strftime(datetime.now(), dt_format), "message": str(obj) }) - encodeStatus = encodeStatus[-10:] + encodeStatus = encodeStatus[-config["l_c"]:] def modifyLastEncodeStatus(obj): @@ -94,7 +108,7 @@ def appendDownloadStatus(obj): "datetime": datetime.strftime(datetime.now(), dt_format), "message": str(obj) }) - downloadStatus = downloadStatus[-10:] + downloadStatus = downloadStatus[-config["l_c"]:] def modifyLastDownloadStatus(obj): @@ -119,7 +133,7 @@ def appendError(obj): "datetime": datetime.strftime(datetime.now(), dt_format), "message": str(obj) }) - errors = errors[-10:] + errors = errors[-config["elc"]:] class downloader(XiGuaLiveApi): @@ -127,7 +141,7 @@ class downloader(XiGuaLiveApi): playlist = None def updRoomInfo(self): - global broadcaster, isBroadcasting, updateTime + global broadcaster, isBroadcasting, updateTime, forceNotBroadcasting, forceStopDownload super(downloader, self).updRoomInfo() updateTime = datetime.strftime(datetime.now(), dt_format) broadcaster = self.roomLiver @@ -135,6 +149,9 @@ class downloader(XiGuaLiveApi): if self.isLive: self.updPlayList() else: + forceStopDownload = False + forceNotBroadcasting = False + self.playlist = False self.files = [] def updPlayList(self): diff --git a/WebMain.py b/WebMain.py index 713f394..22a9204 100644 --- a/WebMain.py +++ b/WebMain.py @@ -1,6 +1,8 @@ +import os +from glob import glob from time import sleep from flask_cors import CORS -from flask import Flask, jsonify, request, redirect, url_for +from flask import Flask, jsonify, request, redirect, render_template, Response import Common import threading from liveDownloader import run as RUN @@ -58,19 +60,25 @@ def getAllStats(): return jsonify({"message":"ok","code":200,"status":0,"data":{ "download":Common.downloadStatus, "encode": Common.encodeStatus, + "encodeQueueSize": Common.encodeQueue.qsize(), "upload": Common.uploadStatus, + "uploadQueueSize": Common.uploadQueue.qsize(), "error": Common.errors, "broadcast": { "broadcaster": Common.broadcaster.__str__(), "isBroadcasting": Common.isBroadcasting, "streamUrl": Common.streamUrl, "updateTime": Common.updateTime + }, + "config": { + "forceNotBroadcasting": Common.forceNotBroadcasting, + "forceStopDownload": Common.forceStopDownload, } }}) @app.route("/stats/broadcast", methods=["GET"]) -def geBroadcastStats(): +def getBroadcastStats(): return jsonify({"message":"ok","code":200,"status":0,"data":{ "broadcast": { "broadcaster": Common.broadcaster.__str__(), @@ -81,8 +89,18 @@ def geBroadcastStats(): }}) +@app.route("/stats/config", methods=["GET"]) +def getConfigStats(): + return jsonify({"message":"ok","code":200,"status":0,"data":{ + "config": { + "forceNotBroadcasting": Common.forceNotBroadcasting, + "forceStopDownload": Common.forceStopDownload, + } + }}) + + @app.route("/stats/download", methods=["GET"]) -def geDownloadStats(): +def getDownloadStats(): return jsonify({"message":"ok","code":200,"status":0,"data":{ "download":Common.downloadStatus, }}) @@ -92,6 +110,7 @@ def geDownloadStats(): def getEncodeStats(): return jsonify({"message":"ok","code":200,"status":0,"data":{ "encode": Common.encodeStatus, + "encodeQueueSize": Common.encodeQueue.qsize(), }}) @@ -99,9 +118,30 @@ def getEncodeStats(): def getUploadStats(): return jsonify({"message":"ok","code":200,"status":0,"data":{ "upload": Common.uploadStatus, + "uploadQueueSize": Common.uploadQueue.qsize(), }}) +@app.route("/files/", methods=["GET"]) +def fileIndex(): + a = [] + for i in (glob("*.mp4") + glob("*.flv")): + a.append({ + "name": i, + "size": Common.parseSize(os.path.getsize(i)) + }) + return render_template("files.html",files=a) + + +@app.route("/files/download/", methods=["GET"]) +def fileDownload(path): + def generate(file): + with open(file, "rb") as f: + for row in f: + yield row + return Response(generate(path), mimetype='application/octet-stream') + + def SubThread(): t = threading.Thread(target=RUN, args=(Common.config['l_u'],)) t.setDaemon(True) diff --git a/bilibili.py b/bilibili.py index 22c9dd7..001b698 100644 --- a/bilibili.py +++ b/bilibili.py @@ -4,6 +4,7 @@ import os import re import json as JSON from datetime import datetime +from time import sleep from Common import appendUploadStatus, modifyLastUploadStatus import rsa import math @@ -11,7 +12,7 @@ import base64 import hashlib import requests from urllib import parse -requests.adapters.DEFAULT_RETRIES = 10 + class VideoPart: def __init__(self, path, title='', desc=''): @@ -25,6 +26,7 @@ class Bilibili: self.files = [] self.videos = [] self.session = requests.session() + self.session.keep_alive = False if cookie: self.session.headers["cookie"] = cookie self.csrf = re.search('bili_jct=(.*?);', cookie).group(1) @@ -203,6 +205,7 @@ class Bilibili: filepath = part.path filename = os.path.basename(filepath) filesize = os.path.getsize(filepath) + appendUploadStatus("Upload >{}< Started".format(filepath)) self.files.append(part) r = self.session.get('https://member.bilibili.com/preupload?' 'os=upos&upcdn=ws&name={name}&size={size}&r=upos&profile=ugcupos%2Fyb&ssl=0' @@ -237,12 +240,13 @@ class Bilibili: # {"upload_id":"72eb747b9650b8c7995fdb0efbdc2bb6","key":"\/i181012ws2wg1tb7tjzswk2voxrwlk1u.mp4","OK":1,"bucket":"ugc"} json = r.json() upload_id = json['upload_id'] - appendUploadStatus("Upload >{}< Started".format(filepath)) with open(filepath, 'rb') as f: chunks_num = math.ceil(filesize / chunk_size) chunks_index = 0 chunks_data = f.read(chunk_size) + modifyLastUploadStatus("Uploading >{}< @ {:.2f}%".format(filepath, 100.0 * chunks_index / chunks_num)) while True: + _d = datetime.now() if not chunks_data: break r = self.session.put('https:{endpoint}/{upos_uri}?' @@ -265,6 +269,8 @@ class Bilibili: chunks_data = f.read(chunk_size) chunks_index += 1 # start with 0 modifyLastUploadStatus("Uploading >{}< @ {:.2f}%".format(filepath, 100.0*chunks_index/chunks_num)) + if (datetime.now()-_d).seconds < 2: + sleep(1) # NOT DELETE! Refer to https://github.com/comwrg/bilibiliupload/issues/15#issuecomment-424379769 self.session.post('https:{endpoint}/{upos_uri}?' @@ -330,7 +336,7 @@ class Bilibili: "order_id": 0, "videos": self.videos} ) - appendUploadStatus(">{}< Published | Result : {}".format(title, r.text)) + appendUploadStatus("[{}] Published | Result : {}".format(title, r.text)) def reloadFromPrevious(self): if os.path.exists("uploaded.json"): diff --git a/liveDownloader.py b/liveDownloader.py index 5829dcf..cb284d0 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -17,20 +17,20 @@ def download(url): path = datetime.strftime(datetime.now(), "%Y%m%d_%H%M.flv") p = requests.get(url, stream=True) if p.status_code != 200: - appendDownloadStatus("Download [{}] Response 404 ,will stop looping".format(url)) + appendDownloadStatus("Download with Response 404, maybe broadcaster is not broadcasting") return True isDownload = True - appendDownloadStatus("Starting Download >{}<".format(path)) + appendDownloadStatus("Download >{}< Start".format(path)) f = open(path, "wb") try: for t in p.iter_content(chunk_size=64 * 1024): if t: f.write(t) _size = os.path.getsize(path) - modifyLastDownloadStatus("Download >{}< @ {:.2f}%".format(path, 100.0 * _size/config["p_s"])) + modifyLastDownloadStatus("Downloading >{}< @ {:.2f}%".format(path, 100.0 * _size/config["p_s"])) if _size > config["p_s"] or forceStopDownload: break - modifyLastDownloadStatus("Finished Download >{}<".format(path)) + modifyLastDownloadStatus("Download >{}< Finished".format(path)) except Exception as e: appendError("Download >{}< With Exception {}".format(path, datetime.strftime(datetime.now(), "%y%m%d %H%M"), e.__str__())) @@ -53,10 +53,10 @@ def encode(): i = encodeQueue.get() if os.path.exists(i): isEncode = True - appendEncodeStatus("Start Encoding >{}<".format(i)) + appendEncodeStatus("Encoding >{}< Start".format(i)) os.system("ffmpeg -i {} -c:v copy -c:a copy -f mp4 {} -y".format(i, i[:13] + ".mp4")) uploadQueue.put(i[:13] + ".mp4") - modifyLastEncodeStatus("Finished Encoding >{}<".format(i)) + modifyLastEncodeStatus("Encode >{}< Finished".format(i)) if config["mv"]: shutil.move(i, config["mtd"]) elif config["del"]: @@ -107,10 +107,10 @@ def run(name): _count = 0 _count_error = 0 while True: - if api.isLive: + if api.isLive and not forceNotBroadcasting: if d is None: d = datetime.strftime(datetime.now(), "%Y_%m_%d") - if not t.is_alive(): + if not t.is_alive() and not forceStopDownload: _count_error += 1 _preT = api.playlist t = threading.Thread(target=download, args=(_preT,)) diff --git a/static/index.html b/static/index.html new file mode 100644 index 0000000..a457651 --- /dev/null +++ b/static/index.html @@ -0,0 +1,73 @@ + + + + + 录播 + + + + +
+

基本信息

+ + + + + + + + + + + + + + + + + +
主播名
是否正在直播
直播视频流地址
信息更新时间
+

特殊设置

+ + + + + + + + + +
是否设置强制认为不直播
是否设置强制不下载
+
+

当前状态

+ + + + + + + + + + + + + + + + + + + + + +
下载日志
转码日志
上传日志
错误日志
操作日志
+
+
+ + + \ No newline at end of file diff --git a/static/index.js b/static/index.js new file mode 100644 index 0000000..86f19b6 --- /dev/null +++ b/static/index.js @@ -0,0 +1,45 @@ +function update(){ + $.ajax( + "/stats", + { + success: function (res){ + $("#broadcaster").text(res.data.broadcast.broadcaster) + $("#isBroadcasting").text(res.data.broadcast.isBroadcasting) + $("#streamUrl").text(res.data.broadcast.streamUrl) + $("#forceNotBroadcasting").text(res.data.config.forceNotBroadcasting) + $("#forceStopDownload").text(res.data.config.forceStopDownload) + $("#updateTime").text(res.data.broadcast.updateTime) + $("#download").html(function(){ + var ret = "" + res.data.download.reverse().forEach(function(obj){ + ret += "" + obj.datetime + "" + obj.message + "" + }) + return "" + ret + "
" + }) + $("#encode").html(function(){ + var ret = "" + res.data.encode.reverse().forEach(function(obj){ + ret += "" + obj.datetime + "" + obj.message + "" + }) + return "" + ret + "
" + }) + $("#upload").html(function(){ + var ret = "" + res.data.upload.reverse().forEach(function(obj){ + ret += "" + obj.datetime + "" + obj.message + "" + }) + return "" + ret + "
" + }) + $("#error").html(function(){ + var ret = "" + res.data.error.reverse().forEach(function(obj){ + ret += "" + obj.datetime + "" + obj.message + "" + }) + return "" + ret + "
" + }) + } + } + ) +} +update() +setInterval(update,10000) \ No newline at end of file diff --git a/templates/files.html b/templates/files.html new file mode 100644 index 0000000..891e537 --- /dev/null +++ b/templates/files.html @@ -0,0 +1,25 @@ + + + + + 文件 + + + +

所有录像文件

+ + + + + {%for i in files %} + + + + {% endfor %} +
文件名文件大小链接
{{i.name}}{{i.size}}下载文件
+ + \ No newline at end of file From b55686d09508fda17e3bd5ce697b14b0e5a48d42 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Mon, 8 Apr 2019 18:32:23 +0800 Subject: [PATCH 09/58] =?UTF-8?q?=E6=94=AF=E6=8C=81=E4=B8=8D=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E8=A7=86=E9=A2=91=EF=BC=88=E5=BE=85=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Common.py | 1 + liveDownloader.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/Common.py b/Common.py index 7c42f59..69abff7 100644 --- a/Common.py +++ b/Common.py @@ -25,6 +25,7 @@ updateTime = "" forceStopDownload = False forceNotBroadcasting = False +forceNotUpload = False uploadQueue = queue.Queue() encodeQueue = queue.Queue() diff --git a/liveDownloader.py b/liveDownloader.py index cb284d0..dba5b45 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -66,7 +66,12 @@ def encode(): def upload(date=datetime.strftime(datetime.now(), "%Y_%m_%d")): i = uploadQueue.get() + appendUploadStatus("Upload Daemon Starting") while True: + if forceNotUpload: + modifyLastUploadStatus("设置了不上传，所以不会上传了") + i = uploadQueue.get() + continue if isinstance(i, bool): if i is True: b.finishUpload(config["t_t"].format(date), 17, config["tag"], config["des"], From 5e3f9e9aee188a1013627846d7233b9528b935bd Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Mon, 8 Apr 2019 18:33:29 +0800 Subject: [PATCH 10/58] =?UTF-8?q?=E6=94=AF=E6=8C=81=E4=B8=8D=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E8=A7=86=E9=A2=91=EF=BC=88=E5=BE=85=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- WebMain.py | 2 ++ static/index.html | 6 +++++- static/index.js | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/WebMain.py b/WebMain.py index 22a9204..46107eb 100644 --- a/WebMain.py +++ b/WebMain.py @@ -73,6 +73,7 @@ def getAllStats(): "config": { "forceNotBroadcasting": Common.forceNotBroadcasting, "forceStopDownload": Common.forceStopDownload, + "forceNotUpload": Common.forceNotUpload, } }}) @@ -95,6 +96,7 @@ def getConfigStats(): "config": { "forceNotBroadcasting": Common.forceNotBroadcasting, "forceStopDownload": Common.forceStopDownload, + "forceNotUpload": Common.forceNotUpload, } }}) diff --git a/static/index.html b/static/index.html index a457651..37e2266 100644 --- a/static/index.html +++ b/static/index.html @@ -31,6 +31,7 @@ +

特殊设置

@@ -41,6 +42,10 @@ + + + +
是否设置强制不下载
是否设置强制不上传

当前状态

@@ -66,7 +71,6 @@ -
diff --git a/static/index.js b/static/index.js index 86f19b6..e1f72f8 100644 --- a/static/index.js +++ b/static/index.js @@ -8,6 +8,7 @@ function update(){ $("#streamUrl").text(res.data.broadcast.streamUrl) $("#forceNotBroadcasting").text(res.data.config.forceNotBroadcasting) $("#forceStopDownload").text(res.data.config.forceStopDownload) + $("#forceNotUpload").text(res.data.config.forceNotUpload) $("#updateTime").text(res.data.broadcast.updateTime) $("#download").html(function(){ var ret = "" From 14430e79eff8c2353b23b4c8aa5a3ce2a019e821 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Mon, 8 Apr 2019 21:11:22 +0800 Subject: [PATCH 11/58] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=EF=BC=9A=E7=B3=BB?= =?UTF-8?q?=E7=BB=9F=E4=BF=A1=E6=81=AF=E6=9F=A5=E7=9C=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Common.py | 3 ++- WebMain.py | 1 + liveDownloader.py | 6 +++++- static/index.html | 25 ++++++++++++++++++++++++- static/index.js | 28 +++++++++++++++++++++++++--- 5 files changed, 57 insertions(+), 6 deletions(-) diff --git a/Common.py b/Common.py index 69abff7..00138de 100644 --- a/Common.py +++ b/Common.py @@ -25,7 +25,8 @@ updateTime = "" forceStopDownload = False forceNotBroadcasting = False -forceNotUpload = False +forceNotUpload = True +forceNotEncode = True uploadQueue = queue.Queue() encodeQueue = queue.Queue() diff --git a/WebMain.py b/WebMain.py index 46107eb..ff679b5 100644 --- a/WebMain.py +++ b/WebMain.py @@ -74,6 +74,7 @@ def getAllStats(): "forceNotBroadcasting": Common.forceNotBroadcasting, "forceStopDownload": Common.forceStopDownload, "forceNotUpload": Common.forceNotUpload, + "forceNotEncode": Common.forceNotEncode, } }}) diff --git a/liveDownloader.py b/liveDownloader.py index dba5b45..703c844 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -49,8 +49,12 @@ def download(url): def encode(): global isEncode + appendEncodeStatus("Encode Daemon Start") while True: i = encodeQueue.get() + if forceNotEncode: + modifyLastUploadStatus("设置了不编码，所以[{}]不会编码".format(i)) + continue if os.path.exists(i): isEncode = True appendEncodeStatus("Encoding >{}< Start".format(i)) @@ -69,7 +73,7 @@ def upload(date=datetime.strftime(datetime.now(), "%Y_%m_%d")): appendUploadStatus("Upload Daemon Starting") while True: if forceNotUpload: - modifyLastUploadStatus("设置了不上传，所以不会上传了") + modifyLastUploadStatus("设置了不上传，所以[{}]不会上传了".format(i)) i = uploadQueue.get() continue if isinstance(i, bool): diff --git a/static/index.html b/static/index.html index 37e2266..e84c01b 100644 --- a/static/index.html +++ b/static/index.html @@ -5,7 +5,7 @@ 录播 @@ -46,6 +46,10 @@ 是否设置强制不上传 + + 是否设置强制不转码 + +

当前状态

@@ -71,6 +75,25 @@ +
+

机器状态

+ + + + + + + + + + + + + + + + +
CPU使用率%
内存使用率/(%)
磁盘使用率/(%)
diff --git a/static/index.js b/static/index.js index e1f72f8..2fbeda0 100644 --- a/static/index.js +++ b/static/index.js @@ -1,4 +1,4 @@ -function update(){ +function taskUpdate(){ $.ajax( "/stats", { @@ -9,6 +9,7 @@ function update(){ $("#forceNotBroadcasting").text(res.data.config.forceNotBroadcasting) $("#forceStopDownload").text(res.data.config.forceStopDownload) $("#forceNotUpload").text(res.data.config.forceNotUpload) + $("#forceNotEncode").text(res.data.config.forceNotEncode) $("#updateTime").text(res.data.broadcast.updateTime) $("#download").html(function(){ var ret = "" @@ -42,5 +43,26 @@ function update(){ } ) } -update() -setInterval(update,10000) \ No newline at end of file +function deviceUpdate(){ + $.ajax( + "/stats/device", + { + success: function (res){ + $("#memTotal").text(res.data.status.memTotal) + $("#memUsed").text(res.data.status.memUsed) + $("#memUsage").text(res.data.status.memUsage) + $("#diskTotal").text(res.data.status.diskTotal) + $("#diskUsed").text(res.data.status.diskUsed) + $("#diskUsage").text(res.data.status.diskUsage) + $("#cpu").text(res.data.status.cpu) + $("#memUsageP").val(res.data.status.memUsage) + $("#diskUsageP").val(res.data.status.diskUsage) + $("#cpuP").val(res.data.status.cpu) + } + } + ) +} +taskUpdate() +deviceUpdate() +setInterval(taskUpdate,10000) +setInterval(deviceUpdate,5000) \ No newline at end of file From 78fe5582e6e990a2ef547ede4e6ec526523db2db Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Mon, 8 Apr 2019 21:13:42 +0800 Subject: [PATCH 12/58] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=EF=BC=9A=E7=B3=BB?= =?UTF-8?q?=E7=BB=9F=E4=BF=A1=E6=81=AF=E6=9F=A5=E7=9C=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- WebMain.py | 22 ++++++++++++++++++++-- liveDownloader.py | 2 +- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/WebMain.py b/WebMain.py index ff679b5..2001edd 100644 --- a/WebMain.py +++ b/WebMain.py @@ -6,6 +6,7 @@ from flask import Flask, jsonify, request, redirect, render_template, Response import Common import threading from liveDownloader import run as RUN +import psutil app = Flask("liveStatus") app.config['JSON_AS_ASCII'] = False @@ -57,6 +58,7 @@ def finishUpload(): @app.route("/stats", methods=["GET"]) def getAllStats(): + _disk = psutil.disk_usage("/") return jsonify({"message":"ok","code":200,"status":0,"data":{ "download":Common.downloadStatus, "encode": Common.encodeStatus, @@ -74,8 +76,24 @@ def getAllStats(): "forceNotBroadcasting": Common.forceNotBroadcasting, "forceStopDownload": Common.forceStopDownload, "forceNotUpload": Common.forceNotUpload, - "forceNotEncode": Common.forceNotEncode, - } + }, + }}) + + +@app.route("/stats/device", methods=["GET"]) +def getDeviceStatus(): + _disk = psutil.disk_usage("/") + _mem = psutil.virtual_memory() + return jsonify({"message":"ok","code":200,"status":0,"data":{ + "status": { + "memTotal": Common.parseSize(_mem.total), + "memUsed": Common.parseSize(_mem.used), + "memUsage": _mem.percent, + "diskTotal": Common.parseSize(_disk.total), + "diskUsed": Common.parseSize(_disk.used), + "diskUsage": _disk.percent, + "cpu": psutil.cpu_percent(), + }, }}) diff --git a/liveDownloader.py b/liveDownloader.py index 703c844..036ea67 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -69,8 +69,8 @@ def encode(): def upload(date=datetime.strftime(datetime.now(), "%Y_%m_%d")): - i = uploadQueue.get() appendUploadStatus("Upload Daemon Starting") + i = uploadQueue.get() while True: if forceNotUpload: modifyLastUploadStatus("设置了不上传，所以[{}]不会上传了".format(i)) From 9f999c013e53e5434b82b36fcb07b5b5f6c202cb Mon Sep 17 00:00:00 2001 From: JerryYan <792602257@qq.com> Date: Mon, 8 Apr 2019 21:19:13 +0800 Subject: [PATCH 13/58] Typo Error Fix --- WebMain.py | 1 + liveDownloader.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/WebMain.py b/WebMain.py index 2001edd..7643d58 100644 --- a/WebMain.py +++ b/WebMain.py @@ -76,6 +76,7 @@ def getAllStats(): "forceNotBroadcasting": Common.forceNotBroadcasting, "forceStopDownload": Common.forceStopDownload, "forceNotUpload": Common.forceNotUpload, + "forceNotEncode": Common.forceNotEncode, }, }}) diff --git a/liveDownloader.py b/liveDownloader.py index 036ea67..bdfa4ed 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -49,7 +49,7 @@ def download(url): def encode(): global isEncode - appendEncodeStatus("Encode Daemon Start") + appendEncodeStatus("Encode Daemon Starting") while True: i = encodeQueue.get() if forceNotEncode: From ef7d4c8d3ec82e50508dfb6536f1b8003e7fbc2a Mon Sep 17 00:00:00 2001 From: JerryYan <792602257@qq.com> Date: Mon, 8 Apr 2019 22:01:38 +0800 Subject: [PATCH 14/58] Typo Error Fix --- liveDownloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liveDownloader.py b/liveDownloader.py index bdfa4ed..8828013 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -53,7 +53,7 @@ def encode(): while True: i = encodeQueue.get() if forceNotEncode: - modifyLastUploadStatus("设置了不编码，所以[{}]不会编码".format(i)) + modifyLastEncodeStatus("设置了不编码，所以[{}]不会编码".format(i)) continue if os.path.exists(i): isEncode = True From 86fbf31120f8b4b5983331de17b0cba10ac25a14 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Mon, 8 Apr 2019 23:05:06 +0800 Subject: [PATCH 15/58] Fix download a file wasnt exists --- WebMain.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/WebMain.py b/WebMain.py index 7643d58..543d08b 100644 --- a/WebMain.py +++ b/WebMain.py @@ -161,7 +161,10 @@ def fileDownload(path): with open(file, "rb") as f: for row in f: yield row - return Response(generate(path), mimetype='application/octet-stream') + if os.path.exists(path): + return Response(generate(path), mimetype='application/octet-stream') + else: + return Response(status=404) def SubThread(): From a6662450de135622b1d3e17bbede0d05f519666d Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Tue, 9 Apr 2019 08:11:09 +0800 Subject: [PATCH 16/58] =?UTF-8?q?=E9=83=A8=E5=88=86=E9=80=BB=E8=BE=91?= =?UTF-8?q?=EF=BC=8C=E6=9B=B4=E6=94=B9=E5=90=8D=E5=AD=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Common.py | 6 +++--- WebMain.py | 36 ++++++++++++++++++++++++++++++++++-- liveDownloader.py | 18 +++++++++--------- static/index.html | 2 +- static/index.js | 2 +- 5 files changed, 48 insertions(+), 16 deletions(-) diff --git a/Common.py b/Common.py index 00138de..0288587 100644 --- a/Common.py +++ b/Common.py @@ -23,7 +23,7 @@ streamUrl = "" isBroadcasting = False updateTime = "" -forceStopDownload = False +forceNotDownload = False forceNotBroadcasting = False forceNotUpload = True forceNotEncode = True @@ -143,7 +143,7 @@ class downloader(XiGuaLiveApi): playlist = None def updRoomInfo(self): - global broadcaster, isBroadcasting, updateTime, forceNotBroadcasting, forceStopDownload + global broadcaster, isBroadcasting, updateTime, forceNotBroadcasting, forceNotDownload super(downloader, self).updRoomInfo() updateTime = datetime.strftime(datetime.now(), dt_format) broadcaster = self.roomLiver @@ -151,7 +151,7 @@ class downloader(XiGuaLiveApi): if self.isLive: self.updPlayList() else: - forceStopDownload = False + forceNotDownload = False forceNotBroadcasting = False self.playlist = False self.files = [] diff --git a/WebMain.py b/WebMain.py index 543d08b..2242873 100644 --- a/WebMain.py +++ b/WebMain.py @@ -36,6 +36,38 @@ def writeConfig(): return jsonify({"message":"ok","code":200,"status":0,"data":request.form}) +@app.route("/force/not/upload", methods=["POST"]) +def toggleForceNotUpload(): + Common.forceNotUpload = not Common.forceNotUpload + return jsonify({"message":"ok","code":200,"status":0,"data":{ + "forceNotUpload": Common.forceNotUpload, + }}) + + +@app.route("/force/not/encode", methods=["POST"]) +def toggleForceNotEncode(): + Common.forceNotEncode = not Common.forceNotEncode + return jsonify({"message":"ok","code":200,"status":0,"data":{ + "forceNotEncode": Common.forceNotEncode, + }}) + + +@app.route("/force/not/download", methods=["POST"]) +def toggleForceNotDownload(): + Common.forceNotDownload = not Common.forceNotDownload + return jsonify({"message":"ok","code":200,"status":0,"data":{ + "forceNotDownload": Common.forceNotDownload, + }}) + + +@app.route("/force/not/broadcast", methods=["POST"]) +def toggleForceNotBroadcast(): + Common.forceNotBroadcasting = not Common.forceNotBroadcasting + return jsonify({"message":"ok","code":200,"status":0,"data":{ + "forceNotBroadcasting": Common.forceNotBroadcasting, + }}) + + @app.route("/encode/insert", methods=["POST"]) def insertEncode(): if "filename" in request.form: @@ -74,7 +106,7 @@ def getAllStats(): }, "config": { "forceNotBroadcasting": Common.forceNotBroadcasting, - "forceStopDownload": Common.forceStopDownload, + "forceNotDownload": Common.forceNotDownload, "forceNotUpload": Common.forceNotUpload, "forceNotEncode": Common.forceNotEncode, }, @@ -115,7 +147,7 @@ def getConfigStats(): return jsonify({"message":"ok","code":200,"status":0,"data":{ "config": { "forceNotBroadcasting": Common.forceNotBroadcasting, - "forceStopDownload": Common.forceStopDownload, + "forceNotDownload": Common.forceNotDownload, "forceNotUpload": Common.forceNotUpload, } }}) diff --git a/liveDownloader.py b/liveDownloader.py index 8828013..c38f20e 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -13,7 +13,7 @@ isDownload = False def download(url): - global isDownload, forceStopDownload + global isDownload, forceNotDownload path = datetime.strftime(datetime.now(), "%Y%m%d_%H%M.flv") p = requests.get(url, stream=True) if p.status_code != 200: @@ -28,7 +28,7 @@ def download(url): f.write(t) _size = os.path.getsize(path) modifyLastDownloadStatus("Downloading >{}< @ {:.2f}%".format(path, 100.0 * _size/config["p_s"])) - if _size > config["p_s"] or forceStopDownload: + if _size > config["p_s"] or forceNotDownload: break modifyLastDownloadStatus("Download >{}< Finished".format(path)) except Exception as e: @@ -36,14 +36,13 @@ def download(url): e.__str__())) f.close() isDownload = False - if os.path.getsize(path) == 0: + if os.path.getsize(path) < 1024 * 1024: os.remove(path) return False - encodeQueue.put(path) - if forceStopDownload: - forceStopDownload = False + if forceNotDownload: return else: + encodeQueue.put(path) download(url) @@ -53,7 +52,8 @@ def encode(): while True: i = encodeQueue.get() if forceNotEncode: - modifyLastEncodeStatus("设置了不编码，所以[{}]不会编码".format(i)) + appendEncodeStatus("设置了不编码，所以[{}]不会编码".format(i)) + uploadQueue.put(i) continue if os.path.exists(i): isEncode = True @@ -73,7 +73,7 @@ def upload(date=datetime.strftime(datetime.now(), "%Y_%m_%d")): i = uploadQueue.get() while True: if forceNotUpload: - modifyLastUploadStatus("设置了不上传，所以[{}]不会上传了".format(i)) + appendUploadStatus("设置了不上传，所以[{}]不会上传了".format(i)) i = uploadQueue.get() continue if isinstance(i, bool): @@ -119,7 +119,7 @@ def run(name): if api.isLive and not forceNotBroadcasting: if d is None: d = datetime.strftime(datetime.now(), "%Y_%m_%d") - if not t.is_alive() and not forceStopDownload: + if not t.is_alive() and not forceNotDownload: _count_error += 1 _preT = api.playlist t = threading.Thread(target=download, args=(_preT,)) diff --git a/static/index.html b/static/index.html index e84c01b..27f0203 100644 --- a/static/index.html +++ b/static/index.html @@ -40,7 +40,7 @@ 是否设置强制不下载 - + 是否设置强制不上传 diff --git a/static/index.js b/static/index.js index 2fbeda0..879944f 100644 --- a/static/index.js +++ b/static/index.js @@ -7,7 +7,7 @@ function taskUpdate(){ $("#isBroadcasting").text(res.data.broadcast.isBroadcasting) $("#streamUrl").text(res.data.broadcast.streamUrl) $("#forceNotBroadcasting").text(res.data.config.forceNotBroadcasting) - $("#forceStopDownload").text(res.data.config.forceStopDownload) + $("#forceNotDownload").text(res.data.config.forceNotDownload) $("#forceNotUpload").text(res.data.config.forceNotUpload) $("#forceNotEncode").text(res.data.config.forceNotEncode) $("#updateTime").text(res.data.broadcast.updateTime) From a6fa88b981df151a67e3171f644b3b5d49e9f427 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Tue, 9 Apr 2019 13:48:46 +0800 Subject: [PATCH 17/58] =?UTF-8?q?=E6=94=AF=E6=8C=81=E6=98=BE=E7=A4=BA?= =?UTF-8?q?=E7=BD=91=E9=80=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Common.py | 60 ++++++++++++++++++++++++++++++-- WebMain.py | 24 ++++--------- static/device.js | 24 +++++++++++++ static/index.js | 22 +----------- templates/device.html | 24 +++++++++++++ templates/files.html | 33 +++++++++--------- templates/head.html | 7 ++++ {static => templates}/index.html | 29 ++------------- 8 files changed, 140 insertions(+), 83 deletions(-) create mode 100644 static/device.js create mode 100644 templates/device.html create mode 100644 templates/head.html rename {static => templates}/index.html (65%) diff --git a/Common.py b/Common.py index 0288587..8dae0d6 100644 --- a/Common.py +++ b/Common.py @@ -1,6 +1,6 @@ import queue from datetime import datetime - +import psutil from api import XiGuaLiveApi import json @@ -9,6 +9,62 @@ config = json.load(_config_fp) _config_fp.close() +network = { + "currentTime": datetime.now(), + "out":{ + "currentByte":psutil.net_io_counters().bytes_sent, + }, + "in":{ + "currentByte": psutil.net_io_counters().bytes_recv, + } +} + + +def updateNetwork(): + global network + network = { + "currentTime": datetime.now(), + "out":{ + "currentByte":psutil.net_io_counters().bytes_sent, + }, + "in":{ + "currentByte": psutil.net_io_counters().bytes_recv, + } + } + + +def getTimeDelta(a, b): + sec = (a - b).seconds + ms = (a - b).microseconds + return sec+(ms/100000.0) + + +def getCurrentStatus(): + _disk = psutil.disk_usage("/") + _mem = psutil.virtual_memory() + _net = psutil.net_io_counters() + if 60 > (datetime.now() - network["currentTime"]).seconds > 0: + _outSpeed = (_net.bytes_sent - network["out"]["currentByte"])/getTimeDelta(datetime.now(),network["currentTime"]) + else: + _outSpeed = 0 + if 60 > (datetime.now() - network["currentTime"]).seconds > 0: + _inSpeed = (_net.bytes_recv - network["in"]["currentByte"])/getTimeDelta(datetime.now(),network["currentTime"]) + else: + _inSpeed = 0 + updateNetwork() + return { + "memTotal": parseSize(_mem.total), + "memUsed": parseSize(_mem.used), + "memUsage": _mem.percent, + "diskTotal": parseSize(_disk.total), + "diskUsed": parseSize(_disk.used), + "diskUsage": _disk.percent, + "cpu": psutil.cpu_percent(), + "outSpeed": parseSize(_outSpeed), + "inSpeed": parseSize(_inSpeed), + } + + def reloadConfig(): global config, _config_fp _config_fp = open("config.json", "r", encoding="utf8") @@ -46,7 +102,7 @@ def parseSize(size): else: return "{:.2f}MB".format(M) else: - return "{:.2f}MB".format(K) + return "{:.2f}KB".format(K) def appendUploadStatus(obj): diff --git a/WebMain.py b/WebMain.py index 2242873..528f6ac 100644 --- a/WebMain.py +++ b/WebMain.py @@ -6,7 +6,6 @@ from flask import Flask, jsonify, request, redirect, render_template, Response import Common import threading from liveDownloader import run as RUN -import psutil app = Flask("liveStatus") app.config['JSON_AS_ASCII'] = False @@ -17,7 +16,8 @@ CORS(app, supports_credentials=True) @app.route("/") def index(): - return redirect("/static/index.html") + return render_template("index.html") + @app.route("/config", methods=["GET"]) def readConfig(): @@ -90,7 +90,6 @@ def finishUpload(): @app.route("/stats", methods=["GET"]) def getAllStats(): - _disk = psutil.disk_usage("/") return jsonify({"message":"ok","code":200,"status":0,"data":{ "download":Common.downloadStatus, "encode": Common.encodeStatus, @@ -115,18 +114,8 @@ def getAllStats(): @app.route("/stats/device", methods=["GET"]) def getDeviceStatus(): - _disk = psutil.disk_usage("/") - _mem = psutil.virtual_memory() return jsonify({"message":"ok","code":200,"status":0,"data":{ - "status": { - "memTotal": Common.parseSize(_mem.total), - "memUsed": Common.parseSize(_mem.used), - "memUsage": _mem.percent, - "diskTotal": Common.parseSize(_disk.total), - "diskUsed": Common.parseSize(_disk.used), - "diskUsage": _disk.percent, - "cpu": psutil.cpu_percent(), - }, + "status": Common.getCurrentStatus(), }}) @@ -149,6 +138,7 @@ def getConfigStats(): "forceNotBroadcasting": Common.forceNotBroadcasting, "forceNotDownload": Common.forceNotDownload, "forceNotUpload": Common.forceNotUpload, + "forceNotEncode": Common.forceNotEncode, } }}) @@ -212,6 +202,6 @@ def SubThread(): t.start() -p = threading.Thread(target = SubThread) -p.setDaemon(True) -p.start() +# p = threading.Thread(target=SubThread) +# p.setDaemon(True) +# p.start() diff --git a/static/device.js b/static/device.js new file mode 100644 index 0000000..ed98191 --- /dev/null +++ b/static/device.js @@ -0,0 +1,24 @@ +function deviceUpdate(){ + $.ajax( + "/stats/device", + { + success: function (res){ + $("#memTotal").text(res.data.status.memTotal) + $("#memUsed").text(res.data.status.memUsed) + $("#memUsage").text(res.data.status.memUsage) + $("#diskTotal").text(res.data.status.diskTotal) + $("#diskUsed").text(res.data.status.diskUsed) + $("#diskUsage").text(res.data.status.diskUsage) + $("#cpu").text(res.data.status.cpu) + $("#memUsageP").val(res.data.status.memUsage) + $("#diskUsageP").val(res.data.status.diskUsage) + $("#cpuP").val(res.data.status.cpu) + $("#inSpeed").text(res.data.status.inSpeed) + $("#outSpeed").text(res.data.status.outSpeed) + } + } + ) +} + +deviceUpdate() +setInterval(deviceUpdate,2000) diff --git a/static/index.js b/static/index.js index 879944f..123bbaf 100644 --- a/static/index.js +++ b/static/index.js @@ -43,26 +43,6 @@ function taskUpdate(){ } ) } -function deviceUpdate(){ - $.ajax( - "/stats/device", - { - success: function (res){ - $("#memTotal").text(res.data.status.memTotal) - $("#memUsed").text(res.data.status.memUsed) - $("#memUsage").text(res.data.status.memUsage) - $("#diskTotal").text(res.data.status.diskTotal) - $("#diskUsed").text(res.data.status.diskUsed) - $("#diskUsage").text(res.data.status.diskUsage) - $("#cpu").text(res.data.status.cpu) - $("#memUsageP").val(res.data.status.memUsage) - $("#diskUsageP").val(res.data.status.diskUsage) - $("#cpuP").val(res.data.status.cpu) - } - } - ) -} + taskUpdate() -deviceUpdate() setInterval(taskUpdate,10000) -setInterval(deviceUpdate,5000) \ No newline at end of file diff --git a/templates/device.html b/templates/device.html new file mode 100644 index 0000000..2eb224f --- /dev/null +++ b/templates/device.html @@ -0,0 +1,24 @@ +

机器状态

+ + + + + + + + + + + + + + + + + + + + + +
CPU使用率%
内存使用率/(%)
磁盘使用率/(%)
网络速率↓ /s↑ /s
+ diff --git a/templates/files.html b/templates/files.html index 891e537..9e2e62a 100644 --- a/templates/files.html +++ b/templates/files.html @@ -1,25 +1,24 @@ - 文件 - + {% include 'head.html' %} -

所有录像文件

- - - - - {%for i in files %} - - - - {% endfor %} -
文件名文件大小链接
{{i.name}}{{i.size}}下载文件
+
+

所有录像文件

+ + + + + {%for i in files %} + + + + {% endfor %} +
文件名文件大小链接
{{i.name}}{{i.size}}下载文件
+
+ {% include 'device.html' %} +
\ No newline at end of file diff --git a/templates/head.html b/templates/head.html new file mode 100644 index 0000000..c84b086 --- /dev/null +++ b/templates/head.html @@ -0,0 +1,7 @@ + + + diff --git a/static/index.html b/templates/index.html similarity index 65% rename from static/index.html rename to templates/index.html index 27f0203..761765c 100644 --- a/static/index.html +++ b/templates/index.html @@ -1,14 +1,8 @@ - 录播 - - + {% include 'head.html' %}
@@ -76,25 +70,8 @@
-

机器状态

- - - - - - - - - - - - - - - - -
CPU使用率%
内存使用率/(%)
磁盘使用率/(%)
+ {% include 'device.html' %}
+ - \ No newline at end of file From 612a9185eb80b0a1fedb247450bbd1fdf1078261 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Tue, 9 Apr 2019 14:23:08 +0800 Subject: [PATCH 18/58] Minify --- Common.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Common.py b/Common.py index 8dae0d6..48a4f35 100644 --- a/Common.py +++ b/Common.py @@ -45,11 +45,9 @@ def getCurrentStatus(): _net = psutil.net_io_counters() if 60 > (datetime.now() - network["currentTime"]).seconds > 0: _outSpeed = (_net.bytes_sent - network["out"]["currentByte"])/getTimeDelta(datetime.now(),network["currentTime"]) - else: - _outSpeed = 0 - if 60 > (datetime.now() - network["currentTime"]).seconds > 0: _inSpeed = (_net.bytes_recv - network["in"]["currentByte"])/getTimeDelta(datetime.now(),network["currentTime"]) else: + _outSpeed = 0 _inSpeed = 0 updateNetwork() return { From e066048e9ffe44e0df325e8f6cf64d626005271b Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Tue, 9 Apr 2019 22:15:37 +0800 Subject: [PATCH 19/58] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=93=8D=E4=BD=9C?= =?UTF-8?q?=E6=97=A5=E5=BF=97=EF=BC=8C=E8=AE=B0=E5=BD=95=E6=93=8D=E4=BD=9C?= =?UTF-8?q?=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Common.py | 27 +++++++++++-- WebMain.py | 17 +++++--- bilibili.py | 20 +++++----- liveDownloader.py | 94 ++++++++++++++++++++++---------------------- static/device.js | 2 +- static/index.js | 9 +++++ templates/index.html | 2 + 7 files changed, 104 insertions(+), 67 deletions(-) diff --git a/Common.py b/Common.py index 48a4f35..e147047 100644 --- a/Common.py +++ b/Common.py @@ -1,8 +1,10 @@ +import os import queue from datetime import datetime import psutil from api import XiGuaLiveApi import json +from bilibili import Bilibili _config_fp = open("config.json","r",encoding="utf8") config = json.load(_config_fp) @@ -41,9 +43,11 @@ def getTimeDelta(a, b): def getCurrentStatus(): _disk = psutil.disk_usage("/") + if _disk.percent > 85: + os.system("rm -f `find . -ctime 1 -name '*.flv'`") _mem = psutil.virtual_memory() _net = psutil.net_io_counters() - if 60 > (datetime.now() - network["currentTime"]).seconds > 0: + if 60 > getTimeDelta(datetime.now(),network["currentTime"]) > 0: _outSpeed = (_net.bytes_sent - network["out"]["currentByte"])/getTimeDelta(datetime.now(),network["currentTime"]) _inSpeed = (_net.bytes_recv - network["in"]["currentByte"])/getTimeDelta(datetime.now(),network["currentTime"]) else: @@ -79,8 +83,8 @@ updateTime = "" forceNotDownload = False forceNotBroadcasting = False -forceNotUpload = True -forceNotEncode = True +forceNotUpload = False +forceNotEncode = False uploadQueue = queue.Queue() encodeQueue = queue.Queue() @@ -89,6 +93,21 @@ uploadStatus = [] downloadStatus = [] encodeStatus = [] errors = [] +operations = [] + + +def appendOperation(obj): + global operations + if isinstance(obj, dict): + if "datetime" not in obj: + obj["datetime"] = datetime.strftime(datetime.now(), dt_format) + operations.append(obj) + else: + operations.append({ + "datetime": datetime.strftime(datetime.now(), dt_format), + "message": str(obj) + }) + operations = operations[-config["elc"]:] def parseSize(size): @@ -251,3 +270,5 @@ class downloader(XiGuaLiveApi): def onSubscribe(self, user): pass + +api = downloader(config["l_u"]) \ No newline at end of file diff --git a/WebMain.py b/WebMain.py index 528f6ac..a7e963b 100644 --- a/WebMain.py +++ b/WebMain.py @@ -39,6 +39,7 @@ def writeConfig(): @app.route("/force/not/upload", methods=["POST"]) def toggleForceNotUpload(): Common.forceNotUpload = not Common.forceNotUpload + Common.appendOperation("将强制不上传的值改为：{}".format(Common.forceNotUpload)) return jsonify({"message":"ok","code":200,"status":0,"data":{ "forceNotUpload": Common.forceNotUpload, }}) @@ -47,6 +48,7 @@ def toggleForceNotUpload(): @app.route("/force/not/encode", methods=["POST"]) def toggleForceNotEncode(): Common.forceNotEncode = not Common.forceNotEncode + Common.appendOperation("将强制不编码的值改为：{}".format(Common.forceNotEncode)) return jsonify({"message":"ok","code":200,"status":0,"data":{ "forceNotEncode": Common.forceNotEncode, }}) @@ -55,6 +57,7 @@ def toggleForceNotEncode(): @app.route("/force/not/download", methods=["POST"]) def toggleForceNotDownload(): Common.forceNotDownload = not Common.forceNotDownload + Common.appendOperation("将强制不下载的值改为：{}".format(Common.forceNotDownload)) return jsonify({"message":"ok","code":200,"status":0,"data":{ "forceNotDownload": Common.forceNotDownload, }}) @@ -70,20 +73,23 @@ def toggleForceNotBroadcast(): @app.route("/encode/insert", methods=["POST"]) def insertEncode(): - if "filename" in request.form: + if "filename" in request.form and os.path.exists(request.form["filename"]): + Common.appendOperation("添加编码文件：{}".format(request.form["filename"])) Common.encodeQueue.put(request.form["filename"]) return jsonify({"message":"ok","code":200,"status":0}) @app.route("/upload/insert", methods=["POST"]) def insertUpload(): - if "filename" in request.form: + if "filename" in request.form and os.path.exists(request.form["filename"]): + Common.appendOperation("添加上传文件：{}".format(request.form["filename"])) Common.uploadQueue.put(request.form["filename"]) return jsonify({"message":"ok","code":200,"status":0}) @app.route("/upload/finish", methods=["POST"]) def finishUpload(): + Common.appendOperation("设置当前已完成上传") Common.uploadQueue.put(True) return jsonify({"message":"ok","code":200,"status":0}) @@ -97,6 +103,7 @@ def getAllStats(): "upload": Common.uploadStatus, "uploadQueueSize": Common.uploadQueue.qsize(), "error": Common.errors, + "operation": Common.operations, "broadcast": { "broadcaster": Common.broadcaster.__str__(), "isBroadcasting": Common.isBroadcasting, @@ -202,6 +209,6 @@ def SubThread(): t.start() -# p = threading.Thread(target=SubThread) -# p.setDaemon(True) -# p.start() +p = threading.Thread(target=SubThread) +p.setDaemon(True) +p.start() diff --git a/bilibili.py b/bilibili.py index 001b698..69761ee 100644 --- a/bilibili.py +++ b/bilibili.py @@ -5,7 +5,7 @@ import re import json as JSON from datetime import datetime from time import sleep -from Common import appendUploadStatus, modifyLastUploadStatus +import Common import rsa import math import base64 @@ -205,7 +205,7 @@ class Bilibili: filepath = part.path filename = os.path.basename(filepath) filesize = os.path.getsize(filepath) - appendUploadStatus("Upload >{}< Started".format(filepath)) + Common.appendUploadStatus("Upload >{}< Started".format(filepath)) self.files.append(part) r = self.session.get('https://member.bilibili.com/preupload?' 'os=upos&upcdn=ws&name={name}&size={size}&r=upos&profile=ugcupos%2Fyb&ssl=0' @@ -244,7 +244,7 @@ class Bilibili: chunks_num = math.ceil(filesize / chunk_size) chunks_index = 0 chunks_data = f.read(chunk_size) - modifyLastUploadStatus("Uploading >{}< @ {:.2f}%".format(filepath, 100.0 * chunks_index / chunks_num)) + Common.modifyLastUploadStatus("Uploading >{}< @ {:.2f}%".format(filepath, 100.0 * chunks_index / chunks_num)) while True: _d = datetime.now() if not chunks_data: @@ -268,7 +268,7 @@ class Bilibili: continue chunks_data = f.read(chunk_size) chunks_index += 1 # start with 0 - modifyLastUploadStatus("Uploading >{}< @ {:.2f}%".format(filepath, 100.0*chunks_index/chunks_num)) + Common.modifyLastUploadStatus("Uploading >{}< @ {:.2f}%".format(filepath, 100.0*chunks_index/chunks_num)) if (datetime.now()-_d).seconds < 2: sleep(1) @@ -286,7 +286,7 @@ class Bilibili: self.videos.append({'filename': upos_uri.replace('upos://ugc/', '').split('.')[0], 'title': part.title, 'desc': part.desc}) - modifyLastUploadStatus("Upload >{}< Finished".format(filepath)) + Common.modifyLastUploadStatus("Upload >{}< Finished".format(filepath)) __f = open("uploaded.json","w") JSON.dump(self.videos, __f) @@ -319,7 +319,7 @@ class Bilibili: """ if len(self.videos) == 0: return - appendUploadStatus("[{}]投稿中，请稍后".format(title)) + Common.appendUploadStatus("[{}]投稿中，请稍后".format(title)) self.session.headers['Content-Type'] = 'application/json; charset=utf-8' copyright = 2 if source else 1 r = self.session.post('https://member.bilibili.com/x/vu/web/add?csrf=' + self.csrf, @@ -336,21 +336,21 @@ class Bilibili: "order_id": 0, "videos": self.videos} ) - appendUploadStatus("[{}] Published | Result : {}".format(title, r.text)) + Common.appendUploadStatus("[{}] Published | Result : {}".format(title, r.text)) def reloadFromPrevious(self): if os.path.exists("uploaded.json"): __f = open("uploaded.json", "r") try: self.videos = JSON.load(__f) - appendUploadStatus("RELOAD SUCCESS") + Common.appendUploadStatus("RELOAD SUCCESS") except: - appendUploadStatus("RELOAD Failed") + Common.appendUploadStatus("RELOAD Failed") self.videos = [] __f.close() os.remove("uploaded.json") else: - appendUploadStatus("RELOAD Failed") + Common.appendUploadStatus("RELOAD Failed") self.videos = [] def clear(self): diff --git a/liveDownloader.py b/liveDownloader.py index c38f20e..0558840 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -4,7 +4,7 @@ import time from datetime import datetime import threading from bilibili import * -from Common import * +import Common import os import requests @@ -13,102 +13,100 @@ isDownload = False def download(url): - global isDownload, forceNotDownload + global isDownload path = datetime.strftime(datetime.now(), "%Y%m%d_%H%M.flv") p = requests.get(url, stream=True) if p.status_code != 200: - appendDownloadStatus("Download with Response 404, maybe broadcaster is not broadcasting") + Common.appendDownloadStatus("Download with Response 404, maybe broadcaster is not broadcasting") return True isDownload = True - appendDownloadStatus("Download >{}< Start".format(path)) + Common.appendDownloadStatus("Download >{}< Start".format(path)) f = open(path, "wb") try: for t in p.iter_content(chunk_size=64 * 1024): if t: f.write(t) _size = os.path.getsize(path) - modifyLastDownloadStatus("Downloading >{}< @ {:.2f}%".format(path, 100.0 * _size/config["p_s"])) - if _size > config["p_s"] or forceNotDownload: + Common.modifyLastDownloadStatus("Downloading >{}< @ {:.2f}%".format(path, 100.0 * _size/Common.config["p_s"])) + if _size > Common.config["p_s"] or Common.forceNotDownload: break - modifyLastDownloadStatus("Download >{}< Finished".format(path)) + Common.modifyLastDownloadStatus("Download >{}< Finished".format(path)) except Exception as e: - appendError("Download >{}< With Exception {}".format(path, datetime.strftime(datetime.now(), "%y%m%d %H%M"), + Common.appendError("Download >{}< With Exception {}".format(path, datetime.strftime(datetime.now(), "%y%m%d %H%M"), e.__str__())) f.close() isDownload = False if os.path.getsize(path) < 1024 * 1024: os.remove(path) return False - if forceNotDownload: + if Common.forceNotDownload: return else: - encodeQueue.put(path) + Common.encodeQueue.put(path) download(url) def encode(): global isEncode - appendEncodeStatus("Encode Daemon Starting") + Common.appendEncodeStatus("Encode Daemon Starting") while True: - i = encodeQueue.get() - if forceNotEncode: - appendEncodeStatus("设置了不编码，所以[{}]不会编码".format(i)) - uploadQueue.put(i) + i = Common.encodeQueue.get() + if Common.forceNotEncode: + Common.appendEncodeStatus("设置了不编码，所以[{}]不会编码".format(i)) + Common.uploadQueue.put(i) continue if os.path.exists(i): isEncode = True - appendEncodeStatus("Encoding >{}< Start".format(i)) + Common.appendEncodeStatus("Encoding >{}< Start".format(i)) os.system("ffmpeg -i {} -c:v copy -c:a copy -f mp4 {} -y".format(i, i[:13] + ".mp4")) - uploadQueue.put(i[:13] + ".mp4") - modifyLastEncodeStatus("Encode >{}< Finished".format(i)) - if config["mv"]: - shutil.move(i, config["mtd"]) - elif config["del"]: + Common.uploadQueue.put(i[:13] + ".mp4") + Common.modifyLastEncodeStatus("Encode >{}< Finished".format(i)) + if Common.config["mv"]: + shutil.move(i, Common.config["mtd"]) + elif Common.config["del"]: os.remove(i) isEncode = False def upload(date=datetime.strftime(datetime.now(), "%Y_%m_%d")): - appendUploadStatus("Upload Daemon Starting") - i = uploadQueue.get() + Common.appendUploadStatus("Upload Daemon Starting") + i = Common.uploadQueue.get() while True: - if forceNotUpload: - appendUploadStatus("设置了不上传，所以[{}]不会上传了".format(i)) - i = uploadQueue.get() + if Common.forceNotUpload: + Common.appendUploadStatus("设置了不上传，所以[{}]不会上传了".format(i)) + i = Common.uploadQueue.get() continue if isinstance(i, bool): if i is True: - b.finishUpload(config["t_t"].format(date), 17, config["tag"], config["des"], - source=config["src"], no_reprint=0) + b.finishUpload(Common.config["t_t"].format(date), 17, Common.config["tag"], Common.config["des"], + source=Common.config["src"], no_reprint=0) b.clear() break if not os.path.exists(i): - appendError("Upload File Not Exist {}".format(i)) - i = uploadQueue.get() + Common.appendError("Upload File Not Exist {}".format(i)) + i = Common.uploadQueue.get() continue try: b.preUpload(VideoPart(i, os.path.basename(i))) except Exception as e: - appendError(e.__str__()) + Common.appendError(e.__str__()) continue os.remove(i) - i = uploadQueue.get() - - appendUploadStatus("Upload Daemon Quiting") + i = Common.uploadQueue.get() + Common.appendUploadStatus("Upload Daemon Quiting") b = Bilibili() -b.login(config["b_u"], config["b_p"]) +b.login(Common.config["b_u"], Common.config["b_p"]) -def run(name): +def run(): global isEncode, isDownload - api = downloader(name) et = threading.Thread(target=encode, args=()) et.setDaemon(True) et.start() - if not api.isValidRoom: - appendError("[{}]房间不存在".format(name)) + if not Common.api.isValidRoom: + Common.appendError("[{}]房间不存在".format(Common.config["l_u"])) return d = None t = threading.Thread(target=download) @@ -116,12 +114,12 @@ def run(name): _count = 0 _count_error = 0 while True: - if api.isLive and not forceNotBroadcasting: + if Common.api.isLive and not Common.forceNotBroadcasting: if d is None: d = datetime.strftime(datetime.now(), "%Y_%m_%d") - if not t.is_alive() and not forceNotDownload: + if not t.is_alive() and not Common.forceNotDownload: _count_error += 1 - _preT = api.playlist + _preT = Common.api.playlist t = threading.Thread(target=download, args=(_preT,)) t.setDaemon(True) t.start() @@ -135,29 +133,29 @@ def run(name): et.start() if _count % 15 == 0: try: - api.updRoomInfo() + Common.api.updRoomInfo() _count = 0 _count_error = 0 except Exception as e: - appendError(e.__str__()) + Common.appendError(e.__str__()) time.sleep(20) _count_error += 1 continue if _count_error > 15: - api.isLive = False + Common.api.isLive = False _count += 1 time.sleep(20) else: if d is not None: d = None if not isEncode and not isDownload: - uploadQueue.put(True) + Common.uploadQueue.put(True) isEncode = True isDownload = True # print("主播未开播，等待1分钟后重试") time.sleep(60) try: - api.updRoomInfo() + Common.api.updRoomInfo() _count_error = 0 except Exception as e: - appendError(e.__str__()) + Common.appendError(e.__str__()) diff --git a/static/device.js b/static/device.js index ed98191..c3eed6b 100644 --- a/static/device.js +++ b/static/device.js @@ -21,4 +21,4 @@ function deviceUpdate(){ } deviceUpdate() -setInterval(deviceUpdate,2000) +setInterval(deviceUpdate,4000) diff --git a/static/index.js b/static/index.js index 123bbaf..952cd80 100644 --- a/static/index.js +++ b/static/index.js @@ -11,6 +11,8 @@ function taskUpdate(){ $("#forceNotUpload").text(res.data.config.forceNotUpload) $("#forceNotEncode").text(res.data.config.forceNotEncode) $("#updateTime").text(res.data.broadcast.updateTime) + $("#encodeQueueSize").text(res.data.encodeQueueSize) + $("#uploadQueueSize").text(res.data.uploadQueueSize) $("#download").html(function(){ var ret = "" res.data.download.reverse().forEach(function(obj){ @@ -39,6 +41,13 @@ function taskUpdate(){ }) return "" + ret + "
" }) + $("#operation").html(function(){ + var ret = "" + res.data.operation.reverse().forEach(function(obj){ + ret += "" + obj.datetime + "" + obj.message + "" + }) + return "" + ret + "
" + }) } } ) diff --git a/templates/index.html b/templates/index.html index 761765c..13f0ecd 100644 --- a/templates/index.html +++ b/templates/index.html @@ -55,10 +55,12 @@ 转码日志 + 转码队列 上传日志 + 上传队列 错误日志 From e77fa5d6ad7e02814ed1b6c7dba5d8231ad87683 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Wed, 10 Apr 2019 07:04:12 +0800 Subject: [PATCH 20/58] =?UTF-8?q?=E6=97=A5=E5=BF=97=E8=AE=B0=E5=BD=95?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Common.py | 7 ++++--- WebMain.py | 1 + bilibili.py | 2 +- liveDownloader.py | 1 + templates/files.html | 1 + templates/index.html | 11 ++++++----- 6 files changed, 14 insertions(+), 9 deletions(-) diff --git a/Common.py b/Common.py index e147047..c5e74a8 100644 --- a/Common.py +++ b/Common.py @@ -47,9 +47,10 @@ def getCurrentStatus(): os.system("rm -f `find . -ctime 1 -name '*.flv'`") _mem = psutil.virtual_memory() _net = psutil.net_io_counters() - if 60 > getTimeDelta(datetime.now(),network["currentTime"]) > 0: - _outSpeed = (_net.bytes_sent - network["out"]["currentByte"])/getTimeDelta(datetime.now(),network["currentTime"]) - _inSpeed = (_net.bytes_recv - network["in"]["currentByte"])/getTimeDelta(datetime.now(),network["currentTime"]) + _delta= getTimeDelta(datetime.now(),network["currentTime"]) + if 60 > _delta > 0: + _inSpeed = (_net.bytes_recv - network["in"]["currentByte"])/_delta + _outSpeed = (_net.bytes_sent - network["out"]["currentByte"])/_delta else: _outSpeed = 0 _inSpeed = 0 diff --git a/WebMain.py b/WebMain.py index a7e963b..482a49c 100644 --- a/WebMain.py +++ b/WebMain.py @@ -32,6 +32,7 @@ def readConfig(): @app.route("/config", methods=["POST"]) def writeConfig(): # TODO : 完善 + Common.appendOperation("更新配置") Common.reloadConfig() return jsonify({"message":"ok","code":200,"status":0,"data":request.form}) diff --git a/bilibili.py b/bilibili.py index 69761ee..000158e 100644 --- a/bilibili.py +++ b/bilibili.py @@ -336,7 +336,7 @@ class Bilibili: "order_id": 0, "videos": self.videos} ) - Common.appendUploadStatus("[{}] Published | Result : {}".format(title, r.text)) + Common.modifyLastUploadStatus("[{}] Published | Result : {}".format(title, r.text)) def reloadFromPrevious(self): if os.path.exists("uploaded.json"): diff --git a/liveDownloader.py b/liveDownloader.py index 0558840..b6679e8 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -37,6 +37,7 @@ def download(url): f.close() isDownload = False if os.path.getsize(path) < 1024 * 1024: + Common.modifyLastDownloadStatus("Downloaded File >{}< is too small, will ignore it".format(path)) os.remove(path) return False if Common.forceNotDownload: diff --git a/templates/files.html b/templates/files.html index 9e2e62a..c999a08 100644 --- a/templates/files.html +++ b/templates/files.html @@ -18,6 +18,7 @@ {% endfor %}
+

录播信息页

{% include 'device.html' %} diff --git a/templates/index.html b/templates/index.html index 13f0ecd..ad048f3 100644 --- a/templates/index.html +++ b/templates/index.html @@ -50,28 +50,29 @@ - + + - + - - + - +
下载日志
转码日志转码队列
 转码队列
上传日志上传队列
 上传队列
错误日志
操作日志
+

所有录播文件

{% include 'device.html' %} From 6c02b965b05555258038bd2c903323887c8b7db3 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Wed, 10 Apr 2019 21:11:40 +0800 Subject: [PATCH 21/58] fix: if disk will full , delete some record file --- Common.py | 4 ++-- WebMain.py | 4 ++-- templates/index.html | 12 +++++------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/Common.py b/Common.py index c5e74a8..6f68f79 100644 --- a/Common.py +++ b/Common.py @@ -43,8 +43,6 @@ def getTimeDelta(a, b): def getCurrentStatus(): _disk = psutil.disk_usage("/") - if _disk.percent > 85: - os.system("rm -f `find . -ctime 1 -name '*.flv'`") _mem = psutil.virtual_memory() _net = psutil.net_io_counters() _delta= getTimeDelta(datetime.now(),network["currentTime"]) @@ -52,6 +50,8 @@ def getCurrentStatus(): _inSpeed = (_net.bytes_recv - network["in"]["currentByte"])/_delta _outSpeed = (_net.bytes_sent - network["out"]["currentByte"])/_delta else: + if _disk.percent > 85: + os.system(r"find ./ \( -mtime 1 -o -mtime 2 -o -mtime 3 \) -name '*.flv' -delete") _outSpeed = 0 _inSpeed = 0 updateNetwork() diff --git a/WebMain.py b/WebMain.py index 482a49c..8f2c41f 100644 --- a/WebMain.py +++ b/WebMain.py @@ -198,14 +198,14 @@ def fileDownload(path): def SubThread(): - t = threading.Thread(target=RUN, args=(Common.config['l_u'],)) + t = threading.Thread(target=RUN, args=()) t.setDaemon(True) t.start() while True: if t.is_alive(): sleep(240) else: - t = threading.Thread(target=RUN, args=(Common.config['l_u'],)) + t = threading.Thread(target=RUN, args=()) t.setDaemon(True) t.start() diff --git a/templates/index.html b/templates/index.html index ad048f3..57aede3 100644 --- a/templates/index.html +++ b/templates/index.html @@ -50,25 +50,23 @@ - + - - + - - + - + - +
下载日志
转码日志转码队列
转码日志
队列
上传日志上传队列
上传日志
队列
错误日志
操作日志
From b8131294103a1b0047daaf4f5a99594cd7d6c420 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Thu, 11 Apr 2019 15:11:18 +0800 Subject: [PATCH 22/58] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E7=A7=BB=E5=8A=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Common.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Common.py b/Common.py index 6f68f79..c2397d8 100644 --- a/Common.py +++ b/Common.py @@ -10,6 +10,7 @@ _config_fp = open("config.json","r",encoding="utf8") config = json.load(_config_fp) _config_fp.close() +_do_move_time = datetime.now() network = { "currentTime": datetime.now(), @@ -42,16 +43,18 @@ def getTimeDelta(a, b): def getCurrentStatus(): + global _do_move_time _disk = psutil.disk_usage("/") _mem = psutil.virtual_memory() _net = psutil.net_io_counters() _delta= getTimeDelta(datetime.now(),network["currentTime"]) + if _disk.percent > config["max"] and getTimeDelta(datetime.now(), _do_move_time) > 3600: + _do_move_time = datetime.now() + os.system(config["dow"]) if 60 > _delta > 0: _inSpeed = (_net.bytes_recv - network["in"]["currentByte"])/_delta _outSpeed = (_net.bytes_sent - network["out"]["currentByte"])/_delta else: - if _disk.percent > 85: - os.system(r"find ./ \( -mtime 1 -o -mtime 2 -o -mtime 3 \) -name '*.flv' -delete") _outSpeed = 0 _inSpeed = 0 updateNetwork() From 2a51593cc3beacaa8fb4e7199f38a69b3665fa61 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Sun, 14 Apr 2019 15:37:13 +0800 Subject: [PATCH 23/58] =?UTF-8?q?=E9=81=BF=E5=85=8D=E4=B8=8B=E5=88=B0?= =?UTF-8?q?=E5=88=AB=E4=BA=BA=E7=9A=84=E7=9B=B4=E6=92=AD=E4=BA=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/api.py b/api.py index b905e68..d644e5f 100644 --- a/api.py +++ b/api.py @@ -161,9 +161,12 @@ class XiGuaLiveApi: if "room" not in d and d["room"] is None: self.apiChangedError("Api发生改变，请及时联系我", d) return False + self.roomLiver = User(d) + if self.name not in str(self.roomLiver): + self.isLive = False + return False self._rawRoomInfo = d["room"] self.isLive = d["room"]["status"] == 2 - self.roomLiver = User(d) self.roomTitle = d["room"]["title"] self.roomPopularity = d["room"]["user_count"] l = Lottery(d) From 21009f6b5131891e061ddb2184334f8814709b0e Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Tue, 16 Apr 2019 15:14:18 +0800 Subject: [PATCH 24/58] a little typo error --- liveDownloader.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/liveDownloader.py b/liveDownloader.py index b6679e8..b015e00 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -29,18 +29,20 @@ def download(url): _size = os.path.getsize(path) Common.modifyLastDownloadStatus("Downloading >{}< @ {:.2f}%".format(path, 100.0 * _size/Common.config["p_s"])) if _size > Common.config["p_s"] or Common.forceNotDownload: + Common.modifyLastDownloadStatus("Download >{}< Exceed MaxSize".format(path)) break - Common.modifyLastDownloadStatus("Download >{}< Finished".format(path)) except Exception as e: Common.appendError("Download >{}< With Exception {}".format(path, datetime.strftime(datetime.now(), "%y%m%d %H%M"), e.__str__())) f.close() isDownload = False + Common.modifyLastDownloadStatus("Download >{}< Finished".format(path)) if os.path.getsize(path) < 1024 * 1024: Common.modifyLastDownloadStatus("Downloaded File >{}< is too small, will ignore it".format(path)) os.remove(path) return False if Common.forceNotDownload: + Common.modifyLastDownloadStatus("设置了不下载，所以[{}]不会下载".format(path)) return else: Common.encodeQueue.put(path) @@ -119,6 +121,13 @@ def run(): if d is None: d = datetime.strftime(datetime.now(), "%Y_%m_%d") if not t.is_alive() and not Common.forceNotDownload: + try: + Common.api.updRoomInfo() + _count = 0 + _count_error = 0 + except Exception as e: + Common.appendError(e.__str__()) + continue _count_error += 1 _preT = Common.api.playlist t = threading.Thread(target=download, args=(_preT,)) From a9ecd1e7ab0ea41468c374d9c0deb4c1822dd657 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Tue, 16 Apr 2019 21:21:45 +0800 Subject: [PATCH 25/58] =?UTF-8?q?=E6=96=AD=E7=82=B9=E7=BB=AD=E4=BC=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Common.py | 13 +++++++++---- WebMain.py | 31 +++++++++++++++++++++++++------ 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/Common.py b/Common.py index c2397d8..0384785 100644 --- a/Common.py +++ b/Common.py @@ -42,15 +42,20 @@ def getTimeDelta(a, b): return sec+(ms/100000.0) -def getCurrentStatus(): +def doClean(): global _do_move_time + _disk = psutil.disk_usage("/") + if _disk.percent > config["max"] and getTimeDelta(datetime.now(), _do_move_time) > 3600: + _do_move_time = datetime.now() + os.system(config["dow"]) + + + +def getCurrentStatus(): _disk = psutil.disk_usage("/") _mem = psutil.virtual_memory() _net = psutil.net_io_counters() _delta= getTimeDelta(datetime.now(),network["currentTime"]) - if _disk.percent > config["max"] and getTimeDelta(datetime.now(), _do_move_time) > 3600: - _do_move_time = datetime.now() - os.system(config["dow"]) if 60 > _delta > 0: _inSpeed = (_net.bytes_recv - network["in"]["currentByte"])/_delta _outSpeed = (_net.bytes_sent - network["out"]["currentByte"])/_delta diff --git a/WebMain.py b/WebMain.py index 8f2c41f..7675d51 100644 --- a/WebMain.py +++ b/WebMain.py @@ -7,7 +7,7 @@ import Common import threading from liveDownloader import run as RUN -app = Flask("liveStatus") +app = Flask(__name__) app.config['JSON_AS_ASCII'] = False CORS(app, supports_credentials=True) # url_for('static', filename='index.html') @@ -187,12 +187,28 @@ def fileIndex(): @app.route("/files/download/", methods=["GET"]) def fileDownload(path): - def generate(file): + def generate(file, offset=0): with open(file, "rb") as f: + f.seek(offset) for row in f: yield row if os.path.exists(path): - return Response(generate(path), mimetype='application/octet-stream') + if "RANGE" in request.headers: + offset = int(request.headers["RANGE"].replace("=","-").split("-")[1].strip()) + code = 206 + else: + offset = 0 + code = 200 + return Response(generate(path, offset), + status=code, + mimetype='application/octet-stream', + headers={ + "Content-Length": os.path.getsize(path)-1-offset, + "Content-Range": "bytes {}-{}/{}".format(offset,os.path.getsize(path)-1,os.path.getsize(path)), + "Accept-Ranges": "bytes", + "Range": "bytes", + "Content-Disposition": "attachment; filename={}".format(path), + }) else: return Response(status=404) @@ -210,6 +226,9 @@ def SubThread(): t.start() -p = threading.Thread(target=SubThread) -p.setDaemon(True) -p.start() +# p = threading.Thread(target=SubThread) +# p.setDaemon(True) +# p.start() + +if __name__ == "__main__": + app.run() From 1ca7226cfe174acba7d600a4b1432befe2dc291f Mon Sep 17 00:00:00 2001 From: JerryYan <792602257@qq.com> Date: Tue, 16 Apr 2019 21:28:22 +0800 Subject: [PATCH 26/58] =?UTF-8?q?=E4=B8=BB=E5=8A=A8=E6=B8=85=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Common.py | 1 - WebMain.py | 3 +-- liveDownloader.py | 4 +++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Common.py b/Common.py index 0384785..34afc1e 100644 --- a/Common.py +++ b/Common.py @@ -50,7 +50,6 @@ def doClean(): os.system(config["dow"]) - def getCurrentStatus(): _disk = psutil.disk_usage("/") _mem = psutil.virtual_memory() diff --git a/WebMain.py b/WebMain.py index 7675d51..b4336a0 100644 --- a/WebMain.py +++ b/WebMain.py @@ -203,11 +203,10 @@ def fileDownload(path): status=code, mimetype='application/octet-stream', headers={ - "Content-Length": os.path.getsize(path)-1-offset, + "Content-Length": os.path.getsize(path), "Content-Range": "bytes {}-{}/{}".format(offset,os.path.getsize(path)-1,os.path.getsize(path)), "Accept-Ranges": "bytes", "Range": "bytes", - "Content-Disposition": "attachment; filename={}".format(path), }) else: return Response(status=404) diff --git a/liveDownloader.py b/liveDownloader.py index b015e00..690c674 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -94,7 +94,9 @@ def upload(date=datetime.strftime(datetime.now(), "%Y_%m_%d")): except Exception as e: Common.appendError(e.__str__()) continue - os.remove(i) + if not Common.forceNotEncode: + os.remove(i) + Common.doClean() i = Common.uploadQueue.get() Common.appendUploadStatus("Upload Daemon Quiting") From ba401342198ce91e142cbc636ddd4776b3aeeda6 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Wed, 17 Apr 2019 07:59:14 +0800 Subject: [PATCH 27/58] Limit upload too small file --- liveDownloader.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/liveDownloader.py b/liveDownloader.py index 690c674..7d95f2c 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -53,6 +53,7 @@ def encode(): global isEncode Common.appendEncodeStatus("Encode Daemon Starting") while True: + isEncode = False i = Common.encodeQueue.get() if Common.forceNotEncode: Common.appendEncodeStatus("设置了不编码，所以[{}]不会编码".format(i)) @@ -60,6 +61,9 @@ def encode(): continue if os.path.exists(i): isEncode = True + if os.path.getsize(path) < 8 * 1024 * 1024: + Common.appendEncodeStatus("Encoded File >{}< is too small, will ignore it".format(path)) + continue Common.appendEncodeStatus("Encoding >{}< Start".format(i)) os.system("ffmpeg -i {} -c:v copy -c:a copy -f mp4 {} -y".format(i, i[:13] + ".mp4")) Common.uploadQueue.put(i[:13] + ".mp4") @@ -68,7 +72,6 @@ def encode(): shutil.move(i, Common.config["mtd"]) elif Common.config["del"]: os.remove(i) - isEncode = False def upload(date=datetime.strftime(datetime.now(), "%Y_%m_%d")): From 9f075077baa65a0fd89f42dcbb1d565c1508d5af Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Wed, 17 Apr 2019 22:49:50 +0800 Subject: [PATCH 28/58] =?UTF-8?q?=E9=81=BF=E5=85=8D=E6=B2=99=E9=9B=95?= =?UTF-8?q?=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- WebMain.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/WebMain.py b/WebMain.py index b4336a0..df26d30 100644 --- a/WebMain.py +++ b/WebMain.py @@ -225,9 +225,10 @@ def SubThread(): t.start() -# p = threading.Thread(target=SubThread) -# p.setDaemon(True) -# p.start() +if not app.debug: + p = threading.Thread(target=SubThread) + p.setDaemon(True) + p.start() if __name__ == "__main__": app.run() From 1ec9f3c9222524f09fa5df9e2e438f57bed73679 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Thu, 18 Apr 2019 07:57:27 +0800 Subject: [PATCH 29/58] =?UTF-8?q?=E4=BF=AE=E6=94=B9i=E6=B8=85=E7=90=86?= =?UTF-8?q?=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Common.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Common.py b/Common.py index 34afc1e..9bb2a11 100644 --- a/Common.py +++ b/Common.py @@ -5,6 +5,7 @@ import psutil from api import XiGuaLiveApi import json from bilibili import Bilibili +import threading _config_fp = open("config.json","r",encoding="utf8") config = json.load(_config_fp) @@ -55,6 +56,10 @@ def getCurrentStatus(): _mem = psutil.virtual_memory() _net = psutil.net_io_counters() _delta= getTimeDelta(datetime.now(),network["currentTime"]) + if getTimeDelta(datetime.now(), _do_move_time) > 3600: + p = threading.Thread(target=doClean) + p.setDaemon(True) + p.start() if 60 > _delta > 0: _inSpeed = (_net.bytes_recv - network["in"]["currentByte"])/_delta _outSpeed = (_net.bytes_sent - network["out"]["currentByte"])/_delta From 17d2cc4ebda2ce4a2f6f0b289ca77c64a1aee55e Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Thu, 18 Apr 2019 08:02:54 +0800 Subject: [PATCH 30/58] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E9=83=A8=E5=88=86?= =?UTF-8?q?=E6=8F=90=E9=86=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- WebMain.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/WebMain.py b/WebMain.py index df26d30..3f6ae5e 100644 --- a/WebMain.py +++ b/WebMain.py @@ -78,6 +78,8 @@ def insertEncode(): Common.appendOperation("添加编码文件：{}".format(request.form["filename"])) Common.encodeQueue.put(request.form["filename"]) return jsonify({"message":"ok","code":200,"status":0}) + else: + return jsonify({"message":"no filename specific","code":400,"status":1}) @app.route("/upload/insert", methods=["POST"]) @@ -86,6 +88,8 @@ def insertUpload(): Common.appendOperation("添加上传文件：{}".format(request.form["filename"])) Common.uploadQueue.put(request.form["filename"]) return jsonify({"message":"ok","code":200,"status":0}) + else: + return jsonify({"message":"no filename specific","code":400,"status":1}) @app.route("/upload/finish", methods=["POST"]) From 86e4aa483f39fcf0cdb930bc297af0fc8a287c9d Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Thu, 18 Apr 2019 10:37:13 +0800 Subject: [PATCH 31/58] =?UTF-8?q?=E5=BC=BA=E5=88=B6=E7=BC=96=E7=A0=81?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Common.py | 2 ++ WebMain.py | 16 ++++++++++++++++ liveDownloader.py | 13 +++++++++++++ 3 files changed, 31 insertions(+) diff --git a/Common.py b/Common.py index 9bb2a11..ad6328f 100644 --- a/Common.py +++ b/Common.py @@ -98,6 +98,8 @@ forceNotDownload = False forceNotBroadcasting = False forceNotUpload = False forceNotEncode = False +forceStartEncodeThread = False +forceStartUploadThread = False uploadQueue = queue.Queue() encodeQueue = queue.Queue() diff --git a/WebMain.py b/WebMain.py index 3f6ae5e..e95e7fd 100644 --- a/WebMain.py +++ b/WebMain.py @@ -72,6 +72,22 @@ def toggleForceNotBroadcast(): }}) +@app.route("/force/start/encode", methods=["POST"]) +def toggleForceStartEncodeThread(): + Common.forceStartEncodeThread = True + Common.appendOperation("强制运行编码线程") + return jsonify({"message":"ok","code":200,"status":0,"data":{ + }}) + + +@app.route("/force/start/upload", methods=["POST"]) +def toggleForceStartEncodeThread(): + Common.forceStartUploadThread = True + Common.appendOperation("强制运行上传线程") + return jsonify({"message":"ok","code":200,"status":0,"data":{ + }}) + + @app.route("/encode/insert", methods=["POST"]) def insertEncode(): if "filename" in request.form and os.path.exists(request.form["filename"]): diff --git a/liveDownloader.py b/liveDownloader.py index 7d95f2c..f3aa933 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -174,3 +174,16 @@ def run(): _count_error = 0 except Exception as e: Common.appendError(e.__str__()) + if Common.forceStartEncodeThread: + if not et.is_alive(): + et = threading.Thread(target=encode, args=()) + et.setDaemon(True) + et.start() + Common.forceStartEncodeThread = False + if Common.forceStartUploadThread: + if not ut.is_alive(): + d = datetime.strftime(datetime.now(), "%Y_%m_%d") + ut = threading.Thread(target=upload, args=(d,)) + ut.setDaemon(True) + ut.start() + Common.forceStartEncodeThread = False From 69b594cf8ee33ebb31a783423035b127f313f6c8 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Thu, 18 Apr 2019 10:39:25 +0800 Subject: [PATCH 32/58] ...Typo Error --- WebMain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WebMain.py b/WebMain.py index e95e7fd..bd856b7 100644 --- a/WebMain.py +++ b/WebMain.py @@ -81,7 +81,7 @@ def toggleForceStartEncodeThread(): @app.route("/force/start/upload", methods=["POST"]) -def toggleForceStartEncodeThread(): +def toggleForceStartUploadThread(): Common.forceStartUploadThread = True Common.appendOperation("强制运行上传线程") return jsonify({"message":"ok","code":200,"status":0,"data":{ From b3162b1ed5677009aa9721547c2dd4a5c5b2733b Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Thu, 18 Apr 2019 10:43:04 +0800 Subject: [PATCH 33/58] ...Typo Error --- liveDownloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liveDownloader.py b/liveDownloader.py index f3aa933..b037a7e 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -61,7 +61,7 @@ def encode(): continue if os.path.exists(i): isEncode = True - if os.path.getsize(path) < 8 * 1024 * 1024: + if os.path.getsize(i) < 8 * 1024 * 1024: Common.appendEncodeStatus("Encoded File >{}< is too small, will ignore it".format(path)) continue Common.appendEncodeStatus("Encoding >{}< Start".format(i)) From e520e39b721490c99a541650fa2ae5c5930caa52 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Thu, 18 Apr 2019 10:43:27 +0800 Subject: [PATCH 34/58] ...Typo Error --- liveDownloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liveDownloader.py b/liveDownloader.py index b037a7e..68a44c4 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -62,7 +62,7 @@ def encode(): if os.path.exists(i): isEncode = True if os.path.getsize(i) < 8 * 1024 * 1024: - Common.appendEncodeStatus("Encoded File >{}< is too small, will ignore it".format(path)) + Common.appendEncodeStatus("Encoded File >{}< is too small, will ignore it".format(i)) continue Common.appendEncodeStatus("Encoding >{}< Start".format(i)) os.system("ffmpeg -i {} -c:v copy -c:a copy -f mp4 {} -y".format(i, i[:13] + ".mp4")) From d17d0bd5347c03e6990edc09531f642331513753 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Thu, 18 Apr 2019 10:47:29 +0800 Subject: [PATCH 35/58] ...Typo Error --- liveDownloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liveDownloader.py b/liveDownloader.py index 68a44c4..ca646b5 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -186,4 +186,4 @@ def run(): ut = threading.Thread(target=upload, args=(d,)) ut.setDaemon(True) ut.start() - Common.forceStartEncodeThread = False + Common.forceStartUploadThread = False From 5c1f712900a54ca6882dfaf395d303d344fc36b7 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Mon, 22 Apr 2019 05:59:22 +0800 Subject: [PATCH 36/58] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=A1=B5=E9=9D=A2?= =?UTF-8?q?=E6=A0=B7=E5=BC=8F=EF=BC=8C=E4=BF=AE=E5=A4=8D=E7=BD=91=E9=80=9F?= =?UTF-8?q?=E7=9A=84=E5=87=86=E7=A1=AE=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Common.py | 10 +++++----- liveDownloader.py | 3 +-- static/index.js | 10 +++++----- templates/device.html | 8 ++++---- templates/head.html | 6 ++++++ templates/index.html | 10 +++++----- 6 files changed, 26 insertions(+), 21 deletions(-) diff --git a/Common.py b/Common.py index 9bb2a11..26a0d7c 100644 --- a/Common.py +++ b/Common.py @@ -54,18 +54,18 @@ def doClean(): def getCurrentStatus(): _disk = psutil.disk_usage("/") _mem = psutil.virtual_memory() - _net = psutil.net_io_counters() _delta= getTimeDelta(datetime.now(),network["currentTime"]) - if getTimeDelta(datetime.now(), _do_move_time) > 3600: - p = threading.Thread(target=doClean) - p.setDaemon(True) - p.start() + _net = psutil.net_io_counters() if 60 > _delta > 0: _inSpeed = (_net.bytes_recv - network["in"]["currentByte"])/_delta _outSpeed = (_net.bytes_sent - network["out"]["currentByte"])/_delta else: _outSpeed = 0 _inSpeed = 0 + if getTimeDelta(datetime.now(), _do_move_time) > 3600: + p = threading.Thread(target=doClean) + p.setDaemon(True) + p.start() updateNetwork() return { "memTotal": parseSize(_mem.total), diff --git a/liveDownloader.py b/liveDownloader.py index 7d95f2c..ba5409e 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -32,8 +32,7 @@ def download(url): Common.modifyLastDownloadStatus("Download >{}< Exceed MaxSize".format(path)) break except Exception as e: - Common.appendError("Download >{}< With Exception {}".format(path, datetime.strftime(datetime.now(), "%y%m%d %H%M"), - e.__str__())) + Common.appendError("Download >{}< With Exception {}".format(path, e.__str__())) f.close() isDownload = False Common.modifyLastDownloadStatus("Download >{}< Finished".format(path)) diff --git a/static/index.js b/static/index.js index 952cd80..2d54541 100644 --- a/static/index.js +++ b/static/index.js @@ -16,35 +16,35 @@ function taskUpdate(){ $("#download").html(function(){ var ret = "" res.data.download.reverse().forEach(function(obj){ - ret += "" + obj.datetime + "" + obj.message + "" + ret += "" + obj.datetime + "" + obj.message + "" }) return "" + ret + "
" }) $("#encode").html(function(){ var ret = "" res.data.encode.reverse().forEach(function(obj){ - ret += "" + obj.datetime + "" + obj.message + "" + ret += "" + obj.datetime + "" + obj.message + "" }) return "" + ret + "
" }) $("#upload").html(function(){ var ret = "" res.data.upload.reverse().forEach(function(obj){ - ret += "" + obj.datetime + "" + obj.message + "" + ret += "" + obj.datetime + "" + obj.message + "" }) return "" + ret + "
" }) $("#error").html(function(){ var ret = "" res.data.error.reverse().forEach(function(obj){ - ret += "" + obj.datetime + "" + obj.message + "" + ret += "" + obj.datetime + "" + obj.message + "" }) return "" + ret + "
" }) $("#operation").html(function(){ var ret = "" res.data.operation.reverse().forEach(function(obj){ - ret += "" + obj.datetime + "" + obj.message + "" + ret += "" + obj.datetime + "" + obj.message + "" }) return "" + ret + "
" }) diff --git a/templates/device.html b/templates/device.html index 2eb224f..0c8783e 100644 --- a/templates/device.html +++ b/templates/device.html @@ -1,22 +1,22 @@

机器状态

- + - + - + - + diff --git a/templates/head.html b/templates/head.html index c84b086..c9f310b 100644 --- a/templates/head.html +++ b/templates/head.html @@ -4,4 +4,10 @@ td{ border: solid 1px lightgray; } + .title{ + width: 6em; + } + .time{ + width: 10em; + } diff --git a/templates/index.html b/templates/index.html index 57aede3..0c350ad 100644 --- a/templates/index.html +++ b/templates/index.html @@ -49,23 +49,23 @@

当前状态

CPU使用率CPU使用率 %
内存使用率内存使用率 /(%)
磁盘使用率磁盘使用率 /(%)
网络速率网络速率 ↓ /s ↑ /s
- + - + - + - + - +
下载日志下载日志
转码日志
队列		转码日志
队列
上传日志
队列		上传日志
队列
错误日志错误日志
操作日志操作日志
From 99926827f1188371ae41d71a53703c4f7e9c6f39 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Wed, 24 Apr 2019 11:09:17 +0800 Subject: [PATCH 37/58] small fix --- Common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common.py b/Common.py index 7894a22..0d14c5b 100644 --- a/Common.py +++ b/Common.py @@ -62,11 +62,11 @@ def getCurrentStatus(): else: _outSpeed = 0 _inSpeed = 0 + updateNetwork() if getTimeDelta(datetime.now(), _do_move_time) > 3600: p = threading.Thread(target=doClean) p.setDaemon(True) p.start() - updateNetwork() return { "memTotal": parseSize(_mem.total), "memUsed": parseSize(_mem.used), From c4c6b1a752b2ec780d5bd837d80a2cdc1f246c93 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Wed, 24 Apr 2019 13:05:59 +0800 Subject: [PATCH 38/58] fix: while set not upload ,it will not move any file to free disk space --- liveDownloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liveDownloader.py b/liveDownloader.py index d53fedd..e702a85 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -77,6 +77,7 @@ def upload(date=datetime.strftime(datetime.now(), "%Y_%m_%d")): Common.appendUploadStatus("Upload Daemon Starting") i = Common.uploadQueue.get() while True: + Common.doClean() if Common.forceNotUpload: Common.appendUploadStatus("设置了不上传，所以[{}]不会上传了".format(i)) i = Common.uploadQueue.get() @@ -98,7 +99,6 @@ def upload(date=datetime.strftime(datetime.now(), "%Y_%m_%d")): continue if not Common.forceNotEncode: os.remove(i) - Common.doClean() i = Common.uploadQueue.get() Common.appendUploadStatus("Upload Daemon Quiting") From f607f11b82dfb99ba34b946cfd4f25e3628bbc05 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Fri, 26 Apr 2019 14:48:11 +0800 Subject: [PATCH 39/58] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=99=BE=E5=BA=A6?= =?UTF-8?q?=E4=BA=91=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- templates/files.html | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/files.html b/templates/files.html index c999a08..40221cb 100644 --- a/templates/files.html +++ b/templates/files.html @@ -7,6 +7,7 @@

所有录像文件

+

部分录像文件已转移至百度云，请在这里下载 提取码: ddxt

From 1644f52eb0c97f613c691bfd6b30e441119bc919 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Fri, 26 Apr 2019 14:49:28 +0800 Subject: [PATCH 40/58] =?UTF-8?q?=E9=81=BF=E5=85=8D=E5=9C=A8=E6=97=A0?= =?UTF-8?q?=E6=B3=95=E8=8E=B7=E5=8F=96=E4=B8=8B=E8=BD=BD=E9=93=BE=E6=8E=A5?= =?UTF-8?q?=E6=98=AF=E5=B0=86false=E4=BC=A0=E5=85=A5downloader=E5=86=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- liveDownloader.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/liveDownloader.py b/liveDownloader.py index e702a85..5eb5aad 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -134,6 +134,9 @@ def run(): continue _count_error += 1 _preT = Common.api.playlist + if not _preT: + Common.api.updRoomInfo() + continue t = threading.Thread(target=download, args=(_preT,)) t.setDaemon(True) t.start() From 4ea19b4ff4b0631c317d2996d95620919e699ec3 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Sat, 27 Apr 2019 22:08:16 +0800 Subject: [PATCH 41/58] Fix when program at another disk --- Common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common.py b/Common.py index 0d14c5b..5dfc4b2 100644 --- a/Common.py +++ b/Common.py @@ -45,7 +45,7 @@ def getTimeDelta(a, b): def doClean(): global _do_move_time - _disk = psutil.disk_usage("/") + _disk = psutil.disk_usage(".") if _disk.percent > config["max"] and getTimeDelta(datetime.now(), _do_move_time) > 3600: _do_move_time = datetime.now() os.system(config["dow"]) From 2099321f0bff60c9e6bf841bbfafc56ee737603b Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Sat, 27 Apr 2019 22:10:02 +0800 Subject: [PATCH 42/58] threading doClean --- Common.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Common.py b/Common.py index 5dfc4b2..1311972 100644 --- a/Common.py +++ b/Common.py @@ -43,7 +43,7 @@ def getTimeDelta(a, b): return sec+(ms/100000.0) -def doClean(): +def _doClean(): global _do_move_time _disk = psutil.disk_usage(".") if _disk.percent > config["max"] and getTimeDelta(datetime.now(), _do_move_time) > 3600: @@ -51,6 +51,12 @@ def doClean(): os.system(config["dow"]) +def doClean(): + p = threading.Thread(target=_doClean) + p.setDaemon(True) + p.start() + + def getCurrentStatus(): _disk = psutil.disk_usage("/") _mem = psutil.virtual_memory() @@ -64,9 +70,7 @@ def getCurrentStatus(): _inSpeed = 0 updateNetwork() if getTimeDelta(datetime.now(), _do_move_time) > 3600: - p = threading.Thread(target=doClean) - p.setDaemon(True) - p.start() + doClean() return { "memTotal": parseSize(_mem.total), "memUsed": parseSize(_mem.used), From ed3ab5dc4f10790fe5487f52c92d331ce85d55fa Mon Sep 17 00:00:00 2001 From: JerryYan <792602257@qq.com> Date: Sat, 27 Apr 2019 22:11:51 +0800 Subject: [PATCH 43/58] time fix --- Common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common.py b/Common.py index 1311972..ef82ba9 100644 --- a/Common.py +++ b/Common.py @@ -46,7 +46,7 @@ def getTimeDelta(a, b): def _doClean(): global _do_move_time _disk = psutil.disk_usage(".") - if _disk.percent > config["max"] and getTimeDelta(datetime.now(), _do_move_time) > 3600: + if _disk.percent > config["max"] and getTimeDelta(datetime.now(), _do_move_time) > 7200: _do_move_time = datetime.now() os.system(config["dow"]) From 43e161ebb66cc2a4659e626b18c8ec5ca4617e5c Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Sat, 27 Apr 2019 22:17:31 +0800 Subject: [PATCH 44/58] typo fix --- Common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common.py b/Common.py index ef82ba9..5e06ed4 100644 --- a/Common.py +++ b/Common.py @@ -58,7 +58,7 @@ def doClean(): def getCurrentStatus(): - _disk = psutil.disk_usage("/") + _disk = psutil.disk_usage(".") _mem = psutil.virtual_memory() _delta= getTimeDelta(datetime.now(),network["currentTime"]) _net = psutil.net_io_counters() From f1b4a878a56d94ddd721883ca4c7a54577e971a9 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Mon, 29 Apr 2019 09:50:26 +0800 Subject: [PATCH 45/58] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=BC=BA=E5=88=B6?= =?UTF-8?q?=E6=B8=85=E7=90=86=E5=8A=9F=E8=83=BD=EF=BC=8C=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E7=99=BE=E5=BA=A6=E4=BA=91=E4=B8=8A=E4=BC=A0=E5=8A=9F=E8=83=BD?= =?UTF-8?q?=EF=BC=8C=E9=83=A8=E5=88=86=E9=80=BB=E8=BE=91=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Common.py | 63 ++++++++++++++++++++++++++++--------------- WebMain.py | 10 +++++-- liveDownloader.py | 4 --- static/device.js | 2 ++ templates/device.html | 5 ++++ 5 files changed, 56 insertions(+), 28 deletions(-) diff --git a/Common.py b/Common.py index 5e06ed4..4628cfc 100644 --- a/Common.py +++ b/Common.py @@ -1,24 +1,27 @@ import os import queue from datetime import datetime +from glob import glob + import psutil from api import XiGuaLiveApi import json from bilibili import Bilibili import threading +from bypy import ByPy -_config_fp = open("config.json","r",encoding="utf8") +_config_fp = open("config.json", "r", encoding="utf8") config = json.load(_config_fp) _config_fp.close() - -_do_move_time = datetime.now() +bypy = ByPy() +doCleanTime = datetime.now() network = { "currentTime": datetime.now(), - "out":{ - "currentByte":psutil.net_io_counters().bytes_sent, + "out": { + "currentByte": psutil.net_io_counters().bytes_sent, }, - "in":{ + "in": { "currentByte": psutil.net_io_counters().bytes_recv, } } @@ -28,10 +31,10 @@ def updateNetwork(): global network network = { "currentTime": datetime.now(), - "out":{ - "currentByte":psutil.net_io_counters().bytes_sent, + "out": { + "currentByte": psutil.net_io_counters().bytes_sent, }, - "in":{ + "in": { "currentByte": psutil.net_io_counters().bytes_recv, } } @@ -43,16 +46,30 @@ def getTimeDelta(a, b): return sec+(ms/100000.0) -def _doClean(): - global _do_move_time +def _doClean(_force=False): + global doCleanTime _disk = psutil.disk_usage(".") - if _disk.percent > config["max"] and getTimeDelta(datetime.now(), _do_move_time) > 7200: - _do_move_time = datetime.now() - os.system(config["dow"]) + if (_disk.percent > config["max"] and getTimeDelta(datetime.now(), doCleanTime) > 7200) or _force: + doCleanTime = datetime.now() + _list = sorted(glob("*.flv"), key=lambda x: datetime.utcfromtimestamp(os.path.getmtime(x))) + for _i in _list: + if not os.path.exists(_i): + break + doCleanTime = datetime.now() + if (datetime.now() - datetime.utcfromtimestamp(os.path.getmtime(_i))).days > config["exp"]: + if config["dow"] == "bypy": + _res = bypy.upload(_i) + if _res == 0: + os.remove(_i) + else: + os.system(config["dow"]) + else: + break + doCleanTime = datetime.now() -def doClean(): - p = threading.Thread(target=_doClean) +def doClean(_force=False): + p = threading.Thread(target=_doClean, args=(_force,)) p.setDaemon(True) p.start() @@ -63,13 +80,13 @@ def getCurrentStatus(): _delta= getTimeDelta(datetime.now(),network["currentTime"]) _net = psutil.net_io_counters() if 60 > _delta > 0: - _inSpeed = (_net.bytes_recv - network["in"]["currentByte"])/_delta - _outSpeed = (_net.bytes_sent - network["out"]["currentByte"])/_delta + _inSpeed = (_net.bytes_recv - network["in"]["currentByte"]) / _delta + _outSpeed = (_net.bytes_sent - network["out"]["currentByte"]) / _delta else: _outSpeed = 0 _inSpeed = 0 updateNetwork() - if getTimeDelta(datetime.now(), _do_move_time) > 3600: + if getTimeDelta(datetime.now(), doCleanTime) > 3600: doClean() return { "memTotal": parseSize(_mem.total), @@ -81,6 +98,8 @@ def getCurrentStatus(): "cpu": psutil.cpu_percent(), "outSpeed": parseSize(_outSpeed), "inSpeed": parseSize(_inSpeed), + "doCleanTime": doCleanTime, + "fileExpire": config["exp"], } @@ -91,7 +110,7 @@ def reloadConfig(): _config_fp.close() -dt_format="%Y/%m/%d %H:%M:%S" +dt_format = "%Y/%m/%d %H:%M:%S" broadcaster = "" streamUrl = "" @@ -130,9 +149,9 @@ def appendOperation(obj): def parseSize(size): - K = size/1024.0 + K = size / 1024.0 if K > 1000: - M = K/1024.0 + M = K / 1024.0 if M > 1000: return "{:.2f}GB".format(M / 1024.0) else: diff --git a/WebMain.py b/WebMain.py index bd856b7..1516464 100644 --- a/WebMain.py +++ b/WebMain.py @@ -23,8 +23,6 @@ def index(): def readConfig(): config = Common.config.copy() config.pop("b_p") - config.pop("mtd") - config.pop("del") config.pop("mv") return jsonify(config) @@ -88,6 +86,14 @@ def toggleForceStartUploadThread(): }}) +@app.route("/force/start/clean", methods=["POST"]) +def toggleForceStartUploadThread(): + Common.doClean(True) + Common.appendOperation("强制执行清理程序") + return jsonify({"message":"ok","code":200,"status":0,"data":{ + }}) + + @app.route("/encode/insert", methods=["POST"]) def insertEncode(): if "filename" in request.form and os.path.exists(request.form["filename"]): diff --git a/liveDownloader.py b/liveDownloader.py index 5eb5aad..a87f7d0 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -67,10 +67,6 @@ def encode(): os.system("ffmpeg -i {} -c:v copy -c:a copy -f mp4 {} -y".format(i, i[:13] + ".mp4")) Common.uploadQueue.put(i[:13] + ".mp4") Common.modifyLastEncodeStatus("Encode >{}< Finished".format(i)) - if Common.config["mv"]: - shutil.move(i, Common.config["mtd"]) - elif Common.config["del"]: - os.remove(i) def upload(date=datetime.strftime(datetime.now(), "%Y_%m_%d")): diff --git a/static/device.js b/static/device.js index c3eed6b..2d311b8 100644 --- a/static/device.js +++ b/static/device.js @@ -15,6 +15,8 @@ function deviceUpdate(){ $("#cpuP").val(res.data.status.cpu) $("#inSpeed").text(res.data.status.inSpeed) $("#outSpeed").text(res.data.status.outSpeed) + $("#doCleanTime").text(res.data.status.doCleanTime) + $("#fileExpire").text(res.data.status.fileExpire) } } ) diff --git a/templates/device.html b/templates/device.html index 0c8783e..6007c62 100644 --- a/templates/device.html +++ b/templates/device.html @@ -20,5 +20,10 @@ + + + + +
文件名文件大小链接↓ /s ↑ /s
文件清理时间@ 清理超过天的文件
From 4389016b95d7a4a619290f33bede319d7c312bcc Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Mon, 29 Apr 2019 10:08:18 +0800 Subject: [PATCH 46/58] Typo Error Fix --- WebMain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WebMain.py b/WebMain.py index 1516464..70b9481 100644 --- a/WebMain.py +++ b/WebMain.py @@ -87,7 +87,7 @@ def toggleForceStartUploadThread(): @app.route("/force/start/clean", methods=["POST"]) -def toggleForceStartUploadThread(): +def startForceCleanDisk(): Common.doClean(True) Common.appendOperation("强制执行清理程序") return jsonify({"message":"ok","code":200,"status":0,"data":{ From 84a283b4c28268e70805e0a73c49b8ab9e85773d Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Mon, 29 Apr 2019 10:09:56 +0800 Subject: [PATCH 47/58] Time Format --- Common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common.py b/Common.py index 4628cfc..ee5f0d0 100644 --- a/Common.py +++ b/Common.py @@ -98,7 +98,7 @@ def getCurrentStatus(): "cpu": psutil.cpu_percent(), "outSpeed": parseSize(_outSpeed), "inSpeed": parseSize(_inSpeed), - "doCleanTime": doCleanTime, + "doCleanTime": datetime.strftime(doCleanTime, dt_format), "fileExpire": config["exp"], } From 251c1e53fc41a54e71b0411e3c75832ed90ba30b Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Mon, 29 Apr 2019 10:27:52 +0800 Subject: [PATCH 48/58] simple doClean Flag --- Common.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Common.py b/Common.py index ee5f0d0..fd03ea9 100644 --- a/Common.py +++ b/Common.py @@ -15,6 +15,7 @@ config = json.load(_config_fp) _config_fp.close() bypy = ByPy() doCleanTime = datetime.now() +_clean_flag = None network = { "currentTime": datetime.now(), @@ -47,7 +48,7 @@ def getTimeDelta(a, b): def _doClean(_force=False): - global doCleanTime + global doCleanTime, _clean_flag _disk = psutil.disk_usage(".") if (_disk.percent > config["max"] and getTimeDelta(datetime.now(), doCleanTime) > 7200) or _force: doCleanTime = datetime.now() @@ -57,6 +58,7 @@ def _doClean(_force=False): break doCleanTime = datetime.now() if (datetime.now() - datetime.utcfromtimestamp(os.path.getmtime(_i))).days > config["exp"]: + _clean_flag = True if config["dow"] == "bypy": _res = bypy.upload(_i) if _res == 0: @@ -66,9 +68,13 @@ def _doClean(_force=False): else: break doCleanTime = datetime.now() + _clean_flag = False def doClean(_force=False): + if _clean_flag: + appendError("doClean request on cleaning, will ignore it") + return p = threading.Thread(target=_doClean, args=(_force,)) p.setDaemon(True) p.start() From 1f07e1046c170bc93989f5e16a683c9ebbfd0f71 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Mon, 29 Apr 2019 11:44:58 +0800 Subject: [PATCH 49/58] some text changes --- templates/device.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/device.html b/templates/device.html index 6007c62..a10011a 100644 --- a/templates/device.html +++ b/templates/device.html @@ -21,9 +21,9 @@ ↑ /s - 文件清理时间 + 文件清理 + 清理天前的文件 @ - 清理超过天的文件 From 46b3500faa6754ac97c8ce61b54686f43cfbaf70 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Tue, 30 Apr 2019 10:09:42 +0800 Subject: [PATCH 50/58] more exactly time settings --- Common.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Common.py b/Common.py index fd03ea9..96e727b 100644 --- a/Common.py +++ b/Common.py @@ -6,7 +6,6 @@ from glob import glob import psutil from api import XiGuaLiveApi import json -from bilibili import Bilibili import threading from bypy import ByPy @@ -57,7 +56,7 @@ def _doClean(_force=False): if not os.path.exists(_i): break doCleanTime = datetime.now() - if (datetime.now() - datetime.utcfromtimestamp(os.path.getmtime(_i))).days > config["exp"]: + if (datetime.now() - datetime.utcfromtimestamp(os.path.getmtime(_i))).days >= config["exp"]: _clean_flag = True if config["dow"] == "bypy": _res = bypy.upload(_i) From ef0fe2cb0477c58b52b682d4ff5b95dd2d118758 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Sun, 5 May 2019 14:29:58 +0800 Subject: [PATCH 51/58] trying fixing stuck at downloading --- liveDownloader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/liveDownloader.py b/liveDownloader.py index a87f7d0..3f6c762 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -26,6 +26,8 @@ def download(url): for t in p.iter_content(chunk_size=64 * 1024): if t: f.write(t) + else: + raise Exception("`t` is not valid") _size = os.path.getsize(path) Common.modifyLastDownloadStatus("Downloading >{}< @ {:.2f}%".format(path, 100.0 * _size/Common.config["p_s"])) if _size > Common.config["p_s"] or Common.forceNotDownload: From c1141c366aadb26d9b5added04dfb85fa4ac1a34 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Fri, 10 May 2019 23:21:03 +0800 Subject: [PATCH 52/58] Try to add a new Curses Version --- CursesDownload.py | 70 ++++++++++++++++++++++++++++++ CursesMain.py | 107 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 CursesDownload.py create mode 100644 CursesMain.py diff --git a/CursesDownload.py b/CursesDownload.py new file mode 100644 index 0000000..ba31106 --- /dev/null +++ b/CursesDownload.py @@ -0,0 +1,70 @@ +import curses +import Common + + +widths = [ + (126, 1), (159, 0), (687, 1), (710, 0), (711, 1), + (727, 0), (733, 1), (879, 0), (1154, 1), (1161, 0), + (4347, 1), (4447, 2), (7467, 1), (7521, 0), (8369, 1), + (8426, 0), (9000, 1), (9002, 2), (11021, 1), (12350, 2), + (12351, 1), (12438, 2), (12442, 0), (19893, 2), (19967, 1), + (55203, 2), (63743, 1), (64106, 2), (65039, 1), (65059, 0), + (65131, 2), (65279, 1), (65376, 2), (65500, 1), (65510, 2), + (120831, 1), (262141, 2), (1114109, 1), +] + + +def get_width(o): + global widths + if o == 0xe or o == 0xf: + return 0 + for num, wid in widths: + if o <= num: + return wid + return 1 + + +def c_print(handle, y, x, string, style=curses.A_NORMAL): + if type(string) != str: + string = str(string) + for _i in string: + _w = get_width(ord(_i)) + if(_w>1): + handle.addch(y, x+1, " ", style) + handle.addch(y, x, ord(_i), style) + x += _w + + +def render(screen): + _style = curses.A_DIM + if Common.api.isLive: + _style = curses.A_BOLD | curses.A_BLINK | curses.A_ITALIC | curses.A_UNDERLINE + c_print(screen, 1, 3, Common.api.roomLiver, _style) + screen.refresh() + +def main(stdscr): + global screen + screen = stdscr.subwin(23, 79, 0, 0) + screen.timeout(2000) + screen.box() + screen.hline(2, 1, curses.ACS_HLINE, 77) + c_print(screen, 1, 2, " "*45 + " 西瓜录播助手 -- by JerryYan ", curses.A_STANDOUT) + render(screen) + while True: + c = stdscr.getch() + if c == ord("q"): + break + elif c == ord("f"): + render(screen) + + +stdscr = curses.initscr() +curses.noecho() +curses.cbreak() +stdscr.keypad(1) +curses.wrapper(main) +stdscr.keypad(0) +curses.echo() +curses.nocbreak() +curses.endwin() + diff --git a/CursesMain.py b/CursesMain.py new file mode 100644 index 0000000..4a1f513 --- /dev/null +++ b/CursesMain.py @@ -0,0 +1,107 @@ +import curses + +from Struct.Chat import Chat +from Struct.Gift import Gift +from Struct.MemberMsg import MemberMsg +from Struct.User import User +from api import XiGuaLiveApi + + +class Api(XiGuaLiveApi): + danmakuList = [] + def onAd(self, i): + pass + def onChat(self, chat: Chat): + self.danmakuList.append(str(chat)) + def onLike(self, user: User): + pass + def onEnter(self, msg: MemberMsg): + pass + def onJoin(self, user: User): + self.danmakuList.append(str(user)) + def onSubscribe(self, user: User): + self.danmakuList.append(str(user)) + def onPresent(self, gift: Gift): + pass + def onPresentEnd(self, gift: Gift): + self.danmakuList.append(str(gift)) + + +api = Api() +widths = [ + (126, 1), (159, 0), (687, 1), (710, 0), (711, 1), + (727, 0), (733, 1), (879, 0), (1154, 1), (1161, 0), + (4347, 1), (4447, 2), (7467, 1), (7521, 0), (8369, 1), + (8426, 0), (9000, 1), (9002, 2), (11021, 1), (12350, 2), + (12351, 1), (12438, 2), (12442, 0), (19893, 2), (19967, 1), + (55203, 2), (63743, 1), (64106, 2), (65039, 1), (65059, 0), + (65131, 2), (65279, 1), (65376, 2), (65500, 1), (65510, 2), + (120831, 1), (262141, 2), (1114109, 1), +] + + +def get_width(o): + global widths + if o == 0xe or o == 0xf: + return 0 + for num, wid in widths: + if o <= num: + return wid + return 1 + + +def c_print(handle, y, x, string, style=curses.A_NORMAL): + if type(string) != str: + string = str(string) + for _i in string: + _w = get_width(ord(_i)) + if(_w>1): + handle.addch(y, x+1, " ", style) + if _i != " " or style!=curses.A_NORMAL: + handle.addch(y, x, ord(_i), style) + else: + handle.addch(y, x, 0, style) + x += _w + + + + +def render(screen): + screen.erase() + screen.box() + screen.hline(2, 1, curses.ACS_HLINE, 77) + c_print(screen, 1, 2, " "*45 + " 西瓜弹幕助手 -- by JerryYan ", curses.A_STANDOUT) + _style = curses.A_DIM + if api.isLive: + _style = curses.A_BOLD | curses.A_BLINK | curses.A_ITALIC + c_print(screen, 1, 3, api.roomLiver, _style) + _y = 3 + api.getDanmaku() + for i in api.danmakuList[-10:]: + c_print(screen, _y, 2, i) + _y += 1 + screen.move(0,0) + screen.refresh() + +def main(stdscr): + global screen + screen = stdscr.subwin(23, 79, 0, 0) + screen.timeout(2000) + render(screen) + while True: + c = screen.getch() + if c == ord("q"): + break + render(screen) + + +stdscr = curses.initscr() +curses.noecho() +curses.cbreak() +stdscr.keypad(1) +curses.wrapper(main) +stdscr.keypad(0) +curses.echo() +curses.nocbreak() +curses.endwin() + From f4bc6d847d0c579b377c831c62d89bc7b4373652 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Fri, 10 May 2019 23:22:53 +0800 Subject: [PATCH 53/58] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=B8=80=E4=B8=8B?= =?UTF-8?q?=E5=9B=A0=E4=B8=BA=E4=B8=8D=E9=80=80=E5=87=BA=E8=80=8C=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E4=B9=8B=E5=90=8E=E7=9A=84=E5=8F=91=E5=B8=83=E6=97=A5?= =?UTF-8?q?=E6=9C=9F=E4=B8=8D=E5=AF=B9=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- liveDownloader.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/liveDownloader.py b/liveDownloader.py index 3f6c762..e10d4ae 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -77,6 +77,9 @@ def upload(date=datetime.strftime(datetime.now(), "%Y_%m_%d")): while True: Common.doClean() if Common.forceNotUpload: + if isinstance(i, bool): + Common.appendUploadStatus("设置了不上传，不会发布了") + return Common.appendUploadStatus("设置了不上传，所以[{}]不会上传了".format(i)) i = Common.uploadQueue.get() continue From b61bd4c70c4046c12c5162e58b5bda6b8a7eca45 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Sat, 11 May 2019 21:21:43 +0800 Subject: [PATCH 54/58] Avoid loop in Boardcaster is not found and stop download --- Common.py | 7 ++++++- liveDownloader.py | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Common.py b/Common.py index 96e727b..265d968 100644 --- a/Common.py +++ b/Common.py @@ -314,4 +314,9 @@ class downloader(XiGuaLiveApi): pass -api = downloader(config["l_u"]) \ No newline at end of file +api = downloader(config["l_u"]) + + +def refreshDownloader(): + global api + api = downloader(config["l_u"]) diff --git a/liveDownloader.py b/liveDownloader.py index e10d4ae..43c45a7 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -177,6 +177,9 @@ def run(): _count_error = 0 except Exception as e: Common.appendError(e.__str__()) + _count_error += 1 + if not Common.api.roomLiver: + _count_error += 1 if Common.forceStartEncodeThread: if not et.is_alive(): et = threading.Thread(target=encode, args=()) @@ -190,3 +193,5 @@ def run(): ut.setDaemon(True) ut.start() Common.forceStartUploadThread = False + if _count_error>10: + Common.refreshDownloader() From f1615957bce85aa30df5adf73fa49baf2fcfe5f8 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Sat, 11 May 2019 21:27:15 +0800 Subject: [PATCH 55/58] Update README.md --- README.md | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 40bd08b..5189c9e 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,19 @@ # XiguaLiveDanmakuHelper -### 西瓜直播弹幕助手--控制台版 +### 西瓜直播弹幕助手--录播端 -界面版：[q792602257/XiguaDanmakuHelperGUI](https://github.com/q792602257/XiguaDanmakuHelperGUI "C# ver") -### 计划更新： -+ ~~Digg消息（点亮了喜欢）实例化~~ +- 能够自动进行ffmpeg转码 - 现APP端无法发送点亮喜欢消息 - -+ √ ~~可以输入中文用户名进行查询~~ +- 转码后自动上传至B站 - 已实现，但在删除文字时可能会出现错位 +- 顺便还能自己清理录播的文件（移动到一个位置，执行shell命令，上传百度云） -+ √ ~~使用android app协议~~ +- 把录像文件分一定大小保存（B站有限制，但是不知道是多少） - 已使用 - 除从用户ID获取roomID外，其他均改为Android Api +- 少部分错误包容机制 -+ √ ~~闲的无聊的时候看一看有没有好用的GUI轮子可以用用~~ +- 有一个简单的WEB页面，及简单的控制接口 - 已完成 - 已基于BiliLive_dm制作出初代西瓜直播弹幕姬，待处理效率问题 +### 计划更新 + +### 并没有呢，这段时间太忙了 \ No newline at end of file From 22925398dad22bfa8e81b4f77b14aa40fa5c7330 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Sun, 12 May 2019 19:31:43 +0800 Subject: [PATCH 56/58] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=9B=A0=E8=A5=BF?= =?UTF-8?q?=E7=93=9C=E7=9B=B4=E6=92=AD=E6=90=9C=E7=B4=A2=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=AF=BC=E8=87=B4=E7=9A=84=E6=97=A0=E6=B3=95?= =?UTF-8?q?=E8=8E=B7=E5=8F=96=E7=9B=B4=E6=92=AD=E7=94=A8=E6=88=B7=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- WinMain.py | 10 +++++++--- liveDownloader.py | 10 ++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/WinMain.py b/WinMain.py index c5a7011..294e088 100644 --- a/WinMain.py +++ b/WinMain.py @@ -185,10 +185,14 @@ if __name__ == "__main__": else: name = readInput("请输入主播用户名，默认为", name, 3) api = WinMain(name) - print("进入", api.roomLiver, "的直播间") - if not api.isValidRoom: - input("房间不存在") + while not api.isValidRoom: + set_cmd_text_color(FOREGROUND_RED) + print("未找到对应房间或未开播，等待1分钟后重试") + resetColor() + time.sleep(60) + api.updRoomInfo() sys.exit() + print("进入", api.roomLiver, "的直播间") os.system("title {}".format(api.getTitle())) print("=" * 30) while True: diff --git a/liveDownloader.py b/liveDownloader.py index 43c45a7..e827bdc 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -107,14 +107,16 @@ def upload(date=datetime.strftime(datetime.now(), "%Y_%m_%d")): b = Bilibili() b.login(Common.config["b_u"], Common.config["b_p"]) +et = threading.Thread(target=encode, args=()) +et.setDaemon(True) +et.start() + def run(): global isEncode, isDownload - et = threading.Thread(target=encode, args=()) - et.setDaemon(True) - et.start() + Common.refreshDownloader() if not Common.api.isValidRoom: - Common.appendError("[{}]房间不存在".format(Common.config["l_u"])) + Common.appendError("[{}]房间未找到".format(Common.config["l_u"])) return d = None t = threading.Thread(target=download) From 1677afd97565c3eac0ad97e6037fc19d0fb74c56 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Mon, 13 May 2019 23:20:49 +0800 Subject: [PATCH 57/58] Typo Fix --- liveDownloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liveDownloader.py b/liveDownloader.py index e827bdc..734b0a5 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -113,7 +113,7 @@ et.start() def run(): - global isEncode, isDownload + global isEncode, isDownload, et Common.refreshDownloader() if not Common.api.isValidRoom: Common.appendError("[{}]房间未找到".format(Common.config["l_u"])) From f5971c8429d3f0307c77c6226051d6b941d302dd Mon Sep 17 00:00:00 2001 From: JerryYan <792602257@qq.com> Date: Tue, 14 May 2019 20:32:50 +0800 Subject: [PATCH 58/58] =?UTF-8?q?=E3=80=82=E3=80=82=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- liveDownloader.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/liveDownloader.py b/liveDownloader.py index 734b0a5..ff6a461 100644 --- a/liveDownloader.py +++ b/liveDownloader.py @@ -179,9 +179,9 @@ def run(): _count_error = 0 except Exception as e: Common.appendError(e.__str__()) - _count_error += 1 + Common.refreshDownloader() if not Common.api.roomLiver: - _count_error += 1 + Common.refreshDownloader() if Common.forceStartEncodeThread: if not et.is_alive(): et = threading.Thread(target=encode, args=()) @@ -195,5 +195,3 @@ def run(): ut.setDaemon(True) ut.start() Common.forceStartUploadThread = False - if _count_error>10: - Common.refreshDownloader()