From 82110682152a7fcab3d7958aa8f04b22e35e0174 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Mon, 6 Jun 2022 12:25:42 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BD=BF=E7=94=A87.0.x=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E7=9A=84mitmproxy=EF=BC=8C=E5=8E=BB=E9=99=A4=E5=A4=96=E7=BD=AE?= =?UTF-8?q?mitmproxy=E4=BE=9D=E8=B5=96=EF=BC=8C=E5=8F=AF=E7=AE=80=E5=8C=96?= =?UTF-8?q?http=E7=BD=91=E7=BB=9C=E5=BC=80=E9=94=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/settings.yml | 2 +- handler/common.py | 11 ------- main.py | 18 +++++------ output/manager.py | 5 +-- proxy/__init__.py | 0 proxy/addon/__init__.py | 0 proxy/addon/danmaku_ws.py | 24 +++++++++++++++ proxy/addon/userinfo_http.py | 11 +++++++ proxy/common.py | 9 ++++++ proxy/manager.py | 59 ++++++++++++++++++++++++++++++++++++ proxy/queues.py | 4 +++ requirements.txt | 4 +-- 12 files changed, 119 insertions(+), 28 deletions(-) delete mode 100644 handler/common.py create mode 100644 proxy/__init__.py create mode 100644 proxy/addon/__init__.py create mode 100644 proxy/addon/danmaku_ws.py create mode 100644 proxy/addon/userinfo_http.py create mode 100644 proxy/common.py create mode 100644 proxy/manager.py create mode 100644 proxy/queues.py diff --git a/config/settings.yml b/config/settings.yml index 8f53b71..726e3b7 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -1,5 +1,4 @@ mitm: - bin: 'C:\Program Files (x86)\mitmproxy\bin\mitmdump.exe' host: 127.0.0.1 port: 8080 @@ -26,6 +25,7 @@ output: live: rooms: - "585723119943" + - "583853809376" users: - MS4wLjABAAAAzBItqEvCjPryxn_Y6w6LtRBFDOVNfjvYSJg8VVZFwlw diff --git a/handler/common.py b/handler/common.py deleted file mode 100644 index 292b2c6..0000000 --- a/handler/common.py +++ /dev/null @@ -1,11 +0,0 @@ -import time -from queue import SimpleQueue - -MESSAGE_QUEUE: "SimpleQueue[MessagePayload]" = SimpleQueue() - - -class MessagePayload(object): - def __init__(self, body: bytes, timestamp: str = ""): - self.request_timestamp = timestamp - self.body = body - self.curretnt_timestamp = time.time() diff --git a/main.py b/main.py index 768b348..344b17b 100644 --- a/main.py +++ b/main.py @@ -1,19 +1,13 @@ -import threading -import subprocess import atexit +import signal -from config.helper import config -from handler.http_server import app from browser.manager import init_manager as init_browser_manager from output.manager import OutputManager +from proxy.manager import init_manager as init_proxy_manager if __name__ == '__main__': - mitmproxy_process = subprocess.Popen([ - config()["mitm"]["bin"], "-s", "./proxy_script.py", "-q", - "--listen-host", config()["mitm"]["host"], "--listen-port", str(config()["mitm"]["port"]) - ]) - api_thread = threading.Thread(target=app.run, args=(config()["http"]["host"], config()["http"]["port"],)) - api_thread.start() + proxy_manager = init_proxy_manager() + proxy_manager.start_loop() browser_manager = init_browser_manager() output_manager = OutputManager() @@ -25,5 +19,7 @@ if __name__ == '__main__': atexit.register(terminate) + signal.signal(signal.SIGTERM, terminate) + signal.signal(signal.SIGINT, terminate) output_manager.start_loop() - api_thread.join() + proxy_manager.join() diff --git a/output/manager.py b/output/manager.py index 47f637b..8577612 100644 --- a/output/manager.py +++ b/output/manager.py @@ -3,7 +3,7 @@ import threading from typing import TYPE_CHECKING from config.helper import config -from handler.common import MessagePayload, MESSAGE_QUEUE +from proxy.queues import MESSAGE_QUEUE from messages.chat import ChatMessage from messages.control import ControlMessage from messages.gift import GiftMessage @@ -19,6 +19,7 @@ from protobuf import message_pb2, wss_pb2 if TYPE_CHECKING: from typing import Type, Optional from output.IOutput import IOutput + from proxy.common import MessagePayload class OutputManager(): @@ -42,7 +43,7 @@ class OutputManager(): def __del__(self): self.terminate() - def decode_payload(self, message: MessagePayload): + def decode_payload(self, message: "MessagePayload"): try: response = message_pb2.Response() wss = wss_pb2.WssResponse() diff --git a/proxy/__init__.py b/proxy/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/proxy/addon/__init__.py b/proxy/addon/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/proxy/addon/danmaku_ws.py b/proxy/addon/danmaku_ws.py new file mode 100644 index 0000000..4d79d48 --- /dev/null +++ b/proxy/addon/danmaku_ws.py @@ -0,0 +1,24 @@ +import re + +from proxy.common import MessagePayload +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from mitmproxy import http + from queue import SimpleQueue + + +class DanmakuWebsocketAddon: + def __init__(self, queue: "SimpleQueue[MessagePayload]"): + self._queue = queue + + def websocket_message(self, flow: "http.HTTPFlow"): + re_c = re.search('webcast\d-ws-web-.*\.douyin\.com', flow.request.host) + if re_c: + message = flow.websocket.messages[-1] + if message.from_client: + return + payload = MessagePayload(message.content) + payload.request_url = flow.request.url + payload.request_query = flow.request.query + self._queue.put(payload) diff --git a/proxy/addon/userinfo_http.py b/proxy/addon/userinfo_http.py new file mode 100644 index 0000000..e3e3074 --- /dev/null +++ b/proxy/addon/userinfo_http.py @@ -0,0 +1,11 @@ +from mitmproxy import http + + +class UserInfoAddon: + def __init__(self): + ... + + def response(self, flow: http.HTTPFlow): + # /aweme/v1/web/user/profile/other/ 他人主页获取他人信息 + if '/aweme/v1/web/user/profile/other' in flow.request.path: + content = flow.response.content diff --git a/proxy/common.py b/proxy/common.py new file mode 100644 index 0000000..68f00b9 --- /dev/null +++ b/proxy/common.py @@ -0,0 +1,9 @@ +import time + + +class MessagePayload(object): + def __init__(self, body: bytes): + self.body = body + self.timestamp: float = time.time() + self.request_url: str = "" + self.request_query: dict[str, str] = {} diff --git a/proxy/manager.py b/proxy/manager.py new file mode 100644 index 0000000..6c418ff --- /dev/null +++ b/proxy/manager.py @@ -0,0 +1,59 @@ +import asyncio +import threading +from typing import TYPE_CHECKING + +from mitmproxy.options import Options +from mitmproxy.tools.dump import DumpMaster + +from config.helper import config +from proxy.addon.danmaku_ws import DanmakuWebsocketAddon +from proxy.queues import MESSAGE_QUEUE + +if TYPE_CHECKING: + from typing import Optional + +_manager: "Optional[ProxyManager]" = None + + +class ProxyManager: + def __init__(self): + self._mitm_instance = None + opts = Options( + listen_host=config()['mitm']['host'], + listen_port=config()['mitm']['port'], + ) + self._mitm_instance = DumpMaster(options=opts) + self._load_addon() + opts.update_defer( + flow_detail=0, + termlog_verbosity="error", + ) + self._thread = None + + def _load_addon(self): + self._mitm_instance.addons.add(DanmakuWebsocketAddon(MESSAGE_QUEUE)) + + def _start(self): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + self._mitm_instance.run() + + def start_loop(self): + self._thread = threading.Thread(target=self._start) + self._thread.start() + + def join(self): + if self._thread: + self._thread.join() + + +def init_manager(): + global _manager + _manager = ProxyManager() + return _manager + + +def get_manager(): + if _manager is None: + return init_manager() + return _manager diff --git a/proxy/queues.py b/proxy/queues.py new file mode 100644 index 0000000..ac037ba --- /dev/null +++ b/proxy/queues.py @@ -0,0 +1,4 @@ +from queue import SimpleQueue +from proxy.common import MessagePayload + +MESSAGE_QUEUE: "SimpleQueue[MessagePayload]" = SimpleQueue() diff --git a/requirements.txt b/requirements.txt index ee9e40c..dcc86b3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,5 @@ protobuf==3.19.1 PyYAML==6.0 selenium==4.1.0 requests==2.27.1 -scripts==2.0 - -Flask~=2.1.2 \ No newline at end of file +mitmproxy~=7.0.4