使用7.0.x版本的mitmproxy，去除外置mitmproxy依赖，可简化http网络开销

config/settings.yml

@@ -1,5 +1,4 @@
 mitm:
-  bin: 'C:\Program Files (x86)\mitmproxy\bin\mitmdump.exe'
   host: 127.0.0.1
   port: 8080
 
@@ -26,6 +25,7 @@ output:
 live:
   rooms:
     - "585723119943"
+    - "583853809376"
   users:
     - MS4wLjABAAAAzBItqEvCjPryxn_Y6w6LtRBFDOVNfjvYSJg8VVZFwlw
 

handler/common.py

@@ -1,11 +0,0 @@
-import time
-from queue import SimpleQueue
-
-MESSAGE_QUEUE: "SimpleQueue[MessagePayload]" = SimpleQueue()
-
-
-class MessagePayload(object):
-    def __init__(self, body: bytes, timestamp: str = ""):
-        self.request_timestamp = timestamp
-        self.body = body
-        self.curretnt_timestamp = time.time()

main.py

@@ -1,19 +1,13 @@
-import threading
-import subprocess
 import atexit
+import signal
 
-from config.helper import config
-from handler.http_server import app
 from browser.manager import init_manager as init_browser_manager
 from output.manager import OutputManager
+from proxy.manager import init_manager as init_proxy_manager
 
 if __name__ == '__main__':
-    mitmproxy_process = subprocess.Popen([
-        config()["mitm"]["bin"], "-s", "./proxy_script.py", "-q",
-        "--listen-host", config()["mitm"]["host"], "--listen-port", str(config()["mitm"]["port"])
-    ])
-    api_thread = threading.Thread(target=app.run, args=(config()["http"]["host"], config()["http"]["port"],))
-    api_thread.start()
+    proxy_manager = init_proxy_manager()
+    proxy_manager.start_loop()
     browser_manager = init_browser_manager()
     output_manager = OutputManager()
 
@@ -25,5 +19,7 @@ if __name__ == '__main__':
 
 
     atexit.register(terminate)
+    signal.signal(signal.SIGTERM, terminate)
+    signal.signal(signal.SIGINT, terminate)
     output_manager.start_loop()
-    api_thread.join()
+    proxy_manager.join()

output/manager.py

@@ -3,7 +3,7 @@ import threading
 from typing import TYPE_CHECKING
 
 from config.helper import config
-from handler.common import MessagePayload, MESSAGE_QUEUE
+from proxy.queues import MESSAGE_QUEUE
 from messages.chat import ChatMessage
 from messages.control import ControlMessage
 from messages.gift import GiftMessage
@@ -19,6 +19,7 @@ from protobuf import message_pb2, wss_pb2
 if TYPE_CHECKING:
     from typing import Type, Optional
     from output.IOutput import IOutput
+    from proxy.common import MessagePayload
 
 
 class OutputManager():
@@ -42,7 +43,7 @@ class OutputManager():
     def __del__(self):
         self.terminate()
 
-    def decode_payload(self, message: MessagePayload):
+    def decode_payload(self, message: "MessagePayload"):
         try:
             response = message_pb2.Response()
             wss = wss_pb2.WssResponse()

proxy/addon/danmaku_ws.py

@@ -0,0 +1,24 @@
+import re
+
+from proxy.common import MessagePayload
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from mitmproxy import http
+    from queue import SimpleQueue
+
+
+class DanmakuWebsocketAddon:
+    def __init__(self, queue: "SimpleQueue[MessagePayload]"):
+        self._queue = queue
+
+    def websocket_message(self, flow: "http.HTTPFlow"):
+        re_c = re.search('webcast\d-ws-web-.*\.douyin\.com', flow.request.host)
+        if re_c:
+            message = flow.websocket.messages[-1]
+            if message.from_client:
+                return
+            payload = MessagePayload(message.content)
+            payload.request_url = flow.request.url
+            payload.request_query = flow.request.query
+            self._queue.put(payload)

proxy/addon/userinfo_http.py

@@ -0,0 +1,11 @@
+from mitmproxy import http
+
+
+class UserInfoAddon:
+    def __init__(self):
+        ...
+
+    def response(self, flow: http.HTTPFlow):
+        # /aweme/v1/web/user/profile/other/ 他人主页获取他人信息
+        if '/aweme/v1/web/user/profile/other' in flow.request.path:
+            content = flow.response.content

proxy/common.py

@@ -0,0 +1,9 @@
+import time
+
+
+class MessagePayload(object):
+    def __init__(self, body: bytes):
+        self.body = body
+        self.timestamp: float = time.time()
+        self.request_url: str = ""
+        self.request_query: dict[str, str] = {}

proxy/manager.py

@@ -0,0 +1,59 @@
+import asyncio
+import threading
+from typing import TYPE_CHECKING
+
+from mitmproxy.options import Options
+from mitmproxy.tools.dump import DumpMaster
+
+from config.helper import config
+from proxy.addon.danmaku_ws import DanmakuWebsocketAddon
+from proxy.queues import MESSAGE_QUEUE
+
+if TYPE_CHECKING:
+    from typing import Optional
+
+_manager: "Optional[ProxyManager]" = None
+
+
+class ProxyManager:
+    def __init__(self):
+        self._mitm_instance = None
+        opts = Options(
+            listen_host=config()['mitm']['host'],
+            listen_port=config()['mitm']['port'],
+        )
+        self._mitm_instance = DumpMaster(options=opts)
+        self._load_addon()
+        opts.update_defer(
+            flow_detail=0,
+            termlog_verbosity="error",
+        )
+        self._thread = None
+
+    def _load_addon(self):
+        self._mitm_instance.addons.add(DanmakuWebsocketAddon(MESSAGE_QUEUE))
+
+    def _start(self):
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        self._mitm_instance.run()
+
+    def start_loop(self):
+        self._thread = threading.Thread(target=self._start)
+        self._thread.start()
+
+    def join(self):
+        if self._thread:
+            self._thread.join()
+
+
+def init_manager():
+    global _manager
+    _manager = ProxyManager()
+    return _manager
+
+
+def get_manager():
+    if _manager is None:
+        return init_manager()
+    return _manager

proxy/queues.py

@@ -0,0 +1,4 @@
+from queue import SimpleQueue
+from proxy.common import MessagePayload
+
+MESSAGE_QUEUE: "SimpleQueue[MessagePayload]" = SimpleQueue()

requirements.txt

@@ -3,7 +3,5 @@ protobuf==3.19.1
 PyYAML==6.0
 selenium==4.1.0
 requests==2.27.1
-scripts==2.0
 
-
-Flask~=2.1.2
+mitmproxy~=7.0.4