抖音直播间(web版)弹幕抓取

.gitignore

@@ -0,0 +1,2 @@
+evn/*
+.DS_Store

README.md

@@ -0,0 +1,18 @@
+**抖音直播间(web)弹幕抓取**
+
+Pre Requirements
+- Python3
+- Charles
+
+ 1. `git clone https://github.com/gll19920817/tiktok_live`
+ 2. `pip install -r requirements.txt`
+ 3. `Open Charles > Tools > Mirror > Mirrors Setting`
+	 - `Enable Mirror`
+	 - `Save to a folder, eg:/Users/douyin/feeds/`
+	 - `Add location: https://live.douyin.com/webcast/im/fetch/`
+4. `change main.py Watcher directory parameter to folder that Step 3 	  choose, eg: /Users/douyin/feeds/`
+5. `python3 main.py`
+
+Final thoughts :
+ 1. Save data to mongodb
+ 2. Charles alternative: maybe mitmproxy & scapy

main.py

@@ -0,0 +1,6 @@
+from scripts import watcher
+
+
+if __name__ == '__main__':
+    w = watcher.Watcher(directory='/Users/geng/charles/autosaved')
+    w.run()

messages/Base.py

@@ -0,0 +1,13 @@
+class Base:
+
+    instance = None
+
+    def set_payload(self, payload):
+        self.instance.ParseFromString(payload)
+    
+    def user(self):
+        return self.instance.user
+
+    def __str__(self):
+        pass
+

messages/chat.py

@@ -0,0 +1,11 @@
+import time
+
+from protobuf import message_pb2
+from messages.base import Base
+
+class ChatMessage(Base):
+    def __init__(self):
+        self.instance = message_pb2.ChatMessage()
+
+    def __str__(self):
+        return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '【发言】' + self.user().nickname + ': ' + self.instance.content

messages/gift.py

@@ -0,0 +1,11 @@
+import time
+
+from protobuf import message_pb2
+from messages.base import Base
+
+class GiftMessage(Base):
+    def __init__(self):
+        self.instance = message_pb2.GiftMessage()
+
+    def __str__(self):
+        return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '【送礼】' + self.instance.common.describe

messages/like.py

@@ -0,0 +1,11 @@
+import time
+
+from protobuf import message_pb2
+from messages.base import Base
+
+class LikeMessage(Base):
+    def __init__(self):
+        self.instance = message_pb2.LikeMessage()
+
+    def __str__(self):
+        return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '【点赞】' +  self.user().nickname + ' 点赞了直播间（' + str(self.instance.count) + '连赞）'

messages/member.py

@@ -0,0 +1,14 @@
+import time
+
+from protobuf import message_pb2
+from messages.base import Base
+
+class MemberMessage(Base):
+    def __init__(self):
+        self.instance = message_pb2.MemberMessage()
+
+    def __str__(self):
+        template = self.instance.common.displayText.defaultPattern
+        nickname = self.user().nickname
+
+        return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '【进入直播间】' + template.replace('{0:user}', nickname).replace('{1:string}', '')

messages/roomuserseq.py

@@ -0,0 +1,11 @@
+import time
+
+from protobuf import message_pb2
+from messages.base import Base
+
+class RoomUserSeqMessage(Base):
+    def __init__(self):
+        self.instance = message_pb2.RoomUserSeqMessage()
+
+    def __str__(self):
+        return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '【观看人数】' + self.instance.totalUserStr

messages/social.py

@@ -0,0 +1,11 @@
+import time
+
+from protobuf import message_pb2
+from messages.base import Base
+
+class SocialMessage(Base):
+    def __init__(self):
+        self.instance = message_pb2.SocialMessage()
+
+    def __str__(self):
+        return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '【关注】' +  self.user().nickname + ' 关注了主播'

messages/utils.py

@@ -0,0 +1,67 @@
+from protobuf import message_pb2
+
+from messages.member import MemberMessage
+from messages.like import LikeMessage
+from messages.roomuserseq import RoomUserSeqMessage
+from messages.gift import GiftMessage
+from messages.social import SocialMessage
+from messages.chat import ChatMessage
+
+from colorama import init, Fore
+# define colors
+RED   = Fore.RED
+GREEN = Fore.GREEN
+BLUE = Fore.BLUE
+CYAN = Fore.CYAN
+MAGENTA = Fore.MAGENTA
+YELLOW = Fore.YELLOW
+WHITE = Fore.WHITE
+RESET = Fore.RESET
+init()
+
+def unpackMsgBin(filepath):
+    response = message_pb2.Response()
+
+    try:
+        with open(filepath, 'rb') as f:
+            response.ParseFromString(f.read())
+
+            decodeMsg(response.messages)
+    except Exception as e:
+        pass
+
+def decodeMsg(messages):
+    for message in messages:
+        try:
+            if message.method == 'WebcastMemberMessage':
+                member_message = MemberMessage()
+                member_message.set_payload(message.payload)
+                print(f"\n{RED}[+] {member_message} {RESET}")
+
+            elif message.method == 'WebcastSocialMessage':
+                social_message = SocialMessage()
+                social_message.set_payload(message.payload)
+                print(f"\n{GREEN}[+] {social_message} {RESET}")
+
+            elif message.method == 'WebcastChatMessage':
+                chat_message = ChatMessage()
+                chat_message.set_payload(message.payload)
+                print(f"\n{BLUE}[+] {chat_message} {RESET}")
+
+            elif message.method == 'WebcastLikeMessage':
+                like_message = LikeMessage()
+                like_message.set_payload(message.payload)
+                print(f"\n{CYAN}[+] {like_message} {RESET}")
+
+            elif message.method == 'WebcastGiftMessage':
+                gift_message = GiftMessage()
+                gift_message.set_payload(message.payload)
+                print(f"\n{MAGENTA}[+] {gift_message} {RESET}")
+
+            elif message.method == 'WebcastRoomUserSeqMessage':
+                room_user_seq_message = RoomUserSeqMessage() 
+                room_user_seq_message.set_payload(message.payload)
+                print(f"\n{YELLOW}[+] {room_user_seq_message} {RESET}")
+
+        except Exception as e:
+            print(e)

protobuf/message.proto

@@ -0,0 +1,551 @@
+syntax = "proto3";
+
+message Response{
+    repeated Message messages = 1;
+    string cursor = 2;
+    int64 fetchInterval = 3;
+    int64 now = 4;
+    string internalExt = 5;
+    int32 fetchType = 6;
+    map<string, string> routeParams = 7;
+    int64 heartbeatDuration = 8;
+    bool needAck = 9;
+    string pushServer = 10;
+}
+
+message Message{
+    string method = 1;
+    bytes payload = 2;
+    int64 msgId = 3;
+    int32 msgType = 4;
+    int64 offset = 5;
+}
+
+message RoomUserSeqMessage {
+    Common common = 1;
+    repeated Contributor ranks = 2;
+    int64 total = 3;
+    string popStr = 4;
+    repeated Contributor seats = 5;
+    int64 popularity = 6;
+    int64 totalUser = 7;
+    string totalUserStr = 8;
+    string totalStr = 9;
+    string onlineUserForAnchor = 10;
+    string totalPvForAnchor = 11;
+
+    message Contributor {
+        int64 score = 1;
+        User user = 2;
+        int64 rank = 3;
+        int64 delta = 4;
+        bool isHidden = 5;
+        string scoreDescription = 6;
+        string exactlyScore = 7;
+    }
+}
+
+message GiftMessage {
+    Common common = 1;
+    int64 giftId = 2;
+    int64 fanTicketCount = 3;
+    int64 groupCount = 4;
+    int64 repeatCount = 5;
+    int64 comboCount = 6;
+    User user = 7;
+    User toUser = 8;
+    int32 repeatEnd = 9;
+    TextEffect textEffect = 10;
+    int64 groupId = 11;
+    int64 incomeTaskgifts = 12;
+    int64 roomFanTicketCount = 13;
+    GiftIMPriority priority = 14;
+    GiftStruct gift = 15;
+    string logId = 16;
+    int64 sendType = 17;
+    PublicAreaCommon publicAreaCommon = 18;
+    Text trayDisplayText = 19;
+    int64 bannedDisplayEffects = 20;
+    GiftTrayInfo trayInfo = 21;
+    AssetEffectMixInfo assetEffectMixInfo = 24;
+
+    message TextEffect{
+        Detail portrait = 1;
+        Detail landscape = 2;
+
+        message Detail {
+            Text text = 1;
+            int32 textFontSize = 2;
+            Image background = 3;
+            int32 start = 4;
+            int32 duration = 5;
+            int32 x = 6;
+            int32 y = 7;
+            int32 width = 8;
+            int32 height = 9;
+            int32 shadowDx = 10;
+            int32 shadowDy = 11;
+            int32 shadowRadius = 12;
+            string shadowColor = 13;
+            string strokeColor = 14;
+            int32 strokeWidth = 15;
+        }
+    }
+}
+
+message LikeMessage {
+   Common common = 1; 
+   int64 count = 2;
+   int64 total = 3;
+   int64 color = 4;
+   User user = 5;
+   string icon = 6;
+}
+
+message ChatMessage {
+    Common common = 1;
+    User user = 2;
+    string content = 3;
+    bool visibleToSender = 4;
+    Image backgroundImage = 5;
+    string fullScreenTextColor = 6;
+    Image backgroundImageV2 = 7;
+    PublicAreaCommon publicAreaCommon = 9;
+    Image giftImage = 10;
+}
+
+message SocialMessage {
+    Common common = 1;
+    User user = 2;
+    int64 shareType = 3;
+    int64 action = 4;
+    string shareTarget = 5;
+    int64 followCount = 6;
+    PublicAreaCommon publicAreaCommon = 7;
+}
+
+message MemberMessage{
+    Common common = 1;
+    User user = 2;
+    int64 memberCount = 3;
+    User operator = 4;
+    bool isSetToAdmin = 5;
+    bool isTopUser = 6;
+    int64 rankScore = 7;
+    int64 topUserNo = 8;
+    int64 enterType = 9;
+    int64 action = 10;
+    string actionDescription = 11;
+    int64 userId = 12;
+    EffectConfig effectConfig = 13;
+    string popStr = 14;
+    EffectConfig enterEffectConfig = 15;
+    Image backgroundImage = 16;
+    Image backgroundImageV2 = 17;
+    Text anchorDisplayText = 18;
+    PublicAreaCommon publicAreaCommon = 19;
+
+    message EffectConfig{
+        int64 type = 1;
+        Image icon = 2;
+        int64 avatarPos = 3;
+        Text text = 4;
+        Image textIcon = 5;
+        int32 stayTime = 6;
+        int64 animAssetId = 7;
+        Image badge = 8;
+        repeated int64 flexSettingArray = 9;
+        Image textIconOverlay = 10;
+        Image animatedBadge = 11;
+        bool hasSweepLight = 12;
+        repeated int64 textFlexSettingArray = 13;
+        int64 centerAnimAssetId = 14;
+    }
+}
+
+message Common{
+    string method = 1;
+    int64 msgId = 2;
+    int64 roomId = 3;
+    int64 createTime = 4;
+    int32 monitor = 5;
+    bool isShowMsg = 6;
+    string describe = 7;
+    Text displayText = 8;
+    int64 foldType = 9;
+    int64 anchorFoldType = 10;
+    int64 priorityScore = 11;
+    string logId = 12;
+    string msgProcessFilterK = 13;
+    string msgProcessFilterV = 14;
+    User user = 15;
+    Room room = 16;
+    int64 anchorFoldTypeV2 = 17;
+    int64 processAtSeiTimeMs = 18;
+}
+
+message Text{
+    string key = 1;
+    string defaultPattern = 2;
+    TextFormat defaultFormat = 3;
+    repeated TextPiece pieces = 4;
+}
+
+message Room {
+    int64 id = 1;
+    string idStr = 2;
+    int64 status = 3;
+    int64 ownerUserId = 4;
+    string title = 5;
+    int64 userCount = 6;
+    int64 createTime = 7;
+    int64 linkmicLayout = 8;
+    int64 finishTime = 9;
+    RoomExtra extra = 10;
+    string dynamicCoverUri = 11;
+    map<int64, string> dynamicCoverDict = 12;
+    int64 lastPingTime = 13;
+    int64 liveId = 14;
+    int64 streamProvider = 15;
+    int64 osType = 16;
+    int64 clientVersion = 17;
+    bool withLinkmic = 18;
+    bool enableRoomPerspective = 19;
+    Image cover = 20;
+    Image dynamicCover = 21;
+    Image dynamicCoverLow = 22;
+    string shareUrl = 23;
+    string anchorShareText = 24;
+    string userShareText = 25;
+    int64 streamId = 26;
+    string streamIdStr = 27;
+    StreamUrl streamUrl = 28;
+    int64 mosaicStatus = 29;
+    string mosaicTip = 30;
+    int64 cellStyle = 31;
+    LinkMic linkMic = 32;
+    int64 luckymoneyNum = 33;
+    repeated Decoration decoList = 34;
+    repeated TopFan topFans = 35;
+    RoomStats stats = 36;
+    string sunDailyIconContent = 37;
+    string distance = 38;
+    string distanceCity = 39;
+    string location = 40;
+    string realDistance = 41;
+    Image feedRoomLabel = 42;
+    string commonLabelList = 43;
+    RoomUserAttr livingRoomAttrs = 44;
+    repeated int64 adminUserIds = 45;
+    User owner = 46;
+    string privateInfo = 47;
+}
+
+message RoomExtra{
+
+}
+
+message RoomStats{
+
+}
+
+message RoomUserAttr{
+
+}
+
+message StreamUrl{
+
+}
+
+message LinkMic {
+
+}
+
+message Decoration{
+
+}
+
+message TopFan {
+
+}
+
+message User{
+    int64 id = 1;
+    int64 shortId = 2;
+    string nickname = 3;
+    int32 gender = 4;
+    string signature = 5;
+    int32 level = 6;
+    int64 birthday = 7;
+    string telephone = 8;
+    Image avatarThumb = 9;
+    Image avatarMedium = 10;
+    Image avatarLarge = 11;
+    bool verified = 12;
+    int32 experience = 13;
+    string city = 14;
+    int32 status = 15;
+    int64 createTime = 16;
+    int64 modifyTime = 17;
+    int32 secret = 18;
+    string shareQrcodeUri = 19;
+    int32 incomeSharePercent = 20;
+    Image badgeImageList = 21;
+    FollowInfo followInfo = 22;
+    PayGrade payGrade = 23;
+    FansClub fansClub = 24;
+    Border border = 25;
+    string specialId = 26;
+    Image avatarBorder = 27;
+    Image medal = 28;
+    repeated Image realTimeIcons = 29;
+    repeated Image newRealTimeIcons = 30;
+    int64 topVipNo = 31;
+    UserAttr userAttr = 32;
+    OwnRoom ownRoom = 33;
+    int64 payScore = 34;
+    int64 ticketCount = 35;
+    AnchorInfo anchorInfo = 36;
+    int32 linkMicStats = 37;
+    string displayId = 38;
+
+    message UserAttr {
+
+    }
+
+    message OwnRoom {
+
+    }
+
+    message AnchorInfo {
+
+    }
+
+    message FollowInfo {
+        int64 followingCount = 1;
+        int64 followerCount = 2;
+        int64 followStatus = 3;
+        int64 pushStatus = 4;
+        string remarkName = 5;
+    }
+
+    message FansClub{
+        FansClubData data = 1;
+        map<int32, FansClubData> preferData = 2;
+
+        message FansClubData {
+            string clubName = 1;
+            int32 level = 2;
+            int32 userFansClubStatus = 3;
+            UserBadge badge = 4;
+            repeated int64 availableGiftIds = 5;
+            int64 anchorId = 6;
+
+            message UserBadge {
+                map<int32, Image> icons = 1;
+                string title = 2;
+            }
+        }
+    }
+
+    message Border{
+
+    }
+
+    message GradeBuffInfo {
+        int64 buffLevel = 1;
+        int32 status = 2;
+        int64 endTime = 3;
+        map<int64, int64> statsInfo = 4;
+        Image buffBadge = 5;
+    }
+
+    message PayGrade {
+        int64 totalDiamondCount = 1;
+        Image diamondIcon = 2;
+        string name = 3;
+        Image icon = 4;
+        string nextName = 5;
+        int64 level = 6;
+        Image nextIcon = 7;
+        int64 nextDiamond = 8;
+        int64 nowDiamond = 9;
+        int64 thisGradeMinDiamond = 10;
+        int64 thisGradeMaxDiamond = 11;
+        int64 payDiamondBak = 12;
+        string gradeDescribe = 13;
+        repeated GradeIcon gradeIconList = 14;
+        int64 screenChatType = 15;
+        Image imIcon = 16;
+        Image imIconWithLevel = 17;
+        Image liveIcon = 18;
+        Image newImIconWithLevel = 19;
+        Image newLiveIcon = 20;
+        int64 upgradeNeedConsume = 21;
+        string nextPrivileges = 22;
+        Image background = 23;
+        Image backgroundBack = 24;
+        int64 score = 25;
+        GradeBuffInfo buffInfo = 26;
+        string gradeBanner = 1001;
+        Image profileDialogBg = 1002;
+        Image profileDialogBgBack = 1003;
+
+        message GradeIcon{
+            Image icon = 1;
+            int64 iconDiamond = 2;
+            int64 level = 3;
+            string levelStr = 4;
+        }
+    }
+}
+
+message TextFormat{
+    string color = 1;
+    bool bold = 2;
+    bool italic = 3;
+    int32 weight = 4;
+    int32 italicAngle = 5;
+    int32 fontSize = 6;
+    bool userHeightLightColor = 7;
+    bool useRemoteClor = 8;
+}
+
+message TextPiece{
+    int32 type = 1;
+    TextFormat format = 2;
+    string stringValue = 11;
+    TextPieceUser userValue = 21;
+}
+
+message Image{
+}
+
+message TextPieceUser{
+    User user = 1;
+    bool withColon = 2;
+}
+
+message PublicAreaCommon {
+    Image userLabel = 1;
+    int64 userConsumeInRoom = 2;
+    int64 userSendGiftCntInRoom = 3;
+}
+
+message GiftIMPriority {
+    repeated int64 queueSizes = 1;
+    int64 selfQueuePriority = 2;
+    int64 priority = 3;
+}
+
+message GiftTrayInfo{
+    Text trayDisplayText = 1;
+    Image trayBaseImg = 2;
+    Image trayHeadImg = 3;
+    Image trayRightImg = 4;
+    int64 trayLevel = 5;
+    Image trayDynamicImg = 6;
+}
+
+message GiftStruct {
+    Image image = 1;
+    string describe = 2;
+    bool notify = 3;
+    int64 duration = 4;
+    int64 id = 5;
+    GiftStructFansClubInfo fansclubInfo = 6;
+    bool forLinkmic = 7;
+    bool doodle = 8;
+    bool forFansclub = 9;
+    bool combo = 10;
+    int32 type = 11;
+    int32 diamondCount = 12;
+    int32 isDisplayedOnPanel = 13;
+    int64 primaryEffectId = 14;
+    Image giftLabelIcon = 15;
+    string name = 16;
+    string region = 17;
+    string manual = 18;
+    bool forCustom = 19;
+    map<string, int64> specialEffects = 20;
+    Image icon = 21;
+    int32 actionType = 22;
+    int32 watermelonSeeds = 23;
+    string goldEffect = 24;
+    repeated LuckyMoneyGiftMeta subs = 25;
+    int64 goldenBeans = 26;
+    int64 honorLevel = 27;
+    int32 itemType = 28;
+    string schemeUrl = 29;
+    GiftPanelOperation giftOperation = 30;
+    string eventName = 31;
+    int64 nobleLevel = 32; 
+    string guideUrl = 33;
+    bool punishMedicine = 34;
+    bool forPortal = 35;
+    string businessText = 36;
+    bool cnyGift = 37;
+    int64 appId = 38;
+    int64 vipLevel = 39;
+    bool isGray = 40;
+    string graySchemeUrl = 41;
+    int64 giftScene = 42;
+    GiftBanner giftBanner = 43;
+    repeated string triggerWords = 44;
+    repeated GiftBuffInfo giftBuffInfos = 45;
+    bool forFirstRecharge = 46;
+    Image dynamicImgForSelected = 47;
+    int32 afterSendAction = 48;
+    int64 giftOfflineTime = 49;
+    string topBarText = 50;
+    Image topRightAvatar = 51;
+    string bannerSchemeUrl = 52;
+    bool isLocked = 53;
+    int64 reqExtraType = 54;
+    repeated int64 assetIds = 55;
+    GiftPreviewInfo giftPreviewInfo = 56;
+    GiftTip giftTip = 57;
+    int32 needSweepLightCount = 58;
+    repeated GiftGroupInfo groupInfo = 59;
+
+    message GiftStructFansClubInfo {
+        int32 minLevel = 1;
+        int32 insertPos = 2;
+    }
+}
+
+message AssetEffectMixInfo{
+
+}
+
+message LuckyMoneyGiftMeta {
+
+}
+
+message GiftPanelOperation {
+
+}
+
+message GiftBanner {
+
+}
+
+message GiftBuffInfo{
+
+}
+
+message GiftPreviewInfo{
+
+}
+
+message GiftTip {
+
+}
+
+message GiftGroupInfo {
+
+}
+
+message EffectMixImageInfo {
+    
+}

requirements.txt

@@ -0,0 +1,3 @@
+colorama==0.4.4
+protobuf==3.19.1
+watchdog==2.1.6

scripts/watcher.py

@@ -0,0 +1,37 @@
+import time
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+
+from messages.utils import unpackMsgBin
+
+class Watcher:
+    DIRECTORY_TO_WATCH = ""
+
+    def __init__(self, directory):
+        self.observer = Observer()
+        self.DIRECTORY_TO_WATCH = directory
+
+    def run(self):
+        event_handler = Handler()
+        self.observer.schedule(event_handler, self.DIRECTORY_TO_WATCH, recursive=True)
+        self.observer.start()
+
+        try:
+            while True:
+                time.sleep(5)
+        except:
+            self.observer.stop()
+
+        self.observer.join()
+
+
+class Handler(FileSystemEventHandler):
+
+    @staticmethod
+    def on_any_event(event):
+        if event.is_directory:
+            return None
+
+        elif event.event_type == 'created':
+            unpackMsgBin(event.src_path)
+