Merge branch 'release/v0.0.7.2'

v0.0.7.1

pytchat/__init__.py

@@ -2,7 +2,7 @@
 pytchat is a python library for fetching youtube live chat without using yt api, Selenium, or BeautifulSoup.
 """
 __copyright__    = 'Copyright (C) 2019 taizan-hokuto'
-__version__      = '0.0.6.6'
+__version__      = '0.0.7.2'
 __license__      = 'MIT'
 __author__       = 'taizan-hokuto'
 __author_email__ = '55448286+taizan-hokuto@users.noreply.github.com'

pytchat/core_multithread/livechat.py

@@ -72,8 +72,6 @@ class LiveChat:
     '''
 
     _setup_finished = False
-    #チャット監視中のListenerのリスト
-    _listeners = []
 
     def __init__(self, video_id,
                  seektime=0,
@@ -103,19 +101,13 @@ class LiveChat:
         self._parser = Parser(is_replay=self._is_replay)
         self._pauser = Queue()
         self._pauser.put_nowait(None)
-        self._setup()
         self._first_fetch = True
         self._fetch_url = "live_chat/get_live_chat?continuation="
         self._topchat_only = topchat_only
         self._logger = logger
-        LiveChat._logger = logger
-        if not LiveChat._setup_finished:
-            LiveChat._setup_finished = True
         if interruptable:
-                signal.signal(signal.SIGINT,  (lambda a, b:  
+            signal.signal(signal.SIGINT, lambda a, b: self.terminate())
-                (LiveChat.shutdown(None,signal.SIGINT,b))
+        self._setup()
-                ))
-        LiveChat._listeners.append(self)
 
     def _setup(self):
         # direct modeがTrueでcallback未設定の場合例外発生。
@@ -174,7 +166,8 @@ class LiveChat:
                     }
                     time_mark = time.time()
                     if self._direct_mode:
-                        processed_chat = self.processor.process([chat_component])
+                        processed_chat = self.processor.process(
+                            [chat_component])
                         if isinstance(processed_chat, tuple):
                             self._callback(*processed_chat)
                         else:
@@ -319,14 +312,7 @@ class LiveChat:
         '''
         Listenerを終了する。
         '''
+        if self.is_alive():
             self._is_alive = False
-        if self._direct_mode == False:
+            self._buffer.put({})
-            #bufferにダミーオブジェクトを入れてis_alive()を判定させる
+            self._logger.info(f'[{self.video_id}]終了しました')
-            self._buffer.put({'chatdata':'','timeout':0}) 
-        self._logger.info(f'[{self.video_id}]finished.')
-  
-    @classmethod
-    def shutdown(cls, event, sig = None, handler=None):
-        cls._logger.debug("shutdown...")
-        for t in LiveChat._listeners:
-            t._is_alive = False

pytchat/paramgen/arcparam.py

@@ -12,6 +12,7 @@ Author: taizan-hokuto (2019) @taizan205
 ver 0.0.1 2019.10.05
 '''
 
+
 def _gen_vid(video_id):
     """generate video_id parameter.
     Parameter
@@ -40,9 +41,11 @@ def _gen_vid(video_id):
         b64enc(reduce(lambda x, y: x+y, item)).decode()
     ).encode()
 
+
 def _nval(val):
     """convert value to byte array"""
-    if val<0: raise ValueError
+    if val < 0:
+        raise ValueError
     buf = b''
     while val >> 7:
         m = val & 0xFF | 0x80
@@ -51,6 +54,7 @@ def _nval(val):
     buf += val.to_bytes(1, 'big')
     return buf
 
+
 def _build(video_id, seektime, topchat_only):
     switch_01 = b'\x04' if topchat_only else b'\x01'
     if seektime < 0:
@@ -75,7 +79,8 @@ def _build(video_id, seektime, topchat_only):
     sep_3 = b'\x00\x58\x03\x60'
     sep_4 = b'\x68' + parity + b'\x72\x04\x08'
     sep_5 = b'\x10' + parity + b'\x78\x00'
-    body = [
+
+    body = b''.join([
         sep_0,
         _nval(len(vid)),
         vid,
@@ -90,9 +95,7 @@ def _build(video_id, seektime, topchat_only):
         sep_4,
         switch_01,
         sep_5
-    ]
+    ])
-
-    body = reduce(lambda x, y: x+y, body)
 
     return urllib.parse.quote(
         b64enc(header_magic +
@@ -101,6 +104,7 @@ def _build(video_id, seektime, topchat_only):
                ).decode()
     )
 
+
 def getparam(video_id, seektime=0, topchat_only=False):
     '''
     Parameter

pytchat/paramgen/liveparam.py

@@ -11,6 +11,8 @@ Author: taizan-hokuto (2019) @taizan205
 
 ver 0.0.1 2019.10.05
 '''
+
+
 def _gen_vid(video_id):
     """generate video_id parameter.
     Parameter
@@ -44,6 +46,7 @@ def _gen_vid(video_id):
         b64enc(reduce(lambda x, y: x+y, item)).decode()
     ).encode()
 
+
 def _tzparity(video_id, times):
     t = 0
     for i, s in enumerate(video_id):
@@ -55,9 +58,11 @@ def _tzparity(video_id,times):
 
     return ((times ^ t) % 2).to_bytes(1, 'big')
 
+
 def _nval(val):
     """convert value to byte array"""
-    if val<0: raise ValueError
+    if val < 0:
+        raise ValueError
     buf = b''
     while val >> 7:
         m = val & 0xFF | 0x80
@@ -66,6 +71,7 @@ def _nval(val):
     buf += val.to_bytes(1, 'big')
     return buf
 
+
 def _build(video_id, _ts1, _ts2, _ts3, _ts4, _ts5, topchat_only):
     # _short_type2
     switch_01 = b'\x04' if topchat_only else b'\x01'
@@ -99,7 +105,7 @@ def _build(video_id, _ts1, _ts2, _ts3, _ts4, _ts5, topchat_only):
     sep_9 = b'\x88\x01\x00\xA0\x01'
     timestamp5 = _nval(_ts5)
 
-    body = [
+    body = b''.join([
         sep_0,
         _nval(len(vid)),
         vid,
@@ -121,15 +127,13 @@ def _build(video_id, _ts1, _ts2, _ts3, _ts4, _ts5, topchat_only):
         ts_3_end,
         timestamp4,
         sep_6,
-        switch_01,#
+        switch_01,
         sep_7,
-        switch_01,#
+        switch_01,
         sep_8,
         sep_9,
         timestamp5
-    ]
+    ])
-
-    body = reduce(lambda x, y: x+y, body)
 
     return urllib.parse.quote(
         b64enc(header_magic +
@@ -161,4 +165,3 @@ def getparam(video_id, past_sec = 0, topchat_only = False):
         if True, fetch only 'top chat'
     '''
     return _build(video_id, *_times(past_sec), topchat_only)
-

pytchat/processors/compatible/processor.py

@@ -4,10 +4,12 @@ from .renderer.textmessage import LiveChatTextMessageRenderer
 from .renderer.paidmessage import LiveChatPaidMessageRenderer
 from .renderer.paidsticker import LiveChatPaidStickerRenderer
 from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
+from .renderer.membership import LiveChatMembershipItemRenderer
 from .. chat_processor import ChatProcessor
 from ... import config
 logger = config.logger(__name__)
 
+
 class CompatibleProcessor(ChatProcessor):
 
     def process(self, chat_components: list):
@@ -24,11 +26,15 @@ class CompatibleProcessor(ChatProcessor):
                 timeout += chat_component.get('timeout', 0)
                 chatdata = chat_component.get('chatdata')
 
-                if chatdata is None: break
+                if chatdata is None:
+                    break
                 for action in chatdata:
-                    if action is None: continue
+                    if action is None:
-                    if action.get('addChatItemAction') is None: continue
+                        continue
-                    if action['addChatItemAction'].get('item') is None: continue
+                    if action.get('addChatItemAction') is None:
+                        continue
+                    if action['addChatItemAction'].get('item') is None:
+                        continue
 
                     chat = self.parse(action)
                     if chat:
@@ -47,7 +53,8 @@ class CompatibleProcessor(ChatProcessor):
         action = sitem.get("addChatItemAction")
         if action:
             item = action.get("item")
-        if item is None: return None
+        if item is None:
+            return None
         rd = {}
         try:
             renderer = self.get_renderer(item)
@@ -75,7 +82,8 @@ class CompatibleProcessor(ChatProcessor):
             renderer = LiveChatPaidStickerRenderer(item)
         elif item.get("liveChatLegacyPaidMessageRenderer"):
             renderer = LiveChatLegacyPaidMessageRenderer(item)
+        elif item.get("liveChatMembershipItemRenderer"):
+            renderer = LiveChatMembershipItemRenderer(item)
         else:
             renderer = None
         return renderer
-

pytchat/processors/compatible/renderer/membership.py

@@ -0,0 +1,40 @@
+from .base import BaseRenderer
+
+
+class LiveChatMembershipItemRenderer(BaseRenderer):
+    def __init__(self, item):
+        super().__init__(item, "newSponsorEvent")
+
+    def get_snippet(self):
+        message = self.get_message(self.renderer)
+        return {
+            "type": self.chattype,
+            "liveChatId": "",
+            "authorChannelId": self.renderer.get("authorExternalChannelId"),
+            "publishedAt": self.get_publishedat(self.renderer.get("timestampUsec", 0)),
+            "hasDisplayContent": True,
+            "displayMessage": message,
+
+        }
+
+    def get_authordetails(self):
+        authorExternalChannelId = self.renderer.get("authorExternalChannelId")
+        # parse subscriber type
+        isVerified, isChatOwner, _, isChatModerator = (
+            self.get_badges(self.renderer)
+        )
+        return {
+            "channelId": authorExternalChannelId,
+            "channelUrl": "http://www.youtube.com/channel/"+authorExternalChannelId,
+            "displayName": self.renderer["authorName"]["simpleText"],
+            "profileImageUrl": self.renderer["authorPhoto"]["thumbnails"][1]["url"],
+            "isVerified": isVerified,
+            "isChatOwner": isChatOwner,
+            "isChatSponsor": True,
+            "isChatModerator": isChatModerator
+        }
+
+    def get_message(self, renderer):
+        message = ''.join([mes.get("text", "") for mes in renderer["headerSubtext"]["runs"]])
+        return message, [message]
+

pytchat/processors/default/processor.py

@@ -4,10 +4,13 @@ from .renderer.textmessage import LiveChatTextMessageRenderer
 from .renderer.paidmessage import LiveChatPaidMessageRenderer
 from .renderer.paidsticker import LiveChatPaidStickerRenderer
 from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
+from .renderer.membership import LiveChatMembershipItemRenderer
 from .. chat_processor import ChatProcessor
 from ... import config
+
 logger = config.logger(__name__)
 
+
 class Chatdata:
     def __init__(self, chatlist: list, timeout: float):
         self.items = chatlist
@@ -25,6 +28,7 @@ class Chatdata:
             return
         await asyncio.sleep(self.interval/len(self.items))
 
+
 class DefaultProcessor(ChatProcessor):
     def process(self, chat_components: list):
 
@@ -35,24 +39,27 @@ class DefaultProcessor(ChatProcessor):
             for component in chat_components:
                 timeout += component.get('timeout', 0)
                 chatdata = component.get('chatdata')
-                if chatdata is None: continue
+                if chatdata is None:
+                    continue
                 for action in chatdata:
-                    if action is None: continue
+                    if action is None:
-                    if action.get('addChatItemAction') is None: continue
+                        continue
-                    if action['addChatItemAction'].get('item') is None: continue
+                    if action.get('addChatItemAction') is None:
+                        continue
+                    if action['addChatItemAction'].get('item') is None:
+                        continue
 
                     chat = self._parse(action)
                     if chat:
                         chatlist.append(chat)
         return Chatdata(chatlist, float(timeout))
 
-
     def _parse(self, sitem):
-
         action = sitem.get("addChatItemAction")
         if action:
             item = action.get("item")
-        if item is None: return None
+        if item is None:
+            return None
         try:
             renderer = self._get_renderer(item)
             if renderer == None:
@@ -74,6 +81,8 @@ class DefaultProcessor(ChatProcessor):
             renderer = LiveChatPaidStickerRenderer(item)
         elif item.get("liveChatLegacyPaidMessageRenderer"):
             renderer = LiveChatLegacyPaidMessageRenderer(item)
+        elif item.get("liveChatMembershipItemRenderer"):
+            renderer = LiveChatMembershipItemRenderer(item)
         else:
             renderer = None
         return renderer

pytchat/processors/default/renderer/membership.py

@@ -0,0 +1,15 @@
+from .base import BaseRenderer
+
+
+class LiveChatMembershipItemRenderer(BaseRenderer):
+    def __init__(self, item):
+        super().__init__(item, "newSponsor")
+
+    def get_authordetails(self):
+        super().get_authordetails()
+        self.author.isChatSponsor = True
+
+    def get_message(self, renderer):
+        message = ''.join([mes.get("text", "") for mes in renderer["headerSubtext"]["runs"]])
+        return message, [message]
+

pytchat/tool/extract/duplcheck.py

@@ -70,7 +70,8 @@ def check_duplicate_offset(chatdata):
         if is_duplicate(i,i+1)]
 
 def remove_duplicate_head(blocks):
-    if len(blocks) == 1 : return blocks
+    if len(blocks) == 0 or len(blocks) == 1:
+        return blocks
 
     def is_duplicate_head(index):
 
@@ -97,7 +98,8 @@ def remove_duplicate_head(blocks):
     return ret
 
 def remove_duplicate_tail(blocks):
-    if len(blocks) == 1 : return blocks    
+    if len(blocks) == 0 or len(blocks) == 1:
+        return blocks    
 
     def is_duplicate_tail(index):
         if len(blocks[index].chat_data) == 0:
@@ -126,7 +128,8 @@ def remove_overlap(blocks):
     Align the last offset of each block to the first offset 
     of next block (equals `end` offset of each block).
     """
-    if len(blocks) == 1 : return blocks
+    if len(blocks) == 0 or len(blocks) == 1:
+        return blocks    
 
     for block in blocks:
         if block.is_last:

pytchat/tool/extract/extractor.py

@@ -40,6 +40,7 @@ class Extractor:
         return self
 
     def _set_block_end(self):
+        if len(self.blocks) > 0:
             for i in range(len(self.blocks)-1):
                 self.blocks[i].end = self.blocks[i+1].first
             self.blocks[-1].end = self.duration*1000

tests/test_compatible_processor.py

@@ -1,6 +1,7 @@
 import json
 import pytest
-import asyncio,aiohttp
+import asyncio
+import aiohttp
 from pytchat.parser.live import Parser
 from pytchat.processors.compatible.processor import CompatibleProcessor
 from pytchat.exceptions import (
@@ -14,6 +15,7 @@ from pytchat.processors.compatible.renderer.legacypaid import LiveChatLegacyPaid
 
 parser = Parser(is_replay=False)
 
+
 def test_textmessage(mocker):
     '''api互換processorのテスト：通常テキストメッセージ'''
     processor = CompatibleProcessor()
@@ -51,6 +53,7 @@ def test_textmessage(mocker):
     assert "LCC." in ret["items"][0]["id"]
     assert ret["items"][0]["snippet"]["type"] == "textMessageEvent"
 
+
 def test_newsponcer(mocker):
     '''api互換processorのテスト：メンバ新規登録'''
     processor = CompatibleProcessor()
@@ -87,6 +90,42 @@ def test_newsponcer(mocker):
     assert ret["items"][0]["snippet"]["type"] == "newSponsorEvent"
 
 
+def test_newsponcer_rev(mocker):
+    '''api互換processorのテスト：メンバ新規登録'''
+    processor = CompatibleProcessor()
+
+    _json = _open_file("tests/testdata/compatible/newSponsor_rev.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+    ret = processor.process([data])
+
+    assert ret["kind"] == "youtube#liveChatMessageListResponse"
+    assert ret["pollingIntervalMillis"] == data["timeout"]*1000
+    assert ret.keys() == {
+        "kind",   "etag",   "pageInfo",   "nextPageToken", "pollingIntervalMillis", "items"
+    }
+    assert ret["pageInfo"].keys() == {
+        "totalResults",   "resultsPerPage"
+    }
+    assert ret["items"][0].keys() == {
+        "kind",   "etag",   "id",   "snippet", "authorDetails"
+    }
+    assert ret["items"][0]["snippet"].keys() == {
+        'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage'
+
+    }
+    assert ret["items"][0]["authorDetails"].keys() == {
+        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator'
+    }
+    assert "LCC." in ret["items"][0]["id"]
+    assert ret["items"][0]["snippet"]["type"] == "newSponsorEvent"
+
+
 def test_superchat(mocker):
     '''api互換processorのテスト：スパチャ'''
     processor = CompatibleProcessor()
@@ -124,6 +163,7 @@ def test_superchat(mocker):
     assert "LCC." in ret["items"][0]["id"]
     assert ret["items"][0]["snippet"]["type"] == "superChatEvent"
 
+
 def test_unregistered_currency(mocker):
     processor = CompatibleProcessor()