Implement Superchat Calculator

pytchat/core_async/livechat.py

@@ -178,20 +178,20 @@ class LiveChatAsync:
                     }
                     time_mark =time.time()
                     if self._direct_mode:
-                        await self._callback(
+                        processed_chat = self.processor.process([chat_component])
-                            self.processor.process([chat_component])
+                        if isinstance(processed_chat,tuple):
-                            )
+                            await self._callback(*processed_chat)
+                        else:
+                            await self._callback(processed_chat)
                     else:
                         await self._buffer.put(chat_component)
                     diff_time = timeout - (time.time()-time_mark)
                     await asyncio.sleep(diff_time)        
                     continuation = metadata.get('continuation')     
         except ChatParseException as e:
-            #self.terminate()
             self._logger.debug(f"[{self.video_id}]{str(e)}")
             return            
         except (TypeError , json.JSONDecodeError) :
-            #self.terminate()
             self._logger.error(f"{traceback.format_exc(limit = -1)}")
             return
         
@@ -211,13 +211,13 @@ class LiveChatAsync:
         return continuation
 
     async def _get_contents(self, continuation, session, headers):
-        '''Get 'contents' dict from livechat json.
+        '''Get 'continuationContents' from livechat json.
            If contents is None at first fetching, 
            try to fetch archive chat data.
 
           Return:
           -------
-            'contents' dict which includes metadata & chatdata.
+            'continuationContents' which includes metadata & chatdata.
         '''
         livechat_json = (await 
             self._get_livechat_json(continuation, session, headers)
@@ -275,8 +275,11 @@ class LiveChatAsync:
         """
         while self.is_alive():
             items = await self._buffer.get()
-            data = self.processor.process(items)
+            processed_chat = self.processor.process(items)
-            await callback(data)
+            if isinstance(processed_chat, tuple):
+                await self._callback(*processed_chat)
+            else:
+                await self._callback(processed_chat)
 
     async def get(self):
         """ bufferからデータを取り出し、processorに投げ、

pytchat/core_multithread/livechat.py

@@ -174,9 +174,11 @@ class LiveChat:
                     }
                     time_mark =time.time()
                     if self._direct_mode:
-                        self._callback(
+                        processed_chat = self.processor.process([chat_component])
-                            self.processor.process([chat_component])
+                        if isinstance(processed_chat,tuple):
-                            )
+                            self._callback(*processed_chat)
+                        else:
+                            self._callback(processed_chat)
                     else:
                         self._buffer.put(chat_component)
                     diff_time = timeout - (time.time()-time_mark)
@@ -204,13 +206,13 @@ class LiveChat:
         return continuation
 
     def _get_contents(self, continuation, session, headers):
-        '''Get 'contents' dict from livechat json.
+        '''Get 'continuationContents' from livechat json.
            If contents is None at first fetching, 
            try to fetch archive chat data.
 
           Return:
           -------
-            'contents' dict which includes metadata & chatdata.
+            'continuationContents' which includes metadata & chat data.
         '''
         livechat_json = ( 
             self._get_livechat_json(continuation, session, headers)
@@ -268,8 +270,11 @@ class LiveChat:
         """
         while self.is_alive():
             items = self._buffer.get()
-            data = self.processor.process(items)
+            processed_chat = self.processor.process(items)
-            callback(data)
+            if isinstance(processed_chat, tuple):
+                self._callback(*processed_chat)
+            else:
+                self._callback(processed_chat)
 
     def get(self):
         """ bufferからデータを取り出し、processorに投げ、

pytchat/processors/chat_processor.py

@@ -1,23 +1,22 @@
 class ChatProcessor:
     '''
-     Listenerからチャットデータ（actions）を受け取り
+    Abstract class that processes chat data.
-    チャットデータを加工するクラスの抽象クラス
+    Receive chat data (actions) from Listener.
     '''
     def process(self, chat_components: list):
         '''
-        チャットデータの加工を表すインターフェース。
+        Interface that represents processing of chat data.
-        LiveChatオブジェクトから呼び出される。 
+        Called from LiveChat object. 
                
         Parameter
         ----------
         chat_components: List[component]
             component : dict {
                 "video_id" : str
-                    動画ID
                 "timeout"  : int
-                    次のチャットの再読み込みまでの時間(秒）
+                    Time to fetch next chat (seconds)
                 "chatdata" : List[dict]
-                    チャットデータのリスト
+                    List of chat data.
             }
         '''
         pass

pytchat/processors/default/processor.py

@@ -35,7 +35,6 @@ class DefaultProcessor(ChatProcessor):
             for component in chat_components:
                 timeout += component.get('timeout', 0)
                 chatdata = component.get('chatdata')
-             
                 if chatdata is None: continue
                 for action in chatdata:
                     if action is None: continue

pytchat/processors/superchat/calculator.py

@@ -0,0 +1,73 @@
+import re
+from pytchat.processors.chat_processor import ChatProcessor
+
+superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
+
+items_paid = [
+    'addChatItemAction',
+    'item',
+    'liveChatPaidMessageRenderer'
+]
+
+items_sticker = [
+    'addChatItemAction',
+    'item',
+    'liveChatPaidStickerRenderer'
+]
+class Calculator(ChatProcessor):
+    """
+    Calculate the amount of SuperChat by currency.
+    """
+    def __init__(self):
+        self.results = {}
+
+    def process(self, chat_components: list):
+        """
+        Return
+        ------------
+        results : dict :
+            List of amount by currency.
+            key: currency symbol, value: total amount.
+        """
+        if chat_components is None:
+            return self.results
+        for component in chat_components:
+            chatdata = component.get('chatdata')
+            if chatdata is None: continue
+            for action in chatdata:
+                renderer = self._get_item(action, items_paid) or \
+                           self._get_item(action, items_sticker)
+                if renderer is None: continue
+                symbol, amount = self._parse(renderer)
+                self.results.setdefault(symbol,0)
+                self.results[symbol]+=amount
+        return self.results
+    
+    def _parse(self, renderer):
+        purchase_amount_text = renderer["purchaseAmountText"]["simpleText"]
+        m = superchat_regex.search(purchase_amount_text)
+        if m:
+            symbol = m.group(1)
+            amount = float(m.group(2).replace(',',''))
+        else:
+            symbol = ""
+            amount = 0.0
+        return symbol, amount
+
+    def _get_item(self, dict_body, items: list):
+        for item in items:
+            if dict_body is None:
+                break
+            if isinstance(dict_body, dict):
+                dict_body = dict_body.get(item)
+                continue
+            if isinstance(item, int) and \
+               isinstance(dict_body, list) and \
+               len(dict_body) > item:
+                dict_body = dict_body[item]
+                continue
+            return None
+        return dict_body
+
+
+

pytchat/tool/mining/block.py

@@ -38,6 +38,9 @@ class Block:
     during_split : bool :
         whether this block is in the process of during_split.
         while True, this block is excluded from duplicate split procedure.
+    
+    seektime : float :
+        the last position of this block(seconds) already fetched.
     """
     
     __slots__ = ['first','last','end','continuation','chat_data','remaining',
@@ -45,7 +48,7 @@ class Block:
 
     def __init__(self, first = 0, last = 0, end = 0,
                 continuation = '', chat_data = [], is_last = False,
-                during_split = False,seektime = None):
+                during_split = False, seektime = None):
         self.first = first
         self.last = last
         self.end = end

pytchat/tool/mining/dlworker.py

@@ -34,7 +34,6 @@ class DownloadWorker:
         self.video_id:str = video_id
         self.parent_block:Block = None
 
-
     async def run(self, session):
         while self.block.continuation:
             patch = await self.fetch(
@@ -44,11 +43,3 @@ class DownloadWorker:
         self.block.done = True    
 
 
-def fd(name,mes,src,patch,end):
-    def offset(chats):
-        if len(chats)==0:
-            return None,None
-        return parser.get_offset(chats[0]),parser.get_offset(chats[-1])
-
-    with open("v://tlog.csv",encoding="utf-8",mode="a") as f:
-        f.write(f"WORKER,{name},mes,{mes},edge,{offset(src)[1]},first,{offset(patch)[0]},last,{offset(patch)[1]},end,{end}\n")

pytchat/tool/mining/parser.py

@@ -35,10 +35,7 @@ def parse(jsn):
     metadata = cont.get('liveChatReplayContinuationData')
     if metadata:
         continuation = metadata.get("continuation")
-        #print(continuation)
         actions = contents['liveChatContinuation'].get('actions')
-        # print(list(actions[0]['replayChatItemAction']["actions"][0].values()
-        #         )[0]['item'].get("liveChatPaidMessageRenderer"))
         if continuation:
             return continuation, [action["replayChatItemAction"]["actions"][0] 
                     for action in actions 

tests/test_arcparam.py

@@ -8,7 +8,6 @@ def test_arcparam_0(mocker):
     param = arcparam.getparam("01234567890",-1)
     assert param == "op2w0wRyGjxDZzhhRFFvTE1ERXlNelExTmpjNE9UQWFFLXFvM2JrQkRRb0xNREV5TXpRMU5qYzRPVEFnQVElM0QlM0QoADAAOABAAEgEUhwIABAAGAAgACoOc3RhdGljY2hlY2tzdW1AAFgDYAFoAHIECAEQAHgA" 
 
-
 def test_arcparam_1(mocker):
     param = arcparam.getparam("01234567890", seektime = 100000)
     assert param == "op2w0wR3GjxDZzhhRFFvTE1ERXlNelExTmpjNE9UQWFFLXFvM2JrQkRRb0xNREV5TXpRMU5qYzRPVEFnQVElM0QlM0QogNDbw_QCMAA4AEAASANSHAgAEAAYACAAKg5zdGF0aWNjaGVja3N1bUAAWANgAWgAcgQIARAAeAA%3D"

tests/test_calculator_get_item.py

@@ -0,0 +1,138 @@
+from pytchat.processors.superchat.calculator import Calculator
+
+get_item = Calculator()._get_item
+
+dict_test = {
+    'root':{
+        'node0' : 'value0',
+        'node1' : 'value1',
+        'node2' : {
+            'node2-0' : 'value2-0'
+        },
+    
+        'node3' : [
+            {'node3-0' : 'value3-0'},
+            {'node3-1' : 
+                {'node3-1-0' : 'value3-1-0'}
+            }
+        ],
+        'node4' : [],
+        'node5' : [
+            [
+                {'node5-1-0' : 'value5-1-0'},
+                {'node5-1-1' : 'value5-1-1'},
+            ],
+                {'node5-0' : 'value5-0'},
+            
+        ]
+    }
+}
+
+items_test0 = [
+    'root',
+    'node1'
+]
+
+
+items_test_not_found0 = [
+    'root',
+    'other_data'
+]
+
+
+items_test_nest = [
+    'root',
+    'node2',
+    'node2-0'
+]
+
+items_test_list0 = [
+    'root',
+    'node3',
+    1,
+    'node3-1'
+]
+
+items_test_list1 = [
+    'root',
+    'node3',
+    1,
+    'node3-1',
+    'node3-1-0'
+]
+
+items_test_list2 = [
+    'root',
+    'node4',
+    None
+]
+
+items_test_list3 = [
+    'root',
+    'node4'
+]
+items_test_list_nest = [
+    'root',
+    'node5',
+    0,
+    1,
+    'node5-1-1'
+]
+
+items_test_list_nest_not_found1 = [
+    'root',
+    'node5',
+    0,
+    1,
+    'node5-1-1',
+    'nodez'
+]
+
+items_test_not_found1 = [
+    'root',
+    'node3',
+    2,
+    'node3-1',
+    'node3-1-0'
+]
+
+items_test_not_found2 = [
+    'root',
+    'node3',
+    2,
+    'node3-1',
+    'node3-1-0',
+    'nodex'
+]
+def test_get_items_0():
+    assert get_item(dict_test, items_test0) == 'value1'
+
+def test_get_items_1():
+    assert get_item(dict_test, items_test_not_found0) is None
+
+def test_get_items_2():
+    assert get_item(dict_test, items_test_nest) == 'value2-0'
+
+def test_get_items_3():
+    assert get_item(dict_test, items_test_list0) ==  {'node3-1-0' : 'value3-1-0'}
+
+def test_get_items_4():
+    assert get_item(dict_test, items_test_list1) ==  'value3-1-0'
+
+def test_get_items_5():
+    assert get_item(dict_test, items_test_not_found1) ==  None
+
+def test_get_items_6():
+    assert get_item(dict_test, items_test_not_found2) ==  None
+
+def test_get_items_7():
+    assert get_item(dict_test, items_test_list2) ==  None
+
+def test_get_items_8():
+    assert get_item(dict_test, items_test_list_nest) ==  'value5-1-1'
+
+def test_get_items_9():
+    assert get_item(dict_test, items_test_list_nest_not_found1) ==  None
+
+def test_get_items_10():
+    assert get_item(dict_test, items_test_list3) ==  []

tests/test_calculator_parse.py

@@ -0,0 +1,68 @@
+import json
+from pytchat.parser.live import Parser
+from pytchat.processors.superchat.calculator import Calculator
+from pytchat.exceptions import ChatParseException
+parse = Calculator()._parse
+
+
+def _open_file(path):
+    with open(path,mode ='r',encoding = 'utf-8') as f:
+        return f.read()
+
+def load_chatdata(filepath):
+    parser = Parser(is_replay=True)
+    #print(json.loads(_open_file(filepath)))
+    contents = parser.get_contents( json.loads(_open_file(filepath)))
+    return parser.parse(contents)[1]
+
+
+
+def test_parse_1():
+    renderer ={"purchaseAmountText":{"simpleText":"￥2,000"}}
+    symbol ,amount = parse(renderer)
+    assert symbol == '￥'
+    assert amount == 2000.0
+
+def test_parse_2():
+    renderer ={"purchaseAmountText":{"simpleText":"ABC\x0a200"}}
+    symbol ,amount = parse(renderer)
+    assert symbol == 'ABC\x0a'
+    assert amount == 200.0
+
+def test_process_0():
+    """
+    parse superchat data
+    """    
+    chat_component = {
+        'video_id':'',
+        'timeout':10,
+        'chatdata':load_chatdata(r"tests\testdata\calculator\superchat_0.json")
+    }
+    assert Calculator().process([chat_component])=={'￥': 6800.0, '€': 2.0}
+
+def test_process_1():
+    """
+    parse no superchat data
+    """
+    chat_component = {
+        'video_id':'',
+        'timeout':10,
+        'chatdata':load_chatdata(r"tests\testdata\calculator\text_only.json")
+    }
+    assert Calculator().process([chat_component])=={}
+
+def test_process_2():
+    """
+    try to parse after replay end
+    """
+    try:
+        chat_component = {
+            'video_id':'',
+            'timeout':10,
+            'chatdata':load_chatdata(r"tests\testdata\calculator\replay_end.json")
+        }
+        assert False
+        Calculator().process([chat_component])
+    except ChatParseException:
+        assert True
+    

tests/testdata/calculator/replay_end.json

@@ -0,0 +1,18 @@
+{
+ "response": {
+  "responseContext": {
+   "webResponseContextExtensionData": ""
+  },
+  "continuationContents": {
+   "liveChatContinuation": {
+    "continuations": [
+     {
+      "playerSeekContinuationData": {
+       "continuation": "___reload_continuation___"
+      }
+     }
+    ]
+   }
+  }
+ }
+}

tests/testdata/calculator/text_only.json

@@ -0,0 +1,89 @@
+{
+ "response": {
+  "responseContext": {
+   "webResponseContextExtensionData": ""
+  },
+  "continuationContents": {
+   "liveChatContinuation": {
+    "continuations": [
+     {
+      "invalidationContinuationData": {
+       "invalidationId": {
+        "objectSource": 1000,
+        "objectId": "___objectId___",
+        "topic": "chat~00000000000~0000000",
+        "subscribeToGcmTopics": true,
+        "protoCreationTimestampMs": "1577804400000"
+       },
+       "timeoutMs": 10000,
+       "continuation": "___continuation___"
+      }
+     }
+    ],
+    "actions": [
+     {
+      "replayChatItemAction": {
+       "actions": [
+        {
+         "addChatItemAction": {
+          "item": {
+           "liveChatTextMessageRenderer": {
+            "message": {
+             "runs": [
+              {
+               "text": "dummy_message"
+              }
+             ]
+            },
+            "authorName": {
+             "simpleText": "author_name"
+            },
+            "authorPhoto": {
+             "thumbnails": [
+              {
+               "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+               "width": 32,
+               "height": 32
+              },
+              {
+               "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+               "width": 64,
+               "height": 64
+              }
+             ]
+            },
+            "contextMenuEndpoint": {
+             "commandMetadata": {
+              "webCommandMetadata": {
+               "ignoreNavigation": true
+              }
+             },
+             "liveChatItemContextMenuEndpoint": {
+              "params": "___params___"
+             }
+            },
+            "id": "dummy_id",
+            "timestampUsec": 0,
+            "authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url",
+            "contextMenuAccessibility": {
+             "accessibilityData": {
+              "label": "コメントの操作"
+             }
+            },
+            "timestampText": {
+             "simpleText": "0:00"
+            }
+           }
+          },
+          "clientId": "dummy_client_id"
+         }
+        }
+       ],
+       "videoOffsetTimeMsec": "10000"
+      }
+     }
+    ]
+   }
+  }
+ }
+}