Merge branch 'hotfix'

README.md

@@ -8,7 +8,7 @@ pytchat is a python library for fetching youtube live chat
 without using youtube api, Selenium or BeautifulSoup.
 
 Other features:
-+ Customizable chat data processors including yt api compatible one.
++ Customizable chat data processors including youtube api compatible one.
 + Available on asyncio context. 
 + Quick fetching of initial chat data by generating continuation params
 instead of web scraping.
@@ -27,7 +27,7 @@ pip install pytchat
 ```python
 from pytchat import LiveChat
 
-chat = LiveChat("G1w62uEMZ74")
+chat = LiveChat("rsHWP7IjMiw")
 while chat.is_alive():
   data = chat.get()
   for c in data.items:
@@ -40,17 +40,17 @@ while chat.is_alive():
 from pytchat import LiveChat
 import time
 
-def main()
-    chat = LiveChat("G1w62uEMZ74", callback = func)
-    while chat.is_alive():
-      time.sleep(3)
-      #other background operation.
-   
-#callback function is automatically called periodically.
-def func(data):
+#callback function is automatically called.
+def display(data):
   for c in data.items:
     print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}")
     data.tick()
+
+#entry point
+chat = LiveChat("rsHWP7IjMiw", callback = display)
+while chat.is_alive():
+  time.sleep(3)
+  #other background operation.
 ```
 
 ### asyncio context:
@@ -59,61 +59,57 @@ from pytchat import LiveChatAsync
 import asyncio
 
 async def main():
-    chat = LiveChatAsync("G1w62uEMZ74", callback = func)
+  chat = LiveChatAsync("rsHWP7IjMiw", callback = func)
   while chat.is_alive():
     await asyncio.sleep(3)
     #other background operation.
 
-#callback function is automatically called periodically.
+#callback function is automatically called.
 async def func(data):
   for c in data.items:
     print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}")
     await data.tick_async()
 
+try:
   loop = asyncio.get_event_loop()
   loop.run_until_complete(main())
+except CancelledError:
+  pass
 ```
 
 
 ### youtube api compatible processor:
 ```python
 from pytchat import LiveChat, CompatibleProcessor
+import time
 
-chat = LiveChat("G1w62uEMZ74", 
+chat = LiveChat("rsHWP7IjMiw", 
   processor = CompatibleProcessor() )
 
 while chat.is_alive():
   data = chat.get()
-    polling = data["pollingIntervalMillis"]/1000
-    for c in data["items"]:
-        if c.get("snippet"):
+  polling = data['pollingIntervalMillis']/1000
+  for c in data['items']:
+    if c.get('snippet'):
       print(f"[{c['authorDetails']['displayName']}]"
             f"-{c['snippet']['displayMessage']}")
-            time.sleep(polling/len(data["items"]))
+      time.sleep(polling/len(data['items']))
 
 ```
 ### replay:
 ```python
-from pytchat import ReplayChatAsync
-import asyncio
+from pytchat import ReplayChat
 
-async def main():
-    chat = ReplayChatAsync("G1w62uEMZ74", seektime = 1000, callback = func)
-        #other background operation here.
+def main():
+  #seektime (seconds): start position of chat.
+  chat = ReplayChat("ojes5ULOqhc", seektime = 60*30)
+  while chat.is_alive():
+    data = chat.get()
+    for c in data.items:
+      print(f"{c.elapsedTime} [{c.author.name}]-{c.message} {c.amountString}")
+      data.tick()
 
-#callback function is automatically called periodically.
-async def func(data):
-    for count in range(0,len(data.items)):
-        c= data.items[count]
-        if count!=len(data.items):
-            tick=data.items[count+1].timestamp -data.items[count].timestamp
-        else:
-            tick=0
-        print(f"<{c.elapsedTime}> [{c.author.name}]-{c.message} {c.amountString}")
-        await asyncio.sleep(tick/1000)
-
-loop = asyncio.get_event_loop()
-loop.run_until_complete(main())
+main()
 ```
 
 ## Structure of Default Processor
@@ -152,26 +148,26 @@ Each item can be got with items() function.
   <tr>
     <td>datetime</td>
     <td>str</td>
-    <td>ex. "2019-10-10 12:34:56"</td>
+    <td>e.g. "2019-10-10 12:34:56"</td>
   </tr>
     <td>elapsedTime</td>
     <td>str</td>
-    <td>elapsed time. (ex. "1:02:27") *Replay Only.</td>
+    <td>elapsed time. (e.g. "1:02:27") *Replay Only.</td>
   </tr>
   <tr>
     <td>amountValue</td>
     <td>float</td>
-    <td>ex. 1,234.0</td>
+    <td>e.g. 1,234.0</td>
   </tr>
   <tr>
     <td>amountString</td>
     <td>str</td>
-    <td>ex. "$ 1,234"</td>
+    <td>e.g. "$ 1,234"</td>
   </tr>
   <tr>
     <td>currency</td>
     <td>str</td>
-    <td><a href="https://en.wikipedia.org/wiki/ISO_4217">ISO 4217 currency codes</a> (ex. "USD")</td>
+    <td><a href="https://en.wikipedia.org/wiki/ISO_4217">ISO 4217 currency codes</a> (e.g. "USD")</td>
   </tr>
   <tr>
     <td>bgColor</td>
@@ -200,7 +196,7 @@ Structure of author object.
   <tr>
     <td>channelId</td>
     <td>str</td>
-    <td>*chatter's channel ID. NOT broadcasting video's channel ID.</td>
+    <td>*chatter's channel ID.</td>
   </tr>
   <tr>
     <td>channelUrl</td>

pytchat/__init__.py

@@ -2,7 +2,7 @@
 pytchat is a python library for fetching youtube live chat without using yt api, Selenium, or BeautifulSoup.
 """
 __copyright__    = 'Copyright (C) 2019 taizan-hokuto'
-__version__      = '0.0.3.5'
+__version__      = '0.0.4.1'
 __license__      = 'MIT'
 __author__       = 'taizan-hokuto'
 __author_email__ = '55448286+taizan-hokuto@users.noreply.github.com'
@@ -11,12 +11,16 @@ __url__          = 'https://github.com/taizan-hokuto/pytchat'
 __all__ = ["core_async","core_multithread","processors"]
 
 from .api import (
+    config,
     LiveChat,
     LiveChatAsync,
     ReplayChat,
     ReplayChatAsync,
     ChatProcessor,
     CompatibleProcessor,
+    DefaultProcessor, 
     SimpleDisplayProcessor,
-    JsonfileArchiveProcessor
+    JsonfileArchiveProcessor,
+    SpeedCalculator,
+    DummyProcessor
 )

pytchat/api.py

@@ -7,4 +7,6 @@ from .processors.default.processor import DefaultProcessor
 from .processors.compatible.processor import CompatibleProcessor
 from .processors.simple_display_processor import SimpleDisplayProcessor
 from .processors.jsonfile_archive_processor import JsonfileArchiveProcessor
-
+from .processors.speed_calculator import SpeedCalculator
+from .processors.dummy_processor import DummyProcessor
+from . import config

pytchat/config/__init__.py

@@ -1,4 +1,13 @@
 import logging
-LOGGER_MODE = logging.ERROR
+from . import mylogger
+
+LOGGER_MODE = None
+
 headers = {
         'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'}
+
+def logger(module_name: str):
+    module_logger = mylogger.get_logger(module_name, mode = LOGGER_MODE)
+    return module_logger
+
+

pytchat/config/mylogger.py

@@ -0,0 +1,32 @@
+from logging import NullHandler, getLogger, StreamHandler, FileHandler, Formatter
+import logging
+import datetime
+
+
+def get_logger(modname,mode=logging.DEBUG):
+    logger = getLogger(modname)
+    if mode == None:
+        logger.addHandler(NullHandler())
+        return logger
+    logger.setLevel(mode)
+    #create handler1 for showing info
+    handler1 = StreamHandler()
+    my_formatter  = MyFormatter()
+    handler1.setFormatter(my_formatter)
+
+    handler1.setLevel(mode) 
+    logger.addHandler(handler1)
+    #create handler2 for recording log file
+    if mode <= logging.DEBUG:
+        handler2 = FileHandler(filename="log.txt", encoding='utf-8')
+        handler2.setLevel(logging.ERROR)
+        handler2.setFormatter(my_formatter)
+
+
+        logger.addHandler(handler2)
+    return logger
+
+class MyFormatter(logging.Formatter):
+    def format(self, record):
+        s =(datetime.datetime.fromtimestamp(record.created)).strftime("%m-%d %H:%M:%S")+'| '+ (record.module).ljust(15)+(' { '+record.funcName).ljust(20) +":"+str(record.lineno).rjust(4)+'} - '+record.getMessage()
+        return s

pytchat/core_async/livechat.py

@@ -11,15 +11,14 @@ from concurrent.futures import CancelledError
 from .buffer import Buffer
 from ..parser.live import Parser
 from .. import config
-from .. import mylogger
 from ..exceptions  import ChatParseException,IllegalFunctionCall
 from ..paramgen    import liveparam
 from ..processors.default.processor import DefaultProcessor
+from ..processors.combinator import Combinator
 
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
-MAX_RETRY = 10
+logger = config.logger(__name__)
 headers = config.headers
-
+MAX_RETRY = 10
 
 
 class LiveChatAsync:
@@ -72,6 +71,9 @@ class LiveChatAsync:
                 exception_handler = None,
                 direct_mode = False): 
         self.video_id  = video_id
+        if isinstance(processor, tuple):
+            self.processor = Combinator(processor)
+        else:
             self.processor = processor
         self._buffer = buffer
         self._callback = callback
@@ -148,7 +150,7 @@ class LiveChatAsync:
 
     async def _listen(self, continuation):
         ''' continuationに紐付いたチャットデータを取得し
-        チャットデータを格納、
+        Bufferにチャットデータを格納、
         次のcontinuaitonを取得してループする。
 
         Parameter
@@ -180,9 +182,11 @@ class LiveChatAsync:
                     await asyncio.sleep(diff_time)        
                     continuation = metadata.get('continuation')     
         except ChatParseException as e:
-            logger.info(f"{str(e)}（video_id:\"{self.video_id}\"）")
+            self.terminate()
+            logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）")
             return            
         except (TypeError , json.JSONDecodeError) :
+            self.terminate()
             logger.error(f"{traceback.format_exc(limit = -1)}")
             return
         
@@ -211,6 +215,7 @@ class LiveChatAsync:
         else:
             logger.error(f"[{self.video_id}]"
                     f"Exceeded retry count. status_code={status_code}")
+            self.terminate()
             return None
         return livechat_json
 

pytchat/core_async/replaychat.py

@@ -12,12 +12,12 @@ from queue import Queue
 from .buffer import Buffer
 from ..parser.replay import Parser
 from .. import config
-from .. import mylogger
 from ..exceptions  import ChatParseException,IllegalFunctionCall
 from ..paramgen    import arcparam
 from ..processors.default.processor import DefaultProcessor
+from ..processors.combinator import Combinator
 
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
+logger = config.logger(__name__)
 MAX_RETRY = 10
 headers = config.headers
 
@@ -78,6 +78,9 @@ class ReplayChatAsync:
                 direct_mode = False):
         self.video_id  = video_id
         self.seektime = seektime
+        if isinstance(processor, tuple):
+            self.processor = Combinator(processor)
+        else:
             self.processor = processor
         self._buffer = buffer
         self._callback = callback
@@ -194,13 +197,15 @@ class ReplayChatAsync:
                     await asyncio.sleep(diff_time)       
                     continuation = metadata.get('continuation')  
         except ChatParseException as e:
-            logger.info(f"{str(e)}（video_id:\"{self.video_id}\"）")
+            logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）")
             return            
         except (TypeError , json.JSONDecodeError) :
             logger.error(f"{traceback.format_exc(limit = -1)}")
+            self.terminate()
             return
         
         logger.debug(f"[{self.video_id}]チャット取得を終了しました。")
+        self.terminate()
 
     async def _get_livechat_json(self, continuation, session, headers):
         '''
@@ -282,7 +287,7 @@ class ReplayChatAsync:
         if self._direct_mode == False:
             #bufferにダミーオブジェクトを入れてis_alive()を判定させる
             self._buffer.put_nowait({'chatdata':'','timeout':1}) 
-        logger.info(f'終了しました:[{self.video_id}]')
+        logger.info(f'[{self.video_id}]終了しました')
   
     @classmethod
     def _set_exception_handler(cls, handler):

pytchat/core_multithread/livechat.py

@@ -10,15 +10,14 @@ from concurrent.futures import CancelledError, ThreadPoolExecutor
 from .buffer import Buffer
 from ..parser.live import Parser
 from .. import config
-from .. import mylogger
 from ..exceptions  import ChatParseException,IllegalFunctionCall
 from ..paramgen    import liveparam
 from ..processors.default.processor import DefaultProcessor
+from ..processors.combinator import Combinator
 
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
-MAX_RETRY = 10
+logger = config.logger(__name__)
 headers = config.headers
-
+MAX_RETRY = 10
 
 
 class LiveChat:
@@ -65,13 +64,16 @@ class LiveChat:
     _listeners= []
     def __init__(self, video_id,
                 processor = DefaultProcessor(),
-                buffer = Buffer(maxsize = 20),
+                buffer = None,
                 interruptable = True,
                 callback = None,
                 done_callback = None,
                 direct_mode = False
                 ):
         self.video_id  = video_id
+        if isinstance(processor, tuple):
+            self.processor = Combinator(processor)
+        else:
             self.processor = processor
         self._buffer = buffer
         self._callback = callback
@@ -142,8 +144,8 @@ class LiveChat:
 
     def _listen(self, continuation):
         ''' continuationに紐付いたチャットデータを取得し
-        BUfferにチャットデータを格納、
-        次のcontinuaitonを取得してループする
+        Bufferにチャットデータを格納、
+        次のcontinuaitonを取得してループする。
 
         Parameter
         ---------
@@ -175,9 +177,11 @@ class LiveChat:
                     time.sleep(diff_time)        
                     continuation = metadata.get('continuation')  
         except ChatParseException as e:
-            logger.info(f"{str(e)}（video_id:\"{self.video_id}\"）")
+            self.terminate()
+            logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）")
             return            
         except (TypeError , json.JSONDecodeError) :
+            self.terminate()
             logger.error(f"{traceback.format_exc(limit = -1)}")
             return
         
@@ -206,6 +210,7 @@ class LiveChat:
         else:
             logger.error(f"[{self.video_id}]"
                     f"Exceeded retry count. status_code={status_code}")
+            self.terminate()
             return None
         return livechat_json
   
@@ -254,18 +259,10 @@ class LiveChat:
         if self._direct_mode == False:
             #bufferにダミーオブジェクトを入れてis_alive()を判定させる
             self._buffer.put({'chatdata':'','timeout':1}) 
-        logger.info(f'終了しました:[{self.video_id}]')
+        logger.info(f'[{self.video_id}]終了しました')
   
     @classmethod
     def shutdown(cls, event, sig = None, handler=None):
         logger.debug("シャットダウンしています")
         for t in LiveChat._listeners:
             t._is_alive = False
-
-
-
-    
-
-
-
-

pytchat/core_multithread/replaychat.py

@@ -11,15 +11,14 @@ from queue import Queue
 from .buffer import Buffer
 from ..parser.replay import Parser
 from .. import config
-from .. import mylogger
 from ..exceptions  import ChatParseException,IllegalFunctionCall
 from ..paramgen    import arcparam
 from ..processors.default.processor import DefaultProcessor
+from ..processors.combinator import Combinator
 
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
-MAX_RETRY = 10
+logger = config.logger(__name__)
 headers = config.headers
-
+MAX_RETRY = 10
 
 
 class ReplayChat:
@@ -61,7 +60,7 @@ class ReplayChat:
         チャットデータ取得ループ（_listen）用のスレッド
 
     _is_alive : bool
-        チャット取得を終了したか
+        チャット取得を停止するためのフラグ
     '''
 
     _setup_finished = False
@@ -70,7 +69,7 @@ class ReplayChat:
     def __init__(self, video_id,
                 seektime = 0,
                 processor = DefaultProcessor(),
-                buffer = Buffer(maxsize = 20),
+                buffer = None,
                 interruptable = True,
                 callback = None,
                 done_callback = None,
@@ -78,6 +77,9 @@ class ReplayChat:
                 ):
         self.video_id  = video_id
         self.seektime = seektime
+        if isinstance(processor, tuple):
+            self.processor = Combinator(processor)
+        else:
             self.processor = processor
         self._buffer = buffer
         self._callback = callback
@@ -90,6 +92,7 @@ class ReplayChat:
         self._pauser.put_nowait(None)
        
         self._setup()
+
         if not ReplayChat._setup_finished:
             ReplayChat._setup_finished = True
             if interruptable:
@@ -150,7 +153,7 @@ class ReplayChat:
 
     def _listen(self, continuation):
         ''' continuationに紐付いたチャットデータを取得し
-        にチャットデータを格納、
+        BUfferにチャットデータを格納、
         次のcontinuaitonを取得してループする
 
         Parameter
@@ -189,9 +192,11 @@ class ReplayChat:
                     time.sleep(diff_time)        
                     continuation = metadata.get('continuation')  
         except ChatParseException as e:
-            logger.error(f"{str(e)}（動画ID:\"{self.video_id}\"）")
+            self.terminate()
+            logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）")
             return            
         except (TypeError , json.JSONDecodeError) :
+            self.terminate()
             logger.error(f"{traceback.format_exc(limit = -1)}")
             return
         
@@ -220,6 +225,7 @@ class ReplayChat:
         else:
             logger.error(f"[{self.video_id}]"
                     f"Exceeded retry count. status_code={status_code}")
+            self.terminate()
             return None
         return livechat_json
   
@@ -266,7 +272,7 @@ class ReplayChat:
         '''Listener終了時のコールバック'''
         try: 
             self.terminate()
-        except CancelledError:
+        except RuntimeError:
             logger.debug(f'[{self.video_id}]cancelled:{sender}')
 
     def terminate(self):
@@ -277,7 +283,7 @@ class ReplayChat:
         if self._direct_mode == False:
             #bufferにダミーオブジェクトを入れてis_alive()を判定させる
             self._buffer.put({'chatdata':'','timeout':1}) 
-        logger.info(f'終了しました:[{self.video_id}]')
+        logger.info(f'[{self.video_id}]終了しました')
   
     @classmethod
     def shutdown(cls, event, sig = None, handler=None):

pytchat/parser/live.py

@@ -6,14 +6,13 @@ This module is parser of live chat JSON.
 
 import json
 from .. import config
-from .. import mylogger
 from .. exceptions import ( 
     ResponseContextError, 
     NoContentsException, 
     NoContinuationsException )
 
 
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
+logger = config.logger(__name__)
 
 
 class Parser:
@@ -59,7 +58,7 @@ class Parser:
         if metadata is None:
             unknown = list(cont.keys())[0]
             if unknown:
-                logger.error(f"Received unknown continuation type:{unknown}")
+                logger.debug(f"Received unknown continuation type:{unknown}")
                 metadata = cont.get(unknown)
         metadata.setdefault('timeoutMs', 10000)
         chatdata = contents['liveChatContinuation'].get('actions')

pytchat/parser/replay.py

@@ -1,14 +1,12 @@
 import json
 from .. import config
-from .. import mylogger
 from .. exceptions import ( 
     ResponseContextError, 
     NoContentsException, 
     NoContinuationsException )
 
 
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
-
+logger = config.logger(__name__)
 
 class Parser:
     def parse(self, jsn):
@@ -53,12 +51,13 @@ class Parser:
         metadata = cont.get('liveChatReplayContinuationData')
         if metadata is None:
             unknown = list(cont.keys())[0]
-            if unknown:
+            if unknown != "playerSeekContinuationData":
+                logger.debug(f"Received unknown continuation type:{unknown}")
                 metadata = cont.get(unknown)
-        
         actions = contents['liveChatContinuation'].get('actions')
         if actions is None:
-            raise NoContentsException('チャットデータを取得できませんでした。')
+            #後続のチャットデータなし
+            return {"continuation":None,"timeout":0,"chatdata":[]}
         interval = self.get_interval(actions)
         metadata.setdefault("timeoutMs",interval)
         """アーカイブ済みチャットはライブチャットと構造が異なっているため、以下の行により

pytchat/processors/chat_processor.py

@@ -10,14 +10,14 @@ class ChatProcessor:
                
         Parameter
         ----------
-        chat_components: [LIST:component]
+        chat_components: List[component]
             component : dict {
                 "video_id" : str
                     動画ID
                 "timeout"  : int
                     次のチャットの再読み込みまでの時間(秒）
-                "chatdata" : list<object>
-                    チャットデータ（actions）のリスト
+                "chatdata" : List[dict]
+                    チャットデータのリスト
             }
         '''
         pass

pytchat/processors/combinator.py

@@ -0,0 +1,39 @@
+from .chat_processor import ChatProcessor
+
+class Combinator(ChatProcessor):
+    '''
+    Combinator combines multiple chat processors.
+    Specify processors as tuple at `processor` params of LiveChat object.
+
+    For example:
+        [constructor]
+        chat = LiveChat("video_id", processor = ( Processor1(), Processor2(), Processor3() )
+        
+        [receive return values]
+        ret1, ret2, ret3 = chat.get()
+    
+    The return values are tuple of processed chat data, 
+    the order of return depends on parameter order.
+
+    Parameter
+    ---------
+    processors : Tuple[ChatProcessor]
+        multiple processors for processing chat data
+    '''
+
+    def __init__(self, processors: tuple):
+        self.processors = processors
+
+    def process(self, chat_components: list):
+        '''
+        Called from LiveChat.get() function by user,
+        or LiveChat._listen() automatically.
+
+        Returns
+        -------
+            Tuple of chat data processed by each chat processor.
+        '''
+        return tuple([processor.process(chat_components)
+            for processor in self.processors])
+            
+

pytchat/processors/compatible/processor.py

@@ -4,11 +4,11 @@ from .renderer.textmessage import LiveChatTextMessageRenderer
 from .renderer.paidmessage import LiveChatPaidMessageRenderer
 from .renderer.paidsticker import LiveChatPaidStickerRenderer
 from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
-from ... import mylogger
+from .. chat_processor import ChatProcessor
 from ... import config
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
+logger = config.logger(__name__)
 
-class CompatibleProcessor:
+class CompatibleProcessor(ChatProcessor):
 
     def process(self, chat_components: list):
 

pytchat/processors/default/processor.py

@@ -4,9 +4,9 @@ from .renderer.textmessage import LiveChatTextMessageRenderer
 from .renderer.paidmessage import LiveChatPaidMessageRenderer
 from .renderer.paidsticker import LiveChatPaidStickerRenderer
 from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
+from .. chat_processor import ChatProcessor
 from ... import config
-from ... import mylogger
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
+logger = config.logger(__name__)
 
 class Chatdata:
     def __init__(self,chatlist:list, timeout:float):
@@ -25,7 +25,7 @@ class Chatdata:
             return
         await asyncio.sleep(self.interval/len(self.items))
 
-class DefaultProcessor:
+class DefaultProcessor(ChatProcessor):
     def process(self, chat_components: list):
 
         chatlist = []

pytchat/processors/default/renderer/base.py

@@ -19,8 +19,7 @@ class BaseRenderer:
         else:
             self.elapsedTime = ""
         self.datetime = self.get_datetime(timestampUsec)
-        self.message = self.get_message(self.renderer)
-        self.messageEx = self.get_message_ex(self.renderer)
+        self.message ,self.messageEx = self.get_message(self.renderer)
         self.id =  self.renderer.get('id')
         self.amountValue= 0.0
         self.amountString = ""
@@ -44,6 +43,7 @@ class BaseRenderer:
 
     def get_message(self,renderer):
         message = ''
+        message_ex = []
         if renderer.get("message"):
             runs=renderer["message"].get("runs")
             if runs:
@@ -51,22 +51,13 @@ class BaseRenderer:
                     if r:
                         if r.get('emoji'):
                             message += r['emoji'].get('shortcuts',[''])[0]
+                            message_ex.append(r['emoji']['image']['thumbnails'][1].get('url'))
                         else:
                             message += r.get('text','')
-        return message
+                            message_ex.append(r.get('text',''))
+        return message, message_ex
+
 
-    def get_message_ex(self,renderer):
-        message = []
-        if renderer.get("message"):
-            runs=renderer["message"].get("runs")
-            if runs:
-                for r in runs:
-                    if r:
-                        if r.get('emoji'):
-                            message.append(r['emoji']['image']['thumbnails'][1].get('url'))
-                        else:
-                            message.append(r.get('text',''))
-        return message
 
     def get_badges(self,renderer):
         isVerified = False

pytchat/processors/dummy_processor.py

@@ -0,0 +1,8 @@
+from .chat_processor import ChatProcessor
+
+class DummyProcessor(ChatProcessor):
+    '''
+    Dummy processor just returns received chat_components directly.
+    '''
+    def process(self, chat_components: list):
+        return chat_components

pytchat/processors/speed_calculator.py

@@ -4,7 +4,7 @@ speedmeter.py
 Calculate speed of chat.
 """
 import calendar, datetime, pytz
-
+from .chat_processor import ChatProcessor
 class RingQueue:
     """
     リング型キュー
@@ -21,7 +21,7 @@ class RingQueue:
         キュー内に余裕があるか。キュー内のアイテム個数が、キューの最大個数未満であればTrue。
     """
 
-    def __init__(self, capacity = 10):    
+    def __init__(self, capacity):    
         """
         コンストラクタ
         
@@ -77,34 +77,39 @@ class RingQueue:
     def item_count(self):
         return len(self.items)
         
-class SpeedCalculator(RingQueue):
+class SpeedCalculator(ChatProcessor, RingQueue):
     """
-    チャットの勢いを計算するクラス
+    チャットの勢いを計算する。
+    
+    一定期間のチャットデータのうち、最初のチャットの投稿時刻と
+    最後のチャットの投稿時刻の差を、チャット数で割り返し
+    1分あたりの速度に換算する。
+
     Parameter
     ----------
-        格納するチャットブロックの数
+    capacity : int
+        RingQueueに格納するチャット勢い算出用データの最大数
     """
 
-    def __init__(self, capacity, video_id):
+    def __init__(self, capacity = 10):
         super().__init__(capacity)
-        self.video_id=video_id
         self.speed = 0
 
     def process(self, chat_components: list):
+        chatdata = []
         if chat_components:
             for component in chat_components:
+                if component.get("chatdata"):
+                    chatdata.extend(component.get("chatdata"))
 
-                chatdata = component.get('chatdata')
-             
-                if chatdata is None:
-                    return self.speed
-                self.speed = self.calc(chatdata)
+            self._put_chatdata(chatdata)
+            self.speed = self._calc_speed()
         return self.speed
                 
-    def _value(self):
 
+    def _calc_speed(self):
         """
-        ActionsQueue内のチャットデータリストから、
+        RingQueue内のチャット勢い算出用データリストを元に、
         チャット速度を計算して返す
 
         Return
@@ -112,7 +117,7 @@ class SpeedCalculator(RingQueue):
             チャット速度（１分間で換算したチャット数）
         """
         try:        
-            #キュー内のactionsの総チャット数
+            #キュー内の総チャット数
             total = sum(item['chat_count'] for item in self.items)
             #キュー内の最初と最後のチャットの時間差
             duration = (self.items[self.last_pos]['endtime'] 
@@ -123,24 +128,20 @@ class SpeedCalculator(RingQueue):
         except IndexError:
             return 0
 
-    def _get_timestamp(self, action :dict):
+    def _put_chatdata(self, actions):
         """
-        チャットデータのtimestampUsecを読み取る
-        liveChatTickerSponsorItemRenderer等のtickerデータは時刻格納位置が
-        異なるため、時刻データなしとして扱う
+        チャットデータからタイムスタンプを読み取り、勢い測定用のデータを組み立て、
+        RingQueueに投入する。
+        200円以上のスパチャはtickerとmessageの2つのデータが生成されるが、
+        tickerの方は時刻データの場所が異なることを利用し、勢いの集計から除外している。
+        Parameter
+        ---------
+        actions : List[dict]
+            チャットデータ(addChatItemAction) のリスト
         """
-        try:
-            item = action['addChatItemAction']['item']
-            timestamp = int(item[list(item.keys())[0]]['timestampUsec'])
-        except (KeyError,TypeError):
-            return None
-        return timestamp
-
-    def calc(self,actions):
-
-        def empty_data():
+        def _put_emptydata():
             '''
-            データがない場合にゼロのデータをリングキューに入れる
+            チャットデータがない場合に空のデータをキューに投入する。
             '''
             timestamp_now =  calendar.timegm(datetime.datetime.
             now(pytz.utc).utctimetuple())
@@ -149,12 +150,23 @@ class SpeedCalculator(RingQueue):
                 'starttime':int(timestamp_now),
                 'endtime':int(timestamp_now)
             })
-            return self._value()
+
+        def _get_timestamp(action :dict):
+            """
+            チャットデータから時刻データを取り出す。
+            """
+            try:
+                item = action['addChatItemAction']['item']
+                timestamp = int(item[list(item.keys())[0]]['timestampUsec'])
+            except (KeyError,TypeError):
+                return None
+            return timestamp
 
         if actions is None or len(actions)==0:
-            return empty_data
+            _put_emptydata()
+            return 
            
-        #actions内の時刻データを持つチャットデータの数（tickerは除く）
+        #actions内の時刻データを持つチャットデータの数
         counter=0        
         #actions内の最初のチャットデータの時刻
         starttime= None
@@ -163,7 +175,7 @@ class SpeedCalculator(RingQueue):
         
         for action in actions:
             #チャットデータからtimestampUsecを読み取る
-            gettime = self._get_timestamp(action)
+            gettime = _get_timestamp(action)
             
             #時刻のないデータだった場合は次の行のデータで読み取り試行
             if gettime is None:
@@ -179,9 +191,10 @@ class SpeedCalculator(RingQueue):
             #チャットの数をインクリメント
             counter += 1
 
-        #チャット速度用のデータをリングキューに送る
+        #チャット速度用のデータをRingQueueに送る
         if starttime is None or endtime is None:
-            return empty_data
+            _put_emptydata()
+            return 
             
         self.put({
             'chat_count':counter,
@@ -189,4 +202,3 @@ class SpeedCalculator(RingQueue):
             'endtime':int(endtime/1000000)
         })
 
-        return self._value()

pytchat/util/__init__.py

@@ -9,7 +9,7 @@ def download(url):
         json.dump(html.json(),f,ensure_ascii=False)
 
 
-def save(data,filename):
-    with open(str(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')
-                            )+filename,mode ='w',encoding='utf-8') as f:
+def save(data,filename,extention):
+    with open(filename+"_"+(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')
+                            )+extention,mode ='w',encoding='utf-8') as f:
         f.writelines(data)

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages, Command
-from codecs import open
-from os import path, system
+#from codecs import open as open_c
+from os import path, system, remove, rename
 import re
 
 package_name = "pytchat"
@@ -28,6 +28,16 @@ assert author
 assert author_email
 assert url
 
+
+
+with open('README.MD', 'r', encoding='utf-8') as f:
+    txt = f.read()
+
+with open('README1.MD', 'w', encoding='utf-8', newline='\n') as f:
+    f.write(txt)
+    
+remove("README.MD")
+rename("README1.MD","README.MD")
 with open('README.md', encoding='utf-8') as f:
     long_description = f.read()
 

tests/test_speed_calculator.py

@@ -0,0 +1,68 @@
+import json
+import pytest
+import asyncio,aiohttp
+from pytchat.parser.live import Parser
+from pytchat.processors.compatible.processor import CompatibleProcessor
+from pytchat.exceptions import (
+    NoLivechatRendererException,NoYtinitialdataException,
+    ResponseContextError, NoContentsException)
+
+from pytchat.processors.speed_calculator import SpeedCalculator
+
+parser = Parser()
+
+def test_speed_1(mocker):
+    '''test speed calculation with normal json.
+    test json has 15 chatdata, duration is 30 seconds,
+    so the speed of chatdata is 30 chats/minute.
+    '''
+
+    processor = SpeedCalculator(capacity=30)
+
+    _json = _open_file("tests/testdata/speed/speedtest_normal.json")
+
+    _, chatdata = parser.parse(json.loads(_json))
+    data = {
+        "video_id" : "",
+        "timeout" : 10,
+        "chatdata" : chatdata
+    }
+    ret = processor.process([data])
+    assert 30 == ret
+
+def test_speed_2(mocker):
+    '''test speed calculation with no valid chat data.
+    '''
+    processor = SpeedCalculator(capacity=30)
+
+    _json = _open_file("tests/testdata/speed/speedtest_undefined.json")
+
+    _, chatdata = parser.parse(json.loads(_json))
+    data = {
+        "video_id" : "",
+        "timeout" : 10,
+        "chatdata" : chatdata
+    }
+    ret = processor.process([data])
+    assert 0 == ret
+    
+def test_speed_3(mocker):
+    '''test speed calculation with empty data.
+    '''
+    processor = SpeedCalculator(capacity=30)
+
+    _json = _open_file("tests/testdata/speed/speedtest_empty.json")
+
+    _, chatdata = parser.parse(json.loads(_json))
+    data = {
+        "video_id" : "",
+        "timeout" : 10,
+        "chatdata" : chatdata
+    }
+    ret = processor.process([data])
+    assert 0 == ret
+    
+
+def _open_file(path):
+    with open(path,mode ='r',encoding = 'utf-8') as f:
+        return f.read()

tests/testdata/chat.json

@@ -0,0 +1,164 @@
+{
+  "timing": {
+    "info": {
+      "st": 164
+    }
+  },
+  "csn": "",
+  "response": {
+    "responseContext": {
+      "serviceTrackingParams": [{
+        "service": "CSI",
+        "params": [{
+          "key": "GetLiveChat_rid",
+          "value": ""
+        }, {
+          "key": "c",
+          "value": "WEB"
+        }, {
+          "key": "cver",
+          "value": "2.20191219.03.01"
+        }, {
+          "key": "yt_li",
+          "value": "0"
+        }]
+      }, {
+        "service": "GFEEDBACK",
+        "params": [{
+          "key": "e",
+          "value": ""
+        }, {
+          "key": "logged_in",
+          "value": "0"
+        }]
+      }, {
+        "service": "GUIDED_HELP",
+        "params": [{
+          "key": "logged_in",
+          "value": "0"
+        }]
+      }, {
+        "service": "ECATCHER",
+        "params": [{
+          "key": "client.name",
+          "value": "WEB"
+        }, {
+          "key": "client.version",
+          "value": "2.2"
+        }, {
+          "key": "innertube.build.changelist",
+          "value": "228"
+        }, {
+          "key": "innertube.build.experiments.source_version",
+          "value": "2858"
+        }, {
+          "key": "innertube.build.label",
+          "value": "youtube.ytfe.innertube_"
+        }, {
+          "key": "innertube.build.timestamp",
+          "value": "154"
+        }, {
+          "key": "innertube.build.variants.checksum",
+          "value": "e"
+        }, {
+          "key": "innertube.run.job",
+          "value": "ytfe-innertube-replica-only.ytfe"
+        }]
+      }],
+      "webResponseContextExtensionData": {
+        "ytConfigData": {
+          "csn": "ADw",
+          "visitorData": "%3D%3D"
+        }
+      }
+    },
+    "continuationContents": {
+      "liveChatContinuation": {
+        "continuations": [{
+          "timedContinuationData": {
+            "timeoutMs": 10000,
+            "continuation": "continuation"
+          }
+        }],
+        "actions": [{
+          "addChatItemAction": {
+            "item": {
+              "liveChatTextMessageRenderer": {
+                "message": {
+                  "runs": [{
+                    "text": "message"
+                  }]
+                },
+                "authorName": {
+                  "simpleText": "authorName"
+                },
+                "authorPhoto": {
+                  "thumbnails": [{
+                    "url": "https://yt3.ggpht.com/photo.jpg",
+                    "width": 32,
+                    "height": 32
+                  }, {
+                    "url": "https://yt3.ggpht.com/photo.jpg",
+                    "width": 64,
+                    "height": 64
+                  }]
+                },
+                "contextMenuEndpoint": {
+                  "commandMetadata": {
+                    "webCommandMetadata": {
+                      "ignoreNavigation": true
+                    }
+                  },
+                  "liveChatItemContextMenuEndpoint": {
+                    "params": "params"
+                  }
+                },
+                "id": "id",
+                "timestampUsec": "1576851922945411",
+                "authorBadges": [{
+                  "liveChatAuthorBadgeRenderer": {
+                    "customThumbnail": {
+                      "thumbnails": [{
+                        "url": "https://yt3.ggpht.com/photo.jpg"
+                      }, {
+                        "url": "https://yt3.ggpht.com/photo.jpg"
+                      }]
+                    },
+                    "tooltip": "メンバー（6 か月）",
+                    "accessibility": {
+                      "accessibilityData": {
+                        "label": "メンバー（6 か月）"
+                      }
+                    }
+                  }
+                }],
+                "authorExternalChannelId": "UC",
+                "contextMenuAccessibility": {
+                  "accessibilityData": {
+                    "label": "コメントの操作"
+                  }
+                }
+              }
+            },
+            "clientId": "00000000000000000000"
+          }
+        }
+      ]}
+    },
+        
+    "xsrf_token": "xsrf_token",
+    "url": "/live_chat/get_live_chat?continuation=0",
+    "endpoint": {
+      "commandMetadata": {
+        "webCommandMetadata": {
+          "url": "/live_chat/get_live_chat?continuation=0",
+          "rootVe": 0
+        }
+      },
+      "urlEndpoint": {
+        "url": "/live_chat/get_live_chat?continuation=0"
+      }
+    }
+  }
+}
+    

tests/testdata/speed/speedtest_empty.json

@@ -0,0 +1,24 @@
+{
+  "timing": {
+    "info": {
+      "st": 164
+    }
+  },
+  "csn": "",
+  "response": {
+    "responseContext": {
+    },
+    "continuationContents": {
+      "liveChatContinuation": {
+        "continuations": [{
+          "timedContinuationData": {
+            "timeoutMs": 10000,
+            "continuation": "continuation"
+          }
+        }]
+      }
+    }
+  }
+}
+    
+

tests/testdata/speed/speedtest_normal.json

@@ -0,0 +1,188 @@
+{
+  "timing": {
+    "info": {
+      "st": 164
+    }
+  },
+  "csn": "",
+  "response": {
+    "responseContext": {
+    },
+    "continuationContents": {
+      "liveChatContinuation": {
+        "continuations": [{
+          "timedContinuationData": {
+            "timeoutMs": 10000,
+            "continuation": "continuation"
+          }
+        }],
+        "actions": [{
+          "addChatItemAction": {
+            "item": {
+              "liveChatTextMessageRenderer": {
+                
+                "timestampUsec": "1500000000000000"
+
+              }
+            }
+          }
+        },
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatTextMessageRenderer": {
+                
+                "timestampUsec": "1500000000000000"
+
+              }
+            }
+          }
+        },
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatTextMessageRenderer": {
+                
+                "timestampUsec": "1500000000000000"
+
+              }
+            }
+          }
+        },
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatTextMessageRenderer": {
+                
+                "timestampUsec": "1500000000000000"
+
+              }
+            }
+          }
+        },
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatTextMessageRenderer": {
+                
+                "timestampUsec": "1500000000000000"
+
+              }
+            }
+          }
+        },
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatTextMessageRenderer": {
+                
+                "timestampUsec": "1500000000000000"
+
+              }
+            }
+          }
+        },
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatTextMessageRenderer": {
+                
+                "timestampUsec": "1500000000000000"
+
+              }
+            }
+          }
+        },
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatTextMessageRenderer": {
+                
+                "timestampUsec": "1500000000000000"
+
+              }
+            }
+          }
+        },
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatTextMessageRenderer": {
+                
+                "timestampUsec": "1500000000000000"
+
+              }
+            }
+          }
+        },
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatTextMessageRenderer": {
+                
+                "timestampUsec": "1500000000000000"
+
+              }
+            }
+          }
+        },
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatTextMessageRenderer": {
+                
+                "timestampUsec": "1500000000000000"
+
+              }
+            }
+          }
+        },
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatTextMessageRenderer": {
+                
+                "timestampUsec": "1500000000000000"
+
+              }
+            }
+          }
+        },
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatTextMessageRenderer": {
+                
+                "timestampUsec": "1500000000000000"
+
+              }
+            }
+          }
+        },
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatTextMessageRenderer": {
+                
+                "timestampUsec": "1500000000000000"
+
+              }
+            }
+          }
+        },
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatTextMessageRenderer": {
+                
+                "timestampUsec": "1500000030000000"
+
+              }
+            }
+          }
+        }
+      ]}
+    }
+  }
+}
+    

tests/testdata/speed/speedtest_undefined.json

@@ -0,0 +1,42 @@
+{
+  "timing": {
+    "info": {
+      "st": 164
+    }
+  },
+  "csn": "",
+  "response": {
+    "responseContext": {
+    },
+    "continuationContents": {
+      "liveChatContinuation": {
+        "continuations": [{
+          "timedContinuationData": {
+            "timeoutMs": 10000,
+            "continuation": "continuation"
+          }
+        }],
+        "actions": [{
+          "addChatItemAction": {
+            "liveChatPlaceholderItemRenderer": {
+              "id": "",
+              "timestampUsec": "1500000000000000"
+            }
+          }
+        },
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatPlaceholderItemRenderer": {
+                "id": "",                
+                "timestampUsec": "1500000030000000"
+
+              }
+            }
+          }
+        }
+      ]}
+    }
+  }
+}
+