Merge branch 'release'

README.md

@@ -27,7 +27,7 @@ pip install pytchat
 ```python
 from pytchat import LiveChat
 
-chat = LiveChat("G1w62uEMZ74")
+chat = LiveChat("rsHWP7IjMiw")
 while chat.is_alive():
   data = chat.get()
   for c in data.items:
@@ -40,17 +40,17 @@ while chat.is_alive():
 from pytchat import LiveChat
 import time
 
-def main()
+#callback function is automatically called.
-  chat = LiveChat("G1w62uEMZ74", callback = func)
+def display(data):
-  while chat.is_alive():
-    time.sleep(3)
-    #other background operation.
-   
-#callback function is automatically called periodically.
-def func(data):
   for c in data.items:
     print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}")
     data.tick()
+
+#entry point
+chat = LiveChat("rsHWP7IjMiw", callback = display)
+while chat.is_alive():
+  time.sleep(3)
+  #other background operation.
 ```
 
 ### asyncio context:
@@ -59,63 +59,57 @@ from pytchat import LiveChatAsync
 import asyncio
 
 async def main():
-  chat = LiveChatAsync("G1w62uEMZ74", callback = func)
+  chat = LiveChatAsync("rsHWP7IjMiw", callback = func)
   while chat.is_alive():
     await asyncio.sleep(3)
     #other background operation.
 
-#callback function is automatically called periodically.
+#callback function is automatically called.
 async def func(data):
   for c in data.items:
     print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}")
     await data.tick_async()
 
-loop = asyncio.get_event_loop()
+try:
-loop.run_until_complete(main())
+  loop = asyncio.get_event_loop()
+  loop.run_until_complete(main())
+except CancelledError:
+  pass
 ```
 
 
 ### youtube api compatible processor:
 ```python
 from pytchat import LiveChat, CompatibleProcessor
+import time
 
-chat = LiveChat("G1w62uEMZ74", 
+chat = LiveChat("rsHWP7IjMiw", 
   processor = CompatibleProcessor() )
 
 while chat.is_alive():
   data = chat.get()
-  polling = data["pollingIntervalMillis"]/1000
+  polling = data['pollingIntervalMillis']/1000
-  for c in data["items"]:
+  for c in data['items']:
-    if c.get("snippet"):
+    if c.get('snippet'):
       print(f"[{c['authorDetails']['displayName']}]"
             f"-{c['snippet']['displayMessage']}")
-      time.sleep(polling/len(data["items"]))
+      time.sleep(polling/len(data['items']))
 
 ```
 ### replay:
 ```python
-from pytchat import ReplayChatAsync
+from pytchat import ReplayChat
-import asyncio
 
-async def main():
+def main():
-  chat = ReplayChatAsync("G1w62uEMZ74", seektime = 1000, callback = func)
+  #seektime (seconds): start position of chat.
+  chat = ReplayChat("ojes5ULOqhc", seektime = 60*30)
   while chat.is_alive():
-    await asyncio.sleep(3)
+    data = chat.get()
-    #other background operation here.
+    for c in data.items:
+      print(f"{c.elapsedTime} [{c.author.name}]-{c.message} {c.amountString}")
+      data.tick()
 
-#callback function is automatically called periodically.
+main()
-async def func(data):
-  for count in range(0,len(data.items)):
-    c= data.items[count]
-    if count!=len(data.items):
-      tick=data.items[count+1].timestamp -data.items[count].timestamp
-    else:
-      tick=0
-    print(f"<{c.elapsedTime}> [{c.author.name}]-{c.message} {c.amountString}")
-    await asyncio.sleep(tick/1000)
-
-loop = asyncio.get_event_loop()
-loop.run_until_complete(main())
 ```
 
 ## Structure of Default Processor

pytchat/__init__.py

@@ -2,7 +2,7 @@
 pytchat is a python library for fetching youtube live chat without using yt api, Selenium, or BeautifulSoup.
 """
 __copyright__    = 'Copyright (C) 2019 taizan-hokuto'
-__version__      = '0.0.3.8'
+__version__      = '0.0.4.0'
 __license__      = 'MIT'
 __author__       = 'taizan-hokuto'
 __author_email__ = '55448286+taizan-hokuto@users.noreply.github.com'
@@ -11,6 +11,7 @@ __url__          = 'https://github.com/taizan-hokuto/pytchat'
 __all__ = ["core_async","core_multithread","processors"]
 
 from .api import (
+    config,
     LiveChat,
     LiveChatAsync,
     ReplayChat,
@@ -19,5 +20,6 @@ from .api import (
     CompatibleProcessor, 
     SimpleDisplayProcessor,
     JsonfileArchiveProcessor,
-    SpeedCalculator
+    SpeedCalculator,
-    )
+    DummyProcessor
+)

pytchat/api.py

@@ -8,3 +8,5 @@ from .processors.compatible.processor import CompatibleProcessor
 from .processors.simple_display_processor import SimpleDisplayProcessor
 from .processors.jsonfile_archive_processor import JsonfileArchiveProcessor
 from .processors.speed_calculator import SpeedCalculator
+from .processors.dummy_processor import DummyProcessor
+from . import config

pytchat/config/__init__.py

@@ -1,4 +1,13 @@
 import logging
+from . import mylogger
+
 LOGGER_MODE = None
+
 headers = {
         'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'}
+
+def logger(module_name: str):
+    module_logger = mylogger.get_logger(module_name, mode = LOGGER_MODE)
+    return module_logger
+
+

pytchat/config/mylogger.py

@@ -0,0 +1,32 @@
+from logging import NullHandler, getLogger, StreamHandler, FileHandler, Formatter
+import logging
+import datetime
+
+
+def get_logger(modname,mode=logging.DEBUG):
+    logger = getLogger(modname)
+    if mode == None:
+        logger.addHandler(NullHandler())
+        return logger
+    logger.setLevel(mode)
+    #create handler1 for showing info
+    handler1 = StreamHandler()
+    my_formatter  = MyFormatter()
+    handler1.setFormatter(my_formatter)
+
+    handler1.setLevel(mode) 
+    logger.addHandler(handler1)
+    #create handler2 for recording log file
+    if mode <= logging.DEBUG:
+        handler2 = FileHandler(filename="log.txt", encoding='utf-8')
+        handler2.setLevel(logging.ERROR)
+        handler2.setFormatter(my_formatter)
+
+
+        logger.addHandler(handler2)
+    return logger
+
+class MyFormatter(logging.Formatter):
+    def format(self, record):
+        s =(datetime.datetime.fromtimestamp(record.created)).strftime("%m-%d %H:%M:%S")+'| '+ (record.module).ljust(15)+(' { '+record.funcName).ljust(20) +":"+str(record.lineno).rjust(4)+'} - '+record.getMessage()
+        return s

pytchat/core_async/livechat.py

@@ -11,15 +11,14 @@ from concurrent.futures import CancelledError
 from .buffer import Buffer
 from ..parser.live import Parser
 from .. import config
-from .. import mylogger
 from ..exceptions  import ChatParseException,IllegalFunctionCall
 from ..paramgen    import liveparam
 from ..processors.default.processor import DefaultProcessor
+from ..processors.combinator import Combinator
 
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
+logger = config.logger(__name__)
-MAX_RETRY = 10
 headers = config.headers
-
+MAX_RETRY = 10
 
 
 class LiveChatAsync:
@@ -72,6 +71,9 @@ class LiveChatAsync:
                 exception_handler = None,
                 direct_mode = False): 
         self.video_id  = video_id
+        if isinstance(processor, tuple):
+            self.processor = Combinator(processor)
+        else:
             self.processor = processor
         self._buffer = buffer
         self._callback = callback
@@ -148,7 +150,7 @@ class LiveChatAsync:
 
     async def _listen(self, continuation):
         ''' continuationに紐付いたチャットデータを取得し
-        チャットデータを格納、
+        Bufferにチャットデータを格納、
         次のcontinuaitonを取得してループする。
 
         Parameter
@@ -180,9 +182,11 @@ class LiveChatAsync:
                     await asyncio.sleep(diff_time)        
                     continuation = metadata.get('continuation')     
         except ChatParseException as e:
-            logger.info(f"{str(e)}（video_id:\"{self.video_id}\"）")
+            self.terminate()
+            logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）")
             return            
         except (TypeError , json.JSONDecodeError) :
+            self.terminate()
             logger.error(f"{traceback.format_exc(limit = -1)}")
             return
         
@@ -211,6 +215,7 @@ class LiveChatAsync:
         else:
             logger.error(f"[{self.video_id}]"
                     f"Exceeded retry count. status_code={status_code}")
+            self.terminate()
             return None
         return livechat_json
 

pytchat/core_async/replaychat.py

@@ -12,12 +12,12 @@ from queue import Queue
 from .buffer import Buffer
 from ..parser.replay import Parser
 from .. import config
-from .. import mylogger
 from ..exceptions  import ChatParseException,IllegalFunctionCall
 from ..paramgen    import arcparam
 from ..processors.default.processor import DefaultProcessor
+from ..processors.combinator import Combinator
 
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
+logger = config.logger(__name__)
 MAX_RETRY = 10
 headers = config.headers
 
@@ -78,6 +78,9 @@ class ReplayChatAsync:
                 direct_mode = False):
         self.video_id  = video_id
         self.seektime = seektime
+        if isinstance(processor, tuple):
+            self.processor = Combinator(processor)
+        else:
             self.processor = processor
         self._buffer = buffer
         self._callback = callback
@@ -194,13 +197,15 @@ class ReplayChatAsync:
                     await asyncio.sleep(diff_time)       
                     continuation = metadata.get('continuation')  
         except ChatParseException as e:
-            logger.info(f"{str(e)}（video_id:\"{self.video_id}\"）")
+            logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）")
             return            
         except (TypeError , json.JSONDecodeError) :
             logger.error(f"{traceback.format_exc(limit = -1)}")
+            self.terminate()
             return
         
         logger.debug(f"[{self.video_id}]チャット取得を終了しました。")
+        self.terminate()
 
     async def _get_livechat_json(self, continuation, session, headers):
         '''
@@ -282,7 +287,7 @@ class ReplayChatAsync:
         if self._direct_mode == False:
             #bufferにダミーオブジェクトを入れてis_alive()を判定させる
             self._buffer.put_nowait({'chatdata':'','timeout':1}) 
-        logger.info(f'終了しました:[{self.video_id}]')
+        logger.info(f'[{self.video_id}]終了しました')
   
     @classmethod
     def _set_exception_handler(cls, handler):

pytchat/core_multithread/livechat.py

@@ -10,15 +10,14 @@ from concurrent.futures import CancelledError, ThreadPoolExecutor
 from .buffer import Buffer
 from ..parser.live import Parser
 from .. import config
-from .. import mylogger
 from ..exceptions  import ChatParseException,IllegalFunctionCall
 from ..paramgen    import liveparam
 from ..processors.default.processor import DefaultProcessor
+from ..processors.combinator import Combinator
 
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
+logger = config.logger(__name__)
-MAX_RETRY = 10
 headers = config.headers
-
+MAX_RETRY = 10
 
 
 class LiveChat:
@@ -65,13 +64,16 @@ class LiveChat:
     _listeners= []
     def __init__(self, video_id,
                 processor = DefaultProcessor(),
-                buffer = Buffer(maxsize = 20),
+                buffer = None,
                 interruptable = True,
                 callback = None,
                 done_callback = None,
                 direct_mode = False
                 ):
         self.video_id  = video_id
+        if isinstance(processor, tuple):
+            self.processor = Combinator(processor)
+        else:
             self.processor = processor
         self._buffer = buffer
         self._callback = callback
@@ -142,8 +144,8 @@ class LiveChat:
 
     def _listen(self, continuation):
         ''' continuationに紐付いたチャットデータを取得し
-        BUfferにチャットデータを格納、
+        Bufferにチャットデータを格納、
-        次のcontinuaitonを取得してループする
+        次のcontinuaitonを取得してループする。
 
         Parameter
         ---------
@@ -175,9 +177,11 @@ class LiveChat:
                     time.sleep(diff_time)        
                     continuation = metadata.get('continuation')  
         except ChatParseException as e:
-            logger.info(f"{str(e)}（video_id:\"{self.video_id}\"）")
+            self.terminate()
+            logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）")
             return            
         except (TypeError , json.JSONDecodeError) :
+            self.terminate()
             logger.error(f"{traceback.format_exc(limit = -1)}")
             return
         
@@ -206,6 +210,7 @@ class LiveChat:
         else:
             logger.error(f"[{self.video_id}]"
                     f"Exceeded retry count. status_code={status_code}")
+            self.terminate()
             return None
         return livechat_json
   
@@ -254,18 +259,10 @@ class LiveChat:
         if self._direct_mode == False:
             #bufferにダミーオブジェクトを入れてis_alive()を判定させる
             self._buffer.put({'chatdata':'','timeout':1}) 
-        logger.info(f'終了しました:[{self.video_id}]')
+        logger.info(f'[{self.video_id}]終了しました')
   
     @classmethod
     def shutdown(cls, event, sig = None, handler=None):
         logger.debug("シャットダウンしています")
         for t in LiveChat._listeners:
             t._is_alive = False
-
-
-
-    
-
-
-
-

pytchat/core_multithread/replaychat.py

@@ -11,15 +11,14 @@ from queue import Queue
 from .buffer import Buffer
 from ..parser.replay import Parser
 from .. import config
-from .. import mylogger
 from ..exceptions  import ChatParseException,IllegalFunctionCall
 from ..paramgen    import arcparam
 from ..processors.default.processor import DefaultProcessor
+from ..processors.combinator import Combinator
 
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
+logger = config.logger(__name__)
-MAX_RETRY = 10
 headers = config.headers
-
+MAX_RETRY = 10
 
 
 class ReplayChat:
@@ -61,7 +60,7 @@ class ReplayChat:
         チャットデータ取得ループ（_listen）用のスレッド
 
     _is_alive : bool
-        チャット取得を終了したか
+        チャット取得を停止するためのフラグ
     '''
 
     _setup_finished = False
@@ -70,7 +69,7 @@ class ReplayChat:
     def __init__(self, video_id,
                 seektime = 0,
                 processor = DefaultProcessor(),
-                buffer = Buffer(maxsize = 20),
+                buffer = None,
                 interruptable = True,
                 callback = None,
                 done_callback = None,
@@ -78,6 +77,9 @@ class ReplayChat:
                 ):
         self.video_id  = video_id
         self.seektime = seektime
+        if isinstance(processor, tuple):
+            self.processor = Combinator(processor)
+        else:
             self.processor = processor
         self._buffer = buffer
         self._callback = callback
@@ -90,6 +92,7 @@ class ReplayChat:
         self._pauser.put_nowait(None)
        
         self._setup()
+
         if not ReplayChat._setup_finished:
             ReplayChat._setup_finished = True
             if interruptable:
@@ -150,7 +153,7 @@ class ReplayChat:
 
     def _listen(self, continuation):
         ''' continuationに紐付いたチャットデータを取得し
-        にチャットデータを格納、
+        BUfferにチャットデータを格納、
         次のcontinuaitonを取得してループする
 
         Parameter
@@ -189,9 +192,11 @@ class ReplayChat:
                     time.sleep(diff_time)        
                     continuation = metadata.get('continuation')  
         except ChatParseException as e:
-            logger.error(f"{str(e)}（動画ID:\"{self.video_id}\"）")
+            self.terminate()
+            logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）")
             return            
         except (TypeError , json.JSONDecodeError) :
+            self.terminate()
             logger.error(f"{traceback.format_exc(limit = -1)}")
             return
         
@@ -220,6 +225,7 @@ class ReplayChat:
         else:
             logger.error(f"[{self.video_id}]"
                     f"Exceeded retry count. status_code={status_code}")
+            self.terminate()
             return None
         return livechat_json
   
@@ -266,7 +272,7 @@ class ReplayChat:
         '''Listener終了時のコールバック'''
         try: 
             self.terminate()
-        except CancelledError:
+        except RuntimeError:
             logger.debug(f'[{self.video_id}]cancelled:{sender}')
 
     def terminate(self):
@@ -277,7 +283,7 @@ class ReplayChat:
         if self._direct_mode == False:
             #bufferにダミーオブジェクトを入れてis_alive()を判定させる
             self._buffer.put({'chatdata':'','timeout':1}) 
-        logger.info(f'終了しました:[{self.video_id}]')
+        logger.info(f'[{self.video_id}]終了しました')
   
     @classmethod
     def shutdown(cls, event, sig = None, handler=None):

pytchat/parser/live.py

@@ -6,14 +6,13 @@ This module is parser of live chat JSON.
 
 import json
 from .. import config
-from .. import mylogger
 from .. exceptions import ( 
     ResponseContextError, 
     NoContentsException, 
     NoContinuationsException )
 
 
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
+logger = config.logger(__name__)
 
 
 class Parser:
@@ -59,7 +58,7 @@ class Parser:
         if metadata is None:
             unknown = list(cont.keys())[0]
             if unknown:
-                logger.error(f"Received unknown continuation type:{unknown}")
+                logger.debug(f"Received unknown continuation type:{unknown}")
                 metadata = cont.get(unknown)
         metadata.setdefault('timeoutMs', 10000)
         chatdata = contents['liveChatContinuation'].get('actions')

pytchat/parser/replay.py

@@ -1,14 +1,12 @@
 import json
 from .. import config
-from .. import mylogger
 from .. exceptions import ( 
     ResponseContextError, 
     NoContentsException, 
     NoContinuationsException )
 
 
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
+logger = config.logger(__name__)
-
 
 class Parser:
     def parse(self, jsn):
@@ -53,12 +51,13 @@ class Parser:
         metadata = cont.get('liveChatReplayContinuationData')
         if metadata is None:
             unknown = list(cont.keys())[0]
-            if unknown:
+            if unknown != "playerSeekContinuationData":
+                logger.debug(f"Received unknown continuation type:{unknown}")
                 metadata = cont.get(unknown)
-        
         actions = contents['liveChatContinuation'].get('actions')
         if actions is None:
-            raise NoContentsException('チャットデータを取得できませんでした。')
+            #後続のチャットデータなし
+            return {"continuation":None,"timeout":0,"chatdata":[]}
         interval = self.get_interval(actions)
         metadata.setdefault("timeoutMs",interval)
         """アーカイブ済みチャットはライブチャットと構造が異なっているため、以下の行により

pytchat/processors/combinator.py

@@ -0,0 +1,39 @@
+from .chat_processor import ChatProcessor
+
+class Combinator(ChatProcessor):
+    '''
+    Combinator combines multiple chat processors.
+    Specify processors as tuple at `processor` params of LiveChat object.
+
+    For example:
+        [constructor]
+        chat = LiveChat("video_id", processor = ( Processor1(), Processor2(), Processor3() )
+        
+        [receive return values]
+        ret1, ret2, ret3 = chat.get()
+    
+    The return values are tuple of processed chat data, 
+    the order of return depends on parameter order.
+
+    Parameter
+    ---------
+    processors : Tuple[ChatProcessor]
+        multiple processors for processing chat data
+    '''
+
+    def __init__(self, processors: tuple):
+        self.processors = processors
+
+    def process(self, chat_components: list):
+        '''
+        Called from LiveChat.get() function by user,
+        or LiveChat._listen() automatically.
+
+        Returns
+        -------
+            Tuple of chat data processed by each chat processor.
+        '''
+        return tuple([processor.process(chat_components)
+            for processor in self.processors])
+            
+

pytchat/processors/compatible/processor.py

@@ -5,9 +5,8 @@ from .renderer.paidmessage import LiveChatPaidMessageRenderer
 from .renderer.paidsticker import LiveChatPaidStickerRenderer
 from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
 from .. chat_processor import ChatProcessor
-from ... import mylogger
 from ... import config
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
+logger = config.logger(__name__)
 
 class CompatibleProcessor(ChatProcessor):
 

pytchat/processors/default/processor.py

@@ -6,8 +6,7 @@ from .renderer.paidsticker import LiveChatPaidStickerRenderer
 from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
 from .. chat_processor import ChatProcessor
 from ... import config
-from ... import mylogger
+logger = config.logger(__name__)
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
 
 class Chatdata:
     def __init__(self,chatlist:list, timeout:float):

pytchat/processors/default/renderer/base.py

@@ -19,8 +19,7 @@ class BaseRenderer:
         else:
             self.elapsedTime = ""
         self.datetime = self.get_datetime(timestampUsec)
-        self.message = self.get_message(self.renderer)
+        self.message ,self.messageEx = self.get_message(self.renderer)
-        self.messageEx = self.get_message_ex(self.renderer)
         self.id =  self.renderer.get('id')
         self.amountValue= 0.0
         self.amountString = ""
@@ -44,6 +43,7 @@ class BaseRenderer:
 
     def get_message(self,renderer):
         message = ''
+        message_ex = []
         if renderer.get("message"):
             runs=renderer["message"].get("runs")
             if runs:
@@ -51,22 +51,13 @@ class BaseRenderer:
                     if r:
                         if r.get('emoji'):
                             message += r['emoji'].get('shortcuts',[''])[0]
+                            message_ex.append(r['emoji']['image']['thumbnails'][1].get('url'))
                         else:
                             message += r.get('text','')
-        return message
+                            message_ex.append(r.get('text',''))
+        return message, message_ex
+
 
-    def get_message_ex(self,renderer):
-        message = []
-        if renderer.get("message"):
-            runs=renderer["message"].get("runs")
-            if runs:
-                for r in runs:
-                    if r:
-                        if r.get('emoji'):
-                            message.append(r['emoji']['image']['thumbnails'][1].get('url'))
-                        else:
-                            message.append(r.get('text',''))
-        return message
 
     def get_badges(self,renderer):
         isVerified = False

pytchat/processors/dummy_processor.py

@@ -0,0 +1,8 @@
+from .chat_processor import ChatProcessor
+
+class DummyProcessor(ChatProcessor):
+    '''
+    Dummy processor just returns received chat_components directly.
+    '''
+    def process(self, chat_components: list):
+        return chat_components

pytchat/util/__init__.py

@@ -9,7 +9,7 @@ def download(url):
         json.dump(html.json(),f,ensure_ascii=False)
 
 
-def save(data,filename):
+def save(data,filename,extention):
-    with open(str(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')
+    with open(filename+"_"+(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')
-                            )+filename,mode ='w',encoding='utf-8') as f:
+                            )+extention,mode ='w',encoding='utf-8') as f:
         f.writelines(data)

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages, Command
-from codecs import open
+#from codecs import open as open_c
-from os import path, system
+from os import path, system, remove, rename
 import re
 
 package_name = "pytchat"
@@ -28,6 +28,16 @@ assert author
 assert author_email
 assert url
 
+
+
+with open('README.MD', 'r', encoding='utf-8') as f:
+    txt = f.read()
+
+with open('README1.MD', 'w', encoding='utf-8', newline='\n') as f:
+    f.write(txt)
+    
+remove("README.MD")
+rename("README1.MD","README.MD")
 with open('README.md', encoding='utf-8') as f:
     long_description = f.read()