Merge branch 'release/v0.2.0'

v0.1.9

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 taizan-hokuto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -7,7 +7,7 @@ pytchat is a python library for fetching youtube live chat.
 pytchat is a python library for fetching youtube live chat
 without using youtube api, Selenium or BeautifulSoup.
 
-pytchatはAPIを使わずにYouTubeチャットを取得するためのpythonライブラリです。
+pytchatは、YouTubeチャットを閲覧するためのpythonライブラリです。
 
 Other features:
 + Customizable [chat data processors](https://github.com/taizan-hokuto/pytchat/wiki/ChatProcessor) including youtube api compatible one.
@@ -27,13 +27,12 @@ pip install pytchat
 ### CLI
 
 One-liner command.
-Save chat data to html.
+Save chat data to html, with embedded custom emojis.
 
 ```bash
-$ pytchat -v ZJ6Q4U_Vg6s -o "c:/temp/"
+$ pytchat -v https://www.youtube.com/watch?v=ZJ6Q4U_Vg6s -o "c:/temp/"
-
 # options:
-#  -v : video_id
+#  -v : Video ID or URL that includes ID
 #  -o : output directory (default path: './')
 # saved filename is [video_id].html
 ```
@@ -43,7 +42,8 @@ $ pytchat -v ZJ6Q4U_Vg6s -o "c:/temp/"
 ```python
 from pytchat import LiveChat
 livechat = LiveChat(video_id = "Zvp1pJpie4I")
-
+# It is also possible to specify a URL that includes the video ID:
+# livechat = LiveChat("https://www.youtube.com/watch?v=Zvp1pJpie4I")
 while livechat.is_alive():
   try:
     chatdata = livechat.get()
@@ -148,6 +148,20 @@ def main():
 if __name__ == '__main__':
   main()
 ```
+### Extract archived chat data as [HTML](https://github.com/taizan-hokuto/pytchat/wiki/HTMLArchiver) or [tab separated values](https://github.com/taizan-hokuto/pytchat/wiki/TSVArchiver).
+```python
+from pytchat import HTMLArchiver, Extractor
+
+video_id = "*******"
+ex = Extractor(
+    video_id,
+    div=10,
+    processor=HTMLArchiver("c:/test.html")
+)
+
+ex.extract()
+print("finished.")
+```
 
 ## Structure of Default Processor
 Each item can be got with `items` function.
@@ -175,7 +189,7 @@ Each item can be got with `items` function.
   <tr>
     <td>messageEx</td>
     <td>str</td>
-    <td>list of message texts and emoji URLs.</td>
+    <td>list of message texts and emoji dicts(id, txt, url).</td>
   </tr>
   <tr>
     <td>timestamp</td>

pytchat/__init__.py

@@ -1,8 +1,8 @@
 """
-pytchat is a python library for fetching youtube live chat without using yt api, Selenium, or BeautifulSoup.
+pytchat is a lightweight python library to browse youtube livechat without Selenium or BeautifulSoup.
 """
 __copyright__    = 'Copyright (C) 2019 taizan-hokuto'
-__version__      = '0.0.7.1'
+__version__      = '0.2.0'
 __license__      = 'MIT'
 __author__       = 'taizan-hokuto'
 __author_email__ = '55448286+taizan-hokuto@users.noreply.github.com'
@@ -27,4 +27,6 @@ from .api import (
     SpeedCalculator,
     SuperchatCalculator,
     VideoInfo
-)
+)
+
+# flake8: noqa

pytchat/api.py

@@ -14,3 +14,5 @@ from .processors.speed.calculator import SpeedCalculator
 from .processors.superchat.calculator import SuperchatCalculator
 from .tool.extract.extractor import Extractor
 from .tool.videoinfo import VideoInfo
+
+# flake8: noqa

pytchat/cli/__init__.py

@@ -1,60 +1,126 @@
 import argparse
-import os
-from pathlib import Path
-from typing import List, Callable
-from .arguments import Arguments
 
-from .. exceptions import InvalidVideoIdException, NoContentsException
+import os
-from .. processors.tsv_archiver import TSVArchiver
+import sys
+import signal
+from json.decoder import JSONDecodeError
+from pathlib import Path
+from .arguments import Arguments
+from .. exceptions import InvalidVideoIdException, NoContents, PatternUnmatchError
 from .. processors.html_archiver import HTMLArchiver
 from .. tool.extract.extractor import Extractor
 from .. tool.videoinfo import VideoInfo
+from .. util.extract_video_id import extract_video_id
+from .. import util
 from .. import __version__
 
 '''
-Most of CLI modules refer to 
+Most of CLI modules refer to
 Petter Kraabøl's Twitch-Chat-Downloader
 https://github.com/PetterKraabol/Twitch-Chat-Downloader
 (MIT License)
 
 '''
+
+
 def main():
-    # Arguments 
+    # Arguments
     parser = argparse.ArgumentParser(description=f'pytchat v{__version__}')
-    parser.add_argument('-v', f'--{Arguments.Name.VIDEO}', type=str,
+    parser.add_argument('-v', f'--{Arguments.Name.VIDEO_IDS}', type=str,
-         help='Video IDs separated by commas without space.\n'
+                        help='Video ID (or URL that includes Video ID). You can specify multiple video IDs by '
-         'If ID starts with a hyphen (-), enclose the ID in square brackets.')
+                        'separating them with commas without spaces.\n'
+                        'If ID starts with a hyphen (-), enclose the ID in square brackets.')
     parser.add_argument('-o', f'--{Arguments.Name.OUTPUT}', type=str,
-         help='Output directory (end with "/"). default="./"', default='./')
+                        help='Output directory (end with "/"). default="./"', default='./')
+    parser.add_argument(f'--{Arguments.Name.PBAR}', action='store_true',
+                        help='Display rich progress bar')
+    parser.add_argument(f'--{Arguments.Name.SAVE_ERROR_DATA}', action='store_true',
+                        help='Save error data when error occurs(".dat" file)')
     parser.add_argument(f'--{Arguments.Name.VERSION}', action='store_true',
-         help='Settings version')
+                        help='Show version')
     Arguments(parser.parse_args().__dict__)
+
+    if Arguments().pbar:
+        from .progressbar_rich import ProgressBar
+    else:
+        from .progressbar_simple import ProgressBar
     if Arguments().print_version:
-        print(f'pytchat v{__version__}')
+        print(f'pytchat v{__version__}     © 2019 taizan-hokuto')
         return
 
     # Extractor
-    if Arguments().video_ids:
+    if not Arguments().video_ids:
-        for video_id in Arguments().video_ids:
+        parser.print_help()
-            if '[' in video_id:
-                video_id = video_id.replace('[','').replace(']','')
-            try:
-                info = VideoInfo(video_id)
-                print(f"Extracting...\n"
-                      f" video_id: {video_id}\n"
-                      f" channel:  {info.get_channel_name()}\n"
-                      f" title:    {info.get_title()}")
-                path = Path(Arguments().output+video_id+'.html')
-                print(f"output path: {path.resolve()}")
-                Extractor(video_id, 
-                  processor = HTMLArchiver(Arguments().output+video_id+'.html'),
-                  callback = _disp_progress
-                ).extract()
-                print("\nExtraction end.\n")
-            except (InvalidVideoIdException, NoContentsException) as e:
-                print(e)
         return
-    parser.print_help()
+    for video_id in Arguments().video_ids:
+        if '[' in video_id:
+            video_id = video_id.replace('[', '').replace(']', '')
+        try:
+            video_id = extract_video_id(video_id)
+            if os.path.exists(Arguments().output):
+                path = Path(Arguments().output + video_id + '.html')
+            else:
+                raise FileNotFoundError
+            info = VideoInfo(video_id)
+            print(f"Extracting...\n"
+                  f" video_id: {video_id}\n"
+                  f" channel:  {info.get_channel_name()}\n"
+                  f" title:    {info.get_title()}")
 
-def _disp_progress(a,b):
+            print(f" output path: {path.resolve()}")
-    print('.',end="",flush=True)
+            duration = info.get_duration()
+            pbar = ProgressBar(total=(duration * 1000), status="Extracting")
+            ex = Extractor(video_id,                  
+                    callback=pbar._disp,
+                    div=10)
+            signal.signal(signal.SIGINT, (lambda a, b: cancel(ex, pbar)))
+            data = ex.extract()
+            if data == []:
+                return False
+            if Arguments().pbar:
+                pbar.reset("#", "=", total=len(data), status="Rendering  ")
+            else:
+                pbar.reset("=", "", total=len(data), status="Rendering  ")
+            processor = HTMLArchiver(Arguments().output + video_id + '.html', callback=pbar._disp)
+            processor.process(
+                [{'video_id': None,
+                'timeout': 1,
+                'chatdata': (action["replayChatItemAction"]["actions"][0] for action in data)}]
+            )
+            processor.finalize()
+            if Arguments().pbar:
+                pbar.reset('#', '#', status='Completed   ')
+                pbar.close()
+            else:
+                pbar.close()
+                print("\nCompleted")
+            
+            print()
+            if pbar.is_cancelled():
+                print("\nThe extraction process has been discontinued.\n")
+                return False
+            return True
+
+        except InvalidVideoIdException:
+            print("Invalid Video ID or URL:", video_id)
+        except NoContents as e:
+            print(e)
+        except FileNotFoundError:
+            print("The specified directory does not exist.:{}".format(Arguments().output))
+        except JSONDecodeError as e:
+            print(e.msg)
+            print("Cannot parse video information.:{}".format(video_id))
+            if Arguments().save_error_data:
+                util.save(e.doc, "ERR_JSON_DECODE", ".dat")
+        except PatternUnmatchError as e:
+            print(e.msg)
+            print("Cannot parse video information.:{}".format(video_id))
+            if Arguments().save_error_data:
+                util.save(e.doc, "ERR_PATTERN_UNMATCH", ".dat")
+
+    return
+
+
+def cancel(ex, pbar):
+    ex.cancel()
+    pbar.cancel()

pytchat/cli/arguments.py

@@ -2,12 +2,13 @@ from typing import Optional, Dict, Union, List
 from .singleton import Singleton
 
 '''
-This modules refer to 
+This modules refer to
 Petter Kraabøl's Twitch-Chat-Downloader
 https://github.com/PetterKraabol/Twitch-Chat-Downloader
 (MIT License)
 '''
 
+
 class Arguments(metaclass=Singleton):
     """
     Arguments singleton
@@ -15,14 +16,16 @@ class Arguments(metaclass=Singleton):
 
     class Name:
         VERSION: str = 'version'
-        OUTPUT: str = 'output'
+        OUTPUT: str = 'output_dir'
-        VIDEO: str = 'video'
+        VIDEO_IDS: str = 'video_id'
+        SAVE_ERROR_DATA: bool = 'save_error_data'
+        PBAR: bool ='pbar'
 
-    def __init__(self, 
+    def __init__(self,
-        arguments: Optional[Dict[str, Union[str, bool, int]]] = None):
+                 arguments: Optional[Dict[str, Union[str, bool, int]]] = None):
         """
         Initialize arguments
-        :param arguments: Arguments from cli 
+        :param arguments: Arguments from cli
         (Optional to call singleton instance without parameters)
         """
 
@@ -33,7 +36,9 @@ class Arguments(metaclass=Singleton):
         self.print_version: bool = arguments[Arguments.Name.VERSION]
         self.output: str = arguments[Arguments.Name.OUTPUT]
         self.video_ids: List[int] = []
+        self.save_error_data: bool = arguments[Arguments.Name.SAVE_ERROR_DATA]
+        self.pbar: bool = arguments[Arguments.Name.PBAR]
         # Videos
-        if arguments[Arguments.Name.VIDEO]:
+        if arguments[Arguments.Name.VIDEO_IDS]:
-            self.video_ids = [video_id 
+            self.video_ids = [video_id
-                for video_id in arguments[Arguments.Name.VIDEO].split(',')]
+                              for video_id in arguments[Arguments.Name.VIDEO_IDS].split(',')]

pytchat/cli/progressbar_rich.py

@@ -0,0 +1,52 @@
+'''
+This code for this progress bar is based on
+vladignatyev/progress.py
+https://gist.github.com/vladignatyev/06860ec2040cb497f0f3
+(MIT License)
+'''
+import sys
+
+
+class ProgressBar:
+    def __init__(self, total, status):
+        self._bar_len = 60
+        self._cancelled = False
+        self.reset(total=total, status=status)
+        self._blinker = 0
+
+    def reset(self, symbol_done="=", symbol_space=" ", total=100, status=''):
+        self._symbol_done = symbol_done
+        self._symbol_space = symbol_space
+        self._total = total
+        self._status = status
+        self._count = 0
+
+    def _disp(self, _, fetched):
+        self._progress(fetched, self._total)
+
+    def _progress(self, fillin, total):
+        if total == 0 or self._cancelled:
+            return
+        self._count += fillin
+        filled_len = int(round(self._bar_len * self._count / float(total)))
+        percents = round(100.0 * self._count / float(total), 1)
+        if percents > 100:
+            percents = 100.0
+        if filled_len > self._bar_len:
+            filled_len = self._bar_len
+            
+        bar = self._symbol_done * filled_len + \
+              self._symbol_space * (self._bar_len - filled_len)
+        sys.stdout.write(' [%s] %s%s ...%s \r' % (bar, percents, '%', self._status))
+        sys.stdout.flush()
+        self._blinker += 1
+
+    def close(self):
+        if not self._cancelled:
+            self._progress(self._total, self._total)
+
+    def cancel(self):
+        self._cancelled = True
+    
+    def is_cancelled(self):
+        return self._cancelled

pytchat/cli/progressbar_simple.py

@@ -0,0 +1,49 @@
+'''
+This code for this progress bar is based on
+vladignatyev/progress.py
+https://gist.github.com/vladignatyev/06860ec2040cb497f0f3
+(MIT License)
+'''
+import sys
+
+
+class ProgressBar:
+    def __init__(self, total, status):
+        self._bar_len = 60
+        self._cancelled = False
+        print(''.join([' ' * 10, '|', '-' * (self._bar_len), '|']), end="")
+        self.reset(total=total, status=status)
+
+    def reset(self, symbol_done="=", symbol_space=" ", total=100, status=''):
+        self._symbol_done = symbol_done
+        self._symbol_space = symbol_space
+        self._total = total
+        self._status = status
+        self._old_len = 0
+        self._count = 0
+        print()
+        print(f'{status:<11}', end='')
+
+    def _disp(self, _, fetched):
+        self._progress(fetched, self._total)
+
+    def _progress(self, fillin, total):
+        if total == 0 or self._cancelled:
+            return
+        self._count += fillin
+        filled_len = int(round(self._bar_len * self._count / float(total)))
+        if filled_len > self._bar_len:
+            filled_len = self._bar_len
+        print((filled_len - self._old_len) * self._symbol_done, end="")
+        sys.stdout.flush()
+        self._old_len = filled_len
+
+    def close(self):
+        if not self._cancelled:
+            self._progress(self._total, self._total)
+
+    def cancel(self):
+        self._cancelled = True
+    
+    def is_cancelled(self):
+        return self._cancelled

pytchat/cli/singleton.py

@@ -1,9 +1,11 @@
 '''
-This modules refer to 
+This modules refer to
 Petter Kraabøl's Twitch-Chat-Downloader
 https://github.com/PetterKraabol/Twitch-Chat-Downloader
 (MIT License)
 '''
+
+
 class Singleton(type):
     """
     Abstract class for singletons
@@ -16,4 +18,4 @@ class Singleton(type):
         return cls._instances[cls]
 
     def get_instance(cls, *args, **kwargs):
-        cls.__call__(*args, **kwargs)
+        cls.__call__(*args, **kwargs)

pytchat/config/__init__.py

@@ -1,11 +1,10 @@
 import logging
 from . import mylogger
-
 headers = {
-        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'}
+    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36',
+}
 
-def logger(module_name: str, loglevel = None):
+
-    module_logger = mylogger.get_logger(module_name, loglevel = loglevel)
+def logger(module_name: str, loglevel=None):
+    module_logger = mylogger.get_logger(module_name, loglevel=loglevel)
     return module_logger
-
-

pytchat/config/mylogger.py

@@ -1,31 +1,31 @@
-from logging import NullHandler, getLogger, StreamHandler, FileHandler, Formatter
+from logging import NullHandler, getLogger, StreamHandler, FileHandler
 import logging
 from datetime import datetime
 
 
-def get_logger(modname,loglevel=logging.DEBUG):
+def get_logger(modname, loglevel=logging.DEBUG):
     logger = getLogger(modname)
-    if loglevel == None:
+    if loglevel is None:
         logger.addHandler(NullHandler())
         return logger
     logger.setLevel(loglevel)
-    #create handler1 for showing info
+    # create handler1 for showing info
     handler1 = StreamHandler()
-    my_formatter  = MyFormatter()
+    my_formatter = MyFormatter()
     handler1.setFormatter(my_formatter)
 
-    handler1.setLevel(loglevel) 
+    handler1.setLevel(loglevel)
     logger.addHandler(handler1)
-    #create handler2 for recording log file
+    # create handler2 for recording log file
     if loglevel <= logging.DEBUG:
         handler2 = FileHandler(filename="log.txt", encoding='utf-8')
         handler2.setLevel(logging.ERROR)
         handler2.setFormatter(my_formatter)
 
-
         logger.addHandler(handler2)
     return logger
 
+
 class MyFormatter(logging.Formatter):
     def format(self, record):
         timestamp = (
@@ -35,4 +35,4 @@ class MyFormatter(logging.Formatter):
         lineno = str(record.lineno).rjust(4)
         message = record.getMessage()
 
-        return timestamp+'| '+module+' { '+funcname+':'+lineno+'} - '+message 
+        return timestamp + '| ' + module + ' { ' + funcname + ':' + lineno + '} - ' + message

pytchat/core_async/buffer.py

@@ -1,5 +1,7 @@
 
 import asyncio
+
+
 class Buffer(asyncio.Queue):
     '''
     チャットデータを格納するバッファの役割を持つFIFOキュー
@@ -10,19 +12,20 @@ class Buffer(asyncio.Queue):
         格納するチャットブロックの最大個数。0の場合は無限。
         最大値を超える場合は古いチャットブロックから破棄される。
     '''
-    def __init__(self,maxsize = 0):
+
+    def __init__(self, maxsize=0):
         super().__init__(maxsize)
-    
+
-    async def put(self,item):
+    async def put(self, item):
         if item is None:
-            return 
+            return
         if super().full():
             super().get_nowait()
         await super().put(item)
 
-    def put_nowait(self,item):
+    def put_nowait(self, item):
         if item is None:
-            return 
+            return
         if super().full():
             super().get_nowait()
         super().put_nowait(item)
@@ -32,4 +35,4 @@ class Buffer(asyncio.Queue):
         ret.append(await super().get())
         while not super().empty():
             ret.append(super().get_nowait())
-        return ret
+        return ret

pytchat/core_async/livechat.py

@@ -1,28 +1,28 @@
-import aiohttp, asyncio
+
-import datetime
+import asyncio
+import httpx
 import json
-import random
 import signal
 import time
 import traceback
 import urllib.parse
-from aiohttp.client_exceptions import ClientConnectorError
-from concurrent.futures import CancelledError
 from asyncio import Queue
+from concurrent.futures import CancelledError
 from .buffer import Buffer
 from ..parser.live import Parser
 from .. import config
-from ..exceptions  import ChatParseException,IllegalFunctionCall
+from .. import exceptions
-from ..paramgen    import liveparam, arcparam
+from ..paramgen import liveparam, arcparam
 from ..processors.default.processor import DefaultProcessor
 from ..processors.combinator import Combinator
+from ..util.extract_video_id import extract_video_id
 
 headers = config.headers
 MAX_RETRY = 10
 
 
 class LiveChatAsync:
-    '''asyncio(aiohttp)を利用してYouTubeのライブ配信のチャットデータを取得する。
+    '''asyncioを利用してYouTubeのライブ配信のチャットデータを取得する。
 
     Parameter
     ---------
@@ -58,14 +58,14 @@ class LiveChatAsync:
         Trueの場合、bufferを使わずにcallbackを呼ぶ。
         Trueの場合、callbackの設定が必須
         (設定していない場合IllegalFunctionCall例外を発生させる）
-    
+
-    force_replay : bool  
+    force_replay : bool
         Trueの場合、ライブチャットが取得できる場合であっても
         強制的にアーカイブ済みチャットを取得する。
-    
+
     topchat_only : bool
         Trueの場合、上位チャットのみ取得する。
-          
+
     Attributes
     ---------
     _is_alive : bool
@@ -75,19 +75,19 @@ class LiveChatAsync:
     _setup_finished = False
 
     def __init__(self, video_id,
-                seektime = 0,
+                 seektime=-1,
-                processor = DefaultProcessor(),
+                 processor=DefaultProcessor(),
-                buffer = None,
+                 buffer=None,
-                interruptable = True,
+                 interruptable=True,
-                callback = None,
+                 callback=None,
-                done_callback = None,
+                 done_callback=None,
-                exception_handler = None,
+                 exception_handler=None,
-                direct_mode = False,
+                 direct_mode=False,
-                force_replay = False,
+                 force_replay=False,
-                topchat_only = False,
+                 topchat_only=False,
-                logger = config.logger(__name__),
+                 logger=config.logger(__name__),
-                ): 
+                 ):
-        self.video_id  = video_id
+        self._video_id = extract_video_id(video_id)
         self.seektime = seektime
         if isinstance(processor, tuple):
             self.processor = Combinator(processor)
@@ -98,59 +98,57 @@ class LiveChatAsync:
         self._done_callback = done_callback
         self._exception_handler = exception_handler
         self._direct_mode = direct_mode
-        self._is_alive   = True
+        self._is_alive = True
         self._is_replay = force_replay
-        self._parser = Parser(is_replay = self._is_replay)
+        self._parser = Parser(is_replay=self._is_replay)
         self._pauser = Queue()
         self._pauser.put_nowait(None)
-        self._setup()
         self._first_fetch = True
         self._fetch_url = "live_chat/get_live_chat?continuation="
         self._topchat_only = topchat_only
         self._logger = logger
+        self.exception = None
         LiveChatAsync._logger = logger
 
-        if not LiveChatAsync._setup_finished:
+        if exception_handler:
-            LiveChatAsync._setup_finished = True
+            self._set_exception_handler(exception_handler)
-            if exception_handler:
+        if interruptable:
-                self._set_exception_handler(exception_handler)
+            signal.signal(signal.SIGINT,
-            if interruptable:
+                (lambda a, b: asyncio.create_task(
-                signal.signal(signal.SIGINT,  
+                 LiveChatAsync.shutdown(None, signal.SIGINT, b))))
-                    (lambda a, b:asyncio.create_task(
+        self._setup()
-                    LiveChatAsync.shutdown(None,signal.SIGINT,b))
+
-                ))
- 
     def _setup(self):
-        #direct modeがTrueでcallback未設定の場合例外発生。
+        # direct modeがTrueでcallback未設定の場合例外発生。
         if self._direct_mode:
             if self._callback is None:
-                raise IllegalFunctionCall(
+                raise exceptions.IllegalFunctionCall(
                     "When direct_mode=True, callback parameter is required.")
         else:
-            #direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成
+            # direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成
             if self._buffer is None:
-                self._buffer = Buffer(maxsize = 20)
+                self._buffer = Buffer(maxsize=20)
-                #callbackが指定されている場合はcallbackを呼ぶループタスクを作成
+                # callbackが指定されている場合はcallbackを呼ぶループタスクを作成
             if self._callback is None:
-                pass 
+                pass
             else:
-                #callbackを呼ぶループタスクの開始
+                # callbackを呼ぶループタスクの開始
                 loop = asyncio.get_event_loop()
                 loop.create_task(self._callback_loop(self._callback))
-        #_listenループタスクの開始
+        # _listenループタスクの開始
         loop = asyncio.get_event_loop()
-        listen_task = loop.create_task(self._startlisten())
+        self.listen_task = loop.create_task(self._startlisten())
-        #add_done_callbackの登録
+        # add_done_callbackの登録
         if self._done_callback is None:
-            listen_task.add_done_callback(self.finish)
+            self.listen_task.add_done_callback(self._finish)
         else:
-            listen_task.add_done_callback(self._done_callback)
+            self.listen_task.add_done_callback(self._done_callback)
 
     async def _startlisten(self):
         """Fetch first continuation parameter,
         create and start _listen loop.
         """
-        initial_continuation = liveparam.getparam(self.video_id,3)
+        initial_continuation = liveparam.getparam(self._video_id, 3)
         await self._listen(initial_continuation)
 
     async def _listen(self, continuation):
@@ -163,39 +161,41 @@ class LiveChatAsync:
             parameter for next chat data
         '''
         try:
-            async with aiohttp.ClientSession() as session:
+            async with httpx.AsyncClient(http2=True) as client:
                 while(continuation and self._is_alive):
                     continuation = await self._check_pause(continuation)
                     contents = await self._get_contents(
-                        continuation, session, headers)
+                        continuation, client, headers)
-                    metadata, chatdata =  self._parser.parse(contents)
+                    metadata, chatdata = self._parser.parse(contents)
 
-                    timeout = metadata['timeoutMs']/1000
+                    timeout = metadata['timeoutMs'] / 1000
                     chat_component = {
-                        "video_id" : self.video_id,
+                        "video_id": self._video_id,
-                        "timeout"  : timeout,
+                        "timeout": timeout,
-                        "chatdata" : chatdata
+                        "chatdata": chatdata
                     }
-                    time_mark =time.time()
+                    time_mark = time.time()
                     if self._direct_mode:
-                        processed_chat = self.processor.process([chat_component])
+                        processed_chat = self.processor.process(
-                        if isinstance(processed_chat,tuple):
+                            [chat_component])
+                        if isinstance(processed_chat, tuple):
                             await self._callback(*processed_chat)
                         else:
                             await self._callback(processed_chat)
                     else:
                         await self._buffer.put(chat_component)
-                    diff_time = timeout - (time.time()-time_mark)
+                    diff_time = timeout - (time.time() - time_mark)
-                    await asyncio.sleep(diff_time)        
+                    await asyncio.sleep(diff_time)
-                    continuation = metadata.get('continuation')     
+                    continuation = metadata.get('continuation')
-        except ChatParseException as e:
+        except exceptions.ChatParseException as e:
-            self._logger.debug(f"[{self.video_id}]{str(e)}")
+            self._logger.debug(f"[{self._video_id}]{str(e)}")
-            return            
+            raise
-        except (TypeError , json.JSONDecodeError) :
+        except (TypeError, json.JSONDecodeError):
             self._logger.error(f"{traceback.format_exc(limit = -1)}")
-            return
+            raise
-        
+
-        self._logger.debug(f"[{self.video_id}]finished fetching chat.")
+        self._logger.debug(f"[{self._video_id}]finished fetching chat.")
+        raise exceptions.ChatDataFinished
 
     async def _check_pause(self, continuation):
         if self._pauser.empty():
@@ -207,21 +207,19 @@ class LiveChatAsync:
             self._pauser.put_nowait(None)
             if not self._is_replay:
                 continuation = liveparam.getparam(
-                    self.video_id, 3, self._topchat_only)
+                    self._video_id, 3, self._topchat_only)
         return continuation
 
-    async def _get_contents(self, continuation, session, headers):
+    async def _get_contents(self, continuation, client, headers):
         '''Get 'continuationContents' from livechat json.
-           If contents is None at first fetching, 
+           If contents is None at first fetching,
            try to fetch archive chat data.
 
           Return:
           -------
             'continuationContents' which includes metadata & chatdata.
         '''
-        livechat_json = (await 
+        livechat_json = await self._get_livechat_json(continuation, client, headers)
-            self._get_livechat_json(continuation, session, headers)
-        )
         contents = self._parser.get_contents(livechat_json)
         if self._first_fetch:
             if contents is None or self._is_replay:
@@ -229,46 +227,45 @@ class LiveChatAsync:
                 self._parser.is_replay = True
                 self._fetch_url = "live_chat_replay/get_live_chat_replay?continuation="
                 continuation = arcparam.getparam(
-                    self.video_id, self.seektime, self._topchat_only)
+                    self._video_id, self.seektime, self._topchat_only)
-                livechat_json = (await  self._get_livechat_json(
+                livechat_json = (await self._get_livechat_json(
-                    continuation, session, headers))
+                                 continuation, client, headers))
                 reload_continuation = self._parser.reload_continuation(
                     self._parser.get_contents(livechat_json))
                 if reload_continuation:
-                    livechat_json = (await  self._get_livechat_json(
+                    livechat_json = (await self._get_livechat_json(
-                        reload_continuation, session, headers))
+                        reload_continuation, client, headers))
                 contents = self._parser.get_contents(livechat_json)
                 self._is_replay = True
             self._first_fetch = False
         return contents
 
-    async def _get_livechat_json(self, continuation, session, headers):
+    async def _get_livechat_json(self, continuation, client, headers):
         '''
         Get json which includes chat data.
         '''
         continuation = urllib.parse.quote(continuation)
         livechat_json = None
         status_code = 0
-        url =f"https://www.youtube.com/{self._fetch_url}{continuation}&pbj=1"
+        url = f"https://www.youtube.com/{self._fetch_url}{continuation}&pbj=1"
         for _ in range(MAX_RETRY + 1):
-            async with session.get(url ,headers = headers) as resp:
+            try:
-                try:
+                resp = await client.get(url, headers=headers)
-                    text = await resp.text()
+                livechat_json = resp.json()
-                    livechat_json = json.loads(text)
+                break
-                    break
+            except (httpx.HTTPError, json.JSONDecodeError):
-                except (ClientConnectorError,json.JSONDecodeError) :
+                await asyncio.sleep(1)
-                    await asyncio.sleep(1)
+                continue
-                    continue
         else:
-            self._logger.error(f"[{self.video_id}]"
+            self._logger.error(f"[{self._video_id}]"
-                    f"Exceeded retry count. status_code={status_code}")
+                               f"Exceeded retry count. status_code={status_code}")
             return None
         return livechat_json
 
-    async def _callback_loop(self,callback):
+    async def _callback_loop(self, callback):
         """ コンストラクタでcallbackを指定している場合、バックグラウンドで
-        callbackに指定された関数に一定間隔でチャットデータを投げる。        
+        callbackに指定された関数に一定間隔でチャットデータを投げる。
-        
+
         Parameter
         ---------
         callback : func
@@ -285,14 +282,17 @@ class LiveChatAsync:
     async def get(self):
         """ bufferからデータを取り出し、processorに投げ、
         加工済みのチャットデータを返す。
-        
+
         Returns
              : Processorによって加工されたチャットデータ
         """
         if self._callback is None:
-            items = await self._buffer.get()
+            if self.is_alive():
-            return  self.processor.process(items)
+                items = await self._buffer.get()
-        raise IllegalFunctionCall(
+                return self.processor.process(items)
+            else:
+                return []
+        raise exceptions.IllegalFunctionCall(
             "既にcallbackを登録済みのため、get()は実行できません。")
 
     def is_replay(self):
@@ -309,40 +309,55 @@ class LiveChatAsync:
             return
         if self._pauser.empty():
             self._pauser.put_nowait(None)
-        
+
     def is_alive(self):
         return self._is_alive
 
-    def finish(self,sender):
+    def _finish(self, sender):
         '''Listener終了時のコールバック'''
-        try: 
+        try:
-            self.terminate()
+            self._task_finished()
         except CancelledError:
-            self._logger.debug(f'[{self.video_id}]cancelled:{sender}')
+            self._logger.debug(f'[{self._video_id}]cancelled:{sender}')
 
     def terminate(self):
+        if self._pauser.empty():
+            self._pauser.put_nowait(None)
+        self._is_alive = False
+        self._buffer.put_nowait({})
+        self.processor.finalize()
+        
+    def _task_finished(self):
         '''
         Listenerを終了する。
         '''
-        self._is_alive = False
+        if self.is_alive():
-        if self._direct_mode == False:
+            self.terminate()
-            #bufferにダミーオブジェクトを入れてis_alive()を判定させる
+        try:
-            self._buffer.put_nowait({'chatdata':'','timeout':0}) 
+            self.listen_task.result()
-        self._logger.info(f'[{self.video_id}]finished.')
+        except Exception as e:
- 
+            self.exception = e
+            if not isinstance(e, exceptions.ChatParseException):
+                self._logger.error(f'Internal exception - {type(e)}{str(e)}')
+        self._logger.info(f'[{self._video_id}]終了しました')
+
+    def raise_for_status(self):
+        if self.exception is not None:
+            raise self.exception
+
     @classmethod
     def _set_exception_handler(cls, handler):
         loop = asyncio.get_event_loop()
         loop.set_exception_handler(handler)
-    
+
     @classmethod
-    async def shutdown(cls, event, sig = None, handler=None):
+    async def shutdown(cls, event, sig=None, handler=None):
         cls._logger.debug("shutdown...")
         tasks = [t for t in asyncio.all_tasks() if t is not
-        asyncio.current_task()]
+                 asyncio.current_task()]
         [task.cancel() for task in tasks]
 
-        cls._logger.debug(f"complete remaining tasks...")
+        cls._logger.debug("complete remaining tasks...")
-        await asyncio.gather(*tasks,return_exceptions=True)
+        await asyncio.gather(*tasks, return_exceptions=True)
         loop = asyncio.get_event_loop()
-        loop.stop()
+        loop.stop()

pytchat/core_multithread/buffer.py

@@ -1,6 +1,7 @@
 
 import queue
 
+
 class Buffer(queue.Queue):
     '''
     チャットデータを格納するバッファの役割を持つFIFOキュー
@@ -11,28 +12,29 @@ class Buffer(queue.Queue):
         格納するチャットブロックの最大個数。0の場合は無限。
         最大値を超える場合は古いチャットブロックから破棄される。
     '''
-    def __init__(self,maxsize = 0):
+
+    def __init__(self, maxsize=0):
         super().__init__(maxsize=maxsize)
-    
+
-    def put(self,item):
+    def put(self, item):
         if item is None:
-            return 
+            return
         if super().full():
             super().get_nowait()
         else:
             super().put(item)
-            
+
-    def put_nowait(self,item):
+    def put_nowait(self, item):
         if item is None:
-            return 
+            return
         if super().full():
             super().get_nowait()
         else:
             super().put_nowait(item)
-            
+
     def get(self):
         ret = []
         ret.append(super().get())
         while not super().empty():
             ret.append(super().get())
-        return ret
+        return ret

pytchat/core_multithread/livechat.py

@@ -1,20 +1,20 @@
-import requests
+import httpx
-import datetime
 import json
-import random
 import signal
 import time
 import traceback
 import urllib.parse
 from concurrent.futures import CancelledError, ThreadPoolExecutor
 from queue import Queue
+from threading import Event
 from .buffer import Buffer
 from ..parser.live import Parser
 from .. import config
-from ..exceptions import ChatParseException, IllegalFunctionCall
+from .. import exceptions
 from ..paramgen import liveparam, arcparam
 from ..processors.default.processor import DefaultProcessor
 from ..processors.combinator import Combinator
+from ..util.extract_video_id import extract_video_id
 
 headers = config.headers
 MAX_RETRY = 10
@@ -53,9 +53,9 @@ class LiveChat:
     direct_mode : bool
         Trueの場合、bufferを使わずにcallbackを呼ぶ。
         Trueの場合、callbackの設定が必須
-        (設定していない場合IllegalFunctionCall例外を発生させる）  
+        (設定していない場合IllegalFunctionCall例外を発生させる）
 
-    force_replay : bool  
+    force_replay : bool
         Trueの場合、ライブチャットが取得できる場合であっても
         強制的にアーカイブ済みチャットを取得する。
 
@@ -74,7 +74,7 @@ class LiveChat:
     _setup_finished = False
 
     def __init__(self, video_id,
-                 seektime=0,
+                 seektime=-1,
                  processor=DefaultProcessor(),
                  buffer=None,
                  interruptable=True,
@@ -85,7 +85,7 @@ class LiveChat:
                  topchat_only=False,
                  logger=config.logger(__name__)
                  ):
-        self.video_id = video_id
+        self._video_id = extract_video_id(video_id)
         self.seektime = seektime
         if isinstance(processor, tuple):
             self.processor = Combinator(processor)
@@ -104,7 +104,9 @@ class LiveChat:
         self._first_fetch = True
         self._fetch_url = "live_chat/get_live_chat?continuation="
         self._topchat_only = topchat_only
+        self._event = Event()
         self._logger = logger
+        self.exception = None
         if interruptable:
             signal.signal(signal.SIGINT, lambda a, b: self.terminate())
         self._setup()
@@ -113,7 +115,7 @@ class LiveChat:
         # direct modeがTrueでcallback未設定の場合例外発生。
         if self._direct_mode:
             if self._callback is None:
-                raise IllegalFunctionCall(
+                raise exceptions.IllegalFunctionCall(
                     "When direct_mode=True, callback parameter is required.")
         else:
             # direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成
@@ -126,19 +128,19 @@ class LiveChat:
                 # callbackを呼ぶループタスクの開始
                 self._executor.submit(self._callback_loop, self._callback)
         # _listenループタスクの開始
-        listen_task = self._executor.submit(self._startlisten)
+        self.listen_task = self._executor.submit(self._startlisten)
         # add_done_callbackの登録
         if self._done_callback is None:
-            listen_task.add_done_callback(self.finish)
+            self.listen_task.add_done_callback(self._finish)
         else:
-            listen_task.add_done_callback(self._done_callback)
+            self.listen_task.add_done_callback(self._done_callback)
 
     def _startlisten(self):
         time.sleep(0.1)  # sleep shortly to prohibit skipping fetching data
         """Fetch first continuation parameter,
         create and start _listen loop.
         """
-        initial_continuation = liveparam.getparam(self.video_id, 3)
+        initial_continuation = liveparam.getparam(self._video_id, 3)
         self._listen(initial_continuation)
 
     def _listen(self, continuation):
@@ -151,16 +153,14 @@ class LiveChat:
             parameter for next chat data
         '''
         try:
-            with requests.Session() as session:
+            with httpx.Client(http2=True) as client:
                 while(continuation and self._is_alive):
                     continuation = self._check_pause(continuation)
-                    contents = self._get_contents(
+                    contents = self._get_contents(continuation, client, headers)
-                        continuation, session, headers)
                     metadata, chatdata = self._parser.parse(contents)
-
+                    timeout = metadata['timeoutMs'] / 1000
-                    timeout = metadata['timeoutMs']/1000
                     chat_component = {
-                        "video_id": self.video_id,
+                        "video_id": self._video_id,
                         "timeout": timeout,
                         "chatdata": chatdata
                     }
@@ -174,17 +174,18 @@ class LiveChat:
                             self._callback(processed_chat)
                     else:
                         self._buffer.put(chat_component)
-                    diff_time = timeout - (time.time()-time_mark)
+                    diff_time = timeout - (time.time() - time_mark)
-                    time.sleep(diff_time if diff_time > 0 else 0)
+                    self._event.wait(diff_time if diff_time > 0 else 0)
                     continuation = metadata.get('continuation')
-        except ChatParseException as e:
+        except exceptions.ChatParseException as e:
-            self._logger.debug(f"[{self.video_id}]{str(e)}")
+            self._logger.debug(f"[{self._video_id}]{str(e)}")
-            return
+            raise
         except (TypeError, json.JSONDecodeError):
-            self._logger.error(f"{traceback.format_exc(limit = -1)}")
+            self._logger.error(f"{traceback.format_exc(limit=-1)}")
-            return
+            raise
 
-        self._logger.debug(f"[{self.video_id}]finished fetching chat.")
+        self._logger.debug(f"[{self._video_id}]finished fetching chat.")
+        raise exceptions.ChatDataFinished
 
     def _check_pause(self, continuation):
         if self._pauser.empty():
@@ -195,12 +196,12 @@ class LiveChat:
             '''
             self._pauser.put_nowait(None)
             if not self._is_replay:
-                continuation = liveparam.getparam(self.video_id, 3)
+                continuation = liveparam.getparam(self._video_id, 3)
         return continuation
 
-    def _get_contents(self, continuation, session, headers):
+    def _get_contents(self, continuation, client, headers):
         '''Get 'continuationContents' from livechat json.
-           If contents is None at first fetching, 
+           If contents is None at first fetching,
            try to fetch archive chat data.
 
           Return:
@@ -208,7 +209,7 @@ class LiveChat:
             'continuationContents' which includes metadata & chat data.
         '''
         livechat_json = (
-            self._get_livechat_json(continuation, session, headers)
+            self._get_livechat_json(continuation, client, headers)
         )
         contents = self._parser.get_contents(livechat_json)
         if self._first_fetch:
@@ -217,20 +218,19 @@ class LiveChat:
                 self._parser.is_replay = True
                 self._fetch_url = "live_chat_replay/get_live_chat_replay?continuation="
                 continuation = arcparam.getparam(
-                    self.video_id, self.seektime, self._topchat_only)
+                    self._video_id, self.seektime, self._topchat_only)
-                livechat_json = (self._get_livechat_json(
+                livechat_json = (self._get_livechat_json(continuation, client, headers))
-                    continuation, session, headers))
                 reload_continuation = self._parser.reload_continuation(
                     self._parser.get_contents(livechat_json))
                 if reload_continuation:
                     livechat_json = (self._get_livechat_json(
-                        reload_continuation, session, headers))
+                        reload_continuation, client, headers))
                 contents = self._parser.get_contents(livechat_json)
                 self._is_replay = True
             self._first_fetch = False
         return contents
 
-    def _get_livechat_json(self, continuation, session, headers):
+    def _get_livechat_json(self, continuation, client, headers):
         '''
         Get json which includes chat data.
         '''
@@ -239,23 +239,22 @@ class LiveChat:
         status_code = 0
         url = f"https://www.youtube.com/{self._fetch_url}{continuation}&pbj=1"
         for _ in range(MAX_RETRY + 1):
-            with session.get(url, headers=headers) as resp:
+            with client:
                 try:
-                    text = resp.text
+                    livechat_json = client.get(url, headers=headers).json()
-                    livechat_json = json.loads(text)
                     break
                 except json.JSONDecodeError:
                     time.sleep(1)
                     continue
         else:
-            self._logger.error(f"[{self.video_id}]"
+            self._logger.error(f"[{self._video_id}]"
                                f"Exceeded retry count. status_code={status_code}")
-            return None
+            raise exceptions.RetryExceedMaxCount()
         return livechat_json
 
     def _callback_loop(self, callback):
         """ コンストラクタでcallbackを指定している場合、バックグラウンドで
-        callbackに指定された関数に一定間隔でチャットデータを投げる。        
+        callbackに指定された関数に一定間隔でチャットデータを投げる。
 
         Parameter
         ---------
@@ -278,9 +277,12 @@ class LiveChat:
              : Processorによって加工されたチャットデータ
         """
         if self._callback is None:
-            items = self._buffer.get()
+            if self.is_alive():
-            return self.processor.process(items)
+                items = self._buffer.get()
-        raise IllegalFunctionCall(
+                return self.processor.process(items)
+            else:
+                return []
+        raise exceptions.IllegalFunctionCall(
             "既にcallbackを登録済みのため、get()は実行できません。")
 
     def is_replay(self):
@@ -301,18 +303,35 @@ class LiveChat:
     def is_alive(self):
         return self._is_alive
 
-    def finish(self, sender):
+    def _finish(self, sender):
         '''Listener終了時のコールバック'''
         try:
-            self.terminate()
+            self._task_finished()
         except CancelledError:
-            self._logger.debug(f'[{self.video_id}]cancelled:{sender}')
+            self._logger.debug(f'[{self._video_id}]cancelled:{sender}')
 
     def terminate(self):
+        if self._pauser.empty():
+            self._pauser.put_nowait(None)
+        self._is_alive = False
+        self._buffer.put({})
+        self._event.set()
+        self.processor.finalize()
+
+    def _task_finished(self):
         '''
         Listenerを終了する。
         '''
         if self.is_alive():
-            self._is_alive = False
+            self.terminate()
-            self._buffer.put({})
+        try:
-            self._logger.info(f'[{self.video_id}]終了しました')
+            self.listen_task.result()
+        except Exception as e:
+            self.exception = e
+            if not isinstance(e, exceptions.ChatParseException):
+                self._logger.error(f'Internal exception - {type(e)}{str(e)}')
+        self._logger.info(f'[{self._video_id}]終了しました')
+
+    def raise_for_status(self):
+        if self.exception is not None:
+            raise self.exception

pytchat/exceptions.py

@@ -4,11 +4,6 @@ class ChatParseException(Exception):
     '''
     pass
 
-class NoYtinitialdataException(ChatParseException):
-    '''
-    Thrown when the video is not found.
-    '''
-    pass
 
 class ResponseContextError(ChatParseException):
     '''
@@ -16,37 +11,69 @@ class ResponseContextError(ChatParseException):
     '''
     pass
 
-class NoLivechatRendererException(ChatParseException):
-    '''
-    Thrown when livechatRenderer is missing in JSON.
-    '''
-    pass
 
-
+class NoContents(ChatParseException):
-class NoContentsException(ChatParseException):
     '''
     Thrown when ContinuationContents is missing in JSON.
     '''
     pass
 
-class NoContinuationsException(ChatParseException):
+
+class NoContinuation(ChatParseException):
     '''
     Thrown when continuation is missing in ContinuationContents.
     '''
     pass
 
+
 class IllegalFunctionCall(Exception):
     '''
-    Thrown when get () is called even though 
+    Thrown when get() is called even though
-    set_callback () has been executed.
+    set_callback() has been executed.
     '''
     pass
 
+
 class InvalidVideoIdException(Exception):
     '''
     Thrown when the video_id is not exist (VideoInfo).
     '''
     pass
 
+
 class UnknownConnectionError(Exception):
-    pass
+    pass
+
+
+class RetryExceedMaxCount(Exception):
+    '''
+    thrown when the number of retries exceeds the maximum value.
+    '''
+    pass
+
+
+class ChatDataFinished(ChatParseException):
+    pass
+
+
+class ReceivedUnknownContinuation(ChatParseException):
+    pass
+
+
+class FailedExtractContinuation(ChatDataFinished):
+    pass
+
+
+class VideoInfoParseError(Exception):
+    '''
+    thrown when failed to parse video info
+    '''
+
+
+class PatternUnmatchError(VideoInfoParseError):
+    '''
+    thrown when failed to parse video info with unmatched pattern
+    '''
+    def __init__(self, doc):
+        self.msg = "PatternUnmatchError"
+        self.doc = doc

pytchat/paramgen/arcparam.py

@@ -1,111 +1,55 @@
-from base64 import urlsafe_b64encode as b64enc
+from .pb.header_pb2 import Header
-from functools import reduce
+from .pb.replay_pb2 import Continuation
-import math
+from urllib.parse import quote
-import random
+import base64
-import urllib.parse
 
 '''
 Generate continuation parameter of youtube replay chat.
 
-Author: taizan-hokuto (2019) @taizan205
+Author: taizan-hokuto
 
-ver 0.0.1 2019.10.05
+ver 0.0.1 2019.10.05 : Initial release.
+ver 0.0.2 2020.05.30 : Use Protocol Buffers.
 '''
 
 
-def _gen_vid(video_id):
+def _gen_vid(video_id) -> str:
-    """generate video_id parameter.
+    header = Header()
-    Parameter
+    header.info.video.id = video_id
-    ---------
+    header.terminator = 1
-        video_id : str
+    return base64.urlsafe_b64encode(header.SerializeToString()).decode()
-
-    Return
-    ---------
-        bytes : base64 encoded video_id parameter.
-    """
-    header_magic = b'\x0A\x0F\x1A\x0D\x0A'
-    header_id = video_id.encode()
-    header_sep_1 = b'\x1A\x13\xEA\xA8\xDD\xB9\x01\x0D\x0A\x0B'
-    header_terminator = b'\x20\x01'
-
-    item = [
-        header_magic,
-        _nval(len(header_id)),
-        header_id,
-        header_sep_1,
-        header_id,
-        header_terminator
-    ]
-
-    return urllib.parse.quote(
-        b64enc(reduce(lambda x, y: x+y, item)).decode()
-    ).encode()
 
 
-def _nval(val):
+def _build(video_id, seektime, topchat_only) -> str:
-    """convert value to byte array"""
+    chattype = 1
-    if val < 0:
+    timestamp = 0
-        raise ValueError
+    if topchat_only:
-    buf = b''
+        chattype = 4
-    while val >> 7:
-        m = val & 0xFF | 0x80
-        buf += m.to_bytes(1, 'big')
-        val >>= 7
-    buf += val.to_bytes(1, 'big')
-    return buf
 
-
+    fetch_before_start = 3
-def _build(video_id, seektime, topchat_only):
-    switch_01 = b'\x04' if topchat_only else b'\x01'
     if seektime < 0:
-        times = _nval(0)
+        fetch_before_start = 4
-        switch = b'\x04'
     elif seektime == 0:
-        times = _nval(1)
+        timestamp = 1
-        switch = b'\x03'
     else:
-        times = _nval(int(seektime*1000000))
+        timestamp = int(seektime * 1000000)
-        switch = b'\x03'
+    continuation = Continuation()
-    parity = b'\x00'
+    entity = continuation.entity
-
+    entity.header = _gen_vid(video_id)
-    header_magic = b'\xA2\x9D\xB0\xD3\x04'
+    entity.timestamp = timestamp
-    sep_0 = b'\x1A'
+    entity.s6 = 0
-    vid = _gen_vid(video_id)
+    entity.s7 = 0
-    time_tag = b'\x28'
+    entity.s8 = 0
-    timestamp1 = times
+    entity.s9 = fetch_before_start
-    sep_1 = b'\x30\x00\x38\x00\x40\x00\x48'
+    entity.s10 = ''
-    sep_2 = b'\x52\x1C\x08\x00\x10\x00\x18\x00\x20\x00'
+    entity.s12 = chattype
-    chkstr = b'\x2A\x0E\x73\x74\x61\x74\x69\x63\x63\x68\x65\x63\x6B\x73\x75\x6D\x40'
+    entity.chattype.value = chattype
-    sep_3 = b'\x00\x58\x03\x60'
+    entity.s15 = 0
-    sep_4 = b'\x68' + parity + b'\x72\x04\x08'
+    return quote(
-    sep_5 = b'\x10' + parity + b'\x78\x00'
+        base64.urlsafe_b64encode(continuation.SerializeToString()).decode())
-
-    body = b''.join([
-        sep_0,
-        _nval(len(vid)),
-        vid,
-        time_tag,
-        timestamp1,
-        sep_1,
-        switch,
-        sep_2,
-        chkstr,
-        sep_3,
-        switch_01,
-        sep_4,
-        switch_01,
-        sep_5
-    ])
-
-    return urllib.parse.quote(
-        b64enc(header_magic +
-               _nval(len(body)) +
-               body
-               ).decode()
-    )
 
 
-def getparam(video_id, seektime=0, topchat_only=False):
+def getparam(video_id, seektime=-1, topchat_only=False) -> str:
     '''
     Parameter
     ---------

pytchat/paramgen/arcparam_mining.py

@@ -1,7 +1,5 @@
 from base64 import urlsafe_b64encode as b64enc
 from functools import reduce
-import math
-import random
 import urllib.parse
 
 '''
@@ -12,6 +10,7 @@ Author: taizan-hokuto (2019) @taizan205
 ver 0.0.1 2019.10.05
 '''
 
+
 def _gen_vid_long(video_id):
     """generate video_id parameter.
     Parameter
@@ -23,7 +22,7 @@ def _gen_vid_long(video_id):
     byte[] : base64 encoded video_id parameter.
     """
     header_magic = b'\x0A\x0F\x1A\x0D\x0A'
-    header_id =  video_id.encode()
+    header_id = video_id.encode()
     header_sep_1 = b'\x1A\x13\xEA\xA8\xDD\xB9\x01\x0D\x0A\x0B'
     header_terminator = b'\x20\x01'
 
@@ -37,9 +36,10 @@ def _gen_vid_long(video_id):
     ]
 
     return urllib.parse.quote(
-        b64enc(reduce(lambda x, y: x+y, item)).decode()
+        b64enc(reduce(lambda x, y: x + y, item)).decode()
     ).encode()
 
+
 def _gen_vid(video_id):
     """generate video_id parameter.
     Parameter
@@ -51,7 +51,7 @@ def _gen_vid(video_id):
         bytes : base64 encoded video_id parameter.
     """
     header_magic = b'\x0A\x0F\x1A\x0D\x0A'
-    header_id =  video_id.encode()
+    header_id = video_id.encode()
     header_terminator = b'\x20\x01'
 
     item = [
@@ -62,20 +62,23 @@ def _gen_vid(video_id):
     ]
 
     return urllib.parse.quote(
-        b64enc(reduce(lambda x, y: x+y, item)).decode()
+        b64enc(reduce(lambda x, y: x + y, item)).decode()
     ).encode()
 
+
 def _nval(val):
     """convert value to byte array"""
-    if val<0: raise ValueError
+    if val < 0:
+        raise ValueError
     buf = b''
     while val >> 7:
         m = val & 0xFF | 0x80
-        buf += m.to_bytes(1,'big')
+        buf += m.to_bytes(1, 'big')
         val >>= 7
-    buf += val.to_bytes(1,'big')
+    buf += val.to_bytes(1, 'big')
     return buf
 
+
 def _build(video_id, seektime, topchat_only):
     switch_01 = b'\x04' if topchat_only else b'\x01'
     if seektime < 0:
@@ -83,21 +86,19 @@ def _build(video_id, seektime, topchat_only):
     if seektime == 0:
         times = b''
     else:
-        times =_nval(int(seektime*1000))
+        times = _nval(int(seektime * 1000))
     if seektime > 0:
-        _len_time = ( b'\x5A'
+        _len_time = b'\x5A' + (len(times) + 1).to_bytes(1, 'big') + b'\x10'
-                    + (len(times)+1).to_bytes(1,'big')
-                    + b'\x10')
     else:
         _len_time = b''
-    
+
     header_magic = b'\xA2\x9D\xB0\xD3\x04'
-    sep_0        = b'\x1A'
+    sep_0 = b'\x1A'
-    vid          = _gen_vid(video_id)
+    vid = _gen_vid(video_id)
-    _tag         = b'\x40\x01'
+    _tag = b'\x40\x01'
-    timestamp1   = times
+    timestamp1 = times
-    sep_1        = b'\x60\x04\x72\x02\x08'
+    sep_1 = b'\x60\x04\x72\x02\x08'
-    terminator   = b'\x78\x01'
+    terminator = b'\x78\x01'
 
     body = [
         sep_0,
@@ -111,16 +112,15 @@ def _build(video_id, seektime, topchat_only):
         terminator
     ]
 
-    body = reduce(lambda x, y: x+y, body)
+    body = reduce(lambda x, y: x + y, body)
-    
-    return  urllib.parse.quote(
-                b64enc( header_magic +
-                        _nval(len(body)) +
-                        body
-                ).decode()
-            )
 
-def getparam(video_id, seektime = 0.0, topchat_only = False):
+    return urllib.parse.quote(
+        b64enc(header_magic + _nval(len(body)) + body
+               ).decode()
+    )
+
+
+def getparam(video_id, seektime=0.0, topchat_only=False):
     '''
     Parameter
     ---------

pytchat/paramgen/liveparam.py

@@ -1,19 +1,21 @@
-from base64 import urlsafe_b64encode as b64enc
+from .pb.header_pb2 import Header
-from functools import reduce
+from .pb.live_pb2 import Continuation
-import time
+from urllib.parse import quote
+import base64
 import random
-import urllib.parse
+import time
 
 '''
 Generate continuation parameter of youtube live chat.
 
-Author: taizan-hokuto (2019) @taizan205
+Author: taizan-hokuto
 
-ver 0.0.1 2019.10.05
+ver 0.0.1 2019.10.05 : Initial release.
+ver 0.0.2 2020.05.30 : Use Protocol Buffers.
 '''
 
 
-def _gen_vid(video_id):
+def _gen_vid(video_id) -> str:
     """generate video_id parameter.
     Parameter
     ---------
@@ -21,141 +23,60 @@ def _gen_vid(video_id):
 
     Return
     ---------
-    byte[] : base64 encoded video_id parameter.
+    str : base64 encoded video_id parameter.
     """
-    header_magic = b'\x0A\x0F\x0A\x0D\x0A'
+    header = Header()
-    header_id = video_id.encode()
+    header.info.video.id = video_id
-    header_sep_1 = b'\x1A'
+    header.terminator = 1
-    header_sep_2 = b'\x43\xAA\xB9\xC1\xBD\x01\x3D\x0A'
+    return base64.urlsafe_b64encode(header.SerializeToString()).decode()
-    header_suburl = ('https://www.youtube.com/live_chat?v='
-                     f'{video_id}&is_popout=1').encode()
-    header_terminator = b'\x20\x02'
-
-    item = [
-        header_magic,
-        _nval(len(header_id)),
-        header_id,
-        header_sep_1,
-        header_sep_2,
-        _nval(len(header_suburl)),
-        header_suburl,
-        header_terminator
-    ]
-
-    return urllib.parse.quote(
-        b64enc(reduce(lambda x, y: x+y, item)).decode()
-    ).encode()
 
 
-def _tzparity(video_id, times):
+def _build(video_id, ts1, ts2, ts3, ts4, ts5, topchat_only) -> str:
-    t = 0
+    chattype = 1
-    for i, s in enumerate(video_id):
+    if topchat_only:
-        ss = ord(s)
+        chattype = 4
-        if(ss % 2 == 0):
+    continuation = Continuation()
-            t += ss*(12-i)
+    entity = continuation.entity
-        else:
-            t ^= ss*i
 
-    return ((times ^ t) % 2).to_bytes(1, 'big')
+    entity.header = _gen_vid(video_id)
+    entity.timestamp1 = ts1
+    entity.s6 = 0
+    entity.s7 = 0
+    entity.s8 = 1
+    entity.body.b1 = 0
+    entity.body.b2 = 0
+    entity.body.b3 = 0
+    entity.body.b4 = 0
+    entity.body.b7 = ''
+    entity.body.b8 = 0
+    entity.body.b9 = ''
+    entity.body.timestamp2 = ts2
+    entity.body.b11 = 3
+    entity.body.b15 = 0
+    entity.timestamp3 = ts3
+    entity.timestamp4 = ts4
+    entity.s13 = chattype
+    entity.chattype.value = chattype
+    entity.s17 = 0
+    entity.str19.value = 0
+    entity.timestamp5 = ts5
 
-
+    return quote(
-def _nval(val):
+        base64.urlsafe_b64encode(continuation.SerializeToString()).decode()
-    """convert value to byte array"""
-    if val < 0:
-        raise ValueError
-    buf = b''
-    while val >> 7:
-        m = val & 0xFF | 0x80
-        buf += m.to_bytes(1, 'big')
-        val >>= 7
-    buf += val.to_bytes(1, 'big')
-    return buf
-
-
-def _build(video_id, _ts1, _ts2, _ts3, _ts4, _ts5, topchat_only):
-    # _short_type2
-    switch_01 = b'\x04' if topchat_only else b'\x01'
-    parity = _tzparity(video_id, _ts1 ^ _ts2 ^ _ts3 ^ _ts4 ^ _ts5)
-
-    header_magic = b'\xD2\x87\xCC\xC8\x03'
-    sep_0 = b'\x1A'
-    vid = _gen_vid(video_id)
-    time_tag = b'\x28'
-    timestamp1 = _nval(_ts1)
-    sep_1 = b'\x30\x00\x38\x00\x40\x02\x4A'
-    un_len = b'\x2B'
-    sep_2 = b'\x08'+parity+b'\x10\x00\x18\x00\x20\x00'
-    chkstr = b'\x2A\x0E\x73\x74\x61\x74\x69\x63\x63\x68\x65\x63\x6B\x73\x75\x6D'
-    sep_3 = b'\x3A\x00\x40\x00\x4A'
-    sep_4_len = b'\x02'
-    sep_4 = b'\x08\x01'
-    ts_2_start = b'\x50'
-    timestamp2 = _nval(_ts2)
-    ts_2_end = b'\x58'
-    sep_5 = b'\x03'
-    ts_3_start = b'\x50'
-    timestamp3 = _nval(_ts3)
-    ts_3_end = b'\x58'
-    timestamp4 = _nval(_ts4)
-    sep_6 = b'\x68'
-    # switch
-    sep_7 = b'\x82\x01\x04\x08'
-    # switch
-    sep_8 = b'\x10\x00'
-    sep_9 = b'\x88\x01\x00\xA0\x01'
-    timestamp5 = _nval(_ts5)
-
-    body = b''.join([
-        sep_0,
-        _nval(len(vid)),
-        vid,
-        time_tag,
-        timestamp1,
-        sep_1,
-        un_len,
-        sep_2,
-        chkstr,
-        sep_3,
-        sep_4_len,
-        sep_4,
-        ts_2_start,
-        timestamp2,
-        ts_2_end,
-        sep_5,
-        ts_3_start,
-        timestamp3,
-        ts_3_end,
-        timestamp4,
-        sep_6,
-        switch_01,
-        sep_7,
-        switch_01,
-        sep_8,
-        sep_9,
-        timestamp5
-    ])
-
-    return urllib.parse.quote(
-        b64enc(header_magic +
-               _nval(len(body)) +
-               body
-               ).decode()
     )
 
 
 def _times(past_sec):
-
     n = int(time.time())
-
+    _ts1 = n - random.uniform(0, 1 * 3)
-    _ts1 = n - random.uniform(0, 1*3)
     _ts2 = n - random.uniform(0.01, 0.99)
     _ts3 = n - past_sec + random.uniform(0, 1)
-    _ts4 = n - random.uniform(10*60, 60*60)
+    _ts4 = n - random.uniform(10 * 60, 60 * 60)
     _ts5 = n - random.uniform(0.01, 0.99)
-    return list(map(lambda x: int(x*1000000), [_ts1, _ts2, _ts3, _ts4, _ts5]))
+    return list(map(lambda x: int(x * 1000000), [_ts1, _ts2, _ts3, _ts4, _ts5]))
 
 
-def getparam(video_id, past_sec=0, topchat_only=False):
+def getparam(video_id, past_sec=0, topchat_only=False) -> str:
     '''
     Parameter
     ---------

pytchat/paramgen/pb/header_pb2.py

@@ -0,0 +1,159 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: header.proto
+
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='header.proto',
+  package='',
+  syntax='proto3',
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n\x0cheader.proto\"\x13\n\x05Video\x12\n\n\x02id\x18\x01 \x01(\t\"#\n\nHeaderInfo\x12\x15\n\x05video\x18\x01 \x01(\x0b\x32\x06.Video\"7\n\x06Header\x12\x19\n\x04info\x18\x01 \x01(\x0b\x32\x0b.HeaderInfo\x12\x12\n\nterminator\x18\x04 \x01(\x05\x62\x06proto3'
+)
+
+
+
+
+_VIDEO = _descriptor.Descriptor(
+  name='Video',
+  full_name='Video',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='id', full_name='Video.id', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=16,
+  serialized_end=35,
+)
+
+
+_HEADERINFO = _descriptor.Descriptor(
+  name='HeaderInfo',
+  full_name='HeaderInfo',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='video', full_name='HeaderInfo.video', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=37,
+  serialized_end=72,
+)
+
+
+_HEADER = _descriptor.Descriptor(
+  name='Header',
+  full_name='Header',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='info', full_name='Header.info', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='terminator', full_name='Header.terminator', index=1,
+      number=4, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=74,
+  serialized_end=129,
+)
+
+_HEADERINFO.fields_by_name['video'].message_type = _VIDEO
+_HEADER.fields_by_name['info'].message_type = _HEADERINFO
+DESCRIPTOR.message_types_by_name['Video'] = _VIDEO
+DESCRIPTOR.message_types_by_name['HeaderInfo'] = _HEADERINFO
+DESCRIPTOR.message_types_by_name['Header'] = _HEADER
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Video = _reflection.GeneratedProtocolMessageType('Video', (_message.Message,), {
+  'DESCRIPTOR' : _VIDEO,
+  '__module__' : 'header_pb2'
+  # @@protoc_insertion_point(class_scope:Video)
+  })
+_sym_db.RegisterMessage(Video)
+
+HeaderInfo = _reflection.GeneratedProtocolMessageType('HeaderInfo', (_message.Message,), {
+  'DESCRIPTOR' : _HEADERINFO,
+  '__module__' : 'header_pb2'
+  # @@protoc_insertion_point(class_scope:HeaderInfo)
+  })
+_sym_db.RegisterMessage(HeaderInfo)
+
+Header = _reflection.GeneratedProtocolMessageType('Header', (_message.Message,), {
+  'DESCRIPTOR' : _HEADER,
+  '__module__' : 'header_pb2'
+  # @@protoc_insertion_point(class_scope:Header)
+  })
+_sym_db.RegisterMessage(Header)
+
+
+# @@protoc_insertion_point(module_scope)

pytchat/paramgen/pb/live_pb2.py

@@ -0,0 +1,381 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: live.proto
+
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='live.proto',
+  package='live',
+  syntax='proto3',
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n\nlive.proto\x12\x04live\"\x88\x01\n\x04\x42ody\x12\n\n\x02\x62\x31\x18\x01 \x01(\x05\x12\n\n\x02\x62\x32\x18\x02 \x01(\x05\x12\n\n\x02\x62\x33\x18\x03 \x01(\x05\x12\n\n\x02\x62\x34\x18\x04 \x01(\x05\x12\n\n\x02\x62\x37\x18\x07 \x01(\t\x12\n\n\x02\x62\x38\x18\x08 \x01(\x05\x12\n\n\x02\x62\x39\x18\t \x01(\t\x12\x12\n\ntimestamp2\x18\n \x01(\x03\x12\x0b\n\x03\x62\x31\x31\x18\x0b \x01(\x05\x12\x0b\n\x03\x62\x31\x35\x18\x0f \x01(\x05\"\x19\n\x08\x43hatType\x12\r\n\x05value\x18\x01 \x01(\x05\"\x16\n\x05STR19\x12\r\n\x05value\x18\x01 \x01(\x05\"\x8a\x02\n\x12\x43ontinuationEntity\x12\x0e\n\x06header\x18\x03 \x01(\t\x12\x12\n\ntimestamp1\x18\x05 \x01(\x03\x12\n\n\x02s6\x18\x06 \x01(\x05\x12\n\n\x02s7\x18\x07 \x01(\x05\x12\n\n\x02s8\x18\x08 \x01(\x05\x12\x18\n\x04\x62ody\x18\t \x01(\x0b\x32\n.live.Body\x12\x12\n\ntimestamp3\x18\n \x01(\x03\x12\x12\n\ntimestamp4\x18\x0b \x01(\x03\x12\x0b\n\x03s13\x18\r \x01(\x05\x12 \n\x08\x63hattype\x18\x10 \x01(\x0b\x32\x0e.live.ChatType\x12\x0b\n\x03s17\x18\x11 \x01(\x05\x12\x1a\n\x05str19\x18\x13 \x01(\x0b\x32\x0b.live.STR19\x12\x12\n\ntimestamp5\x18\x14 \x01(\x03\";\n\x0c\x43ontinuation\x12+\n\x06\x65ntity\x18\xfa\xc0\x89\x39 \x01(\x0b\x32\x18.live.ContinuationEntityb\x06proto3'
+)
+
+
+
+
+_BODY = _descriptor.Descriptor(
+  name='Body',
+  full_name='live.Body',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='b1', full_name='live.Body.b1', index=0,
+      number=1, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='b2', full_name='live.Body.b2', index=1,
+      number=2, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='b3', full_name='live.Body.b3', index=2,
+      number=3, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='b4', full_name='live.Body.b4', index=3,
+      number=4, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='b7', full_name='live.Body.b7', index=4,
+      number=7, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='b8', full_name='live.Body.b8', index=5,
+      number=8, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='b9', full_name='live.Body.b9', index=6,
+      number=9, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='timestamp2', full_name='live.Body.timestamp2', index=7,
+      number=10, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='b11', full_name='live.Body.b11', index=8,
+      number=11, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='b15', full_name='live.Body.b15', index=9,
+      number=15, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=21,
+  serialized_end=157,
+)
+
+
+_CHATTYPE = _descriptor.Descriptor(
+  name='ChatType',
+  full_name='live.ChatType',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='value', full_name='live.ChatType.value', index=0,
+      number=1, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=159,
+  serialized_end=184,
+)
+
+
+_STR19 = _descriptor.Descriptor(
+  name='STR19',
+  full_name='live.STR19',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='value', full_name='live.STR19.value', index=0,
+      number=1, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=186,
+  serialized_end=208,
+)
+
+
+_CONTINUATIONENTITY = _descriptor.Descriptor(
+  name='ContinuationEntity',
+  full_name='live.ContinuationEntity',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='header', full_name='live.ContinuationEntity.header', index=0,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='timestamp1', full_name='live.ContinuationEntity.timestamp1', index=1,
+      number=5, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s6', full_name='live.ContinuationEntity.s6', index=2,
+      number=6, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s7', full_name='live.ContinuationEntity.s7', index=3,
+      number=7, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s8', full_name='live.ContinuationEntity.s8', index=4,
+      number=8, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='body', full_name='live.ContinuationEntity.body', index=5,
+      number=9, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='timestamp3', full_name='live.ContinuationEntity.timestamp3', index=6,
+      number=10, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='timestamp4', full_name='live.ContinuationEntity.timestamp4', index=7,
+      number=11, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s13', full_name='live.ContinuationEntity.s13', index=8,
+      number=13, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='chattype', full_name='live.ContinuationEntity.chattype', index=9,
+      number=16, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s17', full_name='live.ContinuationEntity.s17', index=10,
+      number=17, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='str19', full_name='live.ContinuationEntity.str19', index=11,
+      number=19, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='timestamp5', full_name='live.ContinuationEntity.timestamp5', index=12,
+      number=20, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=211,
+  serialized_end=477,
+)
+
+
+_CONTINUATION = _descriptor.Descriptor(
+  name='Continuation',
+  full_name='live.Continuation',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='entity', full_name='live.Continuation.entity', index=0,
+      number=119693434, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=479,
+  serialized_end=538,
+)
+
+_CONTINUATIONENTITY.fields_by_name['body'].message_type = _BODY
+_CONTINUATIONENTITY.fields_by_name['chattype'].message_type = _CHATTYPE
+_CONTINUATIONENTITY.fields_by_name['str19'].message_type = _STR19
+_CONTINUATION.fields_by_name['entity'].message_type = _CONTINUATIONENTITY
+DESCRIPTOR.message_types_by_name['Body'] = _BODY
+DESCRIPTOR.message_types_by_name['ChatType'] = _CHATTYPE
+DESCRIPTOR.message_types_by_name['STR19'] = _STR19
+DESCRIPTOR.message_types_by_name['ContinuationEntity'] = _CONTINUATIONENTITY
+DESCRIPTOR.message_types_by_name['Continuation'] = _CONTINUATION
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Body = _reflection.GeneratedProtocolMessageType('Body', (_message.Message,), {
+  'DESCRIPTOR' : _BODY,
+  '__module__' : 'live_pb2'
+  # @@protoc_insertion_point(class_scope:live.Body)
+  })
+_sym_db.RegisterMessage(Body)
+
+ChatType = _reflection.GeneratedProtocolMessageType('ChatType', (_message.Message,), {
+  'DESCRIPTOR' : _CHATTYPE,
+  '__module__' : 'live_pb2'
+  # @@protoc_insertion_point(class_scope:live.ChatType)
+  })
+_sym_db.RegisterMessage(ChatType)
+
+STR19 = _reflection.GeneratedProtocolMessageType('STR19', (_message.Message,), {
+  'DESCRIPTOR' : _STR19,
+  '__module__' : 'live_pb2'
+  # @@protoc_insertion_point(class_scope:live.STR19)
+  })
+_sym_db.RegisterMessage(STR19)
+
+ContinuationEntity = _reflection.GeneratedProtocolMessageType('ContinuationEntity', (_message.Message,), {
+  'DESCRIPTOR' : _CONTINUATIONENTITY,
+  '__module__' : 'live_pb2'
+  # @@protoc_insertion_point(class_scope:live.ContinuationEntity)
+  })
+_sym_db.RegisterMessage(ContinuationEntity)
+
+Continuation = _reflection.GeneratedProtocolMessageType('Continuation', (_message.Message,), {
+  'DESCRIPTOR' : _CONTINUATION,
+  '__module__' : 'live_pb2'
+  # @@protoc_insertion_point(class_scope:live.Continuation)
+  })
+_sym_db.RegisterMessage(Continuation)
+
+
+# @@protoc_insertion_point(module_scope)

pytchat/paramgen/pb/replay_pb2.py

@@ -0,0 +1,215 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: replay.proto
+
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='replay.proto',
+  package='replay',
+  syntax='proto3',
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n\x0creplay.proto\x12\x06replay\"\x19\n\x08\x43hatType\x12\r\n\x05value\x18\x01 \x01(\x05\"\xb2\x01\n\x12\x43ontinuationEntity\x12\x0e\n\x06header\x18\x03 \x01(\t\x12\x11\n\ttimestamp\x18\x05 \x01(\x03\x12\n\n\x02s6\x18\x06 \x01(\x05\x12\n\n\x02s7\x18\x07 \x01(\x05\x12\n\n\x02s8\x18\x08 \x01(\x05\x12\n\n\x02s9\x18\t \x01(\x05\x12\x0b\n\x03s10\x18\n \x01(\t\x12\x0b\n\x03s12\x18\x0c \x01(\x05\x12\"\n\x08\x63hattype\x18\x0e \x01(\x0b\x32\x10.replay.ChatType\x12\x0b\n\x03s15\x18\x0f \x01(\x05\"=\n\x0c\x43ontinuation\x12-\n\x06\x65ntity\x18\xd4\x83\xb6J \x01(\x0b\x32\x1a.replay.ContinuationEntityb\x06proto3'
+)
+
+
+
+
+_CHATTYPE = _descriptor.Descriptor(
+  name='ChatType',
+  full_name='replay.ChatType',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='value', full_name='replay.ChatType.value', index=0,
+      number=1, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=24,
+  serialized_end=49,
+)
+
+
+_CONTINUATIONENTITY = _descriptor.Descriptor(
+  name='ContinuationEntity',
+  full_name='replay.ContinuationEntity',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='header', full_name='replay.ContinuationEntity.header', index=0,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='timestamp', full_name='replay.ContinuationEntity.timestamp', index=1,
+      number=5, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s6', full_name='replay.ContinuationEntity.s6', index=2,
+      number=6, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s7', full_name='replay.ContinuationEntity.s7', index=3,
+      number=7, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s8', full_name='replay.ContinuationEntity.s8', index=4,
+      number=8, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s9', full_name='replay.ContinuationEntity.s9', index=5,
+      number=9, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s10', full_name='replay.ContinuationEntity.s10', index=6,
+      number=10, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s12', full_name='replay.ContinuationEntity.s12', index=7,
+      number=12, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='chattype', full_name='replay.ContinuationEntity.chattype', index=8,
+      number=14, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s15', full_name='replay.ContinuationEntity.s15', index=9,
+      number=15, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=52,
+  serialized_end=230,
+)
+
+
+_CONTINUATION = _descriptor.Descriptor(
+  name='Continuation',
+  full_name='replay.Continuation',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='entity', full_name='replay.Continuation.entity', index=0,
+      number=156074452, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=232,
+  serialized_end=293,
+)
+
+_CONTINUATIONENTITY.fields_by_name['chattype'].message_type = _CHATTYPE
+_CONTINUATION.fields_by_name['entity'].message_type = _CONTINUATIONENTITY
+DESCRIPTOR.message_types_by_name['ChatType'] = _CHATTYPE
+DESCRIPTOR.message_types_by_name['ContinuationEntity'] = _CONTINUATIONENTITY
+DESCRIPTOR.message_types_by_name['Continuation'] = _CONTINUATION
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+ChatType = _reflection.GeneratedProtocolMessageType('ChatType', (_message.Message,), {
+  'DESCRIPTOR' : _CHATTYPE,
+  '__module__' : 'replay_pb2'
+  # @@protoc_insertion_point(class_scope:replay.ChatType)
+  })
+_sym_db.RegisterMessage(ChatType)
+
+ContinuationEntity = _reflection.GeneratedProtocolMessageType('ContinuationEntity', (_message.Message,), {
+  'DESCRIPTOR' : _CONTINUATIONENTITY,
+  '__module__' : 'replay_pb2'
+  # @@protoc_insertion_point(class_scope:replay.ContinuationEntity)
+  })
+_sym_db.RegisterMessage(ContinuationEntity)
+
+Continuation = _reflection.GeneratedProtocolMessageType('Continuation', (_message.Message,), {
+  'DESCRIPTOR' : _CONTINUATION,
+  '__module__' : 'replay_pb2'
+  # @@protoc_insertion_point(class_scope:replay.Continuation)
+  })
+_sym_db.RegisterMessage(Continuation)
+
+
+# @@protoc_insertion_point(module_scope)

pytchat/paramgen/proto/header.proto

@@ -0,0 +1,14 @@
+syntax = "proto3";
+
+message Video {
+    string id = 1;
+}
+
+message HeaderInfo {
+    Video video = 1;
+}
+
+message Header {
+    HeaderInfo info = 1;
+    int32 terminator = 4;
+}

pytchat/paramgen/proto/live.proto

@@ -0,0 +1,45 @@
+syntax = "proto3";
+
+package live;
+
+message Body {
+    int32 b1 = 1;
+    int32 b2 = 2;
+    int32 b3 = 3;
+    int32 b4 = 4;
+    string b7 = 7;
+    int32 b8 = 8;
+    string b9 = 9;
+    int64 timestamp2 = 10;
+    int32 b11 = 11;
+    int32 b15 = 15;
+}
+
+message ChatType {
+    int32 value = 1;
+}
+
+message STR19 {
+    int32 value = 1;
+}
+
+message ContinuationEntity {
+    string header = 3;
+    int64 timestamp1 = 5;
+    int32 s6 = 6;
+    int32 s7 = 7;
+    int32 s8 = 8;
+    Body body = 9;
+    int64 timestamp3 = 10;
+    int64 timestamp4 = 11;
+    int32 s13 = 13;
+    ChatType chattype = 16;
+    int32 s17 = 17;
+    STR19 str19 = 19;
+    int64 timestamp5 = 20;
+}
+
+message Continuation {
+    ContinuationEntity entity = 119693434;
+}
+

pytchat/paramgen/proto/replay.proto

@@ -0,0 +1,24 @@
+syntax = "proto3";
+
+package replay;
+
+message ChatType {
+    int32 value = 1;
+}
+
+message ContinuationEntity {
+    string header = 3;
+    int64 timestamp = 5;
+    int32 s6 = 6;
+    int32 s7 = 7;
+    int32 s8 = 8;
+    int32 s9 = 9;
+    string s10 = 10;
+    int32 s12 = 12;
+    ChatType chattype = 14;
+    int32 s15 = 15;
+}
+
+message Continuation {
+    ContinuationEntity entity = 156074452;
+}  

pytchat/parser/live.py

@@ -4,27 +4,23 @@ pytchat.parser.live
 Parser of live chat JSON.
 """
 
-import json
+from .. import exceptions
-from .. exceptions import ( 
+
-    ResponseContextError, 
-    NoContentsException, 
-    NoContinuationsException,
-    ChatParseException )
 
 class Parser:
 
     __slots__ = ['is_replay']
-    
+
-    def __init__(self, is_replay): 
+    def __init__(self, is_replay):
         self.is_replay = is_replay
 
     def get_contents(self, jsn):
-        if jsn is None: 
+        if jsn is None:
-            raise ChatParseException('Called with none JSON object.')
+            raise exceptions.IllegalFunctionCall('Called with none JSON object.')
         if jsn['response']['responseContext'].get('errors'):
-            raise ResponseContextError('The video_id would be wrong,'
+            raise exceptions.ResponseContextError(
-                'or video is deleted or private.')
+                'The video_id would be wrong, or video is deleted or private.')
-        contents=jsn['response'].get('continuationContents')
+        contents = jsn['response'].get('continuationContents')
         return contents
 
     def parse(self, contents):
@@ -40,58 +36,62 @@ class Parser:
         + metadata : dict
          + timeout
          + video_id
-         + continuation           
+         + continuation
         + chatdata : List[dict]
         """
 
         if contents is None:
             '''Broadcasting end or cannot fetch chat stream'''
-            raise NoContentsException('Chat data stream is empty.')
+            raise exceptions.NoContents('Chat data stream is empty.')
 
         cont = contents['liveChatContinuation']['continuations'][0]
         if cont is None:
-            raise NoContinuationsException('No Continuation')
+            raise exceptions.NoContinuation('No Continuation')
-        metadata = (cont.get('invalidationContinuationData')  or
+        metadata = (cont.get('invalidationContinuationData')
-                    cont.get('timedContinuationData')         or
+                    or cont.get('timedContinuationData')
-                    cont.get('reloadContinuationData')        or
+                    or cont.get('reloadContinuationData')
-                    cont.get('liveChatReplayContinuationData')
+                    or cont.get('liveChatReplayContinuationData')
                     )
         if metadata is None:
             if cont.get("playerSeekContinuationData"):
-                raise ChatParseException('Finished chat data')
+                raise exceptions.ChatDataFinished('Finished chat data')
             unknown = list(cont.keys())[0]
             if unknown:
-                raise ChatParseException(f"Received unknown continuation type:{unknown}")
+                raise exceptions.ReceivedUnknownContinuation(
+                    f"Received unknown continuation type:{unknown}")
             else:
-                raise ChatParseException('Cannot extract continuation data')
+                raise exceptions.FailedExtractContinuation('Cannot extract continuation data')
         return self._create_data(metadata, contents)
 
     def reload_continuation(self, contents):
         """
-        When `seektime = 0` or seektime is abbreviated , 
+        When `seektime == 0` or seektime is abbreviated ,
-        check if fetched chat json has no chat data. 
+        check if fetched chat json has no chat data.
-        If so, try to fetch playerSeekContinuationData. 
+        If so, try to fetch playerSeekContinuationData.
         This function must be run only first fetching.
         """
+        if contents is None:
+            '''Broadcasting end or cannot fetch chat stream'''
+            raise exceptions.NoContents('Chat data stream is empty.')
         cont = contents['liveChatContinuation']['continuations'][0]
         if cont.get("liveChatReplayContinuationData"):
-            #chat data exist.
+            # chat data exist.
             return None
-        #chat data do not exist, get playerSeekContinuationData.
+        # chat data do not exist, get playerSeekContinuationData.
         init_cont = cont.get("playerSeekContinuationData")
         if init_cont:
             return init_cont.get("continuation")
-        raise ChatParseException('Finished chat data')
+        raise exceptions.ChatDataFinished('Finished chat data')
 
-    def _create_data(self, metadata, contents):    
+    def _create_data(self, metadata, contents):
         actions = contents['liveChatContinuation'].get('actions')
-        if self.is_replay:    
+        if self.is_replay:
             interval = self._get_interval(actions)
-            metadata.setdefault("timeoutMs",interval)
+            metadata.setdefault("timeoutMs", interval)
-            """Archived chat has different structures than live chat, 
+            """Archived chat has different structures than live chat,
             so make it the same format."""
             chatdata = [action["replayChatItemAction"]["actions"][0]
-                for action in actions]
+                        for action in actions]
         else:
             metadata.setdefault('timeoutMs', 10000)
             chatdata = actions
@@ -102,4 +102,4 @@ class Parser:
             return 0
         start = int(actions[0]["replayChatItemAction"]["videoOffsetTimeMsec"])
         last = int(actions[-1]["replayChatItemAction"]["videoOffsetTimeMsec"])
-        return (last - start)
+        return (last - start)

pytchat/parser/replay.py

@@ -1,76 +0,0 @@
-import json
-from .. import config
-from .. exceptions import ( 
-    ResponseContextError, 
-    NoContentsException, 
-    NoContinuationsException )
-
-
-logger = config.logger(__name__)
-
-class Parser:
-    def parse(self, jsn):
-        """
-        このparse関数はReplayChat._listen() 関数から定期的に呼び出される。
-        引数jsnはYoutubeから取得したアーカイブ済みチャットデータの生JSONであり、
-        このparse関数によって与えられたJSONを以下に分割して返す。
-         + timeout (次のチャットデータ取得までのインターバル)
-         + chat data（チャットデータ本体）
-         + continuation （次のチャットデータ取得に必要となるパラメータ）.
-        
-        ライブ配信のチャットとアーカイブ済み動画のチャットは構造が若干異なっているが、
-        ライブチャットと同じデータ形式に変換することにより、
-        同じprocessorでライブとリプレイどちらでも利用できるようにしている。
-
-        Parameter
-        ----------
-        + jsn : dict
-            + Youtubeから取得したチャットデータのJSONオブジェクト。
-              （pythonの辞書形式に変換済みの状態で渡される）
-
-        Returns
-        -------
-        + metadata : dict
-            + チャットデータに付随するメタデータ。timeout、 動画ID、continuationパラメータで構成される。           
-        + chatdata : list[dict]
-            + チャットデータ本体のリスト。
-        """
-        if jsn is None: 
-            return {'timeoutMs':0,'continuation':None},[]
-        if jsn['response']['responseContext'].get('errors'):
-            raise ResponseContextError('動画に接続できません。'
-        '動画IDが間違っているか、動画が削除／非公開の可能性があります。')
-        contents=jsn['response'].get('continuationContents')
-        #配信が終了した場合、もしくはチャットデータが取得できない場合
-        if contents is None:
-            raise NoContentsException('チャットデータを取得できませんでした。')
-
-        cont = contents['liveChatContinuation']['continuations'][0]
-        if cont is None:
-            raise NoContinuationsException('Continuationがありません。')
-        metadata = cont.get('liveChatReplayContinuationData')
-        if metadata is None:
-            unknown = list(cont.keys())[0]
-            if unknown != "playerSeekContinuationData":
-                logger.debug(f"Received unknown continuation type:{unknown}")
-                metadata = cont.get(unknown)
-        actions = contents['liveChatContinuation'].get('actions')
-        if actions is None:
-            #後続のチャットデータなし
-            return {"continuation":None,"timeout":0,"chatdata":[]}
-        interval = self.get_interval(actions)
-        metadata.setdefault("timeoutMs",interval)
-        """アーカイブ済みチャットはライブチャットと構造が異なっているため、以下の行により
-        ライブチャットと同じ形式にそろえる"""
-        chatdata = [action["replayChatItemAction"]["actions"][0] for action in actions]
-        return metadata, chatdata
-
-    def get_interval(self, actions: list):
-        if actions is None:
-            return 0
-        start = int(actions[0]["replayChatItemAction"]["videoOffsetTimeMsec"])
-        last = int(actions[-1]["replayChatItemAction"]["videoOffsetTimeMsec"])
-        return (last - start)
-
-
-

pytchat/processors/chat_processor.py

@@ -3,11 +3,12 @@ class ChatProcessor:
     Abstract class that processes chat data.
     Receive chat data (actions) from Listener.
     '''
+
     def process(self, chat_components: list):
         '''
         Interface that represents processing of chat data.
-        Called from LiveChat object. 
+        Called from LiveChat object.
-               
+
         Parameter
         ----------
         chat_components: List[component]
@@ -21,7 +22,9 @@ class ChatProcessor:
         '''
         pass
 
-
+    def finalize(self, *args, **kwargs):
-
+        '''
-
+        Interface for finalizing the process.
-
+        Called when chat fetching finished.
+        '''
+        pass

pytchat/processors/combinator.py

@@ -1,5 +1,6 @@
 from .chat_processor import ChatProcessor
 
+
 class Combinator(ChatProcessor):
     '''
     Combinator combines multiple chat processors.
@@ -8,11 +9,11 @@ class Combinator(ChatProcessor):
     For example:
         [constructor]
         chat = LiveChat("video_id", processor = ( Processor1(), Processor2(), Processor3() ) )
-        
+
         [receive return values]
         ret1, ret2, ret3 = chat.get()
-    
+
-    The return values are tuple of processed chat data, 
+    The return values are tuple of processed chat data,
     the order of return depends on parameter order.
 
     Parameter
@@ -34,6 +35,4 @@ class Combinator(ChatProcessor):
             Tuple of chat data processed by each chat processor.
         '''
         return tuple(processor.process(chat_components)
-            for processor in self.processors)
+                     for processor in self.processors)
-            
-

pytchat/processors/compatible/processor.py

@@ -1,5 +1,3 @@
-import datetime
-import time
 from .renderer.textmessage import LiveChatTextMessageRenderer
 from .renderer.paidmessage import LiveChatPaidMessageRenderer
 from .renderer.paidsticker import LiveChatPaidStickerRenderer
@@ -39,7 +37,7 @@ class CompatibleProcessor(ChatProcessor):
                     chat = self.parse(action)
                     if chat:
                         chatlist.append(chat)
-        ret["pollingIntervalMillis"] = int(timeout*1000)
+        ret["pollingIntervalMillis"] = int(timeout * 1000)
         ret["pageInfo"] = {
             "totalResults": len(chatlist),
             "resultsPerPage": len(chatlist),
@@ -58,7 +56,7 @@ class CompatibleProcessor(ChatProcessor):
         rd = {}
         try:
             renderer = self.get_renderer(item)
-            if renderer == None:
+            if renderer is None:
                 return None
 
             rd["kind"] = "youtube#liveChatMessage"

pytchat/processors/compatible/renderer/base.py

@@ -1,68 +1,67 @@
-import datetime, pytz
+import datetime
+import pytz
+
 
 class BaseRenderer:
     def __init__(self, item, chattype):
         self.renderer = list(item.values())[0]
         self.chattype = chattype
 
-
     def get_snippet(self):
 
         message = self.get_message(self.renderer)
 
         return {
-            "type"              : self.chattype,
+            "type": self.chattype,
-            "liveChatId"        : "",
+            "liveChatId": "",
-            "authorChannelId"   : self.renderer.get("authorExternalChannelId"),
+            "authorChannelId": self.renderer.get("authorExternalChannelId"),
-            "publishedAt"       : self.get_publishedat(self.renderer.get("timestampUsec",0)),
+            "publishedAt": self.get_publishedat(self.renderer.get("timestampUsec", 0)),
-            "hasDisplayContent" : True,
+            "hasDisplayContent": True,
-            "displayMessage"    : message,
+            "displayMessage": message,
             "textMessageDetails": {
-                "messageText"   : message  
+                "messageText": message
             }
         }
 
-
     def get_authordetails(self):
         authorExternalChannelId = self.renderer.get("authorExternalChannelId")
-        #parse subscriber type
+        # parse subscriber type
         isVerified, isChatOwner, isChatSponsor, isChatModerator = (
             self.get_badges(self.renderer)
         )
-        return  {
+        return {
-            "channelId"         : authorExternalChannelId,
+            "channelId": authorExternalChannelId,
-            "channelUrl"        : "http://www.youtube.com/channel/"+authorExternalChannelId,
+            "channelUrl": "http://www.youtube.com/channel/" + authorExternalChannelId,
-            "displayName"       : self.renderer["authorName"]["simpleText"],
+            "displayName": self.renderer["authorName"]["simpleText"],
-            "profileImageUrl"   : self.renderer["authorPhoto"]["thumbnails"][1]["url"] ,
+            "profileImageUrl": self.renderer["authorPhoto"]["thumbnails"][1]["url"],
-            "isVerified"        : isVerified,
+            "isVerified": isVerified,
-            "isChatOwner"       : isChatOwner,
+            "isChatOwner": isChatOwner,
-            "isChatSponsor"     : isChatSponsor,
+            "isChatSponsor": isChatSponsor,
-            "isChatModerator"   : isChatModerator
+            "isChatModerator": isChatModerator
-        }    
+        }
 
-
+    def get_message(self, renderer):
-    def get_message(self,renderer):
         message = ''
         if renderer.get("message"):
-            runs=renderer["message"].get("runs")
+            runs = renderer["message"].get("runs")
             if runs:
                 for r in runs:
                     if r:
                         if r.get('emoji'):
-                            message += r['emoji'].get('shortcuts',[''])[0]
+                            message += r['emoji'].get('shortcuts', [''])[0]
                         else:
-                            message += r.get('text','')
+                            message += r.get('text', '')
         return message
 
-    def get_badges(self,renderer):
+    def get_badges(self, renderer):
         isVerified = False
         isChatOwner = False
         isChatSponsor = False
         isChatModerator = False
-        badges=renderer.get("authorBadges")
+        badges = renderer.get("authorBadges")
         if badges:
             for badge in badges:
-                author_type  = badge["liveChatAuthorBadgeRenderer"]["accessibility"]["accessibilityData"]["label"]
+                author_type = badge["liveChatAuthorBadgeRenderer"]["accessibility"]["accessibilityData"]["label"]
                 if author_type == '確認済み':
                     isVerified = True
                 if author_type == '所有者':
@@ -72,12 +71,11 @@ class BaseRenderer:
                 if author_type == 'モデレーター':
                     isChatModerator = True
         return isVerified, isChatOwner, isChatSponsor, isChatModerator
-    
+
     def get_id(self):
         return self.renderer.get('id')
-    
+
-    def get_publishedat(self,timestamp):
+    def get_publishedat(self, timestamp):
-        dt = datetime.datetime.fromtimestamp(int(timestamp)/1000000)
+        dt = datetime.datetime.fromtimestamp(int(timestamp) / 1000000)
         return dt.astimezone(pytz.utc).isoformat(
-            timespec='milliseconds').replace('+00:00','Z')
+            timespec='milliseconds').replace('+00:00', 'Z')
-        

pytchat/processors/compatible/renderer/currency.py

@@ -35,4 +35,4 @@ symbols = {
     "NOK\xa0": {"fxtext": "NOK", "jptext": "ノルウェー・クローネ"},
     "BAM\xa0": {"fxtext": "BAM", "jptext": "ボスニア・兌換マルカ"},
     "SGD\xa0": {"fxtext": "SGD", "jptext": "シンガポール・ドル"}
-}
+}

pytchat/processors/compatible/renderer/legacypaid.py

@@ -1,4 +1,6 @@
 from .base import BaseRenderer
+
+
 class LiveChatLegacyPaidMessageRenderer(BaseRenderer):
     def __init__(self, item):
         super().__init__(item, "newSponsorEvent")
@@ -8,36 +10,33 @@ class LiveChatLegacyPaidMessageRenderer(BaseRenderer):
         message = self.get_message(self.renderer)
 
         return {
-            "type"              : self.chattype,
+            "type": self.chattype,
-            "liveChatId"        : "",
+            "liveChatId": "",
-            "authorChannelId"   : self.renderer.get("authorExternalChannelId"),
+            "authorChannelId": self.renderer.get("authorExternalChannelId"),
-            "publishedAt"       : self.get_publishedat(self.renderer.get("timestampUsec",0)),
+            "publishedAt": self.get_publishedat(self.renderer.get("timestampUsec", 0)),
-            "hasDisplayContent" : True,
+            "hasDisplayContent": True,
-            "displayMessage"    : message,
+            "displayMessage": message,
-         
+
         }
 
     def get_authordetails(self):
         authorExternalChannelId = self.renderer.get("authorExternalChannelId")
-        #parse subscriber type
+        # parse subscriber type
         isVerified, isChatOwner, _, isChatModerator = (
             self.get_badges(self.renderer)
         )
-        return  {
+        return {
-            "channelId"         : authorExternalChannelId,
+            "channelId": authorExternalChannelId,
-            "channelUrl"        : "http://www.youtube.com/channel/"+authorExternalChannelId,
+            "channelUrl": "http://www.youtube.com/channel/" + authorExternalChannelId,
-            "displayName"       : self.renderer["authorName"]["simpleText"],
+            "displayName": self.renderer["authorName"]["simpleText"],
-            "profileImageUrl"   : self.renderer["authorPhoto"]["thumbnails"][1]["url"] ,
+            "profileImageUrl": self.renderer["authorPhoto"]["thumbnails"][1]["url"],
-            "isVerified"        : isVerified,
+            "isVerified": isVerified,
-            "isChatOwner"       : isChatOwner,
+            "isChatOwner": isChatOwner,
-            "isChatSponsor"     : True,
+            "isChatSponsor": True,
-            "isChatModerator"   : isChatModerator
+            "isChatModerator": isChatModerator
-        }    
+        }
 
-
+    def get_message(self, renderer):
-    def get_message(self,renderer):
         message = (renderer["eventText"]["runs"][0]["text"]
-            )+' / '+(renderer["detailText"]["simpleText"])
+                   ) + ' / ' + (renderer["detailText"]["simpleText"])
         return message
-
-

pytchat/processors/compatible/renderer/membership.py

@@ -25,7 +25,7 @@ class LiveChatMembershipItemRenderer(BaseRenderer):
         )
         return {
             "channelId": authorExternalChannelId,
-            "channelUrl": "http://www.youtube.com/channel/"+authorExternalChannelId,
+            "channelUrl": "http://www.youtube.com/channel/" + authorExternalChannelId,
             "displayName": self.renderer["authorName"]["simpleText"],
             "profileImageUrl": self.renderer["authorPhoto"]["thumbnails"][1]["url"],
             "isVerified": isVerified,
@@ -35,6 +35,6 @@ class LiveChatMembershipItemRenderer(BaseRenderer):
         }
 
     def get_message(self, renderer):
-        message = ''.join([mes.get("text", "") for mes in renderer["headerSubtext"]["runs"]])
+        message = ''.join([mes.get("text", "")
+                           for mes in renderer["headerSubtext"]["runs"]])
         return message, [message]
-

pytchat/processors/compatible/renderer/paidmessage.py

@@ -3,6 +3,7 @@ from . import currency
 from .base import BaseRenderer
 superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
 
+
 class LiveChatPaidMessageRenderer(BaseRenderer):
     def __init__(self, item):
         super().__init__(item, "superChatEvent")
@@ -10,32 +11,32 @@ class LiveChatPaidMessageRenderer(BaseRenderer):
     def get_snippet(self):
         authorName = self.renderer["authorName"]["simpleText"]
         message = self.get_message(self.renderer)
-        amountDisplayString, symbol, amountMicros =(
+        amountDisplayString, symbol, amountMicros = (
             self.get_amountdata(self.renderer)
         )
         return {
-            "type"              : self.chattype,
+            "type": self.chattype,
-            "liveChatId"        : "",
+            "liveChatId": "",
-            "authorChannelId"   : self.renderer.get("authorExternalChannelId"),
+            "authorChannelId": self.renderer.get("authorExternalChannelId"),
-            "publishedAt"       : self.get_publishedat(self.renderer.get("timestampUsec",0)),
+            "publishedAt": self.get_publishedat(self.renderer.get("timestampUsec", 0)),
-            "hasDisplayContent" : True,
+            "hasDisplayContent": True,
-            "displayMessage"    : amountDisplayString+" from "+authorName+': \"'+ message+'\"',
+            "displayMessage": amountDisplayString + " from " + authorName + ': \"' + message + '\"',
-            "superChatDetails"  : {
+            "superChatDetails": {
-                "amountMicros"          : amountMicros,
+                "amountMicros": amountMicros,
-                "currency"              : currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol,
+                "currency": currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol,
-                "amountDisplayString"   : amountDisplayString,
+                "amountDisplayString": amountDisplayString,
-                "tier"                  : 0,
+                "tier": 0,
-                "backgroundColor" : self.renderer.get("bodyBackgroundColor", 0)
+                "backgroundColor": self.renderer.get("bodyBackgroundColor", 0)
             }
         }
 
-    def get_amountdata(self,renderer):
+    def get_amountdata(self, renderer):
         amountDisplayString = renderer["purchaseAmountText"]["simpleText"]
         m = superchat_regex.search(amountDisplayString)
         if m:
             symbol = m.group(1)
-            amountMicros = int(float(m.group(2).replace(',',''))*1000000)
+            amountMicros = int(float(m.group(2).replace(',', '')) * 1000000)
         else:
             symbol = ""
             amountMicros = 0
-        return amountDisplayString, symbol, amountMicros
+        return amountDisplayString, symbol, amountMicros

pytchat/processors/compatible/renderer/paidsticker.py

@@ -3,46 +3,45 @@ from . import currency
 from .base import BaseRenderer
 superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
 
+
 class LiveChatPaidStickerRenderer(BaseRenderer):
     def __init__(self, item):
         super().__init__(item, "superStickerEvent")
 
     def get_snippet(self):
         authorName = self.renderer["authorName"]["simpleText"]
-        amountDisplayString, symbol, amountMicros =(
+        amountDisplayString, symbol, amountMicros = (
             self.get_amountdata(self.renderer)
         )
 
         return {
-            "type"              : self.chattype,
+            "type": self.chattype,
-            "liveChatId"        : "",
+            "liveChatId": "",
-            "authorChannelId"   : self.renderer.get("authorExternalChannelId"),
+            "authorChannelId": self.renderer.get("authorExternalChannelId"),
-            "publishedAt"       : self.get_publishedat(self.renderer.get("timestampUsec",0)),
+            "publishedAt": self.get_publishedat(self.renderer.get("timestampUsec", 0)),
-            "hasDisplayContent" : True,
+            "hasDisplayContent": True,
-            "displayMessage"    : "Super Sticker " + amountDisplayString + " from "+authorName,
+            "displayMessage": "Super Sticker " + amountDisplayString + " from " + authorName,
-            "superStickerDetails"  : {
+            "superStickerDetails": {
-                "superStickerMetaData" : {
+                "superStickerMetaData": {
                     "stickerId": "",
                     "altText": "",
-                    "language": ""                    
+                    "language": ""
                 },
-                "amountMicros"          : amountMicros,
+                "amountMicros": amountMicros,
-                "currency"              : currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol,
+                "currency": currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol,
-                "amountDisplayString"   : amountDisplayString,
+                "amountDisplayString": amountDisplayString,
-                "tier"                  : 0,
+                "tier": 0,
-                "backgroundColor" : self.renderer.get("bodyBackgroundColor", 0)
+                "backgroundColor": self.renderer.get("bodyBackgroundColor", 0)
             }
         }
 
-    def get_amountdata(self,renderer):
+    def get_amountdata(self, renderer):
         amountDisplayString = renderer["purchaseAmountText"]["simpleText"]
         m = superchat_regex.search(amountDisplayString)
         if m:
             symbol = m.group(1)
-            amountMicros = int(float(m.group(2).replace(',',''))*1000000)
+            amountMicros = int(float(m.group(2).replace(',', '')) * 1000000)
         else:
             symbol = ""
             amountMicros = 0
         return amountDisplayString, symbol, amountMicros
- 
-

pytchat/processors/compatible/renderer/textmessage.py

@@ -1,4 +1,6 @@
 from .base import BaseRenderer
+
+
 class LiveChatTextMessageRenderer(BaseRenderer):
     def __init__(self, item):
         super().__init__(item, "textMessageEvent")

pytchat/processors/default/processor.py

@@ -20,13 +20,13 @@ class Chatdata:
         if self.interval == 0:
             time.sleep(1)
             return
-        time.sleep(self.interval/len(self.items))
+        time.sleep(self.interval / len(self.items))
 
     async def tick_async(self):
         if self.interval == 0:
             await asyncio.sleep(1)
             return
-        await asyncio.sleep(self.interval/len(self.items))
+        await asyncio.sleep(self.interval / len(self.items))
 
 
 class DefaultProcessor(ChatProcessor):
@@ -62,7 +62,7 @@ class DefaultProcessor(ChatProcessor):
             return None
         try:
             renderer = self._get_renderer(item)
-            if renderer == None:
+            if renderer is None:
                 return None
 
             renderer.get_snippet()

pytchat/processors/default/renderer/base.py

@@ -1,6 +1,10 @@
 from datetime import datetime
+
+
 class Author:
     pass
+
+
 class BaseRenderer:
     def __init__(self, item, chattype):
         self.renderer = list(item.values())[0]
@@ -10,65 +14,65 @@ class BaseRenderer:
     def get_snippet(self):
         self.type = self.chattype
         self.id = self.renderer.get('id')
-        timestampUsec = int(self.renderer.get("timestampUsec",0))
+        timestampUsec = int(self.renderer.get("timestampUsec", 0))
-        self.timestamp = int(timestampUsec/1000)
+        self.timestamp = int(timestampUsec / 1000)
         tst = self.renderer.get("timestampText")
         if tst:
             self.elapsedTime = tst.get("simpleText")
         else:
             self.elapsedTime = ""
         self.datetime = self.get_datetime(timestampUsec)
-        self.message ,self.messageEx = self.get_message(self.renderer)
+        self.message, self.messageEx = self.get_message(self.renderer)
-        self.id =  self.renderer.get('id')
+        self.id = self.renderer.get('id')
-        self.amountValue= 0.0
+        self.amountValue = 0.0
         self.amountString = ""
-        self.currency= ""
+        self.currency = ""
         self.bgColor = 0
 
     def get_authordetails(self):
         self.author.badgeUrl = ""
-        (self.author.isVerified, 
+        (self.author.isVerified,
-        self.author.isChatOwner, 
+         self.author.isChatOwner,
-        self.author.isChatSponsor, 
+         self.author.isChatSponsor,
-        self.author.isChatModerator) = (
+         self.author.isChatModerator) = (
             self.get_badges(self.renderer)
         )
         self.author.channelId = self.renderer.get("authorExternalChannelId")
-        self.author.channelUrl = "http://www.youtube.com/channel/"+self.author.channelId
+        self.author.channelUrl = "http://www.youtube.com/channel/" + self.author.channelId
-        self.author.name      = self.renderer["authorName"]["simpleText"]
+        self.author.name = self.renderer["authorName"]["simpleText"]
-        self.author.imageUrl= self.renderer["authorPhoto"]["thumbnails"][1]["url"] 
+        self.author.imageUrl = self.renderer["authorPhoto"]["thumbnails"][1]["url"]
-        
 
-
+    def get_message(self, renderer):
-    def get_message(self,renderer):
         message = ''
         message_ex = []
         if renderer.get("message"):
-            runs=renderer["message"].get("runs")
+            runs = renderer["message"].get("runs")
             if runs:
                 for r in runs:
                     if r:
                         if r.get('emoji'):
-                            message += r['emoji'].get('shortcuts',[''])[0]
+                            message += r['emoji'].get('shortcuts', [''])[0]
-                            message_ex.append(r['emoji']['image']['thumbnails'][1].get('url'))
+                            message_ex.append({
+                                'id': r['emoji'].get('emojiId').split('/')[-1],
+                                'txt': r['emoji'].get('shortcuts', [''])[0],
+                                'url': r['emoji']['image']['thumbnails'][0].get('url')
+                            })
                         else:
-                            message += r.get('text','')
+                            message += r.get('text', '')
-                            message_ex.append(r.get('text',''))
+                            message_ex.append(r.get('text', ''))
         return message, message_ex
 
-
+    def get_badges(self, renderer):
-
-    def get_badges(self,renderer):
         self.author.type = ''
         isVerified = False
         isChatOwner = False
         isChatSponsor = False
         isChatModerator = False
-        badges=renderer.get("authorBadges")
+        badges = renderer.get("authorBadges")
         if badges:
             for badge in badges:
                 if badge["liveChatAuthorBadgeRenderer"].get("icon"):
-                    author_type  = badge["liveChatAuthorBadgeRenderer"]["icon"]["iconType"]
+                    author_type = badge["liveChatAuthorBadgeRenderer"]["icon"]["iconType"]
                     self.author.type = author_type
                     if author_type == 'VERIFIED':
                         isVerified = True
@@ -81,13 +85,10 @@ class BaseRenderer:
                     self.author.type = 'MEMBER'
                     self.get_badgeurl(badge)
         return isVerified, isChatOwner, isChatSponsor, isChatModerator
-    
 
-    def get_badgeurl(self,badge):
+    def get_badgeurl(self, badge):
         self.author.badgeUrl = badge["liveChatAuthorBadgeRenderer"]["customThumbnail"]["thumbnails"][0]["url"]
 
-
+    def get_datetime(self, timestamp):
-
+        dt = datetime.fromtimestamp(timestamp / 1000000)
-    def get_datetime(self,timestamp):
+        return dt.strftime('%Y-%m-%d %H:%M:%S')
-        dt = datetime.fromtimestamp(timestamp/1000000)
-        return dt.strftime('%Y-%m-%d %H:%M:%S')

pytchat/processors/default/renderer/currency.py

@@ -35,4 +35,4 @@ symbols = {
     "NOK\xa0": {"fxtext": "NOK", "jptext": "ノルウェー・クローネ"},
     "BAM\xa0": {"fxtext": "BAM", "jptext": "ボスニア・兌換マルカ"},
     "SGD\xa0": {"fxtext": "SGD", "jptext": "シンガポール・ドル"}
-}
+}

pytchat/processors/default/renderer/legacypaid.py

@@ -1,18 +1,15 @@
 from .base import BaseRenderer
+
+
 class LiveChatLegacyPaidMessageRenderer(BaseRenderer):
     def __init__(self, item):
         super().__init__(item, "newSponsor")
 
-        
-
     def get_authordetails(self):
         super().get_authordetails()
-        self.author.isChatSponsor =  True
+        self.author.isChatSponsor = True
 
-
+    def get_message(self, renderer):
-    def get_message(self,renderer):
         message = (renderer["eventText"]["runs"][0]["text"]
-            )+' / '+(renderer["detailText"]["simpleText"])
+                   ) + ' / ' + (renderer["detailText"]["simpleText"])
-        return message
+        return message, [message]
-
-

pytchat/processors/default/renderer/membership.py

@@ -10,6 +10,6 @@ class LiveChatMembershipItemRenderer(BaseRenderer):
         self.author.isChatSponsor = True
 
     def get_message(self, renderer):
-        message = ''.join([mes.get("text", "") for mes in renderer["headerSubtext"]["runs"]])
+        message = ''.join([mes.get("text", "")
+                           for mes in renderer["headerSubtext"]["runs"]])
         return message, [message]
-

pytchat/processors/default/renderer/paidmessage.py

@@ -3,30 +3,44 @@ from . import currency
 from .base import BaseRenderer
 superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
 
+
+class Colors:
+    pass
+
+
 class LiveChatPaidMessageRenderer(BaseRenderer):
     def __init__(self, item):
         super().__init__(item, "superChat")
 
-
     def get_snippet(self):
         super().get_snippet()
-        amountDisplayString, symbol, amount =(
+        amountDisplayString, symbol, amount = (
             self.get_amountdata(self.renderer)
         )
-        self.amountValue= amount
+        self.amountValue = amount
         self.amountString = amountDisplayString
-        self.currency= currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol
+        self.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get(
-        self.bgColor= self.renderer.get("bodyBackgroundColor", 0)
+            symbol) else symbol
-            
+        self.bgColor = self.renderer.get("bodyBackgroundColor", 0)
+        self.colors = self.get_colors()
 
-
+    def get_amountdata(self, renderer):
-    def get_amountdata(self,renderer):
         amountDisplayString = renderer["purchaseAmountText"]["simpleText"]
         m = superchat_regex.search(amountDisplayString)
         if m:
             symbol = m.group(1)
-            amount = float(m.group(2).replace(',',''))
+            amount = float(m.group(2).replace(',', ''))
         else:
             symbol = ""
             amount = 0.0
-        return amountDisplayString, symbol, amount
+        return amountDisplayString, symbol, amount
+
+    def get_colors(self):
+        colors = Colors()
+        colors.headerBackgroundColor = self.renderer.get("headerBackgroundColor", 0)
+        colors.headerTextColor = self.renderer.get("headerTextColor", 0)
+        colors.bodyBackgroundColor = self.renderer.get("bodyBackgroundColor", 0)
+        colors.bodyTextColor = self.renderer.get("bodyTextColor", 0)
+        colors.timestampColor = self.renderer.get("timestampColor", 0)
+        colors.authorNameTextColor = self.renderer.get("authorNameTextColor", 0)
+        return colors

pytchat/processors/default/renderer/paidsticker.py

@@ -3,37 +3,44 @@ from . import currency
 from .base import BaseRenderer
 superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
 
+
+class Colors:
+    pass
+
+
 class LiveChatPaidStickerRenderer(BaseRenderer):
     def __init__(self, item):
         super().__init__(item, "superSticker")
 
-
     def get_snippet(self):
         super().get_snippet()
-        amountDisplayString, symbol, amount =(
+        amountDisplayString, symbol, amount = (
             self.get_amountdata(self.renderer)
         )
         self.amountValue = amount
         self.amountString = amountDisplayString
-        self.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol
+        self.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get(
-        self.bgColor = self.renderer.get("moneyChipBackgroundColor", 0)
+            symbol) else symbol
-        self.sticker = "https:"+self.renderer["sticker"]["thumbnails"][0]["url"]
+        self.bgColor = self.renderer.get("backgroundColor", 0)
-            
+        self.sticker = "".join(("https:",
+            self.renderer["sticker"]["thumbnails"][0]["url"]))
+        self.colors = self.get_colors()
 
-
+    def get_amountdata(self, renderer):
-    def get_amountdata(self,renderer):
         amountDisplayString = renderer["purchaseAmountText"]["simpleText"]
         m = superchat_regex.search(amountDisplayString)
         if m:
             symbol = m.group(1)
-            amount = float(m.group(2).replace(',',''))
+            amount = float(m.group(2).replace(',', ''))
         else:
             symbol = ""
             amount = 0.0
         return amountDisplayString, symbol, amount
 
-
+    def get_colors(self):
- 
+        colors = Colors()
-
+        colors.moneyChipBackgroundColor = self.renderer.get("moneyChipBackgroundColor", 0)
- 
+        colors.moneyChipTextColor = self.renderer.get("moneyChipTextColor", 0)
-
+        colors.backgroundColor = self.renderer.get("backgroundColor", 0)
+        colors.authorNameTextColor = self.renderer.get("authorNameTextColor", 0)
+        return colors

pytchat/processors/default/renderer/textmessage.py

@@ -1,4 +1,6 @@
 from .base import BaseRenderer
+
+
 class LiveChatTextMessageRenderer(BaseRenderer):
     def __init__(self, item):
         super().__init__(item, "textMessage")

pytchat/processors/dummy_processor.py

@@ -1,8 +1,10 @@
 from .chat_processor import ChatProcessor
 
+
 class DummyProcessor(ChatProcessor):
     '''
     Dummy processor just returns received chat_components directly.
     '''
+
     def process(self, chat_components: list):
         return chat_components

pytchat/processors/html_archiver.py

@@ -1,98 +1,141 @@
-import csv
 import os
 import re
+import httpx
+from base64 import standard_b64encode
 from .chat_processor import ChatProcessor
 from .default.processor import DefaultProcessor
 
-PATTERN  = re.compile(r"(.*)\(([0-9]+)\)$")
+
-fmt_headers = ['datetime','elapsed','authorName','message','superchat'
+PATTERN = re.compile(r"(.*)\(([0-9]+)\)$")
-     ,'type','authorChannel']
+
+fmt_headers = ['datetime', 'elapsed', 'authorName',
+               'message', 'superchat', 'type', 'authorChannel']
 
 HEADER_HTML = '''
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html>
+<head>
 <meta http-equiv="Content-Type" content="text/html;charset=UTF-8">
 '''
 
+TABLE_CSS = '''
+table.css {
+ border-collapse: collapse;
+}
+ 
+table.css thead{
+ border-collapse: collapse;
+ border: 1px solid #000
+}
+ 
+table.css tr td{
+ padding: 0.3em;
+ border: 1px solid #000
+}
+
+table.css th{
+ padding: 0.3em;
+ border: 1px solid #000
+}
+'''
+
+
 class HTMLArchiver(ChatProcessor):
     '''
-    HtmlArchiver saves chat data as HTML table format.
+    HTMLArchiver saves chat data as HTML table format.
     '''
-
+    def __init__(self, save_path, callback=None):
-    def __init__(self, save_path):
         super().__init__()
         self.save_path = self._checkpath(save_path)
-        with open(self.save_path, mode='a', encoding = 'utf-8') as f:
-            f.write(HEADER_HTML)
-            f.write('<table border="1" style="border-collapse: collapse">')
-            f.writelines(self._parse_html_header(fmt_headers))
         self.processor = DefaultProcessor()
+        self.emoji_table = {}  # tuble for custom emojis. key: emoji_id, value: base64 encoded image binary.
+        self.header = [HEADER_HTML]
+        self.body = ['<body>\n', '<table class="css">\n', self._parse_table_header(fmt_headers)]
+        self.callback = callback
 
     def _checkpath(self, filepath):
         splitter = os.path.splitext(os.path.basename(filepath))
         body = splitter[0]
         extention = splitter[1]
         newpath = filepath
-        counter = 0
+        counter = 1
         while os.path.exists(newpath):
-            match = re.search(PATTERN,body)
+            match = re.search(PATTERN, body)
             if match:
-                counter=int(match[2])+1
+                counter = int(match[2]) + 1
                 num_with_bracket = f'({str(counter)})'
                 body = f'{match[1]}{num_with_bracket}'
             else:
                 body = f'{body}({str(counter)})'
-            newpath = os.path.join(os.path.dirname(filepath),body+extention)
+            newpath = os.path.join(os.path.dirname(filepath), body + extention)
         return newpath
 
     def process(self, chat_components: list):
         """
         Returns
         ----------
-        dict : 
+        dict :
             save_path : str :
                 Actual save path of file.
             total_lines : int :
                 count of total lines written to the file.
         """
-        if chat_components is None or len (chat_components) == 0:
+        if chat_components is None or len(chat_components) == 0:
             return
-
+        for c in self.processor.process(chat_components).items:
-        with open(self.save_path, mode='a', encoding = 'utf-8') as f:
+            self.body.extend(
-            chats = self.processor.process(chat_components).items
+                self._parse_html_line((
-            for c in chats:
+                    c.datetime,
-                f.writelines(
+                    c.elapsedTime,
-                    self._parse_html_line([
+                    c.author.name,
-                        c.datetime,
+                    self._parse_message(c.messageEx),
-                        c.elapsedTime,
+                    c.amountString,
-                        c.author.name,
+                    c.author.type,
-                        c.message,
+                    c.author.channelId)
-                        c.amountString,
-                        c.author.type,
-                        c.author.channelId]
-                    )
                 )
-            '''
+            )
-            #Palliative treatment#
+            if self.callback:
-            Comment out below line to prevent the table
+                self.callback(None, 1)
-            display from collapsing.
-            '''
-            #f.write('</table>')
 
     def _parse_html_line(self, raw_line):
-        html = ''
+        return ''.join(('<tr>',
-        html+=' <tr>'
+                        ''.join(''.join(('<td>', cell, '</td>')) for cell in raw_line),
-        for cell in raw_line:
+                        '</tr>\n'))
-            html+='<td>'+cell+'</td>'
+
-        html+='</tr>\n'
+    def _parse_table_header(self, raw_line):
-        return html
+        return ''.join(('<thead><tr>',
+                        ''.join(''.join(('<th>', cell, '</th>')) for cell in raw_line),
+                        '</tr></thead>\n'))
+        
+    def _parse_message(self, message_items: list) -> str:
+        return ''.join(''.join(('<span class="', self._set_emoji_table(item), '" title="', item['txt'], '"></span>'))
+                       if type(item) is dict else item
+                       for item in message_items)
+
+    def _encode_img(self, url):
+        resp = httpx.get(url)
+        return standard_b64encode(resp.content).decode()
+
+    def _set_emoji_table(self, item: dict):
+        emoji_id = item['id']
+        if emoji_id not in self.emoji_table:
+            self.emoji_table.setdefault(emoji_id, self._encode_img(item['url']))
+        return emoji_id
+
+    def _stylecode(self, name, code, width, height):
+        return ''.join((".", name, " { display: inline-block; background-image: url(data:image/png;base64,",
+                        code, "); background-repeat: no-repeat; width: ",
+                        str(width), "; height: ", str(height), ";}"))
     
-    def _parse_html_header(self,raw_line):
+    def _create_styles(self):
-        html = ''
+        return '\n'.join(('<style type="text/css">',
-        html+='<thead>\n'
+                          TABLE_CSS,
-        html+=' <tr>'
+                          '\n'.join(self._stylecode(key, self.emoji_table[key], 24, 24)
-        for cell in raw_line:
+                                for key in self.emoji_table.keys()),
-            html+='<th>'+cell+'</th>'
+                          '</style>\n'))
-        html+='</tr>\n'
+    
-        html+='</thead>\n'
+    def finalize(self):
-        return html
+        self.header.extend([self._create_styles(), '</head>\n'])
-    
+        self.body.extend(['</table>\n</body>\n</html>'])
+        with open(self.save_path, mode='a', encoding='utf-8') as f:
+            f.writelines(self.header)
+            f.writelines(self.body)

pytchat/processors/jsonfile_archiver.py

@@ -1,10 +1,10 @@
-import datetime
 import json
 import os
 import re
 from .chat_processor import ChatProcessor
 
-PATTERN  = re.compile(r"(.*)\(([0-9]+)\)$")
+PATTERN = re.compile(r"(.*)\(([0-9]+)\)$")
+
 
 class JsonfileArchiver(ChatProcessor):
     """
@@ -13,39 +13,44 @@ class JsonfileArchiver(ChatProcessor):
     Parameter:
     ----------
     save_path : str :
-        save path of file.If a file with the same name exists, 
+        save path of file.If a file with the same name exists,
         it is automatically saved under a different name
         with suffix '(number)'
     """
-    def __init__(self,save_path):
+
+    def __init__(self, save_path):
         super().__init__()
         self.save_path = self._checkpath(save_path)
         self.line_counter = 0
-                
+
-    def process(self,chat_components: list):
+    def process(self, chat_components: list):
         """
         Returns
         ----------
-        dict : 
+        dict :
             save_path : str :
                 Actual save path of file.
             total_lines : int :
                 count of total lines written to the file.
         """
-        if chat_components is None: return
+        if chat_components is None:
-        with open(self.save_path, mode='a', encoding = 'utf-8') as f:
+            return
+        with open(self.save_path, mode='a', encoding='utf-8') as f:
             for component in chat_components:
-                if component is None: continue
+                if component is None:
+                    continue
                 chatdata = component.get('chatdata')
-                if chatdata is None: continue
+                if chatdata is None:
+                    continue
                 for action in chatdata:
-                    if action is None: continue
+                    if action is None:
-                    json_line = json.dumps(action, ensure_ascii = False)
+                        continue
-                    f.writelines(json_line+'\n')
+                    json_line = json.dumps(action, ensure_ascii=False)
-                    self.line_counter+=1
+                    f.writelines(json_line + '\n')
-            return { "save_path"  : self.save_path,
+                    self.line_counter += 1
-                     "total_lines": self.line_counter }
+            return {"save_path": self.save_path,
-    
+                    "total_lines": self.line_counter}
+
     def _checkpath(self, filepath):
         splitter = os.path.splitext(os.path.basename(filepath))
         body = splitter[0]
@@ -53,14 +58,12 @@ class JsonfileArchiver(ChatProcessor):
         newpath = filepath
         counter = 0
         while os.path.exists(newpath):
-            match = re.search(PATTERN,body)
+            match = re.search(PATTERN, body)
             if match:
-                counter=int(match[2])+1
+                counter = int(match[2]) + 1
                 num_with_bracket = f'({str(counter)})'
                 body = f'{match[1]}{num_with_bracket}'
             else:
                 body = f'{body}({str(counter)})'
-            newpath = os.path.join(os.path.dirname(filepath),body+extention)
+            newpath = os.path.join(os.path.dirname(filepath), body + extention)
         return newpath
-
-

pytchat/processors/simple_display_processor.py

@@ -1,47 +1,49 @@
-import json
-import os
-import traceback
-import datetime
-import time
 from .chat_processor import ChatProcessor
-##version 2
+
+
 class SimpleDisplayProcessor(ChatProcessor):
-    
+
     def process(self, chat_components: list):
         chatlist = []
         timeout = 0
-        
+
         if chat_components is None:
-            return {"timeout":timeout, "chatlist":chatlist}
+            return {"timeout": timeout, "chatlist": chatlist}
         for component in chat_components:
             timeout += component.get('timeout', 0)
             chatdata = component.get('chatdata')
-            
-            if chatdata is None:break
-            for action in chatdata:
-                if action is None:continue
-                if action.get('addChatItemAction') is None:continue
-                if action['addChatItemAction'].get('item') is None:continue
 
-                root = action['addChatItemAction']['item'].get('liveChatTextMessageRenderer')
+            if chatdata is None:
-                
+                break
+            for action in chatdata:
+                if action is None:
+                    continue
+                if action.get('addChatItemAction') is None:
+                    continue
+                if action['addChatItemAction'].get('item') is None:
+                    continue
+
+                root = action['addChatItemAction']['item'].get(
+                    'liveChatTextMessageRenderer')
+
                 if root:
                     author_name = root['authorName']['simpleText']
                     message = self._parse_message(root.get('message'))
                     purchase_amount_text = ''
                 else:
-                    root = ( action['addChatItemAction']['item'].get('liveChatPaidMessageRenderer') or 
+                    root = (action['addChatItemAction']['item'].get('liveChatPaidMessageRenderer')
-                            action['addChatItemAction']['item'].get('liveChatPaidStickerRenderer') )
+                            or action['addChatItemAction']['item'].get('liveChatPaidStickerRenderer'))
                     if root:
                         author_name = root['authorName']['simpleText']
                         message = self._parse_message(root.get('message'))
                         purchase_amount_text = root['purchaseAmountText']['simpleText']
                     else:
                         continue
-                chatlist.append(f'[{author_name}]:  {message}  {purchase_amount_text}')
+                chatlist.append(
-        return {"timeout":timeout, "chatlist":chatlist}
+                    f'[{author_name}]:  {message}  {purchase_amount_text}')
-    
+        return {"timeout": timeout, "chatlist": chatlist}
-    def _parse_message(self,message):
+
+    def _parse_message(self, message):
         if message is None:
             return ''
         if message.get('simpleText'):
@@ -51,11 +53,9 @@ class SimpleDisplayProcessor(ChatProcessor):
             tmp = ''
             for run in runs:
                 if run.get('emoji'):
-                    tmp+=(run['emoji']['shortcuts'][0])
+                    tmp += (run['emoji']['shortcuts'][0])
                 elif run.get('text'):
-                    tmp+=(run['text'])
+                    tmp += (run['text'])
             return tmp
         else:
             return ''
-
-

pytchat/processors/speed/calculator.py

@@ -5,10 +5,12 @@ Calculate speed of chat.
 """
 import time
 from .. chat_processor import ChatProcessor
+
+
 class RingQueue:
     """
     リング型キュー
-    
+
     Attributes
     ----------
     items : list
@@ -21,10 +23,10 @@ class RingQueue:
         キュー内に余裕があるか。キュー内のアイテム個数が、キューの最大個数未満であればTrue。
     """
 
-    def __init__(self, capacity):    
+    def __init__(self, capacity):
         """
         コンストラクタ
-        
+
         Parameter
         ----------
             capacity:このキューに格納するアイテムの最大個数。
@@ -50,17 +52,17 @@ class RingQueue:
         """
         if self.mergin:
             self.items.append(item)
-            self.last_pos = len(self.items)-1
+            self.last_pos = len(self.items) - 1
-            if self.last_pos == self.capacity-1:
+            if self.last_pos == self.capacity - 1:
                 self.mergin = False
             return
         self.last_pos += 1
-        if self.last_pos > self.capacity-1:
+        if self.last_pos > self.capacity - 1:
             self.last_pos = 0
         self.items[self.last_pos] = item
-        
+
         self.first_pos += 1
-        if self.first_pos > self.capacity-1:
+        if self.first_pos > self.capacity - 1:
             self.first_pos = 0
 
     def get(self):
@@ -76,11 +78,12 @@ class RingQueue:
 
     def item_count(self):
         return len(self.items)
-        
+
+
 class SpeedCalculator(ChatProcessor, RingQueue):
     """
     チャットの勢いを計算する。
-    
+
     一定期間のチャットデータのうち、最初のチャットの投稿時刻と
     最後のチャットの投稿時刻の差を、チャット数で割り返し
     1分あたりの速度に換算する。
@@ -91,7 +94,7 @@ class SpeedCalculator(ChatProcessor, RingQueue):
         RingQueueに格納するチャット勢い算出用データの最大数
     """
 
-    def __init__(self, capacity = 10):
+    def __init__(self, capacity=10):
         super().__init__(capacity)
         self.speed = 0
 
@@ -105,7 +108,6 @@ class SpeedCalculator(ChatProcessor, RingQueue):
             self._put_chatdata(chatdata)
             self.speed = self._calc_speed()
         return self.speed
-                
 
     def _calc_speed(self):
         """
@@ -116,14 +118,13 @@ class SpeedCalculator(ChatProcessor, RingQueue):
         ---------------------------
             チャット速度（１分間で換算したチャット数）
         """
-        try:        
+        try:
-            #キュー内の総チャット数
+            # キュー内の総チャット数
             total = sum(item['chat_count'] for item in self.items)
-            #キュー内の最初と最後のチャットの時間差
+            # キュー内の最初と最後のチャットの時間差
-            duration = (self.items[self.last_pos]['endtime'] 
+            duration = (self.items[self.last_pos]['endtime'] - self.items[self.first_pos]['starttime'])
-                        - self.items[self.first_pos]['starttime'])
             if duration != 0:
-                return int(total*60/duration)
+                return int(total * 60 / duration)
             return 0
         except IndexError:
             return 0
@@ -143,61 +144,60 @@ class SpeedCalculator(ChatProcessor, RingQueue):
             '''
             チャットデータがない場合に空のデータをキューに投入する。
             '''
-            timestamp_now =  int(time.time())
+            timestamp_now = int(time.time())
             self.put({
-                'chat_count':0,
+                'chat_count': 0,
-                'starttime':int(timestamp_now),
+                'starttime': int(timestamp_now),
-                'endtime':int(timestamp_now)
+                'endtime': int(timestamp_now)
             })
 
-        def _get_timestamp(action :dict):
+        def _get_timestamp(action: dict):
             """
             チャットデータから時刻データを取り出す。
             """
             try:
                 item = action['addChatItemAction']['item']
                 timestamp = int(item[list(item.keys())[0]]['timestampUsec'])
-            except (KeyError,TypeError):
+            except (KeyError, TypeError):
                 return None
             return timestamp
 
-        if actions is None or len(actions)==0:
+        if actions is None or len(actions) == 0:
             _put_emptydata()
-            return 
+            return
-           
+
-        #actions内の時刻データを持つチャットデータの数
+        # actions内の時刻データを持つチャットデータの数
-        counter=0        
+        counter = 0
-        #actions内の最初のチャットデータの時刻
+        # actions内の最初のチャットデータの時刻
-        starttime= None
+        starttime = None
-        #actions内の最後のチャットデータの時刻
+        # actions内の最後のチャットデータの時刻
-        endtime=None
+        endtime = None
-        
+
         for action in actions:
-            #チャットデータからtimestampUsecを読み取る
+            # チャットデータからtimestampUsecを読み取る
             gettime = _get_timestamp(action)
-            
+
-            #時刻のないデータだった場合は次の行のデータで読み取り試行
+            # 時刻のないデータだった場合は次の行のデータで読み取り試行
             if gettime is None:
                 continue
-            
+
-            #最初に有効な時刻を持つデータのtimestampをstarttimeに設定
+            # 最初に有効な時刻を持つデータのtimestampをstarttimeに設定
             if starttime is None:
                 starttime = gettime
-        
+
-            #最後のtimestampを設定(途中で時刻のないデータの場合もあるので上書きしていく)
+            # 最後のtimestampを設定(途中で時刻のないデータの場合もあるので上書きしていく)
             endtime = gettime
-        
+
-            #チャットの数をインクリメント
+            # チャットの数をインクリメント
             counter += 1
 
-        #チャット速度用のデータをRingQueueに送る
+        # チャット速度用のデータをRingQueueに送る
         if starttime is None or endtime is None:
             _put_emptydata()
-            return 
+            return
-            
-        self.put({
-            'chat_count':counter,
-            'starttime':int(starttime/1000000),
-            'endtime':int(endtime/1000000)
-        })
 
+        self.put({
+            'chat_count': counter,
+            'starttime': int(starttime / 1000000),
+            'endtime': int(endtime / 1000000)
+        })

pytchat/processors/superchat/calculator.py

@@ -15,10 +15,12 @@ items_sticker = [
     'liveChatPaidStickerRenderer'
 ]
 
+
 class SuperchatCalculator(ChatProcessor):
     """
     Calculate the amount of SuperChat by currency.
     """
+
     def __init__(self):
         self.results = {}
 
@@ -34,22 +36,24 @@ class SuperchatCalculator(ChatProcessor):
             return self.results
         for component in chat_components:
             chatdata = component.get('chatdata')
-            if chatdata is None: continue
+            if chatdata is None:
+                continue
             for action in chatdata:
                 renderer = self._get_item(action, items_paid) or \
-                           self._get_item(action, items_sticker)
+                    self._get_item(action, items_sticker)
-                if renderer is None: continue
+                if renderer is None:
+                    continue
                 symbol, amount = self._parse(renderer)
-                self.results.setdefault(symbol,0)
+                self.results.setdefault(symbol, 0)
-                self.results[symbol]+=amount
+                self.results[symbol] += amount
         return self.results
-    
+
     def _parse(self, renderer):
         purchase_amount_text = renderer["purchaseAmountText"]["simpleText"]
         m = superchat_regex.search(purchase_amount_text)
         if m:
             symbol = m.group(1)
-            amount = float(m.group(2).replace(',',''))
+            amount = float(m.group(2).replace(',', ''))
         else:
             symbol = ""
             amount = 0.0
@@ -69,6 +73,3 @@ class SuperchatCalculator(ChatProcessor):
                 continue
             return None
         return dict_body
-
-
-

pytchat/processors/tsv_archiver.py

@@ -4,9 +4,10 @@ import re
 from .chat_processor import ChatProcessor
 from .default.processor import DefaultProcessor
 
-PATTERN  = re.compile(r"(.*)\(([0-9]+)\)$")
+PATTERN = re.compile(r"(.*)\(([0-9]+)\)$")
-fmt_headers = ['datetime','elapsed','authorName','message','superchatAmount'
+fmt_headers = ['datetime', 'elapsed', 'authorName', 'message',
-     ,'authorType','authorChannel']
+               'superchatAmount', 'authorType', 'authorChannel']
+
 
 class TSVArchiver(ChatProcessor):
     '''
@@ -16,7 +17,7 @@ class TSVArchiver(ChatProcessor):
     def __init__(self, save_path):
         super().__init__()
         self.save_path = self._checkpath(save_path)
-        with open(self.save_path, mode='a', encoding = 'utf-8') as f:
+        with open(self.save_path, mode='a', encoding='utf-8') as f:
             writer = csv.writer(f, delimiter='\t')
             writer.writerow(fmt_headers)
         self.processor = DefaultProcessor()
@@ -28,30 +29,30 @@ class TSVArchiver(ChatProcessor):
         newpath = filepath
         counter = 0
         while os.path.exists(newpath):
-            match = re.search(PATTERN,body)
+            match = re.search(PATTERN, body)
             if match:
-                counter=int(match[2])+1
+                counter = int(match[2]) + 1
                 num_with_bracket = f'({str(counter)})'
                 body = f'{match[1]}{num_with_bracket}'
             else:
                 body = f'{body}({str(counter)})'
-            newpath = os.path.join(os.path.dirname(filepath),body+extention)
+            newpath = os.path.join(os.path.dirname(filepath), body + extention)
         return newpath
 
     def process(self, chat_components: list):
         """
         Returns
         ----------
-        dict : 
+        dict :
             save_path : str :
                 Actual save path of file.
             total_lines : int :
                 count of total lines written to the file.
         """
-        if chat_components is None or len (chat_components) == 0:
+        if chat_components is None or len(chat_components) == 0:
             return
 
-        with open(self.save_path, mode='a', encoding = 'utf-8') as f:
+        with open(self.save_path, mode='a', encoding='utf-8') as f:
             writer = csv.writer(f, delimiter='\t')
             chats = self.processor.process(chat_components).items
             for c in chats:
@@ -64,7 +65,3 @@ class TSVArchiver(ChatProcessor):
                     c.author.type,
                     c.author.channelId
                 ])
-
-
-    
-    

pytchat/tool/extract/asyncdl.py

@@ -1,11 +1,10 @@
-import aiohttp
+import httpx
 import asyncio
-import json
 from . import parser
 from . block import Block
 from . worker import ExtractWorker
 from . patch import Patch
-from ... import config 
+from ... import config
 from ... paramgen import arcparam
 from ... exceptions import UnknownConnectionError
 from concurrent.futures import CancelledError
@@ -17,10 +16,11 @@ REPLAY_URL = "https://www.youtube.com/live_chat_replay/" \
              "get_live_chat_replay?continuation="
 MAX_RETRY_COUNT = 3
 
-def _split(start, end, count, min_interval_sec = 120):
+
+def _split(start, end, count, min_interval_sec=120):
     """
     Split section from `start` to `end` into `count` pieces,
-    and returns the beginning of each piece. 
+    and returns the beginning of each piece.
     The `count` is adjusted so that the length of each piece
     is no smaller than `min_interval`.
 
@@ -28,43 +28,44 @@ def _split(start, end, count, min_interval_sec = 120):
     --------
         List of the offset of each block's first chat data.
     """
-    
+    if not (isinstance(start, int) or isinstance(start, float)) or \
-    if not (isinstance(start,int) or isinstance(start,float)) or \
+       not (isinstance(end, int) or isinstance(end, float)):
-       not (isinstance(end,int) or isinstance(end,float)):
         raise ValueError("start/end must be int or float")
-    if not isinstance(count,int):
+    if not isinstance(count, int):
         raise ValueError("count must be int")
-    if start>end:
+    if start > end:
         raise ValueError("end must be equal to or greater than start.")
-    if count<1:
+    if count < 1:
         raise ValueError("count must be equal to or greater than 1.")
-    if (end-start)/count < min_interval_sec:
+    if (end - start) / count < min_interval_sec:
-        count = int((end-start)/min_interval_sec) 
+        count = int((end - start) / min_interval_sec)
-        if count == 0 : count = 1
+        if count == 0:
-    interval= (end-start)/count 
+            count = 1
-    
+    interval = (end - start) / count
+
     if count == 1:
         return [start]
-    return sorted( list(set( [int(start + interval*j)
+    return sorted(list(set([int(start + interval * j)
-        for j in range(count) ])))
+                            for j in range(count)])))
+
 
 def ready_blocks(video_id, duration, div, callback):
-    if div <= 0: raise ValueError
+    if div <= 0:
+        raise ValueError
 
-    async def _get_blocks( video_id, duration, div, callback):
+    async def _get_blocks(video_id, duration, div, callback):
-        async with aiohttp.ClientSession() as session:
+        async with httpx.AsyncClient(http2=True) as session:
-            tasks = [_create_block(session, video_id,  seektime, callback)
+            tasks = [_create_block(session, video_id, seektime, callback)
-                for  seektime in _split(-1, duration, div)]
+                     for seektime in _split(-1, duration, div)]
             return await asyncio.gather(*tasks)
-    
+
     async def _create_block(session, video_id, seektime, callback):
-        continuation = arcparam.getparam(video_id, seektime = seektime)
+        continuation = arcparam.getparam(video_id, seektime=seektime)
         url = f"{REPLAY_URL}{quote(continuation)}&pbj=1"
         for _ in range(MAX_RETRY_COUNT):
-            try :
+            try:
-                async with session.get(url, headers = headers) as resp:
+                resp = await session.get(url, headers=headers)
-                    text = await resp.text()
+                next_continuation, actions = parser.parse(resp.json())
-                next_continuation, actions = parser.parse(json.loads(text))
                 break
             except JSONDecodeError:
                 await asyncio.sleep(3)
@@ -76,43 +77,43 @@ def ready_blocks(video_id, duration, div, callback):
             first = parser.get_offset(actions[0])
             last = parser.get_offset(actions[-1])
             if callback:
-                callback(actions,last-first)
+                callback(actions, last - first)
             return Block(
-                continuation = next_continuation,
+                continuation=next_continuation,
-                chat_data = actions,
+                chat_data=actions,
-                first = first,
+                first=first,
-                last = last
+                last=last
             )
-            
+
     """
     fetch initial blocks.
-    """  
+    """
     loop = asyncio.get_event_loop()
     blocks = loop.run_until_complete(
         _get_blocks(video_id, duration, div, callback))
     return blocks
 
+
 def fetch_patch(callback, blocks, video_id):
 
     async def _allocate_workers():
         workers = [
             ExtractWorker(
-                fetch = _fetch,  block = block,
+                fetch=_fetch, block=block,
-                blocks = blocks, video_id = video_id
+                blocks=blocks, video_id=video_id
             )
             for block in blocks
         ]
-        async with aiohttp.ClientSession() as session:
+        async with httpx.AsyncClient() as session:
             tasks = [worker.run(session) for worker in workers]
-            return await asyncio.gather(*tasks)    
+            return await asyncio.gather(*tasks)
 
-    async def _fetch(continuation,session) -> Patch:
+    async def _fetch(continuation, session) -> Patch:
         url = f"{REPLAY_URL}{quote(continuation)}&pbj=1"
         for _ in range(MAX_RETRY_COUNT):
             try:
-                async with session.get(url,headers = config.headers) as resp:
+                resp = await session.get(url, headers=config.headers)
-                    chat_json = await resp.text()
+                continuation, actions = parser.parse(resp.json())
-                continuation, actions = parser.parse(json.loads(chat_json))
                 break
             except JSONDecodeError:
                 await asyncio.sleep(3)
@@ -126,21 +127,22 @@ def fetch_patch(callback, blocks, video_id):
             if callback:
                 callback(actions, last - first)
             return Patch(actions, continuation, first, last)
-        return Patch(continuation = continuation)
+        return Patch(continuation=continuation)
 
     """
     allocate workers and assign blocks.
-    """   
+    """
     loop = asyncio.get_event_loop()
     try:
         loop.run_until_complete(_allocate_workers())
     except CancelledError:
         pass
 
+
 async def _shutdown():
     print("\nshutdown...")
     tasks = [t for t in asyncio.all_tasks()
-         if t is not asyncio.current_task()]
+             if t is not asyncio.current_task()]
     for task in tasks:
         task.cancel()
         try:
@@ -148,7 +150,7 @@ async def _shutdown():
         except asyncio.CancelledError:
             pass
 
+
 def cancel():
     loop = asyncio.get_event_loop()
     loop.create_task(_shutdown())
-    

pytchat/tool/extract/block.py

@@ -1,14 +1,13 @@
-from . import parser
 class Block:
-    """Block object represents something like a box 
+    """Block object represents something like a box
     to join chunk of chatdata.
 
     Parameter:
     ---------
     first : int :
-        videoOffsetTimeMs of the first chat_data 
+        videoOffsetTimeMs of the first chat_data
         (chat_data[0])
-        
+
     last : int :
         videoOffsetTimeMs of the last chat_data.
         (chat_data[-1])
@@ -23,15 +22,15 @@ class Block:
     continuation : str :
         continuation param of last chat data.
 
-    chat_data : list 
+    chat_data : list
 
     done : bool :
         whether this block has been fetched.
-    
+
     remaining : int :
         remaining data to extract.
         equals end - last.
-    
+
     is_last : bool :
         whether this block is the last one in blocklist.
 
@@ -39,13 +38,13 @@ class Block:
         whether this block is in the process of during_split.
         while True, this block is excluded from duplicate split procedure.
     """
-    
-    __slots__ = ['first','last','end','continuation','chat_data','remaining',
-        'done','is_last','during_split']
 
-    def __init__(self, first = 0, last = 0, end = 0,
+    __slots__ = ['first', 'last', 'end', 'continuation', 'chat_data', 'remaining',
-                continuation = '', chat_data = [], is_last = False,
+                 'done', 'is_last', 'during_split']
-                during_split = False):
+
+    def __init__(self, first=0, last=0, end=0,
+                 continuation='', chat_data=[], is_last=False,
+                 during_split=False):
         self.first = first
         self.last = last
         self.end = end

pytchat/tool/extract/duplcheck.py

@@ -1,7 +1,8 @@
 from . import parser
 
+
 def check_duplicate(chatdata):
-    max_range = len(chatdata)-1 
+    max_range = len(chatdata) - 1
     tbl_offset = [None] * max_range
     tbl_id = [None] * max_range
     tbl_type = [None] * max_range
@@ -9,33 +10,31 @@ def check_duplicate(chatdata):
     def create_table(chatdata, max_range):
         for i in range(max_range):
             tbl_offset[i] = parser.get_offset(chatdata[i])
-            tbl_id[i] = parser.get_id(chatdata[i]) 
+            tbl_id[i] = parser.get_id(chatdata[i])
             tbl_type[i] = parser.get_type(chatdata[i])
 
     def is_duplicate(i, j):
-        return ( 
+        return (
             tbl_offset[i] == tbl_offset[j]
-                and
+            and tbl_id[i] == tbl_id[j]
-            tbl_id[i] == tbl_id[j]
+            and tbl_type[i] == tbl_type[j]
-                and
-            tbl_type[i] == tbl_type[j]
         )
     print("creating table...")
-    create_table(chatdata,max_range)
+    create_table(chatdata, max_range)
     print("searching duplicate data...")
-    return [{ "i":{
+    return [{"i": {
-                "index" : i, "id" : parser.get_id(chatdata[i]),
+        "index": i, "id": parser.get_id(chatdata[i]),
-                "offsetTime" : parser.get_offset(chatdata[i]),
+        "offsetTime": parser.get_offset(chatdata[i]),
-                "type" : parser.get_type(chatdata[i])
+        "type": parser.get_type(chatdata[i])
-                },
+    },
-            "j":{
+        "j":{
-                "index" : j, "id" : parser.get_id(chatdata[j]),
+        "index": j, "id": parser.get_id(chatdata[j]),
-                "offsetTime" : parser.get_offset(chatdata[j]),
+        "offsetTime": parser.get_offset(chatdata[j]),
-                "type" : parser.get_type(chatdata[j])
+        "type": parser.get_type(chatdata[j])
-                }
+    }
-            }
+    }
-        for i in range(max_range) for j in range(i+1,max_range) 
+        for i in range(max_range) for j in range(i + 1, max_range)
-        if is_duplicate(i,j)]
+        if is_duplicate(i, j)]
 
 
 def check_duplicate_offset(chatdata):
@@ -47,91 +46,92 @@ def check_duplicate_offset(chatdata):
     def create_table(chatdata, max_range):
         for i in range(max_range):
             tbl_offset[i] = parser.get_offset(chatdata[i])
-            tbl_id[i] = parser.get_id(chatdata[i]) 
+            tbl_id[i] = parser.get_id(chatdata[i])
             tbl_type[i] = parser.get_type(chatdata[i])
 
     def is_duplicate(i, j):
-        return ( 
+        return (
             tbl_offset[i] == tbl_offset[j]
-                 and
+            and tbl_id[i] == tbl_id[j]
-            tbl_id[i] == tbl_id[j]
         )
 
     print("creating table...")
-    create_table(chatdata,max_range)
+    create_table(chatdata, max_range)
     print("searching duplicate data...")
 
     return [{
-                "index" : i, "id" : tbl_id[i],
+        "index": i, "id": tbl_id[i],
-                "offsetTime" : tbl_offset[i],
+        "offsetTime": tbl_offset[i],
-                "type:" : tbl_type[i]
+        "type:": tbl_type[i]
-            }
+    }
-        for i in range(max_range-1)
+        for i in range(max_range - 1)
-        if is_duplicate(i,i+1)]
+        if is_duplicate(i, i + 1)]
+
 
 def remove_duplicate_head(blocks):
-    if len(blocks) == 1 : return blocks
+    if len(blocks) == 0 or len(blocks) == 1:
+        return blocks
 
     def is_duplicate_head(index):
 
         if len(blocks[index].chat_data) == 0:
             return True
-        elif len(blocks[index+1].chat_data) == 0:
+        elif len(blocks[index + 1].chat_data) == 0:
             return False
-        
+
         id_0 = parser.get_id(blocks[index].chat_data[0])
-        id_1 = parser.get_id(blocks[index+1].chat_data[0])
+        id_1 = parser.get_id(blocks[index + 1].chat_data[0])
         type_0 = parser.get_type(blocks[index].chat_data[0])
-        type_1 = parser.get_type(blocks[index+1].chat_data[0])
+        type_1 = parser.get_type(blocks[index + 1].chat_data[0])
         return (
-            blocks[index].first == blocks[index+1].first
+            blocks[index].first == blocks[index + 1].first
-                and
+            and id_0 == id_1
-            id_0 == id_1
+            and type_0 == type_1
-                and
-            type_0 == type_1
         )
-    ret = [blocks[i] for i in range(len(blocks)-1)
+    ret = [blocks[i] for i in range(len(blocks) - 1)
-        if (len(blocks[i].chat_data)>0 and 
+           if (len(blocks[i].chat_data) > 0
-        not is_duplicate_head(i) )]
+               and not is_duplicate_head(i))]
     ret.append(blocks[-1])
     return ret
 
+
 def remove_duplicate_tail(blocks):
-    if len(blocks) == 1 : return blocks    
+    if len(blocks) == 0 or len(blocks) == 1:
+        return blocks
 
     def is_duplicate_tail(index):
         if len(blocks[index].chat_data) == 0:
             return True
-        elif len(blocks[index-1].chat_data) == 0:
+        elif len(blocks[index - 1].chat_data) == 0:
             return False
-        id_0 = parser.get_id(blocks[index-1].chat_data[-1])
+        id_0 = parser.get_id(blocks[index - 1].chat_data[-1])
         id_1 = parser.get_id(blocks[index].chat_data[-1])
-        type_0 = parser.get_type(blocks[index-1].chat_data[-1])
+        type_0 = parser.get_type(blocks[index - 1].chat_data[-1])
         type_1 = parser.get_type(blocks[index].chat_data[-1])
         return (
-            blocks[index-1].last == blocks[index].last
+            blocks[index - 1].last == blocks[index].last
-                and
+            and id_0 == id_1
-            id_0 == id_1
+            and type_0 == type_1
-                and
-            type_0 == type_1
         )
 
-    ret = [blocks[i] for i in range(0,len(blocks))
+    ret = [blocks[i] for i in range(0, len(blocks))
-        if i == 0 or not  is_duplicate_tail(i) ]
+           if i == 0 or not is_duplicate_tail(i)]
     return ret
 
+
 def remove_overlap(blocks):
     """
     Fix overlapped blocks after ready_blocks().
-    Align the last offset of each block to the first offset 
+    Align the last offset of each block to the first offset
     of next block (equals `end` offset of each block).
     """
-    if len(blocks) == 1 : return blocks
+    if len(blocks) == 0 or len(blocks) == 1:
+        return blocks
 
     for block in blocks:
         if block.is_last:
             break
-        if len(block.chat_data)==0:
+        if len(block.chat_data) == 0:
             continue
         block_end = block.end
         if block.last >= block_end:
@@ -140,14 +140,14 @@ def remove_overlap(blocks):
                     break
                 block.chat_data.pop()
             block.last = parser.get_offset(line)
-            block.remaining=0
+            block.remaining = 0
-            block.done=True
+            block.done = True
             block.continuation = None
     return blocks
-    
+
-        
 
 def _dump(blocks):
-    print(f"----------        first         last         end---")
+    print("----------        first         last         end---")
-    for i,block in enumerate(blocks):
+    for i, block in enumerate(blocks):
-        print(f"block[{i:3}]   {block.first:>10}   {block.last:>10}  {block.end:>10}")
+        print(
+            f"block[{i:3}]   {block.first:>10}   {block.last:>10}  {block.end:>10}")

pytchat/tool/extract/extractor.py

@@ -1,20 +1,21 @@
 from . import asyncdl
-from . import duplcheck 
+from . import duplcheck
-from . import parser
 from .. videoinfo import VideoInfo
 from ... import config
 from ... exceptions import InvalidVideoIdException
+from ... util.extract_video_id import extract_video_id
 
 logger = config.logger(__name__)
-headers=config.headers
+headers = config.headers
+
 
 class Extractor:
-    def __init__(self, video_id, div = 1, callback = None, processor = None):
+    def __init__(self, video_id, div=1, callback=None, processor=None):
-        if not isinstance(div ,int) or div < 1:
+        if not isinstance(div, int) or div < 1:
             raise ValueError('div must be positive integer.')
         elif div > 10:
             div = 10
-        self.video_id = video_id
+        self.video_id = extract_video_id(video_id)
         self.div = div
         self.callback = callback
         self.processor = processor
@@ -33,17 +34,18 @@ class Extractor:
         blocks = asyncdl.ready_blocks(
             self.video_id, self.duration, self.div, self.callback)
         self.blocks = [block for block in blocks if block]
-        return self  
+        return self
 
     def _remove_duplicate_head(self):
         self.blocks = duplcheck.remove_duplicate_head(self.blocks)
         return self
 
     def _set_block_end(self):
-        for i in range(len(self.blocks)-1):
+        if len(self.blocks) > 0:
-            self.blocks[i].end = self.blocks[i+1].first
+            for i in range(len(self.blocks) - 1):
-        self.blocks[-1].end = self.duration*1000
+                self.blocks[i].end = self.blocks[i + 1].first
-        self.blocks[-1].is_last =True
+            self.blocks[-1].end = self.duration * 1000
+            self.blocks[-1].is_last = True
         return self
 
     def _remove_overlap(self):
@@ -61,7 +63,7 @@ class Extractor:
     def _combine(self):
         ret = []
         for block in self.blocks:
-            ret.extend(block.chat_data) 
+            ret.extend(block.chat_data)
         return ret
 
     def _execute_extract_operations(self):
@@ -77,15 +79,18 @@ class Extractor:
 
     def extract(self):
         if self.duration == 0:
-            print("video is not archived.")
+            print("\nCannot extract chat data:\n The specified video has not yet been archived.")
             return []
         data = self._execute_extract_operations()
         if self.processor is None:
-           return data
+            return data
-        return self.processor.process(
+        ret = self.processor.process(
-            [{'video_id':None,'timeout':1,'chatdata' : (action
+            [{'video_id': None,
-            ["replayChatItemAction"]["actions"][0] for action in data)}]
+              'timeout': 1,
-    )
+              'chatdata': (action["replayChatItemAction"]["actions"][0] for action in data)}]
+        )
+        self.processor.finalize()
+        return ret
 
     def cancel(self):
-        asyncdl.cancel()
+        asyncdl.cancel()

pytchat/tool/extract/parser.py

@@ -1,12 +1,9 @@
-import json
 from ... import config
-from ... exceptions import ( 
+from ... import exceptions
-    ResponseContextError, 
-    NoContentsException, 
-    NoContinuationsException )
 
 logger = config.logger(__name__)
 
+
 def parse(jsn):
     """
     Parse replay chat data.
@@ -20,18 +17,18 @@ def parse(jsn):
         actions : list
 
     """
-    if jsn is None: 
+    if jsn is None:
         raise ValueError("parameter JSON is None")
     if jsn['response']['responseContext'].get('errors'):
-        raise ResponseContextError(
+        raise exceptions.ResponseContextError(
-    'video_id is invalid or private/deleted.')
+            'video_id is invalid or private/deleted.')
-    contents=jsn['response'].get('continuationContents')
+    contents = jsn['response'].get('continuationContents')
     if contents is None:
-        raise NoContentsException('No chat data.')
+        raise exceptions.NoContents('No chat data.')
 
     cont = contents['liveChatContinuation']['continuations'][0]
     if cont is None:
-        raise NoContinuationsException('No Continuation')
+        raise exceptions.NoContinuation('No Continuation')
     metadata = cont.get('liveChatReplayContinuationData')
     if metadata:
         continuation = metadata.get("continuation")
@@ -43,12 +40,16 @@ def parse(jsn):
 def get_offset(item):
     return int(item['replayChatItemAction']["videoOffsetTimeMsec"])
 
+
 def get_id(item):
-    return list((list(item['replayChatItemAction']["actions"][0].values()
+    a = list(item['replayChatItemAction']["actions"][0].values())[0].get('item')
-                )[0])['item'].values())[0].get('id')
+    if a:
+        return list(a.values())[0].get('id')
+    return None
+
 
 def get_type(item):
-    return list((list(item['replayChatItemAction']["actions"][0].values()
+    a = list(item['replayChatItemAction']["actions"][0].values())[0].get('item')
-                )[0])['item'].keys())[0]
+    if a:
-
+        return list(a.keys())[0]
-
+    return None

pytchat/tool/extract/patch.py

@@ -2,17 +2,19 @@ from . import parser
 from . block import Block
 from typing import NamedTuple
 
+
 class Patch(NamedTuple):
     """
     Patch represents chunk of chat data
     which is fetched by asyncdl.fetch_patch._fetch().
     """
-    chats : list = []
+    chats: list = []
-    continuation : str = None
+    continuation: str = None
-    first : int = None
+    first: int = None
-    last : int = None
+    last: int = None
 
-def fill(block:Block, patch:Patch):
+
+def fill(block: Block, patch: Patch):
     block_end = block.end
     if patch.last < block_end or block.is_last:
         set_patch(block, patch)
@@ -23,32 +25,31 @@ def fill(block:Block, patch:Patch):
             break
         patch.chats.pop()
     set_patch(block, patch._replace(
-        continuation = None,
+        continuation=None,
-        last = line_offset
+        last=line_offset
-        )
     )
-    block.remaining=0
+    )
-    block.done=True
+    block.remaining = 0
+    block.done = True
 
 
-def split(parent_block:Block, child_block:Block, patch:Patch):
+def split(parent_block: Block, child_block: Block, patch: Patch):
     parent_block.during_split = False
     if patch.first <= parent_block.last:
         ''' When patch overlaps with parent_block,
         discard this block. '''
         child_block.continuation = None
-        ''' Leave child_block.during_split == True 
+        ''' Leave child_block.during_split == True
          to exclude from during_split sequence. '''
-        return    
+        return
     child_block.during_split = False
     child_block.first = patch.first
     parent_block.end = patch.first
     fill(child_block, patch)
-    
 
-def set_patch(block:Block, patch:Patch):
+
+def set_patch(block: Block, patch: Patch):
     block.continuation = patch.continuation
     block.chat_data.extend(patch.chats)
     block.last = patch.last
-    block.remaining = block.end-block.last        
+    block.remaining = block.end - block.last
-

pytchat/tool/extract/worker.py

@@ -1,7 +1,8 @@
-from . import parser
 from . block import Block
-from . patch import Patch, fill, split
+from . patch import fill, split
 from ... paramgen import arcparam
+from typing import Tuple
+
 
 class ExtractWorker:
     """
@@ -17,18 +18,18 @@ class ExtractWorker:
 
     block : Block :
         Block object that includes chat_data
-    
+
     blocks : list :
         List of Block(s)
 
     video_id : str :
 
     parent_block : Block :
-        the block from which current block is splitted 
+        the block from which current block is splitted
     """
     __slots__ = ['block', 'fetch', 'blocks', 'video_id', 'parent_block']
 
-    def __init__(self, fetch, block, blocks, video_id ):
+    def __init__(self, fetch, block, blocks, video_id):
         self.block = block
         self.fetch = fetch
         self.blocks = blocks
@@ -47,34 +48,36 @@ class ExtractWorker:
             if self.parent_block:
                 split(self.parent_block, self.block, patch)
                 self.parent_block = None
-            else:    
+            else:
                 fill(self.block, patch)
             if self.block.continuation is None:
                 """finished fetching this block """
                 self.block.done = True
                 self.block = _search_new_block(self)
 
+
 def _search_new_block(worker) -> Block:
     index, undone_block = _get_undone_block(worker.blocks)
     if undone_block is None:
-        return Block(continuation = None)
+        return Block(continuation=None)
-    mean = (undone_block.last + undone_block.end)/2
+    mean = (undone_block.last + undone_block.end) / 2
-    continuation = arcparam.getparam(worker.video_id, seektime = mean/1000)
+    continuation = arcparam.getparam(worker.video_id, seektime=mean / 1000)
     worker.parent_block = undone_block
     worker.parent_block.during_split = True
     new_block = Block(
-        end =  undone_block.end,
+        end=undone_block.end,
-        chat_data = [], 
+        chat_data=[],
-        continuation = continuation,
+        continuation=continuation,
-        during_split = True,
+        during_split=True,
-        is_last = worker.parent_block.is_last)
+        is_last=worker.parent_block.is_last)
     '''swap last block'''
     if worker.parent_block.is_last:
         worker.parent_block.is_last = False
-    worker.blocks.insert(index+1, new_block)
+    worker.blocks.insert(index + 1, new_block)
     return new_block
 
-def _get_undone_block(blocks) -> (int, Block):
+
+def _get_undone_block(blocks) -> Tuple[int, Block]:
     min_interval_ms = 120000
     max_remaining = 0
     undone_block = None

pytchat/tool/mining/asyncdl.py

@@ -1,141 +0,0 @@
-
-import aiohttp
-import asyncio
-import json
-from . import parser
-from . block import Block
-from . worker import ExtractWorker
-from . patch import Patch
-from ... import config 
-from ... paramgen import arcparam_mining as arcparam
-from concurrent.futures import CancelledError
-from urllib.parse import quote
-
-headers = config.headers
-REPLAY_URL = "https://www.youtube.com/live_chat_replay?continuation="
-INTERVAL = 1
-def _split(start, end, count, min_interval_sec = 120):
-    """
-    Split section from `start` to `end` into `count` pieces,
-    and returns the beginning of each piece. 
-    The `count` is adjusted so that the length of each piece
-    is no smaller than `min_interval`.
-
-    Returns:
-    --------
-        List of the offset of each block's first chat data.
-    """
-    
-    if not (isinstance(start,int) or isinstance(start,float)) or \
-       not (isinstance(end,int) or isinstance(end,float)):
-        raise ValueError("start/end must be int or float")
-    if not isinstance(count,int):
-        raise ValueError("count must be int")
-    if start>end:
-        raise ValueError("end must be equal to or greater than start.")
-    if count<1:
-        raise ValueError("count must be equal to or greater than 1.")
-    if (end-start)/count < min_interval_sec:
-        count = int((end-start)/min_interval_sec) 
-        if count == 0 : count = 1
-    interval= (end-start)/count 
-    
-    if count == 1:
-        return [start]
-    return sorted( list(set( [int(start + interval*j)
-        for j in range(count) ])))
-
-def ready_blocks(video_id, duration, div, callback):
-    if div <= 0: raise ValueError
-
-    async def _get_blocks( video_id, duration, div, callback):
-        async with aiohttp.ClientSession() as session:
-            tasks = [_create_block(session, video_id,  seektime, callback)
-                for  seektime in _split(0, duration, div)]
-            return await asyncio.gather(*tasks)
-
-    
-
-    async def _create_block(session, video_id, seektime, callback):
-        continuation = arcparam.getparam(video_id, seektime = seektime)
-        url=(f"{REPLAY_URL}{quote(continuation)}&playerOffsetMs="
-            f"{int(seektime*1000)}&hidden=false&pbj=1")
-        async with session.get(url, headers = headers) as resp:
-            chat_json = await resp.text()
-        if chat_json is None:
-            return
-        continuation, actions = parser.parse(json.loads(chat_json)[1])
-        first = seektime
-        seektime += INTERVAL
-        if callback:
-            callback(actions, INTERVAL)
-        return Block(
-            continuation = continuation,
-            chat_data = actions,
-            first = first,
-            last = seektime,
-            seektime = seektime
-        )
-    """
-    fetch initial blocks.
-    """  
-    loop = asyncio.get_event_loop()
-    blocks = loop.run_until_complete(
-        _get_blocks(video_id, duration, div, callback))
-    return blocks
-
-def fetch_patch(callback, blocks, video_id):
-
-    async def _allocate_workers():
-        workers = [
-            ExtractWorker(
-                fetch = _fetch,  block = block,
-                blocks = blocks, video_id = video_id
-            )
-            for block in blocks
-        ]
-        async with aiohttp.ClientSession() as session:
-            tasks = [worker.run(session) for worker in workers]
-            return await asyncio.gather(*tasks)    
-
-    async def _fetch(seektime,session) -> Patch:
-        continuation = arcparam.getparam(video_id, seektime = seektime)
-        url=(f"{REPLAY_URL}{quote(continuation)}&playerOffsetMs="
-            f"{int(seektime*1000)}&hidden=false&pbj=1")
-        async with session.get(url,headers = config.headers) as resp:
-            chat_json = await resp.text()
-        actions = []
-        try:
-            if chat_json is None:
-                return Patch()
-            continuation, actions = parser.parse(json.loads(chat_json)[1])
-        except json.JSONDecodeError:
-            pass
-        if callback:
-            callback(actions, INTERVAL)
-        return Patch(chats = actions, continuation = continuation, 
-            seektime = seektime, last = seektime)
-    """
-    allocate workers and assign blocks.
-    """   
-    loop = asyncio.get_event_loop()
-    try:
-        loop.run_until_complete(_allocate_workers())
-    except CancelledError:
-        pass
-
-async def _shutdown():
-    print("\nshutdown...")
-    tasks = [t for t in asyncio.all_tasks()
-         if t is not asyncio.current_task()]
-    for task in tasks:
-        task.cancel()
-        try:
-            await task
-        except asyncio.CancelledError:
-            pass
-
-def cancel():
-    loop = asyncio.get_event_loop()
-    loop.create_task(_shutdown())
-    

pytchat/tool/mining/block.py

@@ -1,62 +0,0 @@
-from . import parser
-class Block:
-    """Block object represents something like a box 
-    to join chunk of chatdata.
-
-    Parameter:
-    ---------
-    first : int :
-        videoOffsetTimeMs of the first chat_data 
-        (chat_data[0])
-        
-    last : int :
-        videoOffsetTimeMs of the last chat_data.
-        (chat_data[-1])
-
-        this value increases as fetching chatdata progresses.
-
-    end : int :
-        target videoOffsetTimeMs of last chat data for extract,
-        equals to first videoOffsetTimeMs of next block.
-        when extract worker reaches this offset, stop fetching.
-
-    continuation : str :
-        continuation param of last chat data.
-
-    chat_data : list 
-
-    done : bool :
-        whether this block has been fetched.
-    
-    remaining : int :
-        remaining data to extract.
-        equals end - last.
-    
-    is_last : bool :
-        whether this block is the last one in blocklist.
-
-    during_split : bool :
-        whether this block is in the process of during_split.
-        while True, this block is excluded from duplicate split procedure.
-    
-    seektime : float :
-        the last position of this block(seconds) already fetched.
-    """
-    
-    __slots__ = ['first','last','end','continuation','chat_data','remaining',
-        'done','is_last','during_split','seektime']
-
-    def __init__(self, first = 0, last = 0, end = 0,
-                continuation = '', chat_data = [], is_last = False,
-                during_split = False, seektime = None):
-        self.first = first
-        self.last = last
-        self.end = end
-        self.continuation = continuation
-        self.chat_data = chat_data
-        self.done = False
-        self.remaining = self.end - self.last
-        self.is_last = is_last
-        self.during_split = during_split
-        self.seektime = seektime
- 

pytchat/tool/mining/parser.py

@@ -1,67 +0,0 @@
-import json
-from ... import config
-from ... exceptions import ( 
-    ResponseContextError, 
-    NoContentsException, 
-    NoContinuationsException )
-
-logger = config.logger(__name__)
-
-def parse(jsn):
-    """
-    Parse replay chat data.
-    Parameter:
-    ----------
-    jsn : dict
-        JSON of replay chat data.
-    Returns:
-    ------
-        continuation : str
-        actions : list
-
-    """
-    if jsn is None: 
-        raise ValueError("parameter JSON is None")
-    if jsn['response']['responseContext'].get('errors'):
-        raise ResponseContextError(
-    'video_id is invalid or private/deleted.')
-    contents=jsn["response"].get('continuationContents')
-    if contents is None:
-        raise NoContentsException('No chat data.')
-
-    cont = contents['liveChatContinuation']['continuations'][0]
-    if cont is None:
-        raise NoContinuationsException('No Continuation')
-    metadata = cont.get('liveChatReplayContinuationData')
-    if metadata:
-        continuation = metadata.get("continuation")
-        actions = contents['liveChatContinuation'].get('actions')
-        if continuation:
-            return continuation, [action["replayChatItemAction"]["actions"][0] 
-                    for action in actions 
-                        if list(action['replayChatItemAction']["actions"][0].values()
-                    )[0]['item'].get("liveChatPaidMessageRenderer")
-                        or list(action['replayChatItemAction']["actions"][0].values()
-                    )[0]['item'].get("liveChatPaidStickerRenderer")
-                    ]
-    return None, []
-
-
-def get_offset(item):
-    return int(item['replayChatItemAction']["videoOffsetTimeMsec"])
-
-def get_id(item):
-    return list((list(item['replayChatItemAction']["actions"][0].values()
-                )[0])['item'].values())[0].get('id')
-
-def get_type(item):
-    return list((list(item['replayChatItemAction']["actions"][0].values()
-                )[0])['item'].keys())[0]
-import re                
-_REGEX_YTINIT = re.compile("window\\[\"ytInitialData\"\\]\\s*=\\s*({.+?});\\s+")
-def extract(text):
-
-    match = re.findall(_REGEX_YTINIT, str(text))
-    if match:
-        return match[0]
-    return None

pytchat/tool/mining/patch.py

@@ -1,27 +0,0 @@
-from . import parser
-from . block import Block
-from typing import NamedTuple
-
-class Patch(NamedTuple):
-    """
-    Patch represents chunk of chat data
-    which is fetched by asyncdl.fetch_patch._fetch().
-    """
-    chats : list = []
-    continuation : str = None
-    seektime : float = None
-    first : int = None
-    last : int = None
-
-def fill(block:Block, patch:Patch):
-    if patch.last < block.end:
-        set_patch(block, patch)
-        return
-    block.continuation = None
-
-def set_patch(block:Block, patch:Patch):
-    block.continuation = patch.continuation
-    block.chat_data.extend(patch.chats)
-    block.last = patch.seektime
-    block.seektime = patch.seektime       
-

pytchat/tool/mining/superchat_miner.py

@@ -1,72 +0,0 @@
-from . import asyncdl
-from . import parser
-from .. videoinfo import VideoInfo
-from ... import config
-from ... exceptions import InvalidVideoIdException
-logger = config.logger(__name__)
-headers=config.headers
-
-class SuperChatMiner:
-    def __init__(self, video_id, duration, div, callback):
-        if not isinstance(div ,int) or div < 1:
-            raise ValueError('div must be positive integer.')
-        elif div > 10:
-            div = 10
-        if not isinstance(duration ,int) or duration < 1:
-            raise ValueError('duration must be positive integer.')
-        self.video_id = video_id
-        self.duration = duration
-        self.div = div
-        self.callback = callback
-        self.blocks = []
-
-    def _ready_blocks(self):
-        blocks = asyncdl.ready_blocks(
-            self.video_id, self.duration, self.div, self.callback)
-        self.blocks = [block for block in blocks if block is not None]
-        return self  
-
-    def _set_block_end(self):
-        for i in range(len(self.blocks)-1):
-            self.blocks[i].end = self.blocks[i+1].first
-        self.blocks[-1].end = self.duration
-        self.blocks[-1].is_last =True
-        return self
-
-    def _download_blocks(self):
-        asyncdl.fetch_patch(self.callback, self.blocks, self.video_id)
-        return self
-
-    def _combine(self):
-        ret = []
-        for block in self.blocks:
-            ret.extend(block.chat_data) 
-        return ret
-
-    def extract(self):
-        return (
-            self._ready_blocks()
-                ._set_block_end()
-                ._download_blocks()
-                ._combine()
-        )
-
-def extract(video_id, div = 1, callback = None, processor = None):
-    duration = 0
-    try:
-        duration = VideoInfo(video_id).get_duration()
-    except InvalidVideoIdException:
-        raise
-    if duration == 0:
-        print("video is live.")
-        return []
-    data = SuperChatMiner(video_id, duration, div, callback).extract()
-    if processor is None:
-        return data
-    return processor.process(
-        [{'video_id':None,'timeout':1,'chatdata' : (action
-        for action in data)}]
-    )
-
-def cancel():
-    asyncdl.cancel()

pytchat/tool/mining/worker.py

@@ -1,45 +0,0 @@
-from . import parser
-from . block import Block
-from . patch import Patch, fill
-from ... paramgen import arcparam
-INTERVAL = 1
-class ExtractWorker:
-    """
-    ExtractWorker associates a download session with a block.
-
-    When the worker finishes fetching, the block
-    being fetched is splitted and assigned the free worker.
-
-    Parameter
-    ----------
-    fetch : func :
-        extract function of asyncdl
-
-    block : Block :
-        Block object that includes chat_data
-    
-    blocks : list :
-        List of Block(s)
-
-    video_id : str :
-
-    parent_block : Block :
-        the block from which current block is splitted 
-    """
-    __slots__ = ['block', 'fetch', 'blocks', 'video_id', 'parent_block']
-    def __init__(self, fetch, block, blocks, video_id ):
-        self.block:Block = block
-        self.fetch = fetch
-        self.blocks:list = blocks
-        self.video_id:str = video_id
-        self.parent_block:Block = None
-
-    async def run(self, session):
-        while self.block.continuation:
-            patch = await self.fetch(
-                self.block.seektime, session)
-            fill(self.block, patch)
-            self.block.seektime += INTERVAL
-        self.block.done = True    
-
-

pytchat/tool/videoinfo.py

@@ -1,15 +1,15 @@
-import json 
+import json
 import re
-import requests
+import httpx
 from .. import config
-from .. import util
+from ..exceptions import InvalidVideoIdException, PatternUnmatchError
-from ..exceptions import InvalidVideoIdException 
+from ..util.extract_video_id import extract_video_id
 
 headers = config.headers
 
-pattern = re.compile(r"yt\.setConfig\({'PLAYER_CONFIG': ({.*})}\);")
+pattern = re.compile(r"'PLAYER_CONFIG': ({.*}}})")
 
-item_channel_id =[
+item_channel_id = [
     "videoDetails",
     "embeddedPlayerOverlayVideoDetailsRenderer",
     "channelThumbnailEndpoint",
@@ -29,7 +29,7 @@ item_response = [
     "embedded_player_response"
 ]
 
-item_author_image =[
+item_author_image = [
     "videoDetails",
     "embeddedPlayerOverlayVideoDetailsRenderer",
     "channelThumbnail",
@@ -63,6 +63,7 @@ item_moving_thumbnail = [
     "url"
 ]
 
+
 class VideoInfo:
     '''
     VideoInfo object retrieves YouTube video information.
@@ -76,20 +77,24 @@ class VideoInfo:
     InvalidVideoIdException :
         Occurs when video_id does not exist on YouTube.
     '''
+
     def __init__(self, video_id):
-        self.video_id = video_id
+        self.video_id = extract_video_id(video_id)
-        text = self._get_page_text(video_id)
+        text = self._get_page_text(self.video_id)
         self._parse(text)
 
     def _get_page_text(self, video_id):
         url = f"https://www.youtube.com/embed/{video_id}"
-        resp = requests.get(url, headers = headers)
+        resp = httpx.get(url, headers=headers)
         resp.raise_for_status()
         return resp.text
 
     def _parse(self, text):
         result = re.search(pattern, text)
-        res= json.loads(result.group(1))
+        if result is None:
+            raise PatternUnmatchError(text)
+        decoder = json.JSONDecoder()
+        res = decoder.raw_decode(result.group(1)[:-1])[0]
         response = self._get_item(res, item_response)
         if response is None:
             self._check_video_is_private(res.get("args"))
@@ -98,7 +103,7 @@ class VideoInfo:
             raise InvalidVideoIdException(
                 f"No renderer found in video_id: [{self.video_id}].")
 
-    def _check_video_is_private(self,args):
+    def _check_video_is_private(self, args):
         if args and args.get("video_id"):
             raise InvalidVideoIdException(
                 f"video_id [{self.video_id}] is private or deleted.")
@@ -130,8 +135,8 @@ class VideoInfo:
 
     def get_title(self):
         if self._renderer.get("title"):
-            return [''.join(run["text"]) 
+            return [''.join(run["text"])
-                for run in self._renderer["title"]["runs"]][0]
+                    for run in self._renderer["title"]["runs"]][0]
         return None
 
     def get_channel_id(self):
@@ -141,13 +146,13 @@ class VideoInfo:
         return None
 
     def get_author_image(self):
-        return self._get_item(self._renderer, item_author_image) 
+        return self._get_item(self._renderer, item_author_image)
 
     def get_thumbnail(self):
         return self._get_item(self._renderer, item_thumbnail)
 
     def get_channel_name(self):
         return self._get_item(self._renderer, item_channel_name)
-    
+
     def get_moving_thumbnail(self):
         return self._get_item(self._renderer, item_moving_thumbnail)

pytchat/util/__init__.py

@@ -1,15 +1,18 @@
-import requests,json,datetime
+import httpx
+import json
+import datetime
 from .. import config
 
+
 def extract(url):
-    _session = requests.Session()
+    _session = httpx.Client(http2=True)
     html = _session.get(url, headers=config.headers)
     with open(str(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')
-                        )+'test.json',mode ='w',encoding='utf-8') as f:
+                  ) + 'test.json', mode='w', encoding='utf-8') as f:
-        json.dump(html.json(),f,ensure_ascii=False)
+        json.dump(html.json(), f, ensure_ascii=False)
 
 
-def save(data,filename,extention):
+def save(data, filename, extention):
-    with open(filename+"_"+(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')
+    with open(filename + "_" + (datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')) + extention,
-                            )+extention,mode ='w',encoding='utf-8') as f:
+              mode='w', encoding='utf-8') as f:
         f.writelines(data)

pytchat/util/extract_video_id.py

@@ -0,0 +1,25 @@
+import re
+from .. exceptions import InvalidVideoIdException
+
+
+PATTERN = re.compile(r"((?<=(v|V)/)|(?<=be/)|(?<=(\?|\&)v=)|(?<=embed/))([\w-]+)")
+YT_VIDEO_ID_LENGTH = 11
+
+
+def extract_video_id(url_or_id: str) -> str:
+    ret = ''
+    if type(url_or_id) != str:
+        raise TypeError(f"{url_or_id}: URL or VideoID must be str, but {type(url_or_id)} is passed.")
+    if len(url_or_id) == YT_VIDEO_ID_LENGTH:
+        return url_or_id
+    match = re.search(PATTERN, url_or_id)
+    if match is None:
+        raise InvalidVideoIdException(url_or_id)
+    try:
+        ret = match.group(4)
+    except IndexError:
+        raise InvalidVideoIdException(url_or_id)
+
+    if ret is None or len(ret) != YT_VIDEO_ID_LENGTH:
+        raise InvalidVideoIdException(url_or_id)
+    return ret

requirements.txt

@@ -1,4 +1,4 @@
-aiohttp
+httpx[http2]==0.14.1
+protobuf==3.13.0
 pytz
-requests
 urllib3

requirements_test.txt

@@ -1,5 +1,4 @@
-aioresponses
 mock
 mocker
 pytest
-pytest-mock
+pytest_httpx

tests/test_arcparam.py

@@ -1,28 +1,32 @@
-import pytest
+import json
-from pytchat.parser.live import Parser
+import httpx
 import pytchat.config as config
-import requests, json
 from pytchat.paramgen import arcparam
+from pytchat.parser.live import Parser
+
 
 def test_arcparam_0(mocker):
-    param = arcparam.getparam("01234567890",-1)
+    param = arcparam.getparam("01234567890", -1)
-    assert param == "op2w0wRyGjxDZzhhRFFvTE1ERXlNelExTmpjNE9UQWFFLXFvM2JrQkRRb0xNREV5TXpRMU5qYzRPVEFnQVElM0QlM0QoADAAOABAAEgEUhwIABAAGAAgACoOc3RhdGljY2hlY2tzdW1AAFgDYAFoAHIECAEQAHgA" 
+    assert param == "op2w0wQmGhxDZzhLRFFvTE1ERXlNelExTmpjNE9UQWdBUT09SARgAXICCAE%3D" 
+
 
 def test_arcparam_1(mocker):
-    param = arcparam.getparam("01234567890", seektime = 100000)
+    param = arcparam.getparam("01234567890", seektime=100000)
-    assert param == "op2w0wR3GjxDZzhhRFFvTE1ERXlNelExTmpjNE9UQWFFLXFvM2JrQkRRb0xNREV5TXpRMU5qYzRPVEFnQVElM0QlM0QogNDbw_QCMAA4AEAASANSHAgAEAAYACAAKg5zdGF0aWNjaGVja3N1bUAAWANgAWgAcgQIARAAeAA%3D"
+    assert param == "op2w0wQtGhxDZzhLRFFvTE1ERXlNelExTmpjNE9UQWdBUT09KIDQ28P0AkgDYAFyAggB"
+
 
 def test_arcparam_2(mocker):
-    param = arcparam.getparam("SsjCnHOk-Sk")
+    param = arcparam.getparam("SsjCnHOk-Sk", seektime=100)
-    url=f"https://www.youtube.com/live_chat_replay/get_live_chat_replay?continuation={param}&pbj=1"
+    url = f"https://www.youtube.com/live_chat_replay/get_live_chat_replay?continuation={param}&pbj=1"
-    resp = requests.Session().get(url,headers = config.headers)
+    resp = httpx.Client(http2=True).get(url, headers=config.headers)
     jsn = json.loads(resp.text)
     parser = Parser(is_replay=True)
-    contents= parser.get_contents(jsn)
+    contents = parser.get_contents(jsn)
-    _ , chatdata = parser.parse(contents)
+    _, chatdata = parser.parse(contents)
     test_id = chatdata[0]["addChatItemAction"]["item"]["liveChatTextMessageRenderer"]["id"]
     assert test_id == "CjoKGkNMYXBzZTdudHVVQ0Zjc0IxZ0FkTnFnQjVREhxDSnlBNHV2bnR1VUNGV0dnd2dvZDd3NE5aZy0w"
 
+
 def test_arcparam_3(mocker):
     param = arcparam.getparam("01234567890")
-    assert param == "op2w0wRyGjxDZzhhRFFvTE1ERXlNelExTmpjNE9UQWFFLXFvM2JrQkRRb0xNREV5TXpRMU5qYzRPVEFnQVElM0QlM0QoATAAOABAAEgDUhwIABAAGAAgACoOc3RhdGljY2hlY2tzdW1AAFgDYAFoAHIECAEQAHgA"
+    assert param == "op2w0wQmGhxDZzhLRFFvTE1ERXlNelExTmpjNE9UQWdBUT09SARgAXICCAE%3D"

tests/test_arcparam_mining.py

@@ -1,40 +0,0 @@
-import pytest
-from pytchat.tool.mining import parser
-import pytchat.config as config
-import requests, json
-from pytchat.paramgen import arcparam_mining as arcparam
-
-def test_arcparam_e(mocker):
-    try:
-        arcparam.getparam("01234567890",-1)
-        assert False
-    except ValueError:
-        assert True
-
-    
-
-
-def test_arcparam_0(mocker):
-    param = arcparam.getparam("01234567890",0)
-
-    assert param =="op2w0wQsGiBDZzhhRFFvTE1ERXlNelExTmpjNE9UQWdBUSUzRCUzREABYARyAggBeAE%3D"
-
-
-def test_arcparam_1(mocker):
-    param = arcparam.getparam("01234567890", seektime = 100000)
-    print(param)
-    assert param == "op2w0wQzGiBDZzhhRFFvTE1ERXlNelExTmpjNE9UQWdBUSUzRCUzREABWgUQgMLXL2AEcgIIAXgB"
-
-def test_arcparam_2(mocker):
-    param = arcparam.getparam("PZz9NB0-Z64",1)
-    url=f"https://www.youtube.com/live_chat_replay?continuation={param}&playerOffsetMs=1000&pbj=1"
-    resp = requests.Session().get(url,headers = config.headers)
-    jsn = json.loads(resp.text)
-    _ , chatdata = parser.parse(jsn[1])
-    test_id = chatdata[0]["addChatItemAction"]["item"]["liveChatPaidMessageRenderer"]["id"]
-    print(test_id)
-    assert test_id == "ChwKGkNKSGE0YnFJeWVBQ0ZWcUF3Z0VkdGIwRm9R"
-
-def test_arcparam_3(mocker):
-    param = arcparam.getparam("01234567890")
-    assert param == "op2w0wQsGiBDZzhhRFFvTE1ERXlNelExTmpjNE9UQWdBUSUzRCUzREABYARyAggBeAE%3D"

tests/test_compatible_processor.py

@@ -1,17 +1,6 @@
 import json
-import pytest
-import asyncio
-import aiohttp
 from pytchat.parser.live import Parser
 from pytchat.processors.compatible.processor import CompatibleProcessor
-from pytchat.exceptions import (
-    NoLivechatRendererException, NoYtinitialdataException,
-    ResponseContextError, NoContentsException)
-
-from pytchat.processors.compatible.renderer.textmessage import LiveChatTextMessageRenderer
-from pytchat.processors.compatible.renderer.paidmessage import LiveChatPaidMessageRenderer
-from pytchat.processors.compatible.renderer.paidsticker import LiveChatPaidStickerRenderer
-from pytchat.processors.compatible.renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
 
 parser = Parser(is_replay=False)
 
@@ -31,21 +20,23 @@ def test_textmessage(mocker):
     ret = processor.process([data])
 
     assert ret["kind"] == "youtube#liveChatMessageListResponse"
-    assert ret["pollingIntervalMillis"] == data["timeout"]*1000
+    assert ret["pollingIntervalMillis"] == data["timeout"] * 1000
     assert ret.keys() == {
-        "kind",   "etag",   "pageInfo",   "nextPageToken", "pollingIntervalMillis", "items"
+        "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
     }
     assert ret["pageInfo"].keys() == {
-        "totalResults",   "resultsPerPage"
+        "totalResults", "resultsPerPage"
     }
     assert ret["items"][0].keys() == {
-        "kind",   "etag",   "id",   "snippet", "authorDetails"
+        "kind", "etag", "id", "snippet", "authorDetails"
     }
     assert ret["items"][0]["snippet"].keys() == {
-        'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage', 'textMessageDetails'
+        'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage',
+        'textMessageDetails'
     }
     assert ret["items"][0]["authorDetails"].keys() == {
-        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator'
+        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor',
+        'isChatModerator'
     }
     assert ret["items"][0]["snippet"]["textMessageDetails"].keys() == {
         'messageText'
@@ -69,22 +60,23 @@ def test_newsponcer(mocker):
     ret = processor.process([data])
 
     assert ret["kind"] == "youtube#liveChatMessageListResponse"
-    assert ret["pollingIntervalMillis"] == data["timeout"]*1000
+    assert ret["pollingIntervalMillis"] == data["timeout"] * 1000
     assert ret.keys() == {
-        "kind",   "etag",   "pageInfo",   "nextPageToken", "pollingIntervalMillis", "items"
+        "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
     }
     assert ret["pageInfo"].keys() == {
-        "totalResults",   "resultsPerPage"
+        "totalResults", "resultsPerPage"
     }
     assert ret["items"][0].keys() == {
-        "kind",   "etag",   "id",   "snippet", "authorDetails"
+        "kind", "etag", "id", "snippet", "authorDetails"
     }
     assert ret["items"][0]["snippet"].keys() == {
         'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage'
 
     }
     assert ret["items"][0]["authorDetails"].keys() == {
-        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator'
+        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor',
+        'isChatModerator'
     }
     assert "LCC." in ret["items"][0]["id"]
     assert ret["items"][0]["snippet"]["type"] == "newSponsorEvent"
@@ -105,22 +97,23 @@ def test_newsponcer_rev(mocker):
     ret = processor.process([data])
 
     assert ret["kind"] == "youtube#liveChatMessageListResponse"
-    assert ret["pollingIntervalMillis"] == data["timeout"]*1000
+    assert ret["pollingIntervalMillis"] == data["timeout"] * 1000
     assert ret.keys() == {
-        "kind",   "etag",   "pageInfo",   "nextPageToken", "pollingIntervalMillis", "items"
+        "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
     }
     assert ret["pageInfo"].keys() == {
-        "totalResults",   "resultsPerPage"
+        "totalResults", "resultsPerPage"
     }
     assert ret["items"][0].keys() == {
-        "kind",   "etag",   "id",   "snippet", "authorDetails"
+        "kind", "etag", "id", "snippet", "authorDetails"
     }
     assert ret["items"][0]["snippet"].keys() == {
         'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage'
 
     }
     assert ret["items"][0]["authorDetails"].keys() == {
-        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator'
+        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor',
+        'isChatModerator'
     }
     assert "LCC." in ret["items"][0]["id"]
     assert ret["items"][0]["snippet"]["type"] == "newSponsorEvent"
@@ -141,21 +134,23 @@ def test_superchat(mocker):
     ret = processor.process([data])
 
     assert ret["kind"] == "youtube#liveChatMessageListResponse"
-    assert ret["pollingIntervalMillis"] == data["timeout"]*1000
+    assert ret["pollingIntervalMillis"] == data["timeout"] * 1000
     assert ret.keys() == {
-        "kind",   "etag",   "pageInfo",   "nextPageToken", "pollingIntervalMillis", "items"
+        "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
     }
     assert ret["pageInfo"].keys() == {
-        "totalResults",   "resultsPerPage"
+        "totalResults", "resultsPerPage"
     }
     assert ret["items"][0].keys() == {
-        "kind",   "etag",   "id",   "snippet", "authorDetails"
+        "kind", "etag", "id", "snippet", "authorDetails"
     }
     assert ret["items"][0]["snippet"].keys() == {
-        'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage', 'superChatDetails'
+        'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage',
+        'superChatDetails'
     }
     assert ret["items"][0]["authorDetails"].keys() == {
-        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator'
+        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor',
+        'isChatModerator'
     }
     assert ret["items"][0]["snippet"]["superChatDetails"].keys() == {
         'amountMicros', 'currency', 'amountDisplayString', 'tier', 'backgroundColor'

tests/test_default_processor.py

@@ -0,0 +1,228 @@
+import json
+from pytchat.parser.live import Parser
+from pytchat.processors.default.processor import DefaultProcessor
+
+
+def test_textmessage(mocker):
+    '''text message'''
+    processor = DefaultProcessor()
+    parser = Parser(is_replay=False)
+    _json = _open_file("tests/testdata/default/textmessage.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+
+    ret = processor.process([data]).items[0]
+    assert ret.chattype == "textMessage"
+    assert ret.id == "dummy_id"
+    assert ret.message == "dummy_message"
+    assert ret.timestamp == 1570678496000
+    assert ret.datetime == "2019-10-10 12:34:56"
+    assert ret.author.name == "author_name"
+    assert ret.author.channelId == "author_channel_id"
+    assert ret.author.channelUrl == "http://www.youtube.com/channel/author_channel_id"
+    assert ret.author.imageUrl == "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg"
+    assert ret.author.badgeUrl == ""
+    assert ret.author.isVerified is False
+    assert ret.author.isChatOwner is False
+    assert ret.author.isChatSponsor is False
+    assert ret.author.isChatModerator is False
+
+
+def test_textmessage_replay_member(mocker):
+    '''text message replay member'''
+    processor = DefaultProcessor()
+    parser = Parser(is_replay=True)
+    _json = _open_file("tests/testdata/default/replay_member_text.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+    
+    ret = processor.process([data]).items[0]
+    assert ret.chattype == "textMessage"
+    assert ret.type == "textMessage"
+    assert ret.id == "dummy_id"
+    assert ret.message == "dummy_message"
+    assert ret.messageEx == ["dummy_message"]
+    assert ret.timestamp == 1570678496000
+    assert ret.datetime == "2019-10-10 12:34:56"
+    assert ret.elapsedTime == "1:23:45"
+    assert ret.author.name == "author_name"
+    assert ret.author.channelId == "author_channel_id"
+    assert ret.author.channelUrl == "http://www.youtube.com/channel/author_channel_id"
+    assert ret.author.imageUrl == "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg"
+    assert ret.author.badgeUrl == "https://yt3.ggpht.com/X=s16-c-k"
+    assert ret.author.isVerified is False
+    assert ret.author.isChatOwner is False
+    assert ret.author.isChatSponsor is True
+    assert ret.author.isChatModerator is False
+
+
+def test_superchat(mocker):
+    '''superchat'''
+    processor = DefaultProcessor()
+    parser = Parser(is_replay=False)
+    _json = _open_file("tests/testdata/default/superchat.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+    
+    ret = processor.process([data]).items[0]
+    print(json.dumps(chatdata, ensure_ascii=False))
+    assert ret.chattype == "superChat"
+    assert ret.type == "superChat"
+    assert ret.id == "dummy_id"
+    assert ret.message == "dummy_message"
+    assert ret.messageEx == ["dummy_message"]
+    assert ret.timestamp == 1570678496000
+    assert ret.datetime == "2019-10-10 12:34:56"
+    assert ret.elapsedTime == ""
+    assert ret.amountValue == 800
+    assert ret.amountString == "￥800"
+    assert ret.currency == "JPY"
+    assert ret.bgColor == 4280150454
+    assert ret.author.name == "author_name"
+    assert ret.author.channelId == "author_channel_id"
+    assert ret.author.channelUrl == "http://www.youtube.com/channel/author_channel_id"
+    assert ret.author.imageUrl == "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg"
+    assert ret.author.badgeUrl == ""
+    assert ret.author.isVerified is False
+    assert ret.author.isChatOwner is False
+    assert ret.author.isChatSponsor is False
+    assert ret.author.isChatModerator is False
+    assert ret.colors.headerBackgroundColor == 4278239141
+    assert ret.colors.headerTextColor == 4278190080
+    assert ret.colors.bodyBackgroundColor == 4280150454
+    assert ret.colors.bodyTextColor == 4278190080
+    assert ret.colors.authorNameTextColor == 2315255808
+    assert ret.colors.timestampColor == 2147483648
+
+
+def test_supersticker(mocker):
+    '''supersticker'''
+    processor = DefaultProcessor()
+    parser = Parser(is_replay=False)
+    _json = _open_file("tests/testdata/default/supersticker.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+   
+    ret = processor.process([data]).items[0]
+    print(json.dumps(chatdata, ensure_ascii=False))
+    assert ret.chattype == "superSticker"
+    assert ret.type == "superSticker"
+    assert ret.id == "dummy_id"
+    assert ret.message == ""
+    assert ret.messageEx == []
+    assert ret.timestamp == 1570678496000
+    assert ret.datetime == "2019-10-10 12:34:56"
+    assert ret.elapsedTime == ""
+    assert ret.amountValue == 200
+    assert ret.amountString == "￥200"
+    assert ret.currency == "JPY"
+    assert ret.bgColor == 4278237396
+    assert ret.sticker == "https://lh3.googleusercontent.com/param_s=s72-rp"
+    assert ret.author.name == "author_name"
+    assert ret.author.channelId == "author_channel_id"
+    assert ret.author.channelUrl == "http://www.youtube.com/channel/author_channel_id"
+    assert ret.author.imageUrl == "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg"
+    assert ret.author.badgeUrl == ""
+    assert ret.author.isVerified is False
+    assert ret.author.isChatOwner is False
+    assert ret.author.isChatSponsor is False
+    assert ret.author.isChatModerator is False
+    assert ret.colors.backgroundColor == 4278237396
+    assert ret.colors.moneyChipBackgroundColor == 4278248959
+    assert ret.colors.moneyChipTextColor == 4278190080
+    assert ret.colors.authorNameTextColor == 3003121664
+
+
+def test_sponsor(mocker):
+    '''sponsor(membership)'''
+    processor = DefaultProcessor()
+    parser = Parser(is_replay=False)
+    _json = _open_file("tests/testdata/default/newSponsor_current.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+    
+    ret = processor.process([data]).items[0]
+    print(json.dumps(chatdata, ensure_ascii=False))
+    assert ret.chattype == "newSponsor"
+    assert ret.type == "newSponsor"
+    assert ret.id == "dummy_id"
+    assert ret.message == "新規メンバー"
+    assert ret.messageEx == ["新規メンバー"]
+    assert ret.timestamp == 1570678496000
+    assert ret.datetime == "2019-10-10 12:34:56"
+    assert ret.elapsedTime == ""
+    assert ret.bgColor == 0
+    assert ret.author.name == "author_name"
+    assert ret.author.channelId == "author_channel_id"
+    assert ret.author.channelUrl == "http://www.youtube.com/channel/author_channel_id"
+    assert ret.author.imageUrl == "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg"
+    assert ret.author.badgeUrl == "https://yt3.ggpht.com/X=s32-c-k"
+    assert ret.author.isVerified is False
+    assert ret.author.isChatOwner is False
+    assert ret.author.isChatSponsor is True
+    assert ret.author.isChatModerator is False
+
+
+def test_sponsor_legacy(mocker):
+    '''lagacy sponsor(membership)'''
+    processor = DefaultProcessor()
+    parser = Parser(is_replay=False)
+    _json = _open_file("tests/testdata/default/newSponsor_lagacy.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+    
+    ret = processor.process([data]).items[0]
+    print(json.dumps(chatdata, ensure_ascii=False))
+    assert ret.chattype == "newSponsor"
+    assert ret.type == "newSponsor"
+    assert ret.id == "dummy_id"
+    assert ret.message == "新規メンバー / ようこそ、author_name！"
+    assert ret.messageEx == ["新規メンバー / ようこそ、author_name！"]
+    assert ret.timestamp == 1570678496000
+    assert ret.datetime == "2019-10-10 12:34:56"
+    assert ret.elapsedTime == ""
+    assert ret.bgColor == 0
+    assert ret.author.name == "author_name"
+    assert ret.author.channelId == "author_channel_id"
+    assert ret.author.channelUrl == "http://www.youtube.com/channel/author_channel_id"
+    assert ret.author.imageUrl == "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg"
+    assert ret.author.badgeUrl == ""
+    assert ret.author.isVerified is False
+    assert ret.author.isChatOwner is False
+    assert ret.author.isChatSponsor is True
+    assert ret.author.isChatModerator is False
+
+
+def _open_file(path):
+    with open(path, mode='r', encoding='utf-8') as f:
+        return f.read()

tests/test_extract_asyncdl.py

@@ -1,77 +0,0 @@
-import aiohttp
-import asyncio
-import json
-from pytchat.tool.extract import parser
-import sys
-import time
-from aioresponses import aioresponses
-from concurrent.futures import CancelledError
-from pytchat.tool.extract import asyncdl
-
-def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
-        return f.read()
-
-
-def test_asyncdl_split():
-
-    ret = asyncdl._split(0,1000,1)
-    assert ret == [0]
-
-    ret = asyncdl._split(1000,1000,10)
-    assert ret == [1000]
-
-    ret = asyncdl._split(0,1000,5)
-    assert ret == [0,200,400,600,800]
-
-    ret = asyncdl._split(10.5, 700.3, 5)
-    assert ret == [10, 148, 286, 424, 562]
-
-
-    ret = asyncdl._split(0,500,5)
-    assert ret == [0,125,250,375]
-    
-    ret = asyncdl._split(0,500,500)
-    assert ret == [0,125,250,375]
-    
-    ret = asyncdl._split(-1,1000,5)
-    assert ret == [-1, 199, 399, 599, 799]
-    
-    """invalid argument order"""
-    try:
-        ret = asyncdl._split(500,0,5)
-        assert False
-    except ValueError:
-        assert True
-
-    """invalid count"""
-    try:
-        ret = asyncdl._split(0,500,-1)
-        assert False
-    except ValueError:
-        assert True
-
-    try:
-        ret = asyncdl._split(0,500,0)
-        assert False
-    except ValueError:
-        assert True
-
-    """invalid argument type"""
-    try:
-        ret = asyncdl._split(0,5000,5.2)
-        assert False
-    except ValueError:
-        assert True
-
-    try:
-        ret = asyncdl._split(0,5000,"test")
-        assert False
-    except ValueError:
-        assert True
-
-    try:
-        ret = asyncdl._split([0,1],5000,5)
-        assert False
-    except ValueError:
-        assert True

tests/test_extract_duplcheck.py

@@ -1,60 +1,66 @@
-import aiohttp
-import asyncio
 import json
-import os, sys
-import time
 from pytchat.tool.extract import duplcheck
 from pytchat.tool.extract import parser
 from pytchat.tool.extract.block import Block
 from pytchat.tool.extract.duplcheck import _dump
-def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
-        return f.read()
 
 
+def _open_file(path):
+    with open(path, mode='r', encoding='utf-8') as f:
+        return f.read()
+
 
 def test_overlap():
     """
-    test overlap data 
+    test overlap data
         operation : [0]  [2] [3]  [4] -> last :align to end
                     [1] , [5] -> no change
-        
+
     """
 
     def load_chatdata(filename):
         return parser.parse(
-            json.loads(_open_file("tests/testdata/extract_duplcheck/overlap/"+filename))
+            json.loads(_open_file(
+                "tests/testdata/extract_duplcheck/overlap/" + filename))
         )[1]
 
     blocks = (
-        Block(first = 0,    last= 12771, end=  9890,chat_data = load_chatdata("dp0-0.json")),     
+        Block(first=0, last=12771, end=9890,
-        Block(first = 9890, last= 15800, end= 20244,chat_data = load_chatdata("dp0-1.json")), 
+              chat_data=load_chatdata("dp0-0.json")),
-        Block(first = 20244,last= 45146, end= 32476,chat_data = load_chatdata("dp0-2.json")), 
+        Block(first=9890, last=15800, end=20244,
-        Block(first = 32476,last= 50520, end= 41380,chat_data = load_chatdata("dp0-3.json")), 
+              chat_data=load_chatdata("dp0-1.json")),
-        Block(first = 41380,last= 62875, end= 52568,chat_data = load_chatdata("dp0-4.json")), 
+        Block(first=20244, last=45146, end=32476,
-        Block(first = 52568,last= 62875, end= 54000,chat_data = load_chatdata("dp0-5.json"),is_last=True)
+              chat_data=load_chatdata("dp0-2.json")),
+        Block(first=32476, last=50520, end=41380,
+              chat_data=load_chatdata("dp0-3.json")),
+        Block(first=41380, last=62875, end=52568,
+              chat_data=load_chatdata("dp0-4.json")),
+        Block(first=52568, last=62875, end=54000,
+              chat_data=load_chatdata("dp0-5.json"), is_last=True)
     )
     result = duplcheck.remove_overlap(blocks)
-    #dp0-0.json has item offset time is 9890 (equals block[0].end = block[1].first), 
+    # dp0-0.json has item offset time is 9890 (equals block[0].end = block[1].first),
-    #but must be aligne to the most close and smaller value:9779.
+    # but must be aligne to the most close and smaller value:9779.
     assert result[0].last == 9779
-    
+
     assert result[1].last == 15800
-    
+
     assert result[2].last == 32196
-    
+
     assert result[3].last == 41116
-    
+
     assert result[4].last == 52384
-    
+
-    #the last block must be always added to result.
+    # the last block must be always added to result.
     assert result[5].last == 62875
-    
+
+
 def test_duplicate_head():
 
     def load_chatdata(filename):
         return parser.parse(
-            json.loads(_open_file("tests/testdata/extract_duplcheck/head/"+filename))
+            json.loads(_open_file(
+                "tests/testdata/extract_duplcheck/head/" + filename))
         )[1]
 
     """
@@ -69,25 +75,26 @@ def test_duplicate_head():
         result    : [2] , [4] , [5] 
     """
 
-    #chat data offsets are ignored.
+    # chat data offsets are ignored.
     blocks = (
-        Block(first = 0, last = 2500, chat_data = load_chatdata("dp0-0.json")),     
+        Block(first=0, last=2500, chat_data=load_chatdata("dp0-0.json")),
-        Block(first = 0, last =38771, chat_data = load_chatdata("dp0-1.json")), 
+        Block(first=0, last=38771, chat_data=load_chatdata("dp0-1.json")),
-        Block(first = 0, last =45146, chat_data = load_chatdata("dp0-2.json")), 
+        Block(first=0, last=45146, chat_data=load_chatdata("dp0-2.json")),
-        Block(first = 20244, last =60520, chat_data = load_chatdata("dp0-3.json")), 
+        Block(first=20244, last=60520, chat_data=load_chatdata("dp0-3.json")),
-        Block(first = 20244, last =62875, chat_data = load_chatdata("dp0-4.json")), 
+        Block(first=20244, last=62875, chat_data=load_chatdata("dp0-4.json")),
-        Block(first = 52568, last =62875, chat_data = load_chatdata("dp0-5.json"))
+        Block(first=52568, last=62875, chat_data=load_chatdata("dp0-5.json"))
     )
     _dump(blocks)
     result = duplcheck.remove_duplicate_head(blocks)
-    
+
     assert len(result) == 3
     assert result[0].first == blocks[2].first
-    assert result[0].last  == blocks[2].last
+    assert result[0].last == blocks[2].last
     assert result[1].first == blocks[4].first
-    assert result[1].last  == blocks[4].last
+    assert result[1].last == blocks[4].last
     assert result[2].first == blocks[5].first
-    assert result[2].last  == blocks[5].last
+    assert result[2].last == blocks[5].last
+
 
 def test_duplicate_tail():
     """
@@ -103,26 +110,25 @@ def test_duplicate_tail():
     """
     def load_chatdata(filename):
         return parser.parse(
-            json.loads(_open_file("tests/testdata/extract_duplcheck/head/"+filename))
+            json.loads(_open_file(
+                "tests/testdata/extract_duplcheck/head/" + filename))
         )[1]
-    #chat data offsets are ignored.    
+    # chat data offsets are ignored.
     blocks = (
-        Block(first = 0,last = 2500, chat_data=load_chatdata("dp0-0.json")),     
+        Block(first=0, last=2500, chat_data=load_chatdata("dp0-0.json")),
-        Block(first = 1500,last = 2500, chat_data=load_chatdata("dp0-1.json")), 
+        Block(first=1500, last=2500, chat_data=load_chatdata("dp0-1.json")),
-        Block(first = 10000,last = 45146, chat_data=load_chatdata("dp0-2.json")), 
+        Block(first=10000, last=45146, chat_data=load_chatdata("dp0-2.json")),
-        Block(first = 20244,last = 45146, chat_data=load_chatdata("dp0-3.json")), 
+        Block(first=20244, last=45146, chat_data=load_chatdata("dp0-3.json")),
-        Block(first = 20244,last = 62875, chat_data=load_chatdata("dp0-4.json")), 
+        Block(first=20244, last=62875, chat_data=load_chatdata("dp0-4.json")),
-        Block(first = 52568,last = 62875, chat_data=load_chatdata("dp0-5.json"))
+        Block(first=52568, last=62875, chat_data=load_chatdata("dp0-5.json"))
     )
 
     result = duplcheck.remove_duplicate_tail(blocks)
     _dump(result)
     assert len(result) == 3
     assert result[0].first == blocks[0].first
-    assert result[0].last  == blocks[0].last
+    assert result[0].last == blocks[0].last
     assert result[1].first == blocks[2].first
-    assert result[1].last  == blocks[2].last
+    assert result[1].last == blocks[2].last
     assert result[2].first == blocks[4].first
-    assert result[2].last  == blocks[4].last
+    assert result[2].last == blocks[4].last
-
-

tests/test_extract_patch.py

@@ -1,23 +1,19 @@
-import aiohttp
-import asyncio
 import json
-import os, sys
+
-import time
-from aioresponses import aioresponses
-from pytchat.tool.extract import duplcheck
 from pytchat.tool.extract import parser
 from pytchat.tool.extract.block import Block
-from pytchat.tool.extract.patch import Patch, fill, split, set_patch
+from pytchat.tool.extract.patch import Patch, split
-from pytchat.tool.extract.duplcheck import _dump
+
 
 def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
+    with open(path, mode='r', encoding='utf-8') as f:
         return f.read()
 
+
 def load_chatdata(filename):
-        return parser.parse(
+    return parser.parse(
-            json.loads(_open_file("tests/testdata/fetch_patch/"+filename))
+        json.loads(_open_file("tests/testdata/fetch_patch/" + filename))
-        )[1]
+    )[1]
 
 
 def test_split_0():
@@ -61,20 +57,23 @@ def test_split_0():
      @fetched patch
                             |-- patch --|
     """
-    parent = Block(first=0, last=4000, end=60000, continuation='parent', during_split=True)
+    parent = Block(first=0, last=4000, end=60000,
-    child = Block(first=0, last=0, end=60000, continuation='mean', during_split=True)
+                   continuation='parent', during_split=True)
+    child = Block(first=0, last=0, end=60000,
+                  continuation='mean', during_split=True)
     patch = Patch(chats=load_chatdata('pt0-5.json'),
-        first=32500, last=34000, continuation='patch')
+                  first=32500, last=34000, continuation='patch')
-    
+
-    split(parent,child,patch)
+    split(parent, child, patch)
 
     assert child.continuation == 'patch'
     assert parent.last < child.first
     assert parent.end == child.first
     assert child.first < child.last
     assert child.last < child.end
-    assert parent.during_split == False
+    assert parent.during_split is False
-    assert child.during_split == False
+    assert child.during_split is False
+
 
 def test_split_1():
     """patch.first <= parent_block.last
@@ -119,14 +118,15 @@ def test_split_1():
     child = Block(first=0, last=0, end=60000, continuation='mean', during_split=True)
     patch = Patch(chats=load_chatdata('pt0-5.json'),
         first=32500, last=34000, continuation='patch')
-    
-    split(parent,child,patch)
 
-    assert parent.last == 33000 #no change
+    split(parent, child, patch)
-    assert parent.end == 60000 #no change
+
+    assert parent.last == 33000  # no change
+    assert parent.end == 60000  # no change
     assert child.continuation is None
-    assert parent.during_split == False
+    assert parent.during_split is False
-    assert child.during_split == True #exclude during_split sequence
+    assert child.during_split is True  # exclude during_split sequence
+
 
 def test_split_2():
     """child_block.end < patch.last:
@@ -174,7 +174,7 @@ def test_split_2():
     patch = Patch(chats=load_chatdata('pt0-5.json'),
         first=32500, last=34000, continuation='patch')
      
-    split(parent,child,patch)
+    split(parent, child, patch)
 
     assert child.continuation is None
     assert parent.last < child.first
@@ -182,8 +182,9 @@ def test_split_2():
     assert child.first < child.last
     assert child.last < child.end
     assert child.continuation is None
-    assert parent.during_split == False
+    assert parent.during_split is False
-    assert child.during_split == False
+    assert child.during_split is False
+
 
 def test_split_none():
     """patch.last <= parent_block.last
@@ -193,7 +194,7 @@ def test_split_none():
     and parent.block.last exceeds patch.first.
 
     In this case, fetched patch is all discarded,
-    and worker searches other processing block again. 
+    and worker searches other processing block again.
     
     ~~~~~~ before ~~~~~~
 
@@ -229,10 +230,10 @@ def test_split_none():
     patch = Patch(chats=load_chatdata('pt0-5.json'),
         first=32500, last=34000, continuation='patch')
     
-    split(parent,child,patch)
+    split(parent, child, patch)
 
-    assert parent.last == 40000 #no change
+    assert parent.last == 40000  # no change
-    assert parent.end == 60000 #no change
+    assert parent.end == 60000  # no change
     assert child.continuation is None
-    assert parent.during_split == False
+    assert parent.during_split is False
-    assert child.during_split == True #exclude during_split sequence
+    assert child.during_split is True  # exclude during_split sequence

tests/test_extract_video_id.py

@@ -0,0 +1,55 @@
+from pytchat.util.extract_video_id import extract_video_id
+from pytchat.exceptions import InvalidVideoIdException
+
+VALID_TEST_PATTERNS = (
+    ("ABC_EFG_IJK", "ABC_EFG_IJK"),
+    ("vid_test_be", "vid_test_be"),
+    ("https://www.youtube.com/watch?v=123_456_789", "123_456_789"),
+    ("https://www.youtube.com/watch?v=123_456_789&t=123s", "123_456_789"),
+    ("www.youtube.com/watch?v=123_456_789", "123_456_789"),
+    ("watch?v=123_456_789", "123_456_789"),
+    ("youtube.com/watch?v=123_456_789", "123_456_789"),
+    ("http://youtu.be/ABC_EFG_IJK", "ABC_EFG_IJK"),
+    ("youtu.be/ABC_EFG_IJK", "ABC_EFG_IJK"),
+    ("https://www.youtube.com/watch?v=ABC_EFG_IJK&list=XYZ_ABC_12345&start_radio=1&t=1", "ABC_EFG_IJK"),
+    ("https://www.youtube.com/embed/ABC_EFG_IJK", "ABC_EFG_IJK"),
+    ("www.youtube.com/embed/ABC_EFG_IJK", "ABC_EFG_IJK"),
+    ("youtube.com/embed/ABC_EFG_IJK", "ABC_EFG_IJK")
+)
+
+INVALID_TEST_PATTERNS = (
+    ("", ""),
+    ("0123456789", "0123456789"),  # less than 11 letters id
+    ("more_than_11_letter_string", "more_than_11_letter_string"),
+    ("https://www.youtube.com/watch?v=more_than_11_letter_string", "more_than_11_letter_string"),
+    ("https://www.youtube.com/channel/123_456_789", "123_456_789"),
+)
+
+TYPEERROR_TEST_PATTERNS = (
+    (100, 100),  # not string
+    (["123_456_789"], "123_456_789"),  # not string
+)
+
+
+def test_extract_valid_pattern():
+    for pattern in VALID_TEST_PATTERNS:
+        ret = extract_video_id(pattern[0])
+        assert ret == pattern[1]
+
+
+def test_extract_invalid_pattern():
+    for pattern in INVALID_TEST_PATTERNS:
+        try:
+            extract_video_id(pattern[0])
+            assert False
+        except InvalidVideoIdException:
+            assert True
+
+
+def test_extract_typeerror_pattern():
+    for pattern in TYPEERROR_TEST_PATTERNS:
+        try:
+            extract_video_id(pattern[0])
+            assert False
+        except TypeError:
+            assert True

tests/test_livechat.py

@@ -1,53 +1,48 @@
-import pytest
+import asyncio
-from pytchat.parser.live import Parser
-
 import json
-import asyncio,aiohttp
+from pytest_httpx import HTTPXMock
-
+from concurrent.futures import CancelledError
-from aioresponses import aioresponses
-from pytchat.core_async.livechat import LiveChatAsync
-from pytchat.exceptions import (
-    NoLivechatRendererException,NoYtinitialdataException,
-    ResponseContextError,NoContentsException)
-
-
 from pytchat.core_multithread.livechat import LiveChat
-import unittest
+from pytchat.core_async.livechat import LiveChatAsync
-from unittest import TestCase
+from pytchat.exceptions import ResponseContextError
+
 
 def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
+    with open(path, mode='r', encoding='utf-8') as f:
         return f.read()
 
-@aioresponses()
+
-def test_Async(*mock):
+def add_response_file(httpx_mock: HTTPXMock, jsonfile_path: str):
-    vid=''
+    testdata = json.loads(_open_file(jsonfile_path))
-    _text = _open_file('tests/testdata/paramgen_firstread.json')
+    httpx_mock.add_response(json=testdata)
-    _text = json.loads(_text)
+
-    mock[0].get(f"https://www.youtube.com/live_chat?v={vid}&is_popout=1", status=200, body=_text)
+
+def test_async(httpx_mock: HTTPXMock):
+    add_response_file(httpx_mock, 'tests/testdata/paramgen_firstread.json')
+
+    async def test_loop():
+        try:
+            chat = LiveChatAsync(video_id='__test_id__')
+            _ = await chat.get()
+            assert chat.is_alive()
+            chat.terminate()
+            assert not chat.is_alive()
+        except ResponseContextError:
+            assert False
+    loop = asyncio.get_event_loop()
     try:
-        chat = LiveChatAsync(video_id='')
+        loop.run_until_complete(test_loop())
+    except CancelledError:
+        assert True
+
+
+def test_multithread(httpx_mock: HTTPXMock):
+    add_response_file(httpx_mock, 'tests/testdata/paramgen_firstread.json')
+    try:
+        chat = LiveChat(video_id='__test_id__')
+        _ = chat.get()
         assert chat.is_alive()
         chat.terminate()
         assert not chat.is_alive()
     except ResponseContextError:
-        assert not chat.is_alive()
+        assert False
-
-def test_MultiThread(mocker):
-    _text = _open_file('tests/testdata/paramgen_firstread.json')
-    _text = json.loads(_text)
-    responseMock = mocker.Mock()
-    responseMock.status_code = 200
-    responseMock.text = _text
-    mocker.patch('requests.Session.get').return_value = responseMock
-    try:
-        chat = LiveChatAsync(video_id='')
-        assert chat.is_alive()
-        chat.terminate()
-        assert not chat.is_alive()
-    except ResponseContextError:
-        chat.terminate()
-        assert not chat.is_alive()
-
-
-

tests/test_livechat_2.py

@@ -1,125 +1,113 @@
-import asyncio, aiohttp
+import asyncio
 import json
-import pytest
+from pytest_httpx import HTTPXMock
-import re
-import requests
-import sys
-import time
-from aioresponses import aioresponses
 from concurrent.futures import CancelledError
-from unittest import TestCase
 from pytchat.core_multithread.livechat import LiveChat
 from pytchat.core_async.livechat import LiveChatAsync
-from pytchat.exceptions import (
-      NoLivechatRendererException,NoYtinitialdataException,
-      ResponseContextError,NoContentsException)
-from pytchat.parser.live import Parser
 from pytchat.processors.dummy_processor import DummyProcessor
 
+
 def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
+    with open(path, mode='r', encoding='utf-8') as f:
         return f.read()
 
-@aioresponses()
-def test_async_live_stream(*mock):
 
-    async def test_loop(*mock):
+def add_response_file(httpx_mock: HTTPXMock, jsonfile_path: str):
-        pattern = re.compile(r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$')
+    testdata = json.loads(_open_file(jsonfile_path))
-        _text = _open_file('tests/testdata/test_stream.json')
+    httpx_mock.add_response(json=testdata)
-        mock[0].get(pattern, status=200, body=_text)
+
-        chat = LiveChatAsync(video_id='', processor = DummyProcessor())
+
+def test_async_live_stream(httpx_mock: HTTPXMock):
+    add_response_file(httpx_mock, 'tests/testdata/test_stream.json')
+
+    async def test_loop():
+        chat = LiveChatAsync(video_id='__test_id__', processor=DummyProcessor())
         chats = await chat.get()
         rawdata = chats[0]["chatdata"]
-        #assert fetching livachat data
+        assert list(rawdata[0]["addChatItemAction"]["item"].keys())[
-        assert list(rawdata[0]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer"
+            0] == "liveChatTextMessageRenderer"
-        assert list(rawdata[1]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer"
+        assert list(rawdata[1]["addChatItemAction"]["item"].keys())[
-        assert list(rawdata[2]["addChatItemAction"]["item"].keys())[0] == "liveChatPlaceholderItemRenderer"
+            0] == "liveChatTextMessageRenderer"
-        assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerPaidMessageItemRenderer"
+        assert list(rawdata[2]["addChatItemAction"]["item"].keys())[
-        assert list(rawdata[4]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer"
+            0] == "liveChatPlaceholderItemRenderer"
-        assert list(rawdata[5]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidStickerRenderer"
+        assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[
-        assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerSponsorItemRenderer"                                
+            0] == "liveChatTickerPaidMessageItemRenderer"
+        assert list(rawdata[4]["addChatItemAction"]["item"].keys())[
+            0] == "liveChatPaidMessageRenderer"
+        assert list(rawdata[5]["addChatItemAction"]["item"].keys())[
+            0] == "liveChatPaidStickerRenderer"
+        assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[
+            0] == "liveChatTickerSponsorItemRenderer"
 
     loop = asyncio.get_event_loop()
     try:
-        loop.run_until_complete(test_loop(*mock))
+        loop.run_until_complete(test_loop())
     except CancelledError:
         assert True
 
-@aioresponses()        
-def test_async_replay_stream(*mock):
 
-    async def test_loop(*mock):
+def test_async_replay_stream(httpx_mock: HTTPXMock):
-        pattern_live = re.compile(r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$')
+    add_response_file(httpx_mock, 'tests/testdata/finished_live.json')
-        pattern_replay =  re.compile(r'^https://www.youtube.com/live_chat_replay/get_live_chat_replay\?continuation=.*$')
+    add_response_file(httpx_mock, 'tests/testdata/chatreplay.json')
-        #empty livechat -> switch to fetch replaychat
-        _text_live = _open_file('tests/testdata/finished_live.json')
-        _text_replay = _open_file('tests/testdata/chatreplay.json')
-        mock[0].get(pattern_live, status=200, body=_text_live)
-        mock[0].get(pattern_replay, status=200, body=_text_replay)
 
-        chat = LiveChatAsync(video_id='', processor = DummyProcessor())
+    async def test_loop():
-        chats = await chat.get()
+        chat = LiveChatAsync(video_id='__test_id__', processor=DummyProcessor())
-        rawdata = chats[0]["chatdata"]
-        #assert fetching replaychat data
-        assert list(rawdata[0]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer"
-        assert list(rawdata[14]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer"
-
-    loop = asyncio.get_event_loop()
-    try:
-        loop.run_until_complete(test_loop(*mock))
-    except CancelledError:
-        assert True
-
-@aioresponses()
-def test_async_force_replay(*mock):
-
-    async def test_loop(*mock):
-        pattern_live = re.compile(r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$')
-        pattern_replay =  re.compile(r'^https://www.youtube.com/live_chat_replay/get_live_chat_replay\?continuation=.*$')
-        #valid live data, but force_replay = True
-        _text_live = _open_file('tests/testdata/test_stream.json')
-        #valid replay data
-        _text_replay = _open_file('tests/testdata/chatreplay.json')
-        
-        mock[0].get(pattern_live, status=200, body=_text_live)
-        mock[0].get(pattern_replay, status=200, body=_text_replay)
-        #force replay
-        chat = LiveChatAsync(video_id='', processor = DummyProcessor(), force_replay = True)
         chats = await chat.get()
         rawdata = chats[0]["chatdata"]
         # assert fetching replaychat data
-        assert list(rawdata[14]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer"
+        assert list(rawdata[0]["addChatItemAction"]["item"].keys())[
-        # assert not mix livechat data
+            0] == "liveChatTextMessageRenderer"
-        assert list(rawdata[2]["addChatItemAction"]["item"].keys())[0] != "liveChatPlaceholderItemRenderer"
+        assert list(rawdata[14]["addChatItemAction"]["item"].keys())[
-    
+            0] == "liveChatPaidMessageRenderer"
+
     loop = asyncio.get_event_loop()
     try:
-        loop.run_until_complete(test_loop(*mock))
+        loop.run_until_complete(test_loop())
     except CancelledError:
         assert True
 
-def test_multithread_live_stream(mocker):
 
-    _text = _open_file('tests/testdata/test_stream.json')
+def test_async_force_replay(httpx_mock: HTTPXMock):
-    responseMock = mocker.Mock()
+    add_response_file(httpx_mock, 'tests/testdata/test_stream.json')
-    responseMock.status_code = 200
+    add_response_file(httpx_mock, 'tests/testdata/chatreplay.json')
-    responseMock.text = _text
-    mocker.patch('requests.Session.get').return_value.__enter__.return_value = responseMock
 
-    chat = LiveChat(video_id='test_id', processor = DummyProcessor())
+    async def test_loop():
+        chat = LiveChatAsync(
+            video_id='__test_id__', processor=DummyProcessor(), force_replay=True)
+        chats = await chat.get()
+        rawdata = chats[0]["chatdata"]
+        # assert fetching replaychat data
+        assert list(rawdata[14]["addChatItemAction"]["item"].keys())[
+            0] == "liveChatPaidMessageRenderer"
+        # assert not mix livechat data
+        assert list(rawdata[2]["addChatItemAction"]["item"].keys())[
+            0] != "liveChatPlaceholderItemRenderer"
+
+    loop = asyncio.get_event_loop()
+    try:
+        loop.run_until_complete(test_loop())
+    except CancelledError:
+        assert True
+
+
+def test_multithread_live_stream(httpx_mock: HTTPXMock):
+    add_response_file(httpx_mock, 'tests/testdata/test_stream.json')
+    chat = LiveChat(video_id='__test_id__', processor=DummyProcessor())
     chats = chat.get()
     rawdata = chats[0]["chatdata"]
-    #assert fetching livachat data
+    # assert fetching livachat data
-    assert list(rawdata[0]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer"
+    assert list(rawdata[0]["addChatItemAction"]["item"].keys())[
-    assert list(rawdata[1]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer"
+        0] == "liveChatTextMessageRenderer"
-    assert list(rawdata[2]["addChatItemAction"]["item"].keys())[0] == "liveChatPlaceholderItemRenderer"
+    assert list(rawdata[1]["addChatItemAction"]["item"].keys())[
-    assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerPaidMessageItemRenderer"
+        0] == "liveChatTextMessageRenderer"
-    assert list(rawdata[4]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer"
+    assert list(rawdata[2]["addChatItemAction"]["item"].keys())[
-    assert list(rawdata[5]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidStickerRenderer"
+        0] == "liveChatPlaceholderItemRenderer"
-    assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerSponsorItemRenderer"
+    assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[
+        0] == "liveChatTickerPaidMessageItemRenderer"
+    assert list(rawdata[4]["addChatItemAction"]["item"].keys())[
+        0] == "liveChatPaidMessageRenderer"
+    assert list(rawdata[5]["addChatItemAction"]["item"].keys())[
+        0] == "liveChatPaidStickerRenderer"
+    assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[
+        0] == "liveChatTickerSponsorItemRenderer"
     chat.terminate()
-
-    
-
-
-

tests/test_liveparam.py

@@ -5,5 +5,5 @@ def test_liveparam_0(mocker):
     _ts1= 1546268400
     param = liveparam._build("01234567890",
     *([_ts1*1000000 for i in range(5)]), topchat_only=False)
-    test_param="0ofMyAPiARp8Q2c4S0RRb0xNREV5TXpRMU5qYzRPVEFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5TURFeU16UTFOamM0T1RBbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiAuNbVqsrfAjAAOABAAkorCAEQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgLjW1arK3wJYA1CAuNbVqsrfAliAuNbVqsrfAmgBggEECAEQAIgBAKABgLjW1arK3wI%3D"
+    test_param="0ofMyANcGhxDZzhLRFFvTE1ERXlNelExTmpjNE9UQWdBUT09KIC41tWqyt8CQAFKC1CAuNbVqsrfAlgDUIC41tWqyt8CWIC41tWqyt8CaAGCAQIIAZoBAKABgLjW1arK3wI%3D"
     assert test_param == param

tests/test_parser.py

@@ -1,44 +1,40 @@
-import pytest
 from pytchat.parser.live import Parser
 import json
-import asyncio,aiohttp
+from pytchat.exceptions import NoContents
-from aioresponses import aioresponses
+
-from pytchat.exceptions import (
+
-    NoLivechatRendererException,NoYtinitialdataException,
+parser = Parser(is_replay=False)
-    ResponseContextError, NoContentsException)
 
 
 def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
+    with open(path, mode='r', encoding='utf-8') as f:
         return f.read()
-parser = Parser(is_replay = False)
 
-@aioresponses()
+
 def test_finishedlive(*mock):
     '''配信が終了した動画を正しく処理できるか'''
-
     _text = _open_file('tests/testdata/finished_live.json')
     _text = json.loads(_text)
 
-    try:    
+    try:
         parser.parse(parser.get_contents(_text))
         assert False
-    except NoContentsException:
+    except NoContents:
         assert True
 
-@aioresponses()
+
 def test_parsejson(*mock):
     '''jsonを正常にパースできるか'''
-
     _text = _open_file('tests/testdata/paramgen_firstread.json')
     _text = json.loads(_text)
 
-    try:    
+    try:
         parser.parse(parser.get_contents(_text))
         jsn = _text
         timeout = jsn["response"]["continuationContents"]["liveChatContinuation"]["continuations"][0]["timedContinuationData"]["timeoutMs"]
-        continuation = jsn["response"]["continuationContents"]["liveChatContinuation"]["continuations"][0]["timedContinuationData"]["continuation"]
+        continuation = jsn["response"]["continuationContents"]["liveChatContinuation"][
-        assert 5035 == timeout
+            "continuations"][0]["timedContinuationData"]["continuation"]
-        assert "0ofMyAPiARp8Q2c4S0RRb0xhelJMZDBsWFQwdERkalFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5YXpSTGQwbFhUMHREZGpRbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiPz5-Os-PkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgJqXjrPj5AJYA1CRwciOs-PkAli3pNq1k-PkAmgBggEECAEQAIgBAKABjbfnjrPj5AI%3D" == continuation
+        assert timeout == 5035
-    except:
+        assert continuation == "0ofMyAPiARp8Q2c4S0RRb0xhelJMZDBsWFQwdERkalFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5YXpSTGQwbFhUMHREZGpRbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiPz5-Os-PkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgJqXjrPj5AJYA1CRwciOs-PkAli3pNq1k-PkAmgBggEECAEQAIgBAKABjbfnjrPj5AI%3D"
-        assert False
+    except Exception:
+        assert False

tests/test_speed_calculator.py

@@ -1,15 +1,9 @@
 import json
-import pytest
-import asyncio,aiohttp
 from pytchat.parser.live import Parser
-from pytchat.processors.compatible.processor import CompatibleProcessor
-from pytchat.exceptions import (
-    NoLivechatRendererException,NoYtinitialdataException,
-    ResponseContextError, NoContentsException)
-
 from pytchat.processors.speed.calculator import SpeedCalculator
 
-parser = Parser(is_replay =False)
+parser = Parser(is_replay=False)
+
 
 def test_speed_1(mocker):
     '''test speed calculation with normal json.
@@ -23,13 +17,14 @@ def test_speed_1(mocker):
 
     _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
     data = {
-        "video_id" : "",
+        "video_id": "",
-        "timeout" : 10,
+        "timeout": 10,
-        "chatdata" : chatdata
+        "chatdata": chatdata
     }
     ret = processor.process([data])
     assert 30 == ret
 
+
 def test_speed_2(mocker):
     '''test speed calculation with no valid chat data.
     '''
@@ -39,13 +34,14 @@ def test_speed_2(mocker):
 
     _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
     data = {
-        "video_id" : "",
+        "video_id": "",
-        "timeout" : 10,
+        "timeout": 10,
-        "chatdata" : chatdata
+        "chatdata": chatdata
     }
     ret = processor.process([data])
-    assert 0 == ret
+    assert ret == 0
-    
+
+
 def test_speed_3(mocker):
     '''test speed calculation with empty data.
     '''
@@ -55,14 +51,14 @@ def test_speed_3(mocker):
 
     _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
     data = {
-        "video_id" : "",
+        "video_id": "",
-        "timeout" : 10,
+        "timeout": 10,
-        "chatdata" : chatdata
+        "chatdata": chatdata
     }
     ret = processor.process([data])
-    assert 0 == ret
+    assert ret == 0
-    
+
 
 def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
+    with open(path, mode='r', encoding='utf-8') as f:
         return f.read()

tests/test_videoinfo.py

@@ -1,35 +1,39 @@
+from json.decoder import JSONDecodeError
 from pytchat.tool.videoinfo import VideoInfo
-from pytchat.exceptions import InvalidVideoIdException
+from pytchat.exceptions import InvalidVideoIdException, PatternUnmatchError
-import pytest
+
 
 def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
+    with open(path, mode='r', encoding='utf-8') as f:
         return f.read()
 
+
 def _set_test_data(filepath, mocker):
     _text = _open_file(filepath)
     response_mock = mocker.Mock()
     response_mock.status_code = 200
     response_mock.text = _text
-    mocker.patch('requests.get').return_value = response_mock
+    mocker.patch('httpx.get').return_value = response_mock
+
 
 def test_archived_page(mocker):
     _set_test_data('tests/testdata/videoinfo/archived_page.txt', mocker)
-    info = VideoInfo('test_id')
+    info = VideoInfo('__test_id__')
-    actual_thumbnail_url =  'https://i.ytimg.com/vi/fzI9FNjXQ0o/hqdefault.jpg'
+    actual_thumbnail_url = 'https://i.ytimg.com/vi/fzI9FNjXQ0o/hqdefault.jpg'
-    assert info.video_id == 'test_id'
+    assert info.video_id == '__test_id__'
     assert info.get_channel_name() == 'GitHub'
     assert info.get_thumbnail() == actual_thumbnail_url
     assert info.get_title() == 'GitHub Arctic Code Vault'
     assert info.get_channel_id() == 'UC7c3Kb6jYCRj4JOHHZTxKsQ'
     assert info.get_duration() == 148
 
+
 def test_live_page(mocker):
     _set_test_data('tests/testdata/videoinfo/live_page.txt', mocker)
-    info = VideoInfo('test_id')
+    info = VideoInfo('__test_id__')
-    '''live page :duration = 0'''        
+    '''live page: duration==0'''
     assert info.get_duration() == 0
-    assert info.video_id == 'test_id'
+    assert info.video_id == '__test_id__'
     assert info.get_channel_name() == 'BGM channel'
     assert info.get_thumbnail() == \
         'https://i.ytimg.com/vi/fEvM-OUbaKs/hqdefault_live.jpg'
@@ -38,25 +42,60 @@ def test_live_page(mocker):
         ' - 24/7 Live Stream - Slow Jazz')
     assert info.get_channel_id() == 'UCQINXHZqCU5i06HzxRkujfg'
 
+
 def test_invalid_video_id(mocker):
     '''Test case invalid video_id is specified.'''
     _set_test_data(
         'tests/testdata/videoinfo/invalid_video_id_page.txt', mocker)
     try:
-        _ = VideoInfo('test_id')
+        _ = VideoInfo('__test_id__')
         assert False
     except InvalidVideoIdException:
         assert True
 
+
 def test_no_info(mocker):
     '''Test case the video page has renderer, but no info.'''
     _set_test_data(
         'tests/testdata/videoinfo/no_info_page.txt', mocker)
-    info = VideoInfo('test_id')
+    info = VideoInfo('__test_id__')
-    assert info.video_id == 'test_id'
+    assert info.video_id == '__test_id__'
     assert info.get_channel_name() is None
     assert info.get_thumbnail() is None
     assert info.get_title() is None
     assert info.get_channel_id() is None
     assert info.get_duration() is None
 
+
+def test_collapsed_data(mocker):
+    '''Test case the video page's info is collapsed.'''
+    _set_test_data(
+        'tests/testdata/videoinfo/collapsed_page.txt', mocker)
+    try:
+        _ = VideoInfo('__test_id__')
+        assert False
+    except JSONDecodeError:
+        assert True
+
+
+def test_pattern_unmatch(mocker):
+    '''Test case the pattern for extraction is unmatched.'''
+    _set_test_data(
+        'tests/testdata/videoinfo/pattern_unmatch.txt', mocker)
+    try:
+        _ = VideoInfo('__test_id__')
+        assert False
+    except PatternUnmatchError:
+        assert True
+
+
+def test_extradata_handling(mocker):
+    '''Test case the extracted data are JSON lines.'''
+    _set_test_data(
+        'tests/testdata/videoinfo/extradata_page.txt', mocker)
+    try:
+        _ = VideoInfo('__test_id__')
+        assert True
+    except JSONDecodeError as e:
+        print(e.doc)
+        assert False

tests/testdata/default/newSponsor_current.json

@@ -0,0 +1,100 @@
+{
+  "response": {
+    "responseContext": {
+      "webResponseContextExtensionData": ""
+    },
+    "continuationContents": {
+      "liveChatContinuation": {
+        "continuations": [
+          {
+            "invalidationContinuationData": {
+              "invalidationId": {
+                "objectSource": 1000,
+                "objectId": "___objectId___",
+                "topic": "chat~00000000000~0000000",
+                "subscribeToGcmTopics": true,
+                "protoCreationTimestampMs": "1577804400000"
+              },
+              "timeoutMs": 10000,
+              "continuation": "___continuation___"
+            }
+          }
+        ],
+        "actions": [
+          {
+            "addChatItemAction": {
+              "item": {
+                "liveChatMembershipItemRenderer": {
+                  "id": "dummy_id",
+                  "timestampUsec": 1570678496000000,
+                  "authorExternalChannelId": "author_channel_id",
+                  "headerSubtext": {
+                    "runs": [
+                      {
+                        "text": "新規メンバー"
+                      }
+                    ]
+                  },
+                  "authorName": {
+                    "simpleText": "author_name"
+                  },
+                  "authorPhoto": {
+                    "thumbnails": [
+                      {
+                        "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                        "width": 32,
+                        "height": 32
+                      },
+                      {
+                        "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                        "width": 64,
+                        "height": 64
+                      }
+                    ]
+                  },
+                  "authorBadges": [
+                    {
+                      "liveChatAuthorBadgeRenderer": {
+                        "customThumbnail": {
+                          "thumbnails": [
+                            {
+                              "url": "https://yt3.ggpht.com/X=s32-c-k"
+                            },
+                            {
+                              "url": "https://yt3.ggpht.com/X=s64-c-k"
+                            }
+                          ]
+                        },
+                        "tooltip": "新規メンバー",
+                        "accessibility": {
+                          "accessibilityData": {
+                            "label": "新規メンバー"
+                          }
+                        }
+                      }
+                    }
+                  ],
+                  "contextMenuEndpoint": {
+                    "commandMetadata": {
+                      "webCommandMetadata": {
+                        "ignoreNavigation": true
+                      }
+                    },
+                    "liveChatItemContextMenuEndpoint": {
+                      "params": "___params___"
+                    }
+                  },
+                  "contextMenuAccessibility": {
+                    "accessibilityData": {
+                      "label": "コメントの操作"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        ]
+      }
+    }
+  }
+}

tests/testdata/default/newSponsor_lagacy.json

@@ -0,0 +1,82 @@
+{
+  "response": {
+    "responseContext": {
+      "webResponseContextExtensionData": ""
+    },
+    "continuationContents": {
+      "liveChatContinuation": {
+        "continuations": [
+          {
+            "invalidationContinuationData": {
+              "invalidationId": {
+                "objectSource": 1000,
+                "objectId": "___objectId___",
+                "topic": "chat~00000000000~0000000",
+                "subscribeToGcmTopics": true,
+                "protoCreationTimestampMs": "1577804400000"
+              },
+              "timeoutMs": 10000,
+              "continuation": "___continuation___"
+            }
+          }
+        ],
+        "actions": [
+          {
+            "addChatItemAction": {
+              "item": {
+                "liveChatLegacyPaidMessageRenderer": {
+                  "id": "dummy_id",
+                  "timestampUsec": 1570678496000000,
+                  "eventText": {
+                    "runs": [
+                      {
+                        "text": "新規メンバー"
+                      }
+                    ]
+                  },
+                  "detailText": {
+                    "simpleText": "ようこそ、author_name！"
+                  },
+                  "authorName": {
+                    "simpleText": "author_name"
+                  },
+                  "authorPhoto": {
+                    "thumbnails": [
+                      {
+                        "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                        "width": 32,
+                        "height": 32
+                      },
+                      {
+                        "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                        "width": 64,
+                        "height": 64
+                      }
+                    ]
+                  },
+                  "authorExternalChannelId": "author_channel_id",
+                  "contextMenuEndpoint": {
+                    "clickTrackingParams": "___clickTrackingParams___",
+                    "commandMetadata": {
+                      "webCommandMetadata": {
+                        "ignoreNavigation": true
+                      }
+                    },
+                    "liveChatItemContextMenuEndpoint": {
+                      "params": "___params___"
+                    }
+                  },
+                  "contextMenuAccessibility": {
+                    "accessibilityData": {
+                      "label": "コメントの操作"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        ]
+      }
+    }
+  }
+}

tests/testdata/default/replay_member_text.json

@@ -0,0 +1,112 @@
+{
+  "response": {
+    "responseContext": {
+      "webResponseContextExtensionData": "data"
+    },
+    "continuationContents": {
+      "liveChatContinuation": {
+        "continuations": [
+          {
+            "liveChatReplayContinuationData": {
+              "invalidationId": {
+                "objectSource": 1000,
+                "objectId": "___objectId___",
+                "topic": "chat~00000000000~0000000",
+                "subscribeToGcmTopics": true,
+                "protoCreationTimestampMs": "1577804400000"
+              },
+              "timeoutMs": 10000,
+              "continuation": "___continuation___"
+            }
+          }
+        ],
+        "actions": [
+          {
+            "replayChatItemAction": {
+              "actions": [
+                {
+                  "addChatItemAction": {
+                    "item": {
+                      "liveChatTextMessageRenderer": {
+                        "message": {
+                          "runs": [
+                            {
+                              "text": "dummy_message"
+                            }
+                          ]
+                        },
+                        "authorName": {
+                          "simpleText": "author_name"
+                        },
+                        "authorPhoto": {
+                          "thumbnails": [
+                            {
+                              "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                              "width": 32,
+                              "height": 32
+                            },
+                            {
+                              "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                              "width": 64,
+                              "height": 64
+                            }
+                          ]
+                        },
+                        "contextMenuEndpoint": {
+                          "clickTrackingParams": "___clickTrackingParams___",
+                          "commandMetadata": {
+                            "webCommandMetadata": {
+                              "ignoreNavigation": true
+                            }
+                          },
+                          "liveChatItemContextMenuEndpoint": {
+                            "params": "___params___"
+                          }
+                        },
+                        "id": "dummy_id",
+                        "timestampUsec": 1570678496000000,
+                        "authorBadges": [
+                          {
+                            "liveChatAuthorBadgeRenderer": {
+                              "customThumbnail": {
+                                "thumbnails": [
+                                  {
+                                    "url": "https://yt3.ggpht.com/X=s16-c-k"
+                                  },
+                                  {
+                                    "url": "https://yt3.ggpht.com/X=s32-c-k"
+                                  }
+                                ]
+                              },
+                              "tooltip": "メンバー（1 か月）",
+                              "accessibility": {
+                                "accessibilityData": {
+                                  "label": "メンバー（1 か月）"
+                                }
+                              }
+                            }
+                          }
+                        ],
+                        "authorExternalChannelId": "author_channel_id",
+                        "contextMenuAccessibility": {
+                          "accessibilityData": {
+                            "label": "コメントの操作"
+                          }
+                        },
+                        "timestampText": {
+                          "simpleText": "1:23:45"
+                        }
+                      }
+                    },
+                    "clientId": "dummy_client_id"
+                  }
+                }
+              ],
+              "videoOffsetTimeMsec": "5025120"
+            }
+          }
+        ]
+      }
+    }
+  }
+}

tests/testdata/default/superchat.json

@@ -0,0 +1,184 @@
+{
+  "response": {
+    "responseContext": {
+      "webResponseContextExtensionData": ""
+    },
+    "continuationContents": {
+      "liveChatContinuation": {
+        "continuations": [
+          {
+            "invalidationContinuationData": {
+              "invalidationId": {
+                "objectSource": 1000,
+                "objectId": "___objectId___",
+                "topic": "chat~00000000000~0000000",
+                "subscribeToGcmTopics": true,
+                "protoCreationTimestampMs": "1577804400000"
+              },
+              "timeoutMs": 10000,
+              "continuation": "___continuation___"
+            }
+          }
+        ],
+        "actions": [
+          {
+            "addChatItemAction": {
+              "item": {
+                "liveChatPaidMessageRenderer": {
+                  "id": "dummy_id",
+                  "timestampUsec": 1570678496000000,
+                  "authorName": {
+                    "simpleText": "author_name"
+                  },
+                  "authorPhoto": {
+                    "thumbnails": [
+                      {
+                        "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                        "width": 32,
+                        "height": 32
+                      },
+                      {
+                        "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                        "width": 64,
+                        "height": 64
+                      }
+                    ]
+                  },
+                  "purchaseAmountText": {
+                    "simpleText": "￥800"
+                  },
+                  "message": {
+                    "runs": [
+                      {
+                        "text": "dummy_message"
+                      }
+                    ]
+                  },
+                  "headerBackgroundColor": 4278239141,
+                  "headerTextColor": 4278190080,
+                  "bodyBackgroundColor": 4280150454,
+                  "bodyTextColor": 4278190080,
+                  "authorExternalChannelId": "author_channel_id",
+                  "authorNameTextColor": 2315255808,
+                  "contextMenuEndpoint": {
+                    "commandMetadata": {
+                      "webCommandMetadata": {
+                        "ignoreNavigation": true
+                      }
+                    },
+                    "liveChatItemContextMenuEndpoint": {
+                      "params": "___params___"
+                    }
+                  },
+                  "timestampColor": 2147483648,
+                  "contextMenuAccessibility": {
+                    "accessibilityData": {
+                      "label": "コメントの操作"
+                    }
+                  }
+                }
+              }
+            }
+          },
+          {
+            "addLiveChatTickerItemAction": {
+              "item": {
+                "liveChatTickerPaidMessageItemRenderer": {
+                  "id": "dummy_id",
+                  "amount": {
+                    "simpleText": "￥846"
+                  },
+                  "amountTextColor": 4278190080,
+                  "startBackgroundColor": 4280150454,
+                  "endBackgroundColor": 4278239141,
+                  "authorPhoto": {
+                    "thumbnails": [
+                      {
+                        "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                        "width": 32,
+                        "height": 32
+                      },
+                      {
+                        "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                        "width": 64,
+                        "height": 64
+                      }
+                    ]
+                  },
+                  "durationSec": 120,
+                  "showItemEndpoint": {
+                    "commandMetadata": {
+                      "webCommandMetadata": {
+                        "ignoreNavigation": true
+                      }
+                    },
+                    "showLiveChatItemEndpoint": {
+                      "renderer": {
+                        "liveChatPaidMessageRenderer": {
+                          "id": "dummy_id",
+                          "timestampUsec": 1570678496000000,
+                          "authorName": {
+                            "simpleText": "author_name"
+                          },
+                          "authorPhoto": {
+                            "thumbnails": [
+                              {
+                                "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                                "width": 32,
+                                "height": 32
+                              },
+                              {
+                                "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                                "width": 64,
+                                "height": 64
+                              }
+                            ]
+                          },
+                          "purchaseAmountText": {
+                            "simpleText": "￥846"
+                          },
+                          "message": {
+                            "runs": [
+                              {
+                                "text": "dummy_message"
+                              }
+                            ]
+                          },
+                          "headerBackgroundColor": 4278239141,
+                          "headerTextColor": 4278190080,
+                          "bodyBackgroundColor": 4280150454,
+                          "bodyTextColor": 4278190080,
+                          "authorExternalChannelId": "author_channel_id",
+                          "authorNameTextColor": 2315255808,
+                          "contextMenuEndpoint": {
+                            "commandMetadata": {
+                              "webCommandMetadata": {
+                                "ignoreNavigation": true
+                              }
+                            },
+                            "liveChatItemContextMenuEndpoint": {
+                              "params": "___params___"
+                            }
+                          },
+                          "timestampColor": 2147483648,
+                          "contextMenuAccessibility": {
+                            "accessibilityData": {
+                              "label": "コメントの操作"
+                            }
+                          }
+                        }
+                      }
+                    }
+                  },
+                  "authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url",
+                  "fullDurationSec": 120
+                }
+              },
+              "durationSec": "120"
+            }
+          }
+        ]
+      }
+    }
+  }
+}

tests/testdata/default/supersticker.json

@@ -0,0 +1,99 @@
+{
+  "response": {
+    "responseContext": {
+      "webResponseContextExtensionData": ""
+    },
+    "continuationContents": {
+      "liveChatContinuation": {
+        "continuations": [
+          {
+            "invalidationContinuationData": {
+              "invalidationId": {
+                "objectSource": 1000,
+                "objectId": "___objectId___",
+                "topic": "chat~00000000000~0000000",
+                "subscribeToGcmTopics": true,
+                "protoCreationTimestampMs": "1577804400000"
+              },
+              "timeoutMs": 10000,
+              "continuation": "___continuation___"
+            }
+          }
+        ],
+        "actions": [
+          {
+            "addChatItemAction": {
+              "item": {
+                "liveChatPaidStickerRenderer": {
+                  "id": "dummy_id",
+                  "contextMenuEndpoint": {
+                    "commandMetadata": {
+                      "webCommandMetadata": {
+                        "ignoreNavigation": true
+                      }
+                    },
+                    "liveChatItemContextMenuEndpoint": {
+                      "params": "___params___"
+                    }
+                  },
+                  "contextMenuAccessibility": {
+                    "accessibilityData": {
+                      "label": "コメントの操作"
+                    }
+                  },
+                  "timestampUsec": 1570678496000000,
+                  "authorPhoto": {
+                    "thumbnails": [
+                      {
+                        "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                        "width": 32,
+                        "height": 32
+                      },
+                      {
+                        "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                        "width": 64,
+                        "height": 64
+                      }
+                    ]
+                  },
+                  "authorName": {
+                    "simpleText": "author_name"
+                  },
+                  "authorExternalChannelId": "author_channel_id",
+                  "sticker": {
+                    "thumbnails": [
+                      {
+                        "url": "//lh3.googleusercontent.com/param_s=s72-rp",
+                        "width": 72,
+                        "height": 72
+                      },
+                      {
+                        "url": "//lh3.googleusercontent.com/param_s=s144-rp",
+                        "width": 144,
+                        "height": 144
+                      }
+                    ],
+                    "accessibility": {
+                      "accessibilityData": {
+                        "label": "___sticker_label___"
+                      }
+                    }
+                  },
+                  "moneyChipBackgroundColor": 4278248959,
+                  "moneyChipTextColor": 4278190080,
+                  "purchaseAmountText": {
+                    "simpleText": "￥200"
+                  },
+                  "stickerDisplayWidth": 72,
+                  "stickerDisplayHeight": 72,
+                  "backgroundColor": 4278237396,
+                  "authorNameTextColor": 3003121664
+                }
+              }
+            }
+          }
+        ]
+      }
+    }
+  }
+}

tests/testdata/default/textmessage.json

@@ -0,0 +1,79 @@
+{
+  "response": {
+    "responseContext": {
+      "webResponseContextExtensionData": ""
+    },
+    "continuationContents": {
+      "liveChatContinuation": {
+        "continuations": [
+          {
+            "invalidationContinuationData": {
+              "invalidationId": {
+                "objectSource": 1000,
+                "objectId": "___objectId___",
+                "topic": "chat~00000000000~0000000",
+                "subscribeToGcmTopics": true,
+                "protoCreationTimestampMs": "1577804400000"
+              },
+              "timeoutMs": 10000,
+              "continuation": "___continuation___"
+            }
+          }
+        ],
+        "actions": [
+          {
+            "addChatItemAction": {
+              "item": {
+                "liveChatTextMessageRenderer": {
+                  "message": {
+                    "runs": [
+                      {
+                        "text": "dummy_message"
+                      }
+                    ]
+                  },
+                  "authorName": {
+                    "simpleText": "author_name"
+                  },
+                  "authorPhoto": {
+                    "thumbnails": [
+                      {
+                        "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                        "width": 32,
+                        "height": 32
+                      },
+                      {
+                        "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                        "width": 64,
+                        "height": 64
+                      }
+                    ]
+                  },
+                  "contextMenuEndpoint": {
+                    "commandMetadata": {
+                      "webCommandMetadata": {
+                        "ignoreNavigation": true
+                      }
+                    },
+                    "liveChatItemContextMenuEndpoint": {
+                      "params": "___params___"
+                    }
+                  },
+                  "id": "dummy_id",
+                  "timestampUsec": 1570678496000000,
+                  "authorExternalChannelId": "author_channel_id",
+                  "contextMenuAccessibility": {
+                    "accessibilityData": {
+                      "label": "コメントの操作"
+                    }
+                  }
+                }
+              },
+              "clientId": "dummy_client_id"
+            }
+          }
+        ]
+      }
+    }
+  }
+}