Merge branch 'hotfix/test_requirements'

v0.3.2

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 taizan-hokuto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

MANIFEST.in

@@ -1 +1,5 @@
 include requirements.txt
+include requirements_test.txt
+include README.md
+global-exclude tests/*
+global-exclude pytchat/testrun*.py

README.md

@@ -5,90 +5,80 @@ pytchat is a python library for fetching youtube live chat.
 
 ## Description
 pytchat is a python library for fetching youtube live chat
-without using youtube api, Selenium or BeautifulSoup.
+without using Selenium or BeautifulSoup.
 
 Other features:
-+ Customizable chat data processors including yt api compatible one.
++ Customizable [chat data processors](https://github.com/taizan-hokuto/pytchat/wiki/ChatProcessor) including youtube api compatible one.
 + Available on asyncio context. 
 + Quick fetching of initial chat data by generating continuation params
 instead of web scraping.
 
+For more detailed information, see [wiki](https://github.com/taizan-hokuto/pytchat/wiki). <br>
+[wiki (Japanese)](https://github.com/taizan-hokuto/pytchat/wiki/Home_jp)
+
 ## Install
 ```python
 pip install pytchat
 ```
-
 ## Examples
-### on-demand mode
-```python
-from pytchat import LiveChat
 
-chat = LiveChat("G1w62uEMZ74")
-while chat.is_alive():
-    data = chat.get()
-    for c in data.items:
-        print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}")
-        data.tick()
+### CLI
+
++ One-liner command.
+
++ Save chat data to html with embedded custom emojis.
+
++ Show chat stream (--echo option).
+
+```bash
+$ pytchat -v uIx8l2xlYVY -o "c:/temp/"
+# options:
+#  -v : Video ID or URL that includes ID
+#  -o : output directory (default path: './')
+#  --echo : Show chats.
+# saved filename is [video_id].html
 ```
 
-### callback mode
+
+### Fetch chat data (see [wiki](https://github.com/taizan-hokuto/pytchat/wiki/PytchatCore))
 ```python
-from pytchat import LiveChat
+import pytchat
+chat = pytchat.create(video_id="uIx8l2xlYVY")
+while chat.is_alive():
+    for c in chat.get().sync_items():
+        print(f"{c.datetime} [{c.author.name}]- {c.message}")
+```
+
+
+### Output JSON format string (feature of [DefaultProcessor](https://github.com/taizan-hokuto/pytchat/wiki/DefaultProcessor))
+```python
+import pytchat
 import time
 
-chat = LiveChat("G1w62uEMZ74", callback = func)
+chat = pytchat.create(video_id="uIx8l2xlYVY")
 while chat.is_alive():
-    #other background operation here.
-    time.sleep(3)
-
-def func(chatdata):
-    for c in chatdata.items:
-        print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}")
-        chat.tick()
-```
-
-### asyncio context:
-```python
-from pytchat import LiveChatAsync
-import asyncio
-
-async def main():
-    chat = LiveChatAsync("G1w62uEMZ74", callback = func)
-    while chat.is_alive():
-        #other background operation here.
-        await asyncio.sleep(3)
-
-async def func(chat)
-    for c in chat.items:
-        print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}")
-        await chat.tick_async()
-
-loop = asyncio.get_event_loop()
-loop.run_until_complete(main())
+    print(chat.get().json())
+    time.sleep(5)
+    '''
+    # Each chat item can also be output in JSON format.
+    for c in chat.get().items:
+        print(c.json())
+    '''     
 ```
 
 
-### yt api compatible processor:
-```python
-from pytchat import LiveChat, CompatibleProcessor
+### other
++ Fetch chat with a buffer ([LiveChat](https://github.com/taizan-hokuto/pytchat/wiki/LiveChat))
 
-chat = LiveChat("G1w62uEMZ74", 
-     processor = CompatibleProcessor() )
++ Use with asyncio ([LiveChatAsync](https://github.com/taizan-hokuto/pytchat/wiki/LiveChatAsync))
 
-while chat.is_alive():
-    data = chat.get()
-    polling = data["pollingIntervalMillis"]/1000
-    for c in data["items"]:
-        if c.get("snippet"):
-            print(f"[{c['authorDetails']['displayName']}]"
-                    f"-{c['snippet']['displayMessage']}")
-            time.sleep(polling/len(data["items"]))
++ YT API compatible chat processor ([CompatibleProcessor](https://github.com/taizan-hokuto/pytchat/wiki/CompatibleProcessor))
 
-```
++ Extract archived chat data ([Extractor](https://github.com/taizan-hokuto/pytchat/wiki/Extractor))
 
 
-## Chatdata Structure of Default Processor
-Structure of each item which got from items() function.
+## Structure of Default Processor
+Each item can be got with `sync_items()` function.
 <table>
   <tr>
     <th>name</th>
@@ -110,6 +100,11 @@ Structure of each item which got from items() function.
     <td>str</td>
     <td>emojis are represented by ":(shortcut text):"</td>
   </tr>
+  <tr>
+    <td>messageEx</td>
+    <td>str</td>
+    <td>list of message texts and emoji dicts(id, txt, url).</td>
+  </tr>
   <tr>
     <td>timestamp</td>
     <td>int</td>
@@ -118,22 +113,31 @@ Structure of each item which got from items() function.
   <tr>
     <td>datetime</td>
     <td>str</td>
-    <td></td>
+    <td>e.g. "2019-10-10 12:34:56"</td>
+  </tr>
+    <td>elapsedTime</td>
+    <td>str</td>
+    <td>elapsed time. (e.g. "1:02:27") *Replay Only.</td>
   </tr>
   <tr>
     <td>amountValue</td>
     <td>float</td>
-    <td>ex. 1,234.0</td>
+    <td>e.g. 1,234.0</td>
   </tr>
   <tr>
     <td>amountString</td>
     <td>str</td>
-    <td>ex. "$ 1,234"</td>
+    <td>e.g. "$ 1,234"</td>
   </tr>
   <tr>
     <td>currency</td>
     <td>str</td>
-    <td>ex. "USD"</td>
+    <td><a href="https://en.wikipedia.org/wiki/ISO_4217">ISO 4217 currency codes</a> (e.g. "USD")</td>
+  </tr>
+  <tr>
+    <td>bgColor</td>
+    <td>int</td>
+    <td>RGB Int</td>
   </tr>
   <tr>
     <td>author</td>
@@ -157,7 +161,7 @@ Structure of author object.
   <tr>
     <td>channelId</td>
     <td>str</td>
-    <td></td>
+    <td>*chatter's channel ID.</td>
   </tr>
   <tr>
     <td>channelUrl</td>
@@ -166,12 +170,12 @@ Structure of author object.
   </tr>
   <tr>
     <td>imageUrl</td>
-    <td></td>
+    <td>str</td>
     <td></td>
   </tr>
   <tr>
     <td>badgeUrl</td>
-    <td></td>
+    <td>str</td>
     <td></td>
   </tr>
   <tr>
@@ -200,8 +204,20 @@ Structure of author object.
 
 [![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](LICENSE)
 
+
+## Contributes
+Great thanks:
+
+Most of source code of CLI refer to:
+
+[PetterKraabol / Twitch-Chat-Downloader](https://github.com/PetterKraabol/Twitch-Chat-Downloader)
+
+Progress bar in CLI is based on:
+
+[vladignatyev/progress.py](https://gist.github.com/vladignatyev/06860ec2040cb497f0f3)
+
 ## Author
 
 [taizan-hokuto](https://github.com/taizan-hokuto)
 
-[twitter:@taizan205](https://twitter.com/taizan205)
+[twitter:@taizan205](https://twitter.com/taizan205)

pytchat/__init__.py

@@ -1,19 +1,47 @@
 """
-pytchat is a python library for fetching youtube live chat.
+pytchat is a lightweight python library to browse youtube livechat without Selenium or BeautifulSoup.
 """
-__copyright__    = 'Copyright (C) 2019 taizan-hokuto'
-__version__      = '0.0.1.8'
+__copyright__    = 'Copyright (C) 2019, 2020 taizan-hokuto'
+__version__      = '0.4.3'
 __license__      = 'MIT'
 __author__       = 'taizan-hokuto'
 __author_email__ = '55448286+taizan-hokuto@users.noreply.github.com'
 __url__          = 'https://github.com/taizan-hokuto/pytchat'
 
-__all__ = ["core_async","core_multithread","processors"]
+
+from .exceptions import (
+    ChatParseException,
+    ResponseContextError,
+    NoContents,
+    NoContinuation,
+    IllegalFunctionCall,
+    InvalidVideoIdException,
+    UnknownConnectionError,
+    RetryExceedMaxCount,
+    ChatDataFinished,
+    ReceivedUnknownContinuation,
+    FailedExtractContinuation,
+    VideoInfoParseError,
+    PatternUnmatchError
+)
 
 from .api import (
+    cli,
+    config,
     LiveChat,
     LiveChatAsync,
-    CompatibleProcessor, 
+    ChatProcessor,
+    CompatibleProcessor,
+    DummyProcessor,
+    DefaultProcessor,
+    Extractor, 
+    HTMLArchiver,
+    TSVArchiver,
+    JsonfileArchiver,
     SimpleDisplayProcessor,
-    JsonfileArchiveProcessor
-    )
+    SpeedCalculator,
+    SuperchatCalculator,
+    VideoInfo,
+    create
+)
+# flake8: noqa

pytchat/api.py

@@ -1,7 +1,39 @@
-from .core_async.livechat import LiveChatAsync
+from . import cli
+from . import config
+from .core import create
 from .core_multithread.livechat import LiveChat
-from .processors.default.processor import DefaultProcessor
+from .core_async.livechat import LiveChatAsync
+from .processors.chat_processor import ChatProcessor
 from .processors.compatible.processor import CompatibleProcessor
+from .processors.default.processor import DefaultProcessor
+from .processors.dummy_processor import DummyProcessor
+from .processors.html_archiver import HTMLArchiver
+from .processors.tsv_archiver import TSVArchiver
+from .processors.jsonfile_archiver import JsonfileArchiver
 from .processors.simple_display_processor import SimpleDisplayProcessor
-from .processors.jsonfile_archive_processor import JsonfileArchiveProcessor
+from .processors.speed.calculator import SpeedCalculator
+from .processors.superchat.calculator import SuperchatCalculator
+from .tool.extract.extractor import Extractor
+from .tool.videoinfo import VideoInfo
 
+__all__ = [
+    cli,
+    config,
+    LiveChat,
+    LiveChatAsync,
+    ChatProcessor,
+    CompatibleProcessor,
+    DummyProcessor,
+    DefaultProcessor,
+    Extractor, 
+    HTMLArchiver,
+    TSVArchiver,
+    JsonfileArchiver,
+    SimpleDisplayProcessor,
+    SpeedCalculator,
+    SuperchatCalculator,
+    VideoInfo,
+    create
+]
+
+# flake8: noqa

pytchat/cli/__init__.py

@@ -0,0 +1,177 @@
+import argparse
+import asyncio
+try:
+    from asyncio import CancelledError
+except ImportError:
+    from asyncio.futures import CancelledError
+import os
+import signal
+from json.decoder import JSONDecodeError
+from pathlib import Path
+from httpcore import ReadTimeout as HCReadTimeout, NetworkError as HCNetworkError
+from .arguments import Arguments
+from .echo import Echo
+from .progressbar import ProgressBar
+from .. exceptions import InvalidVideoIdException, NoContents, PatternUnmatchError, UnknownConnectionError
+from .. processors.html_archiver import HTMLArchiver
+from .. tool.extract.extractor import Extractor
+from .. tool.videoinfo import VideoInfo
+from .. util.extract_video_id import extract_video_id
+from .. import util
+from .. import __version__
+
+'''
+Most of CLI modules refer to
+Petter Kraabøl's Twitch-Chat-Downloader
+https://github.com/PetterKraabol/Twitch-Chat-Downloader
+(MIT License)
+
+'''
+
+
+def main():
+    # Arguments
+    parser = argparse.ArgumentParser(description=f'pytchat v{__version__}')
+    parser.add_argument('-v', f'--{Arguments.Name.VIDEO_IDS}', type=str,
+                        help='Video ID (or URL that includes Video ID). You can specify multiple video IDs by '
+                        'separating them with commas without spaces.\n'
+                        'If ID starts with a hyphen (-), enclose the ID in square brackets.')
+    parser.add_argument('-o', f'--{Arguments.Name.OUTPUT}', type=str,
+                        help='Output directory (end with "/"). default="./"', default='./')
+    parser.add_argument(f'--{Arguments.Name.SAVE_ERROR_DATA}', action='store_true',
+                        help='Save error data when error occurs(".dat" file)')
+    parser.add_argument(f'--{Arguments.Name.VERSION}', action='store_true',
+                        help='Show version')
+    parser.add_argument(f'--{Arguments.Name.ECHO}', action='store_true',
+        help='Show chats of specified video')
+
+    Arguments(parser.parse_args().__dict__)
+
+    if Arguments().print_version:
+        print(f'pytchat v{__version__}     © 2019,2020 taizan-hokuto')
+        return
+
+    # Extractor
+    if not Arguments().video_ids:
+        parser.print_help()
+        return
+
+    # Echo
+    if Arguments().echo:
+        if len(Arguments().video_ids) > 1:
+            print("You can specify only one video ID.")
+            return
+        try:
+            Echo(Arguments().video_ids[0]).run()
+        except InvalidVideoIdException as e:
+            print("Invalid video id:", str(e))
+        except Exception as e:
+            print(type(e), str(e))
+        finally:
+            return
+
+    if not os.path.exists(Arguments().output):
+        print("\nThe specified directory does not exist.:{}\n".format(Arguments().output))
+        return
+    try:
+        Runner().run()
+    except CancelledError as e:
+        print(str(e))
+
+
+class Runner:
+    
+    def run(self) -> None:
+        ex = None
+        pbar = None
+        for counter, video_id in enumerate(Arguments().video_ids):
+            if len(Arguments().video_ids) > 1:
+                print(f"\n{'-' * 10} video:{counter + 1} of {len(Arguments().video_ids)} {'-' * 10}")
+
+            try:
+                video_id = extract_video_id(video_id)
+                separated_path = str(Path(Arguments().output)) + os.path.sep
+                path = util.checkpath(separated_path + video_id + '.html')
+                try:
+                    info = VideoInfo(video_id)
+                except (PatternUnmatchError, JSONDecodeError) as e:
+                    print("Cannot parse video information.:{} {}".format(video_id, type(e)))
+                    if Arguments().save_error_data:
+                        util.save(str(e.doc), "ERR", ".dat")
+                    continue
+                except Exception as e:
+                    print("Cannot parse video information.:{} {}".format(video_id, type(e)))
+                    continue
+
+                print(f"\n"
+                    f" video_id: {video_id}\n"
+                    f" channel:  {info.get_channel_name()}\n"
+                    f" title:    {info.get_title()}\n"
+                    f" output path: {path}")
+
+                duration = info.get_duration()
+                pbar = ProgressBar(total=(duration * 1000), status_txt="Extracting")
+                ex = Extractor(video_id,
+                        callback=pbar.disp,
+                        div=10)
+                signal.signal(signal.SIGINT, (lambda a, b: self.cancel(ex, pbar)))
+
+                data = ex.extract()
+                if data == []:
+                    continue
+                pbar.reset("#", "=", total=len(data), status_txt="Rendering  ")
+                processor = HTMLArchiver(path, callback=pbar.disp)
+                processor.process(
+                    [{'video_id': None,
+                    'timeout': 1,
+                    'chatdata': (action["replayChatItemAction"]["actions"][0] for action in data)}]
+                )
+                processor.finalize()
+                pbar.reset('#', '#', status_txt='Completed   ')
+                pbar.close()
+                print()
+                if pbar.is_cancelled():
+                    print("\nThe extraction process has been discontinued.\n")
+            except InvalidVideoIdException:
+                print("Invalid Video ID or URL:", video_id)
+            except NoContents as e:
+                print(f"Abort:{str(e)}:[{video_id}]")
+            except (JSONDecodeError, PatternUnmatchError) as e:
+                print("{}:{}".format(e.msg, video_id))
+                if Arguments().save_error_data:
+                    util.save(e.doc, "ERR_", ".dat")
+            except (UnknownConnectionError, HCNetworkError, HCReadTimeout) as e:
+                print(f"An unknown network error occurred during the processing of [{video_id}]. : " + str(e))
+            except Exception as e:
+                print(f"Abort:{str(type(e))} {str(e)[:80]}")
+            finally:
+                clear_tasks()
+
+        return
+
+    def cancel(self, ex=None, pbar=None) -> None:
+        '''Called when keyboard interrupted has occurred.
+        '''
+        print("\nKeyboard interrupted.\n")
+        if ex and pbar:
+            ex.cancel()
+            pbar.cancel()
+
+
+def clear_tasks():
+    '''
+    Clear remained tasks.
+    Called when internal exception has occurred or
+    after each extraction process is completed.
+    '''
+    async def _shutdown():
+        tasks = [t for t in asyncio.all_tasks()
+                if t is not asyncio.current_task()]
+        for task in tasks:
+            task.cancel()
+            
+    try:
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(_shutdown())
+    except Exception as e:
+        print(e)

pytchat/cli/arguments.py

@@ -0,0 +1,45 @@
+from typing import Optional, Dict, Union, List
+from .singleton import Singleton
+
+'''
+This modules refer to
+Petter Kraabøl's Twitch-Chat-Downloader
+https://github.com/PetterKraabol/Twitch-Chat-Downloader
+(MIT License)
+'''
+
+
+class Arguments(metaclass=Singleton):
+    """
+    Arguments singleton
+    """
+
+    class Name:
+        VERSION: str = 'version'
+        OUTPUT: str = 'output_dir'
+        VIDEO_IDS: str = 'video_id'
+        SAVE_ERROR_DATA: bool = 'save_error_data'
+        ECHO: bool = 'echo'
+
+    def __init__(self,
+                 arguments: Optional[Dict[str, Union[str, bool, int]]] = None):
+        """
+        Initialize arguments
+        :param arguments: Arguments from cli
+        (Optional to call singleton instance without parameters)
+        """
+
+        if arguments is None:
+            print('Error: arguments were not provided')
+            exit()
+
+        self.print_version: bool = arguments[Arguments.Name.VERSION]
+        self.output: str = arguments[Arguments.Name.OUTPUT]
+        self.video_ids: List[int] = []
+        self.save_error_data: bool = arguments[Arguments.Name.SAVE_ERROR_DATA]
+        self.echo: bool = arguments[Arguments.Name.ECHO]
+        # Videos
+
+        if arguments[Arguments.Name.VIDEO_IDS]:
+            self.video_ids = [video_id
+                              for video_id in arguments[Arguments.Name.VIDEO_IDS].split(',')]

pytchat/cli/echo.py

@@ -0,0 +1,22 @@
+import pytchat
+from ..exceptions import ChatDataFinished, NoContents
+from ..util.extract_video_id import extract_video_id
+
+
+class Echo:
+    def __init__(self, video_id):
+        self.video_id = extract_video_id(video_id)
+    
+    def run(self):
+        livechat = pytchat.create(self.video_id)
+        while livechat.is_alive():
+            chatdata = livechat.get()
+            for c in chatdata.sync_items():
+                print(f"{c.datetime} [{c.author.name}] {c.message} {c.amountString}")
+        
+        try:
+            livechat.raise_for_status()
+        except (ChatDataFinished, NoContents):
+            print("Chat finished.")
+        except Exception as e:
+            print(type(e), str(e))

pytchat/cli/progressbar.py

@@ -0,0 +1,54 @@
+'''
+This code is based on
+vladignatyev/progress.py
+https://gist.github.com/vladignatyev/06860ec2040cb497f0f3
+(MIT License)
+'''
+import shutil
+import sys
+
+
+class ProgressBar:
+    def __init__(self, total, status_txt):
+        self._bar_len = 60
+        self._cancelled = False
+        self.reset(total=total, status_txt=status_txt)
+        
+    def reset(self, symbol_done="=", symbol_space=" ", total=100, status_txt=''):
+        self._console_width = shutil.get_terminal_size(fallback=(80, 24)).columns
+        self._symbol_done = symbol_done
+        self._symbol_space = symbol_space
+        self._total = total
+        self._status_txt = status_txt
+        self._count = 0
+
+    def disp(self, _, fetched):
+        self._progress(fetched, self._total)
+
+    def _progress(self, fillin, total):
+        if total == 0 or self._cancelled:
+            return
+        self._count += fillin
+        filled_len = int(round(self._bar_len * self._count / float(total)))
+        percents = round(100.0 * self._count / float(total), 1)
+        if percents > 100:
+            percents = 100.0
+        if filled_len > self._bar_len:
+            filled_len = self._bar_len
+            
+        bar = self._symbol_done * filled_len + \
+              self._symbol_space * (self._bar_len - filled_len)
+        disp = f" [{bar}] {percents:>5.1f}% ...{self._status_txt} "[:self._console_width - 1] + '\r'
+
+        sys.stdout.write(disp)
+        sys.stdout.flush()
+
+    def close(self):
+        if not self._cancelled:
+            self._progress(self._total, self._total)
+
+    def cancel(self):
+        self._cancelled = True
+    
+    def is_cancelled(self):
+        return self._cancelled

pytchat/cli/singleton.py

@@ -0,0 +1,21 @@
+'''
+This modules refer to
+Petter Kraabøl's Twitch-Chat-Downloader
+https://github.com/PetterKraabol/Twitch-Chat-Downloader
+(MIT License)
+'''
+
+
+class Singleton(type):
+    """
+    Abstract class for singletons
+    """
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super().__call__(*args, **kwargs)
+        return cls._instances[cls]
+
+    def get_instance(cls, *args, **kwargs):
+        cls.__call__(*args, **kwargs)

pytchat/config/__init__.py

@@ -1,4 +1,10 @@
-import logging
-LOGGER_MODE = logging.ERROR
+import logging  # noqa
+from . import mylogger
 headers = {
-        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'}
+    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36',
+}
+
+
+def logger(module_name: str, loglevel=None):
+    module_logger = mylogger.get_logger(module_name, loglevel=loglevel)
+    return module_logger

pytchat/config/mylogger.py

@@ -0,0 +1,38 @@
+from logging import NullHandler, getLogger, StreamHandler, FileHandler
+import logging
+from datetime import datetime
+
+
+def get_logger(modname, loglevel=logging.DEBUG):
+    logger = getLogger(modname)
+    if loglevel is None:
+        logger.addHandler(NullHandler())
+        return logger
+    logger.setLevel(loglevel)
+    # create handler1 for showing info
+    handler1 = StreamHandler()
+    my_formatter = MyFormatter()
+    handler1.setFormatter(my_formatter)
+
+    handler1.setLevel(loglevel)
+    logger.addHandler(handler1)
+    # create handler2 for recording log file
+    if loglevel <= logging.DEBUG:
+        handler2 = FileHandler(filename="log.txt", encoding='utf-8')
+        handler2.setLevel(logging.ERROR)
+        handler2.setFormatter(my_formatter)
+
+        logger.addHandler(handler2)
+    return logger
+
+
+class MyFormatter(logging.Formatter):
+    def format(self, record):
+        timestamp = (
+            datetime.fromtimestamp(record.created)).strftime("%m-%d %H:%M:%S")
+        module = (record.module).ljust(15)
+        funcname = (record.funcName).ljust(18)
+        lineno = str(record.lineno).rjust(4)
+        message = record.getMessage()
+
+        return timestamp + '| ' + module + ' { ' + funcname + ':' + lineno + '} - ' + message

pytchat/core/__init__.py

@@ -0,0 +1,7 @@
+from .pytchat import PytchatCore
+from .. util.extract_video_id import extract_video_id
+
+
+def create(video_id: str, **kwargs):
+    _vid = extract_video_id(video_id)
+    return PytchatCore(_vid, **kwargs)

pytchat/core/pytchat.py

@@ -0,0 +1,207 @@
+import httpx
+import json
+import signal
+import time
+import traceback
+import urllib.parse
+from ..parser.live import Parser
+from .. import config
+from .. import exceptions
+from ..paramgen import liveparam, arcparam
+from ..processors.default.processor import DefaultProcessor
+from ..processors.combinator import Combinator
+from ..util.extract_video_id import extract_video_id
+
+headers = config.headers
+MAX_RETRY = 10
+
+
+class PytchatCore:
+    '''
+
+    Parameter
+    ---------
+    video_id : str
+
+    seektime : int
+        start position of fetching chat (seconds).
+        This option is valid for archived chat only.
+        If negative value, chat data posted before the start of the broadcast
+        will be retrieved as well.
+
+    processor : ChatProcessor
+
+    interruptable : bool
+        Allows keyboard interrupts.
+        Set this parameter to False if your own threading program causes
+        the problem.
+
+    force_replay : bool
+        force to fetch archived chat data, even if specified video is live.
+
+    topchat_only : bool
+        If True, get only top chat.
+
+    hold_exception : bool [default:True]
+        If True, when exceptions occur, the exception is held internally,
+        and can be raised by raise_for_status().
+
+    Attributes
+    ---------
+    _is_alive : bool
+        Flag to stop getting chat.
+    '''
+
+    _setup_finished = False
+
+    def __init__(self, video_id,
+                 seektime=-1,
+                 processor=DefaultProcessor(),
+                 interruptable=True,
+                 force_replay=False,
+                 topchat_only=False,
+                 hold_exception=True,
+                 logger=config.logger(__name__),
+                 ):
+        self._video_id = extract_video_id(video_id)
+        self.seektime = seektime
+        if isinstance(processor, tuple):
+            self.processor = Combinator(processor)
+        else:
+            self.processor = processor
+        self._is_alive = True
+        self._is_replay = force_replay
+        self._hold_exception = hold_exception
+        self._exception_holder = None
+        self._parser = Parser(
+            is_replay=self._is_replay,
+            exception_holder=self._exception_holder
+        )
+        self._first_fetch = True
+        self._fetch_url = "live_chat/get_live_chat?continuation="
+        self._topchat_only = topchat_only
+        self._logger = logger
+        if interruptable:
+            signal.signal(signal.SIGINT, lambda a, b: self.terminate())
+        self._setup()
+
+    def _setup(self):
+        time.sleep(0.1)  # sleep shortly to prohibit skipping fetching data
+        """Fetch first continuation parameter,
+        create and start _listen loop.
+        """
+        self.continuation = liveparam.getparam(self._video_id, 3)
+
+    def _get_chat_component(self):
+        
+        ''' Fetch chat data and store them into buffer,
+        get next continuaiton parameter and loop.
+
+        Parameter
+        ---------
+        continuation : str
+            parameter for next chat data
+        '''
+        try:
+            with httpx.Client(http2=True) as client:
+                if self.continuation and self._is_alive:
+                    contents = self._get_contents(self.continuation, client, headers)
+                    metadata, chatdata = self._parser.parse(contents)
+                    timeout = metadata['timeoutMs'] / 1000
+                    chat_component = {
+                        "video_id": self._video_id,
+                        "timeout": timeout,
+                        "chatdata": chatdata
+                    }
+                    self.continuation = metadata.get('continuation')
+                    return chat_component
+        except exceptions.ChatParseException as e:
+            self._logger.debug(f"[{self._video_id}]{str(e)}")
+            self._raise_exception(e)
+        except (TypeError, json.JSONDecodeError) as e:
+            self._logger.error(f"{traceback.format_exc(limit=-1)}")
+            self._raise_exception(e)
+
+        self._logger.debug(f"[{self._video_id}]finished fetching chat.")
+        self._raise_exception(exceptions.ChatDataFinished)
+
+    def _get_contents(self, continuation, client, headers):
+        '''Get 'continuationContents' from livechat json.
+           If contents is None at first fetching,
+           try to fetch archive chat data.
+
+          Return:
+          -------
+            'continuationContents' which includes metadata & chat data.
+        '''
+        livechat_json = (
+            self._get_livechat_json(continuation, client, headers)
+        )
+        contents = self._parser.get_contents(livechat_json)
+        if self._first_fetch:
+            if contents is None or self._is_replay:
+                '''Try to fetch archive chat data.'''
+                self._parser.is_replay = True
+                self._fetch_url = "live_chat_replay/get_live_chat_replay?continuation="
+                continuation = arcparam.getparam(
+                    self._video_id, self.seektime, self._topchat_only)
+                livechat_json = (self._get_livechat_json(continuation, client, headers))
+                reload_continuation = self._parser.reload_continuation(
+                    self._parser.get_contents(livechat_json))
+                if reload_continuation:
+                    livechat_json = (self._get_livechat_json(
+                        reload_continuation, client, headers))
+                contents = self._parser.get_contents(livechat_json)
+                self._is_replay = True
+            self._first_fetch = False
+        return contents
+
+    def _get_livechat_json(self, continuation, client, headers):
+        '''
+        Get json which includes chat data.
+        '''
+        continuation = urllib.parse.quote(continuation)
+        livechat_json = None
+        err = None
+        url = f"https://www.youtube.com/{self._fetch_url}{continuation}&pbj=1"
+        for _ in range(MAX_RETRY + 1):
+            with client:
+                try:
+                    livechat_json = client.get(url, headers=headers).json()
+                    break
+                except (json.JSONDecodeError, httpx.ConnectTimeout, httpx.ReadTimeout, httpx.ConnectError) as e:
+                    err = e
+                    time.sleep(2)
+                    continue
+        else:
+            self._logger.error(f"[{self._video_id}]"
+                               f"Exceeded retry count. Last error: {str(err)}")
+            self._raise_exception(exceptions.RetryExceedMaxCount())
+        return livechat_json
+    
+    def get(self):
+        if self.is_alive():
+            chat_component = self._get_chat_component()
+            return self.processor.process([chat_component])
+        else:
+            return []
+
+    def is_replay(self):
+        return self._is_replay
+
+    def is_alive(self):
+        return self._is_alive
+
+    def terminate(self):
+        self._is_alive = False
+        self.processor.finalize()
+
+    def raise_for_status(self):
+        if self._exception_holder is not None:
+            raise self._exception_holder
+
+    def _raise_exception(self, exception: Exception = None):
+        self._is_alive = False
+        if self._hold_exception is False:
+            raise exception
+        self._exception_holder = exception

pytchat/core_async/buffer.py

@@ -1,28 +1,38 @@
 
 import asyncio
+
+
 class Buffer(asyncio.Queue):
     '''
-    チャットデータを格納するバッファの役割を持つLIFOキュー
+    Buffer for storing chat data.
 
     Parameter
     ---------
     maxsize : int
-        格納するチャットブロックの最大個数。0の場合は無限。
-        最大値を超える場合は古いチャットブロックから破棄される。
+        Maximum number of chat blocks to be stored.
+        If it exceeds the maximum, the oldest chat block will be discarded.
     '''
-    def __init__(self,maxsize = 0):
+
+    def __init__(self, maxsize=0):
         super().__init__(maxsize)
-    
-    async def put(self,item):
+
+    async def put(self, item):
         if item is None:
-            return 
+            return
         if super().full():
             super().get_nowait()
         await super().put(item)
 
+    def put_nowait(self, item):
+        if item is None:
+            return
+        if super().full():
+            super().get_nowait()
+        super().put_nowait(item)
+
     async def get(self):
         ret = []
         ret.append(await super().get())
         while not super().empty():
             ret.append(super().get_nowait())
-        return ret
+        return ret

pytchat/core_async/listen_manager.py

@@ -1,184 +0,0 @@
-import asyncio
-from .listener import AsyncListener
-from .. import config
-from .. import mylogger
-import datetime
-import os
-import aiohttp
-import signal
-import threading
-from .buffer import Buffer
-from  concurrent.futures import CancelledError
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
-
-class ListenManager:
-
-    '''
-    動画IDまたは動画IDのリストを受け取り、
-    動画IDに対応したListenerを生成・保持する。
-
-    #Attributes
-    ----------
-    _listeners: dict  
-        ListenManegerがつかんでいるListener達のリスト.  
-        key:動画ID value:動画IDに対応するListener
-    _queue: Queue
-        動画IDを外部から受け渡しするためのキュー
-        _queueが空である間は、ノンブロッキングで他のタスクを実行
-        _queueに動画IDが投入されると、_dequeueメソッドで
-        直ちにListenerを生成し返す。
-    _event: threading.Event
-        キーボードのCTRL+Cを検知するためのEventオブジェクト
-    '''
-    def __init__(self,interruptable = True):
-        #チャット監視中の動画リスト
-        self._listeners={}
-        self._tasks = []
-        #外部からvideoを受け取るためのキュー
-        self._queue = asyncio.Queue()
-        self._event = threading.Event()
-        self._ready_queue()
-        self._is_alive = True
-        #キーボードのCtrl+cを押したとき、_hundler関数を呼び出すように設定
-        signal.signal(signal.SIGINT, (lambda a, b: self._handler(self._event, a, b)))
-
-    def is_alive(self)->bool:
-        '''
-        ListenManagerが稼働中であるか。
-            True->稼働中
-            False->Ctrl+Cが押されて終了
-        '''
-        logger.debug(f'check is_alive() :{self._is_alive}')
-        return self._is_alive
-    
-    def _handler(self, event, sig, handler):
-        '''
-        Ctrl+Cが押下されたとき、終了フラグをセットする。
-        '''
-        logger.debug('Ctrl+c pushed')
-        self._is_alive = False
-        logger.debug('terminating listeners.')
-        for listener in self._listeners.values():
-            listener.terminate()
-        logger.debug('end.')
-
-
-    def _ready_queue(self):
-        #loop = asyncio.get_event_loop()
-        self._tasks.append(
-            asyncio.create_task(self._dequeue())
-            )
-
-
-    async def set_video_ids(self,video_ids:list):
-        for video_id in video_ids:
-            if video_id:
-                await self._queue.put(video_id)
-
-
-    async def get_listener(self,video_id) -> AsyncListener:
-        return await self._create_listener(video_id)
-            
-    # async def getlivechat(self,video_id):
-    #     '''
-    #     指定された動画IDのチャットデータを返す
-
-    #     Parameter
-    #     ----------
-    #     video_id: str
-    #         動画ID
-
-    #     Return
-    #     ----------
-    #         引数で受け取った動画IDに対応する
-    #         Listenerオブジェクトへの参照
-            
-    #     '''
-    #     logger.debug('manager get/create listener')
-    #     listener = await self._create_listener(video_id)
-    #     '''
-    #     上が完了しないうちに、下が呼び出される
-    #     '''
-    #     if not listener._initialized:
-    #         await asyncio.sleep(2)
-    #         return []
-    #     if listener:
-    #         #listener._isfirstrun=False
-    #         return await listener.getlivechat()
-        
-
-
-    async def _dequeue(self):
-        '''
-        キューに入った動画IDを
-        Listener登録に回す。
-        
-        '''
-        while True:
-            video_id = await self._queue.get()
-            #listenerを登録、タスクとして実行する
-            logger.debug(f'deque got [{video_id}]')
-            await self._create_listener(video_id)
-
-    async def _create_listener(self, video_id) -> AsyncListener:
-        '''
-        Listenerを作成しチャット取得中リストに加え、
-        Listenerを返す
-        '''
-        if video_id is None or not isinstance(video_id, str):
-            raise TypeError('video_idは文字列でなければなりません')
-        if video_id in self._listeners:
-            return self._listeners[video_id]
-        else:
-            #listenerを登録する
-            listener = AsyncListener(video_id,interruptable = False,buffer = Buffer())
-            self._listeners.setdefault(video_id,listener)
-            #task = asyncio.ensure_future(listener.initialize())
-            #await asyncio.gather(listener.initialize())
-            #task.add_done_callback(self.finish)
-            #await listener.initialize()
-            #self._tasks.append(task)
-
-            return listener
-
- 
-    def finish(self,sender):
-        try:
-            if sender.result():
-                video_id = sender.result()[0]
-                message = sender.result()[1]
-            
-                #listener終了時のコールバック        
-                #sender.result()[]でデータを取得できる
-                logger.info(f'終了しました VIDEO_ID:[{video_id}] message:{message}')
-                #logger.info(f'終了しました')
-                if video_id in self._listeners:
-                    self._listeners.pop(video_id)
-        except CancelledError:
-            logger.debug('cancelled.')
-
-    def get_listeners(self):
-        return self._listeners
-
-    def shutdown(self):
-        '''
-        ListenManegerを終了する
-        '''
-        logger.debug("start shutdown")
-        self._is_alive =False
-        try:
-            #Listenerを停止する。
-            for listener in self._listeners.values():
-                listener.terminate()
-            #taskをキャンセルする。
-            for task in self._tasks:
-                if not task.done():
-                    #print(task)
-                    task.cancel()
-        except Exception as er:
-            logger.info(str(er),type(er))
-
-        logger.debug("finished.")
-
-    def get_tasks(self):
-        return self._tasks

pytchat/core_async/livechat.py

@@ -1,299 +1,353 @@
-import aiohttp, asyncio, async_timeout
-import datetime
+
+import asyncio
+import httpx
 import json
-import random
 import signal
-import threading
 import time
 import traceback
 import urllib.parse
-from aiohttp.client_exceptions import ClientConnectorError
+from asyncio import Queue
 from concurrent.futures import CancelledError
 from .buffer import Buffer
-from .parser import Parser
+from ..parser.live import Parser
 from .. import config
-from .. import mylogger
-from ..exceptions  import ChatParseException,IllegalFunctionCall
-from ..paramgen    import liveparam
-
+from .. import exceptions
+from ..paramgen import liveparam, arcparam
 from ..processors.default.processor import DefaultProcessor
+from ..processors.combinator import Combinator
+from ..util.extract_video_id import extract_video_id
 
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
-MAX_RETRY = 10
 headers = config.headers
-
+MAX_RETRY = 10
 
 
 class LiveChatAsync:
-    '''asyncio(aiohttp)を利用してYouTubeのライブ配信のチャットデータを取得する。
+    '''LiveChatAsync object fetches chat data and stores them
+    in a buffer with asyncio.
 
     Parameter
     ---------
     video_id : str
-        動画ID
+
+    seektime : int
+        start position of fetching chat (seconds).
+        This option is valid for archived chat only.
+        If negative value, chat data posted before the start of the broadcast
+        will be retrieved as well.
 
     processor : ChatProcessor
-        チャットデータを加工するオブジェクト
 
-    buffer : Buffer(maxsize:20[default])
-        チャットデータchat_componentを格納するバッファ。
-        maxsize : 格納できるchat_componentの個数
-        default値20個。1個で約5~10秒分。
+    buffer : Buffer
+        buffer of chat data fetched background.
 
     interruptable : bool
-        Ctrl+Cによる処理中断を行うかどうか。
+        Allows keyboard interrupts.
+        Set this parameter to False if your own threading program causes
+        the problem.
 
     callback : func
-        _listen()関数から一定間隔で自動的に呼びだす関数。
+        function called periodically from _listen().
 
     done_callback : func
-        listener終了時に呼び出すコールバック。
+        function called when listener ends.
 
     exception_handler : func
-        例外を処理する関数
 
     direct_mode : bool
-        Trueの場合、bufferを使わずにcallbackを呼ぶ。
-        Trueの場合、callbackの設定が必須
-        (設定していない場合IllegalFunctionCall例外を発生させる）  
-          
+        If True, invoke specified callback function without using buffer.
+        callback is required. If not, IllegalFunctionCall will be raised.
+
+    force_replay : bool
+        force to fetch archived chat data, even if specified video is live.
+
+    topchat_only : bool
+        If True, get only top chat.
+
     Attributes
     ---------
     _is_alive : bool
-        チャット取得を停止するためのフラグ
+        Flag to stop getting chat.
     '''
 
     _setup_finished = False
 
     def __init__(self, video_id,
-                processor = DefaultProcessor(),
-                buffer = None,
-                interruptable = True,
-                callback = None,
-                done_callback = None,
-                exception_handler = None,
-                direct_mode = False): 
-        self.video_id  = video_id
-        self.processor = processor
+                 seektime=-1,
+                 processor=DefaultProcessor(),
+                 buffer=None,
+                 interruptable=True,
+                 callback=None,
+                 done_callback=None,
+                 exception_handler=None,
+                 direct_mode=False,
+                 force_replay=False,
+                 topchat_only=False,
+                 logger=config.logger(__name__),
+                 ):
+        self._video_id = extract_video_id(video_id)
+        self.seektime = seektime
+        if isinstance(processor, tuple):
+            self.processor = Combinator(processor)
+        else:
+            self.processor = processor
         self._buffer = buffer
         self._callback = callback
         self._done_callback = done_callback
         self._exception_handler = exception_handler
         self._direct_mode = direct_mode
-        self._is_alive   = True
+        self._is_alive = True
+        self._is_replay = force_replay
+        self._parser = Parser(is_replay=self._is_replay)
+        self._pauser = Queue()
+        self._pauser.put_nowait(None)
+        self._first_fetch = True
+        self._fetch_url = "live_chat/get_live_chat?continuation="
+        self._topchat_only = topchat_only
+        self._logger = logger
+        self.exception = None
+        LiveChatAsync._logger = logger
 
+        if exception_handler:
+            self._set_exception_handler(exception_handler)
+        if interruptable:
+            signal.signal(signal.SIGINT,
+                (lambda a, b: self._keyboard_interrupt()))
         self._setup()
-        
-        if not LiveChatAsync._setup_finished:
-            LiveChatAsync._setup_finished = True
-            if exception_handler == None:
-                self._set_exception_handler(self._handle_exception)
-            else:
-                self._set_exception_handler(exception_handler)
-            if interruptable:
-                signal.signal(signal.SIGINT,  
-                    (lambda a, b:asyncio.create_task(
-                    LiveChatAsync.shutdown(None,signal.SIGINT,b))
-                ))
- 
+
     def _setup(self):
-        #direct modeがTrueでcallback未設定の場合例外発生。
+        # An exception is raised when direct mode is true and no callback is set.
         if self._direct_mode:
             if self._callback is None:
-                raise IllegalFunctionCall(
-                    "direct_mode=Trueの場合callbackの設定が必須です。")
+                raise exceptions.IllegalFunctionCall(
+                    "When direct_mode=True, callback parameter is required.")
         else:
-            #direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成
+            # Create a default buffer if `direct_mode` is False and buffer is not set.
             if self._buffer is None:
-                self._buffer = Buffer(maxsize = 20)
-                #callbackが指定されている場合はcallbackを呼ぶループタスクを作成
+                self._buffer = Buffer(maxsize=20)
+                # Create a loop task to call callback if the `callback` param is specified.
             if self._callback is None:
-                pass 
+                pass
             else:
-                #callbackを呼ぶループタスクの開始
+                # Create a loop task to call callback if the `callback` param is specified.
                 loop = asyncio.get_event_loop()
                 loop.create_task(self._callback_loop(self._callback))
-        #_listenループタスクの開始
+        # Start a loop task for _listen()
         loop = asyncio.get_event_loop()
-        listen_task = loop.create_task(self._startlisten())
-        #add_done_callbackの登録
+        self.listen_task = loop.create_task(self._startlisten())
+        # Register add_done_callback
         if self._done_callback is None:
-            listen_task.add_done_callback(self.finish)
+            self.listen_task.add_done_callback(self._finish)
         else:
-            listen_task.add_done_callback(self._done_callback)
+            self.listen_task.add_done_callback(self._done_callback)
 
     async def _startlisten(self):
-        """最初のcontinuationパラメータを取得し、
-        _listenループを開始する
+        """Fetch first continuation parameter,
+        create and start _listen loop.
         """
-        initial_continuation = await self._get_initial_continuation()
-        if initial_continuation is None:
-            self.terminate()
-            logger.debug(f"[{self.video_id}]No initial continuation.")
-            return
+        initial_continuation = liveparam.getparam(self._video_id, 3)
         await self._listen(initial_continuation)
 
-    async def _get_initial_continuation(self):
-        ''' チャットデータ取得に必要な最初のcontinuationを取得する。'''
-        try:    
-            initial_continuation = liveparam.getparam(self.video_id)
-        except ChatParseException as e:
-            self.terminate()
-            logger.debug(f"[{self.video_id}]Error:{str(e)}")
-            return
-        except KeyError:
-            logger.debug(f"[{self.video_id}]KeyError:"
-                         f"{traceback.format_exc(limit = -1)}")
-            self.terminate()
-            return
-        return initial_continuation
-
     async def _listen(self, continuation):
-        ''' continuationに紐付いたチャットデータを取得し
-        チャットデータを格納、
-        次のcontinuaitonを取得してループする。
+        ''' Fetch chat data and store them into buffer,
+        get next continuaiton parameter and loop.
 
         Parameter
         ---------
         continuation : str
-            次のチャットデータ取得に必要なパラメータ
+            parameter for next chat data
         '''
         try:
-            async with aiohttp.ClientSession() as session:
+            async with httpx.AsyncClient(http2=True) as client:
                 while(continuation and self._is_alive):
-                    livechat_json = (await 
-                      self._get_livechat_json(continuation, session, headers)
-                    )
-                    metadata, chatdata =  Parser.parse( livechat_json )
-                    timeout = metadata['timeoutMs']/1000
+                    continuation = await self._check_pause(continuation)
+                    contents = await self._get_contents(
+                        continuation, client, headers)
+                    metadata, chatdata = self._parser.parse(contents)
+
+                    timeout = metadata['timeoutMs'] / 1000
                     chat_component = {
-                        "video_id" : self.video_id,
-                        "timeout"  : timeout,
-                        "chatdata" : chatdata
+                        "video_id": self._video_id,
+                        "timeout": timeout,
+                        "chatdata": chatdata
                     }
-                    time_mark =time.time()
+                    time_mark = time.time()
                     if self._direct_mode:
-                        await self._callback(
-                            self.processor.process([chat_component])
-                            )
+                        processed_chat = self.processor.process(
+                            [chat_component])
+                        if isinstance(processed_chat, tuple):
+                            await self._callback(*processed_chat)
+                        else:
+                            await self._callback(processed_chat)
                     else:
                         await self._buffer.put(chat_component)
-                    diff_time = timeout - (time.time()-time_mark)
-                    await asyncio.sleep(diff_time)        
-                    continuation = metadata.get('continuation')     
-        except ChatParseException as e:
-            logger.info(f"{str(e)}（video_id:\"{self.video_id}\"）")
-            return            
-        except (TypeError , json.JSONDecodeError) :
-            logger.error(f"{traceback.format_exc(limit = -1)}")
-            return
-        
-        logger.debug(f"[{self.video_id}]チャット取得を終了しました。")
+                    diff_time = timeout - (time.time() - time_mark)
+                    await asyncio.sleep(diff_time)
+                    continuation = metadata.get('continuation')
+        except exceptions.ChatParseException as e:
+            self._logger.debug(f"[{self._video_id}]{str(e)}")
+            raise
+        except (TypeError, json.JSONDecodeError):
+            self._logger.error(f"{traceback.format_exc(limit = -1)}")
+            raise
 
-    async def _get_livechat_json(self, continuation, session, headers):
+        self._logger.debug(f"[{self._video_id}] finished fetching chat.")
+        raise exceptions.ChatDataFinished
+
+    async def _check_pause(self, continuation):
+        if self._pauser.empty():
+            '''pause'''
+            await self._pauser.get()
+            '''resume:
+                prohibit from blocking by putting None into _pauser.
+            '''
+            self._pauser.put_nowait(None)
+            if not self._is_replay:
+                continuation = liveparam.getparam(
+                    self._video_id, 3, self._topchat_only)
+        return continuation
+
+    async def _get_contents(self, continuation, client, headers):
+        '''Get 'continuationContents' from livechat json.
+           If contents is None at first fetching,
+           try to fetch archive chat data.
+
+          Return:
+          -------
+            'continuationContents' which includes metadata & chatdata.
         '''
-        チャットデータが格納されたjsonデータを取得する。
+        livechat_json = await self._get_livechat_json(continuation, client, headers)
+        contents = self._parser.get_contents(livechat_json)
+        if self._first_fetch:
+            if contents is None or self._is_replay:
+                '''Try to fetch archive chat data.'''
+                self._parser.is_replay = True
+                self._fetch_url = "live_chat_replay/get_live_chat_replay?continuation="
+                continuation = arcparam.getparam(
+                    self._video_id, self.seektime, self._topchat_only)
+                livechat_json = (await self._get_livechat_json(
+                                 continuation, client, headers))
+                reload_continuation = self._parser.reload_continuation(
+                    self._parser.get_contents(livechat_json))
+                if reload_continuation:
+                    livechat_json = (await self._get_livechat_json(
+                        reload_continuation, client, headers))
+                contents = self._parser.get_contents(livechat_json)
+                self._is_replay = True
+            self._first_fetch = False
+        return contents
+
+    async def _get_livechat_json(self, continuation, client, headers):
+        '''
+        Get json which includes chat data.
         '''
         continuation = urllib.parse.quote(continuation)
         livechat_json = None
-        status_code = 0
-        url =(
-            f"https://www.youtube.com/live_chat/get_live_chat?"
-            f"continuation={continuation}&pbj=1")
+        url = f"https://www.youtube.com/{self._fetch_url}{continuation}&pbj=1"
         for _ in range(MAX_RETRY + 1):
-            async with session.get(url ,headers = headers) as resp:
-                try:
-                    text = await resp.text()
-                    status_code = resp.status
-                    livechat_json = json.loads(text)
-                    break
-                except (ClientConnectorError,json.JSONDecodeError) :
-                    await asyncio.sleep(1)
-                    continue
+            try:
+                resp = await client.get(url, headers=headers)
+                livechat_json = resp.json()
+                break
+            except (json.JSONDecodeError, httpx.HTTPError):
+                await asyncio.sleep(1)
+                continue
         else:
-            logger.error(f"[{self.video_id}]"
-                    f"Exceeded retry count. status_code={status_code}")
+            self._logger.error(f"[{self._video_id}]"
+                               f"Exceeded retry count.")
             return None
         return livechat_json
 
-    async def _callback_loop(self,callback):
-        """ コンストラクタでcallbackを指定している場合、バックグラウンドで
-        callbackに指定された関数に一定間隔でチャットデータを投げる。        
-        
+    async def _callback_loop(self, callback):
+        """ If a callback is specified in the constructor,
+        it throws chat data at regular intervals to the
+        function specified in the callback in the backgroun
+
         Parameter
         ---------
         callback : func
-            加工済みのチャットデータを渡す先の関数。
+            function to which the processed chat data is passed.
         """
         while self.is_alive():
             items = await self._buffer.get()
-            data = self.processor.process(items)
-            await callback(data)
+            processed_chat = self.processor.process(items)
+            if isinstance(processed_chat, tuple):
+                await self._callback(*processed_chat)
+            else:
+                await self._callback(processed_chat)
 
     async def get(self):
-        """ bufferからデータを取り出し、processorに投げ、
-        加工済みのチャットデータを返す。
-        
+        """
+        Retrieves data from the buffer,
+        throws it to the processor,
+        and returns the processed chat data.
+
         Returns
-             : Processorによって加工されたチャットデータ
+             : Chat data processed by the Processor
         """
         if self._callback is None:
-            items = await self._buffer.get()
-            return  self.processor.process(items)
-        raise IllegalFunctionCall(
-            "既にcallbackを登録済みのため、get()は実行できません。")
+            if self.is_alive():
+                items = await self._buffer.get()
+                return self.processor.process(items)
+            else:
+                return []
+        raise exceptions.IllegalFunctionCall(
+            "Callback parameter is already set, so get() cannot be performed.")
+
+    def is_replay(self):
+        return self._is_replay
+
+    def pause(self):
+        if self._callback is None:
+            return
+        if not self._pauser.empty():
+            self._pauser.get_nowait()
+
+    def resume(self):
+        if self._callback is None:
+            return
+        if self._pauser.empty():
+            self._pauser.put_nowait(None)
 
     def is_alive(self):
         return self._is_alive
 
-    def finish(self,sender):
-        '''Listener終了時のコールバック'''
-        try: 
-            self.terminate()
+    def _finish(self, sender):
+        '''Called when the _listen() task finished.'''
+        try:
+            self._task_finished()
         except CancelledError:
-            logger.debug(f'[{self.video_id}]cancelled:{sender}')
+            self._logger.debug(f'[{self._video_id}] cancelled:{sender}')
 
     def terminate(self):
-        '''
-        Listenerを終了する。
-        '''
+        if self._pauser.empty():
+            self._pauser.put_nowait(None)
         self._is_alive = False
-        if self._direct_mode == False:
-            #bufferにダミーオブジェクトを入れてis_alive()を判定させる
-            self._buffer.put_nowait({'chatdata':'','timeout':1}) 
-        logger.info(f'終了しました:[{self.video_id}]')
- 
+        self._buffer.put_nowait({})
+        self.processor.finalize()
+    
+    def _keyboard_interrupt(self):
+        self.exception = exceptions.ChatDataFinished()
+        self.terminate()
+
+    def _task_finished(self):
+        '''
+        Terminate fetching chats.
+        '''
+        if self.is_alive():
+            self.terminate()
+        try:
+            self.listen_task.result()
+        except Exception as e:
+            self.exception = e
+            if not isinstance(e, exceptions.ChatParseException):
+                self._logger.error(f'Internal exception - {type(e)}{str(e)}')
+        self._logger.info(f'[{self._video_id}] finished.')
+
+    def raise_for_status(self):
+        if self.exception is not None:
+            raise self.exception
+
     @classmethod
     def _set_exception_handler(cls, handler):
         loop = asyncio.get_event_loop()
-        #default handler: cls._handle_exception
         loop.set_exception_handler(handler)
-    
-    @classmethod
-    def _handle_exception(cls, loop, context):
-        #msg = context.get("exception", context["message"])
-        if not isinstance(context["exception"],CancelledError):
-            logger.error(f"Caught exception: {context}")
-        loop= asyncio.get_event_loop()
-        loop.create_task(cls.shutdown(None,None,None))
-
-    @classmethod
-    async def shutdown(cls, event, sig = None, handler=None):
-        logger.debug("シャットダウンしています")
-        tasks = [t for t in asyncio.all_tasks() if t is not
-        asyncio.current_task()]
-        [task.cancel() for task in tasks]
-
-        logger.debug(f"残っているタスクを終了しています")
-        await asyncio.gather(*tasks,return_exceptions=True)
-        loop = asyncio.get_event_loop()
-        loop.stop()
-
-
-
-  
-    
-
-
-
-

pytchat/core_async/parser.py

@@ -1,40 +0,0 @@
-import json
-from .. import config
-from .. import mylogger
-from .. exceptions import ( 
-    ResponseContextError, 
-    NoContentsException, 
-    NoContinuationsException )
-
-
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
-
-
-class Parser:
-    @classmethod
-    def parse(cls, jsn):
-        if jsn is None: 
-            return {'timeoutMs':0,'continuation':None},[]
-        if jsn['response']['responseContext'].get('errors'):
-            raise ResponseContextError('動画に接続できません。'
-        '動画IDが間違っているか、動画が削除／非公開の可能性があります。')
-        contents=jsn['response'].get('continuationContents')
-        #配信が終了した場合、もしくはチャットデータが取得できない場合
-        if contents is None:
-            raise NoContentsException('チャットデータを取得できませんでした。')
-
-        cont = contents['liveChatContinuation']['continuations'][0]
-        if cont is None:
-            raise NoContinuationsException('Continuationがありません。')
-        metadata = (cont.get('invalidationContinuationData')  or
-                    cont.get('timedContinuationData')         or
-                    cont.get('reloadContinuationData')
-                    )
-        if metadata is None:
-            unknown = list(cont.keys())[0]
-            if unknown:
-                logger.error(f"Received unknown continuation type:{unknown}")
-                metadata = cont.get(unknown)
-        metadata.setdefault('timeoutMs', 10000)
-        chatdata = contents['liveChatContinuation'].get('actions')
-        return metadata, chatdata

pytchat/core_multithread/buffer.py

@@ -1,31 +1,40 @@
 
 import queue
 
+
 class Buffer(queue.Queue):
     '''
-    チャットデータを格納するバッファの役割を持つLIFOキュー
+    Buffer for storing chat data.
 
     Parameter
     ---------
-    max_size : int
-        格納するチャットブロックの最大個数。0の場合は無限。
-        最大値を超える場合は古いチャットブロックから破棄される。
+    maxsize : int
+        Maximum number of chat blocks to be stored.
+        If it exceeds the maximum, the oldest chat block will be discarded.
     '''
-    def __init__(self,maxsize = 0):
+
+    def __init__(self, maxsize=0):
         super().__init__(maxsize=maxsize)
-    
-    def put(self,item):
+
+    def put(self, item):
         if item is None:
-            return 
+            return
         if super().full():
             super().get_nowait()
         else:
             super().put(item)
-            
+
+    def put_nowait(self, item):
+        if item is None:
+            return
+        if super().full():
+            super().get_nowait()
+        else:
+            super().put_nowait(item)
 
     def get(self):
         ret = []
         ret.append(super().get())
         while not super().empty():
             ret.append(super().get())
-        return ret
+        return ret

pytchat/core_multithread/livechat.py

@@ -1,277 +1,338 @@
-import requests
-import datetime
+import httpx
 import json
-import random
 import signal
-import threading
 import time
 import traceback
 import urllib.parse
 from concurrent.futures import CancelledError, ThreadPoolExecutor
+from queue import Queue
+from threading import Event
 from .buffer import Buffer
-from .parser import Parser
+from ..parser.live import Parser
 from .. import config
-from .. import mylogger
-from ..exceptions  import ChatParseException,IllegalFunctionCall
-from ..paramgen    import liveparam
+from .. import exceptions
+from ..paramgen import liveparam, arcparam
 from ..processors.default.processor import DefaultProcessor
+from ..processors.combinator import Combinator
+from ..util.extract_video_id import extract_video_id
 
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
-MAX_RETRY = 10
 headers = config.headers
-
+MAX_RETRY = 10
 
 
 class LiveChat:
-    ''' スレッドプールを利用してYouTubeのライブ配信のチャットデータを取得する
+    '''
+    LiveChat object fetches chat data and stores them
+    in a buffer with ThreadpoolExecutor.
 
     Parameter
     ---------
     video_id : str
-        動画ID
+
+    seektime : int
+        start position of fetching chat (seconds).
+        This option is valid for archived chat only.
+        If negative value, chat data posted before the start of the broadcast
+        will be retrieved as well.
 
     processor : ChatProcessor
-        チャットデータを加工するオブジェクト
 
-    buffer : Buffer(maxsize:20[default])
-        チャットデータchat_componentを格納するバッファ。
-        maxsize : 格納できるchat_componentの個数
-        default値20個。1個で約5~10秒分。
+    buffer : Buffer
+        buffer of chat data fetched background.
 
     interruptable : bool
-        Ctrl+Cによる処理中断を行うかどうか。
+        Allows keyboard interrupts.
+        Set this parameter to False if your own threading program causes
+        the problem.
 
     callback : func
-        _listen()関数から一定間隔で自動的に呼びだす関数。
+        function called periodically from _listen().
 
     done_callback : func
-        listener終了時に呼び出すコールバック。
+        function called when listener ends.
 
     direct_mode : bool
-        Trueの場合、bufferを使わずにcallbackを呼ぶ。
-        Trueの場合、callbackの設定が必須
-        (設定していない場合IllegalFunctionCall例外を発生させる）  
+        If True, invoke specified callback function without using buffer.
+        callback is required. If not, IllegalFunctionCall will be raised.
+
+    force_replay : bool
+        force to fetch archived chat data, even if specified video is live.
+
+    topchat_only : bool
+        If True, get only top chat.
 
     Attributes
     ---------
     _executor : ThreadPoolExecutor
-        チャットデータ取得ループ（_listen）用のスレッド
+        This is used for _listen() loop.
 
     _is_alive : bool
-        チャット取得を終了したか
+        Flag to stop getting chat.
     '''
 
     _setup_finished = False
-    #チャット監視中のListenerのリスト
-    _listeners= []
+
     def __init__(self, video_id,
-                processor = DefaultProcessor(),
-                buffer = Buffer(maxsize = 20),
-                interruptable = True,
-                callback = None,
-                done_callback = None,
-                direct_mode = False
-                ):
-        self.video_id  = video_id
-        self.processor = processor
+                 seektime=-1,
+                 processor=DefaultProcessor(),
+                 buffer=None,
+                 interruptable=True,
+                 callback=None,
+                 done_callback=None,
+                 direct_mode=False,
+                 force_replay=False,
+                 topchat_only=False,
+                 logger=config.logger(__name__)
+                 ):
+        self._video_id = extract_video_id(video_id)
+        self.seektime = seektime
+        if isinstance(processor, tuple):
+            self.processor = Combinator(processor)
+        else:
+            self.processor = processor
         self._buffer = buffer
         self._callback = callback
         self._done_callback = done_callback
         self._executor = ThreadPoolExecutor(max_workers=2)
         self._direct_mode = direct_mode
-        self._is_alive   = True
-        self._parser = Parser()
+        self._is_alive = True
+        self._is_replay = force_replay
+        self._parser = Parser(is_replay=self._is_replay)
+        self._pauser = Queue()
+        self._pauser.put_nowait(None)
+        self._first_fetch = True
+        self._fetch_url = "live_chat/get_live_chat?continuation="
+        self._topchat_only = topchat_only
+        self._event = Event()
+        self._logger = logger
+        self.exception = None
+        if interruptable:
+            signal.signal(signal.SIGINT, lambda a, b: self.terminate())
         self._setup()
 
-        if not LiveChat._setup_finished:
-            LiveChat._setup_finished = True
-            if interruptable:
-                signal.signal(signal.SIGINT,  (lambda a, b:  
-                (LiveChat.shutdown(None,signal.SIGINT,b))
-                ))
-        LiveChat._listeners.append(self)
-
     def _setup(self):
-        #direct modeがTrueでcallback未設定の場合例外発生。
+        # An exception is raised when direct mode is true and no callback is set.
         if self._direct_mode:
             if self._callback is None:
-                raise IllegalFunctionCall(
-                    "direct_mode=Trueの場合callbackの設定が必須です。")
+                raise exceptions.IllegalFunctionCall(
+                    "When direct_mode=True, callback parameter is required.")
         else:
-            #direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成
+            # Create a default buffer if `direct_mode` is False and buffer is not set.
             if self._buffer is None:
-                self._buffer = Buffer(maxsize = 20)
-                #callbackが指定されている場合はcallbackを呼ぶループタスクを作成
+                self._buffer = Buffer(maxsize=20)
+                # Create a loop task to call callback if the `callback` param is specified.
             if self._callback is None:
-                pass 
+                pass
             else:
-                #callbackを呼ぶループタスクの開始
-                self._executor.submit(self._callback_loop,self._callback)
-        #_listenループタスクの開始
-        listen_task = self._executor.submit(self._startlisten)
-        #add_done_callbackの登録
+                # Start a loop task calling callback function.
+                self._executor.submit(self._callback_loop, self._callback)
+        # Start a loop task for _listen()
+        self.listen_task = self._executor.submit(self._startlisten)
+        # Register add_done_callback
         if self._done_callback is None:
-            listen_task.add_done_callback(self.finish)
+            self.listen_task.add_done_callback(self._finish)
         else:
-            listen_task.add_done_callback(self._done_callback)
+            self.listen_task.add_done_callback(self._done_callback)
 
     def _startlisten(self):
-        """最初のcontinuationパラメータを取得し、
-        _listenループのタスクを作成し開始する
+        time.sleep(0.1)  # sleep shortly to prohibit skipping fetching data
+        """Fetch first continuation parameter,
+        create and start _listen loop.
         """
-        initial_continuation = self._get_initial_continuation()
-        if initial_continuation is None:
-            self.terminate()
-            logger.debug(f"[{self.video_id}]No initial continuation.")
-            return
+        initial_continuation = liveparam.getparam(self._video_id, 3)
         self._listen(initial_continuation)
-   
-    def _get_initial_continuation(self):
-        ''' チャットデータ取得に必要な最初のcontinuationを取得する。'''
-        try:    
-            initial_continuation = liveparam.getparam(self.video_id)
-        except ChatParseException as e:
-            self.terminate()
-            logger.debug(f"[{self.video_id}]Error:{str(e)}")
-            return
-        except KeyError:
-            logger.debug(f"[{self.video_id}]KeyError:"
-                         f"{traceback.format_exc(limit = -1)}")
-            self.terminate()
-            return
-        return initial_continuation
 
     def _listen(self, continuation):
-        ''' continuationに紐付いたチャットデータを取得し
-        にチャットデータを格納、
-        次のcontinuaitonを取得してループする
+        ''' Fetch chat data and store them into buffer,
+        get next continuaiton parameter and loop.
 
         Parameter
         ---------
         continuation : str
-            次のチャットデータ取得に必要なパラメータ
+            parameter for next chat data
         '''
         try:
-            with requests.Session() as session:
+            with httpx.Client(http2=True) as client:
                 while(continuation and self._is_alive):
-                    livechat_json = (
-                      self._get_livechat_json(continuation, session, headers)
-                    )
-                    metadata, chatdata =  self._parser.parse( livechat_json )
-                    #チャットデータを含むコンポーネントを組み立ててbufferに投入する
-                    timeout = metadata['timeoutMs']/1000
+                    continuation = self._check_pause(continuation)
+                    contents = self._get_contents(continuation, client, headers)
+                    metadata, chatdata = self._parser.parse(contents)
+                    timeout = metadata['timeoutMs'] / 1000
                     chat_component = {
-                        "video_id" : self.video_id,
-                        "timeout"  : timeout,
-                        "chatdata" : chatdata
+                        "video_id": self._video_id,
+                        "timeout": timeout,
+                        "chatdata": chatdata
                     }
-                    time_mark =time.time()
+                    time_mark = time.time()
                     if self._direct_mode:
-                        self._callback(
-                            self.processor.process([chat_component])
-                            )
+                        processed_chat = self.processor.process(
+                            [chat_component])
+                        if isinstance(processed_chat, tuple):
+                            self._callback(*processed_chat)
+                        else:
+                            self._callback(processed_chat)
                     else:
                         self._buffer.put(chat_component)
-                    #次のchatを取得するまでsleepする
-                    diff_time = timeout - (time.time()-time_mark)
-                    if diff_time < 0 : diff_time=0
-                    time.sleep(diff_time)        
-                    #次のチャットデータのcontinuationパラメータを取り出す。
-                    continuation = metadata.get('continuation')  
-    
-                    #whileループ先頭に戻る
-        except ChatParseException as e:
-            logger.error(f"{str(e)}（動画ID:\"{self.video_id}\"）")
-            return            
-        except (TypeError , json.JSONDecodeError) :
-            logger.error(f"{traceback.format_exc(limit = -1)}")
-            return
-        
-        logger.debug(f"[{self.video_id}]チャット取得を終了しました。")
+                    diff_time = timeout - (time.time() - time_mark)
+                    self._event.wait(diff_time if diff_time > 0 else 0)
+                    continuation = metadata.get('continuation')
+        except exceptions.ChatParseException as e:
+            self._logger.debug(f"[{self._video_id}]{str(e)}")
+            raise
+        except (TypeError, json.JSONDecodeError):
+            self._logger.error(f"{traceback.format_exc(limit=-1)}")
+            raise
 
-    def _get_livechat_json(self, continuation, session, headers):
+        self._logger.debug(f"[{self._video_id}] finished fetching chat.")
+        raise exceptions.ChatDataFinished
+
+    def _check_pause(self, continuation):
+        if self._pauser.empty():
+            '''pause'''
+            self._pauser.get()
+            '''resume:
+                prohibit from blocking by putting None into _pauser.
+            '''
+            self._pauser.put_nowait(None)
+            if not self._is_replay:
+                continuation = liveparam.getparam(self._video_id, 3)
+        return continuation
+
+    def _get_contents(self, continuation, client, headers):
+        '''Get 'continuationContents' from livechat json.
+           If contents is None at first fetching,
+           try to fetch archive chat data.
+
+          Return:
+          -------
+            'continuationContents' which includes metadata & chat data.
         '''
-        チャットデータが格納されたjsonデータを取得する。
+        livechat_json = (
+            self._get_livechat_json(continuation, client, headers)
+        )
+        contents = self._parser.get_contents(livechat_json)
+        if self._first_fetch:
+            if contents is None or self._is_replay:
+                '''Try to fetch archive chat data.'''
+                self._parser.is_replay = True
+                self._fetch_url = "live_chat_replay/get_live_chat_replay?continuation="
+                continuation = arcparam.getparam(
+                    self._video_id, self.seektime, self._topchat_only)
+                livechat_json = (self._get_livechat_json(continuation, client, headers))
+                reload_continuation = self._parser.reload_continuation(
+                    self._parser.get_contents(livechat_json))
+                if reload_continuation:
+                    livechat_json = (self._get_livechat_json(
+                        reload_continuation, client, headers))
+                contents = self._parser.get_contents(livechat_json)
+                self._is_replay = True
+            self._first_fetch = False
+        return contents
+
+    def _get_livechat_json(self, continuation, client, headers):
+        '''
+        Get json which includes chat data.
         '''
         continuation = urllib.parse.quote(continuation)
         livechat_json = None
-        status_code = 0
-        url =(
-            f"https://www.youtube.com/live_chat/get_live_chat?"
-            f"continuation={continuation}&pbj=1")
+        url = f"https://www.youtube.com/{self._fetch_url}{continuation}&pbj=1"
         for _ in range(MAX_RETRY + 1):
-            with session.get(url ,headers = headers) as resp:
+            with client:
                 try:
-                    text = resp.text
-                    status_code = resp.status_code
-                    livechat_json = json.loads(text)
+                    livechat_json = client.get(url, headers=headers).json()
                     break
-                except json.JSONDecodeError :
-                    time.sleep(1)
+                except (json.JSONDecodeError, httpx.HTTPError):
+                    time.sleep(2)
                     continue
         else:
-            logger.error(f"[{self.video_id}]"
-                    f"Exceeded retry count. status_code={status_code}")
-            return None
+            self._logger.error(f"[{self._video_id}]"
+                               f"Exceeded retry count.")
+            raise exceptions.RetryExceedMaxCount()
         return livechat_json
-  
-    def _callback_loop(self,callback):
-        """ コンストラクタでcallbackを指定している場合、バックグラウンドで
-        callbackに指定された関数に一定間隔でチャットデータを投げる。        
-        
+
+    def _callback_loop(self, callback):
+        """ If a callback is specified in the constructor,
+        it throws chat data at regular intervals to the
+        function specified in the callback in the backgroun
+
         Parameter
         ---------
         callback : func
-            加工済みのチャットデータを渡す先の関数。
+            function to which the processed chat data is passed.
         """
         while self.is_alive():
             items = self._buffer.get()
-            data = self.processor.process(items)
-            callback(data)
+            processed_chat = self.processor.process(items)
+            if isinstance(processed_chat, tuple):
+                self._callback(*processed_chat)
+            else:
+                self._callback(processed_chat)
 
     def get(self):
-        """ bufferからデータを取り出し、processorに投げ、
-        加工済みのチャットデータを返す。
-        
+        """
+        Retrieves data from the buffer,
+        throws it to the processor,
+        and returns the processed chat data.
+
         Returns
-             : Processorによって加工されたチャットデータ
+             : Chat data processed by the Processor
         """
         if self._callback is None:
-            items = self._buffer.get()
-            return  self.processor.process(items)
-        raise IllegalFunctionCall(
-            "既にcallbackを登録済みのため、get()は実行できません。")
+            if self.is_alive():
+                items = self._buffer.get()
+                return self.processor.process(items)
+            else:
+                return []
+        raise exceptions.IllegalFunctionCall(
+            "Callback parameter is already set, so get() cannot be performed.")
+
+    def is_replay(self):
+        return self._is_replay
+
+    def pause(self):
+        if self._callback is None:
+            return
+        if not self._pauser.empty():
+            self._pauser.get()
+
+    def resume(self):
+        if self._callback is None:
+            return
+        if self._pauser.empty():
+            self._pauser.put_nowait(None)
 
     def is_alive(self):
         return self._is_alive
 
-    def finish(self,sender):
-        '''Listener終了時のコールバック'''
-        try: 
-            self.terminate()
+    def _finish(self, sender):
+        '''Called when the _listen() task finished.'''
+        try:
+            self._task_finished()
         except CancelledError:
-            logger.debug(f'[{self.video_id}]cancelled:{sender}')
+            self._logger.debug(f'[{self._video_id}] cancelled:{sender}')
 
     def terminate(self):
         '''
-        Listenerを終了する。
+        Terminate fetching chats.
         '''
+        if self._pauser.empty():
+            self._pauser.put_nowait(None)
         self._is_alive = False
-        if self._direct_mode == False:
-            #bufferにダミーオブジェクトを入れてis_alive()を判定させる
-            self._buffer.put({'chatdata':'','timeout':1}) 
-        logger.info(f'終了しました:[{self.video_id}]')
-  
-    @classmethod
-    def shutdown(cls, event, sig = None, handler=None):
-        logger.debug("シャットダウンしています")
-        for t in LiveChat._listeners:
-            t._is_alive = False
-
-
-
-    
-
-
+        self._buffer.put({})
+        self._event.set()
+        self.processor.finalize()
 
+    def _task_finished(self):
+        if self.is_alive():
+            self.terminate()
+        try:
+            self.listen_task.result()
+        except Exception as e:
+            self.exception = e
+            if not isinstance(e, exceptions.ChatParseException):
+                self._logger.error(f'Internal exception - {type(e)}{str(e)}')
+        self._logger.info(f'[{self._video_id}] finished.')
 
+    def raise_for_status(self):
+        if self.exception is not None:
+            raise self.exception

pytchat/core_multithread/parser.py

@@ -1,39 +0,0 @@
-import json
-from .. import config
-from .. import mylogger
-from .. exceptions import ( 
-    ResponseContextError, 
-    NoContentsException, 
-    NoContinuationsException )
-
-
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
-
-
-class Parser:
-    def parse(self, jsn):
-        if jsn is None: 
-            return {'timeoutMs':0,'continuation':None},[]
-        if jsn['response']['responseContext'].get('errors'):
-            raise ResponseContextError('動画に接続できません。'
-        '動画IDが間違っているか、動画が削除／非公開の可能性があります。')
-        contents=jsn['response'].get('continuationContents')
-        #配信が終了した場合、もしくはチャットデータが取得できない場合
-        if contents is None:
-            raise NoContentsException('チャットデータを取得できませんでした。')
-
-        cont = contents['liveChatContinuation']['continuations'][0]
-        if cont is None:
-            raise NoContinuationsException('Continuationがありません。')
-        metadata = (cont.get('invalidationContinuationData')  or
-                    cont.get('timedContinuationData')         or
-                    cont.get('reloadContinuationData')
-                    )
-        if metadata is None:
-            unknown = list(cont.keys())[0]
-            if unknown:
-                logger.error(f"Received unknown continuation type:{unknown}")
-                metadata = cont.get(unknown)
-        metadata.setdefault('timeoutMs', 10000)
-        chatdata = contents['liveChatContinuation'].get('actions')
-        return metadata, chatdata

pytchat/exceptions.py

@@ -1,43 +1,81 @@
 class ChatParseException(Exception):
     '''
-    チャットデータをパースするライブラリが投げる例外の基底クラス
+    Base exception thrown by the parser
     '''
     pass
 
-class NoYtinitialdataException(ChatParseException):
-    '''
-    配信ページ内にチャットデータurlが見つからないときに投げる例外
-    '''
-    pass
 
 class ResponseContextError(ChatParseException):
     '''
-    配信ページでチャットデータ無効の時に投げる例外
-    '''
-    pass
-
-class NoLivechatRendererException(ChatParseException):
-    '''
-    チャットデータのJSON中にlivechatRendererがない時に投げる例外
+    Thrown when chat data is invalid.
     '''
     pass
 
 
-class NoContentsException(ChatParseException):
+class NoContents(ChatParseException):
     '''
-    チャットデータのJSON中にContinuationContentsがない時に投げる例外
+    Thrown when ContinuationContents is missing in JSON.
     '''
     pass
 
-class NoContinuationsException(ChatParseException):
+
+class NoContinuation(ChatParseException):
     '''
-    チャットデータのContinuationContents中にcontinuationがない時に投げる例外
+    Thrown when continuation is missing in ContinuationContents.
     '''
     pass
 
+
 class IllegalFunctionCall(Exception):
     '''
-    set_callback()を実行済みにもかかわらず
-    get()を呼び出した場合の例外
+    Thrown when get() is called even though
+    set_callback() has been executed.
     '''
     pass
+
+
+class InvalidVideoIdException(Exception):
+    '''
+    Thrown when the video_id is not exist (VideoInfo).
+    '''
+    def __init__(self, doc):
+        self.msg = "InvalidVideoIdException"
+        self.doc = doc
+
+
+class UnknownConnectionError(Exception):
+    pass
+
+
+class RetryExceedMaxCount(Exception):
+    '''
+    Thrown when the number of retries exceeds the maximum value.
+    '''
+    pass
+
+
+class ChatDataFinished(ChatParseException):
+    pass
+
+
+class ReceivedUnknownContinuation(ChatParseException):
+    pass
+
+
+class FailedExtractContinuation(ChatDataFinished):
+    pass
+
+
+class VideoInfoParseError(Exception):
+    '''
+    Base exception when parsing video info.
+    '''
+
+
+class PatternUnmatchError(VideoInfoParseError):
+    '''
+    Thrown when failed to parse video info with unmatched pattern.
+    '''
+    def __init__(self, doc=''):
+        self.msg = "PatternUnmatchError"
+        self.doc = doc

pytchat/mylogger.py

@@ -1,31 +0,0 @@
-import logging
-import datetime
-
-
-def get_logger(modname,mode=logging.DEBUG):
-    logger = logging.getLogger(modname)
-    if mode == None:
-        logger.addHandler(logging.NullHandler())
-        return logger
-    logger.setLevel(mode)
-    #create handler1 for showing info
-    handler1 = logging.StreamHandler()
-    my_formatter  = MyFormatter()
-    handler1.setFormatter(my_formatter)
-
-    handler1.setLevel(mode) 
-    logger.addHandler(handler1)
-    #create handler2 for recording log file
-    if mode <= logging.DEBUG:
-        handler2 = logging.FileHandler(filename="log.txt")
-        handler2.setLevel(logging.ERROR)
-        handler2.setFormatter(my_formatter)
-
-
-        logger.addHandler(handler2)
-    return logger
-
-class MyFormatter(logging.Formatter):
-    def format(self, record):
-        s =(datetime.datetime.fromtimestamp(record.created)).strftime("%m-%d %H:%M:%S")+'| '+ (record.module).ljust(15)+(' { '+record.funcName).ljust(20) +":"+str(record.lineno).rjust(4)+'} - '+record.getMessage()
-        return s

pytchat/paramgen/arcparam.py

@@ -0,0 +1,62 @@
+from .pb.header_pb2 import Header
+from .pb.replay_pb2 import Continuation
+from urllib.parse import quote
+import base64
+
+'''
+Generate continuation parameter of youtube replay chat.
+
+Author: taizan-hokuto
+
+ver 0.0.1 2019.10.05 : Initial release.
+ver 0.0.2 2020.05.30 : Use Protocol Buffers.
+'''
+
+
+def _gen_vid(video_id) -> str:
+    header = Header()
+    header.info.video.id = video_id
+    header.terminator = 1
+    return base64.urlsafe_b64encode(header.SerializeToString()).decode()
+
+
+def _build(video_id, seektime, topchat_only) -> str:
+    chattype = 1
+    timestamp = 0
+    if topchat_only:
+        chattype = 4
+
+    fetch_before_start = 3
+    if seektime < 0:
+        fetch_before_start = 4
+    elif seektime == 0:
+        timestamp = 1
+    else:
+        timestamp = int(seektime * 1000000)
+    continuation = Continuation()
+    entity = continuation.entity
+    entity.header = _gen_vid(video_id)
+    entity.timestamp = timestamp
+    entity.s6 = 0
+    entity.s7 = 0
+    entity.s8 = 0
+    entity.s9 = fetch_before_start
+    entity.s10 = ''
+    entity.s12 = chattype
+    entity.chattype.value = chattype
+    entity.s15 = 0
+    return quote(
+        base64.urlsafe_b64encode(continuation.SerializeToString()).decode())
+
+
+def getparam(video_id, seektime=-1, topchat_only=False) -> str:
+    '''
+    Parameter
+    ---------
+    seektime : int
+        unit:seconds
+        start position of fetching chat data.
+    topchat_only : bool
+        if True, fetch only 'top chat'
+    '''
+    return _build(video_id, seektime, topchat_only)

pytchat/paramgen/liveparam.py

@@ -1,12 +1,21 @@
-from base64 import urlsafe_b64encode as b64enc
-from functools import reduce
-import calendar, datetime, pytz
-import math
+from .pb.header_pb2 import Header
+from .pb.live_pb2 import Continuation
+from urllib.parse import quote
+import base64
 import random
-import urllib.parse
+import time
+
+'''
+Generate continuation parameter of youtube live chat.
+
+Author: taizan-hokuto
+
+ver 0.0.1 2019.10.05 : Initial release.
+ver 0.0.2 2020.05.30 : Use Protocol Buffers.
+'''
 
 
-def _gen_vid(video_id):
+def _gen_vid(video_id) -> str:
     """generate video_id parameter.
     Parameter
     ---------
@@ -14,136 +23,66 @@ def _gen_vid(video_id):
 
     Return
     ---------
-    byte[] : base64 encoded video_id parameter.
+    str : base64 encoded video_id parameter.
     """
-    header_magic = b'\x0A\x0F\x0A\x0D\x0A'
-    header_id =  video_id.encode()
-    header_sep_1 = b'\x1A'
-    header_sep_2 = b'\x43\xAA\xB9\xC1\xBD\x01\x3D\x0A'
-    header_suburl = ('https://www.youtube.com/live_chat?v='
-                    f'{video_id}&is_popout=1').encode()
-    header_terminator = b'\x20\x02'
+    header = Header()
+    header.info.video.id = video_id
+    header.terminator = 1
+    return base64.urlsafe_b64encode(header.SerializeToString()).decode()
 
-    item = [
-        header_magic,
-        _nval(len(header_id)),
-        header_id,
-        header_sep_1,
-        header_sep_2,
-        _nval(len(header_suburl)),
-        header_suburl,
-        header_terminator
-    ]
 
-    return urllib.parse.quote(
-        b64enc(reduce(lambda x, y: x+y, item)).decode()
-    ).encode()
+def _build(video_id, ts1, ts2, ts3, ts4, ts5, topchat_only) -> str:
+    chattype = 1
+    if topchat_only:
+        chattype = 4
+    continuation = Continuation()
+    entity = continuation.entity
 
-def _nval(val):
-    """convert value to byte array"""
-    if val<0: raise ValueError
-    buf = b''
-    while val >> 7:
-        m = val & 0xFF | 0x80
-        buf += m.to_bytes(1,'big')
-        val >>= 7
-    buf += val.to_bytes(1,'big')
-    return buf
+    entity.header = _gen_vid(video_id)
+    entity.timestamp1 = ts1
+    entity.s6 = 0
+    entity.s7 = 0
+    entity.s8 = 1
+    entity.body.b1 = 0
+    entity.body.b2 = 0
+    entity.body.b3 = 0
+    entity.body.b4 = 0
+    entity.body.b7 = ''
+    entity.body.b8 = 0
+    entity.body.b9 = ''
+    entity.body.timestamp2 = ts2
+    entity.body.b11 = 3
+    entity.body.b15 = 0
+    entity.timestamp3 = ts3
+    entity.timestamp4 = ts4
+    entity.s13 = chattype
+    entity.chattype.value = chattype
+    entity.s17 = 0
+    entity.str19.value = 0
+    entity.timestamp5 = ts5
 
-def _build(video_id, _ts1, _ts2, _ts3, _ts4, _ts5, topchatonly = False):
-    #_short_type2
-    switch_01 = b'\x04' if topchatonly else b'\x01'
-    header_magic= b'\xD2\x87\xCC\xC8\x03'
+    return quote(
+        base64.urlsafe_b64encode(continuation.SerializeToString()).decode()
+    )
 
-    sep_0       = b'\x1A'
-    vid         = _gen_vid(video_id)
-    time_tag    = b'\x28'
-    timestamp1  = _nval(_ts1)
-    sep_1       = b'\x30\x00\x38\x00\x40\x02\x4A'
-    un_len      = b'\x2B'
-    sep_2       = b'\x08\x00\x10\x00\x18\x00\x20\x00'
-    chkstr      = b'\x2A\x0E\x73\x74\x61\x74\x69\x63\x63\x68\x65\x63\x6B\x73\x75\x6D'
-    sep_3       = b'\x3A\x00\x40\x00\x4A'
-    sep_4_len   = b'\x02'
-    sep_4       = b'\x08\x01'
-    ts_2_start  = b'\x50'
-    timestamp2  = _nval(_ts2)
-    ts_2_end    = b'\x58'
-    sep_5       = b'\x03'
-    ts_3_start  = b'\x50'
-    timestamp3  = _nval(_ts3)
-    ts_3_end    = b'\x58'
-    timestamp4  = _nval(_ts4)
-    sep_6       = b'\x68'
-    #switch
-    sep_7       = b'\x82\x01\x04\x08'
-    #switch
-    sep_8       = b'\x10\x00'
-    sep_9       = b'\x88\x01\x00\xA0\x01'
-    timestamp5  = _nval(_ts5)
 
-    body = [
-        sep_0,
-        _nval(len(vid)),
-        vid,
-        time_tag,
-        timestamp1,
-        sep_1,
-        un_len,
-        sep_2,
-        chkstr,
-        sep_3,
-        sep_4_len,
-        sep_4,
-        ts_2_start,
-        timestamp2,
-        ts_2_end,
-        sep_5,
-        ts_3_start,
-        timestamp3,
-        ts_3_end,
-        timestamp4,
-        sep_6,
-        switch_01,#
-        sep_7,
-        switch_01,#
-        sep_8,
-        sep_9,
-        timestamp5
-    ]
-
-    body = reduce(lambda x, y: x+y, body)
-    
-    return  urllib.parse.quote(
-                b64enc( header_magic +
-                        _nval(len(body)) +
-                        body
-                ).decode()
-            )
-
-    
 def _times(past_sec):
-    
-    def unixts_now():
-        now = datetime.datetime.now(pytz.utc)
-        return calendar.timegm(now.utctimetuple())
-
-    n = unixts_now()
-    
-    _ts1= n - random.uniform(0,1*3)    
-    _ts2= n - random.uniform(0.01,0.99)    
-    _ts3= n - past_sec + random.uniform(0,1)
-    _ts4= n - random.uniform(10*60,60*60)         
-    _ts5= n - random.uniform(0.01,0.99)
-    return list(map(lambda x:int(x*1000000),[_ts1,_ts2,_ts3,_ts4,_ts5]))
+    n = int(time.time())
+    _ts1 = n - random.uniform(0, 1 * 3)
+    _ts2 = n - random.uniform(0.01, 0.99)
+    _ts3 = n - past_sec + random.uniform(0, 1)
+    _ts4 = n - random.uniform(10 * 60, 60 * 60)
+    _ts5 = n - random.uniform(0.01, 0.99)
+    return list(map(lambda x: int(x * 1000000), [_ts1, _ts2, _ts3, _ts4, _ts5]))
 
 
-def getparam(video_id,past_sec = 60):
+def getparam(video_id, past_sec=0, topchat_only=False) -> str:
     '''
     Parameter
     ---------
     past_sec : int
         seconds to load past chat data
+    topchat_only : bool
+        if True, fetch only 'top chat'
     '''
-    return _build(video_id,*_times(past_sec))
-
+    return _build(video_id, *_times(past_sec), topchat_only)

pytchat/paramgen/pb/header_pb2.py

@@ -0,0 +1,159 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: header.proto
+
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='header.proto',
+  package='',
+  syntax='proto3',
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n\x0cheader.proto\"\x13\n\x05Video\x12\n\n\x02id\x18\x01 \x01(\t\"#\n\nHeaderInfo\x12\x15\n\x05video\x18\x01 \x01(\x0b\x32\x06.Video\"7\n\x06Header\x12\x19\n\x04info\x18\x01 \x01(\x0b\x32\x0b.HeaderInfo\x12\x12\n\nterminator\x18\x04 \x01(\x05\x62\x06proto3'
+)
+
+
+
+
+_VIDEO = _descriptor.Descriptor(
+  name='Video',
+  full_name='Video',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='id', full_name='Video.id', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=16,
+  serialized_end=35,
+)
+
+
+_HEADERINFO = _descriptor.Descriptor(
+  name='HeaderInfo',
+  full_name='HeaderInfo',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='video', full_name='HeaderInfo.video', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=37,
+  serialized_end=72,
+)
+
+
+_HEADER = _descriptor.Descriptor(
+  name='Header',
+  full_name='Header',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='info', full_name='Header.info', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='terminator', full_name='Header.terminator', index=1,
+      number=4, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=74,
+  serialized_end=129,
+)
+
+_HEADERINFO.fields_by_name['video'].message_type = _VIDEO
+_HEADER.fields_by_name['info'].message_type = _HEADERINFO
+DESCRIPTOR.message_types_by_name['Video'] = _VIDEO
+DESCRIPTOR.message_types_by_name['HeaderInfo'] = _HEADERINFO
+DESCRIPTOR.message_types_by_name['Header'] = _HEADER
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Video = _reflection.GeneratedProtocolMessageType('Video', (_message.Message,), {
+  'DESCRIPTOR' : _VIDEO,
+  '__module__' : 'header_pb2'
+  # @@protoc_insertion_point(class_scope:Video)
+  })
+_sym_db.RegisterMessage(Video)
+
+HeaderInfo = _reflection.GeneratedProtocolMessageType('HeaderInfo', (_message.Message,), {
+  'DESCRIPTOR' : _HEADERINFO,
+  '__module__' : 'header_pb2'
+  # @@protoc_insertion_point(class_scope:HeaderInfo)
+  })
+_sym_db.RegisterMessage(HeaderInfo)
+
+Header = _reflection.GeneratedProtocolMessageType('Header', (_message.Message,), {
+  'DESCRIPTOR' : _HEADER,
+  '__module__' : 'header_pb2'
+  # @@protoc_insertion_point(class_scope:Header)
+  })
+_sym_db.RegisterMessage(Header)
+
+
+# @@protoc_insertion_point(module_scope)

pytchat/paramgen/pb/live_pb2.py

@@ -0,0 +1,381 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: live.proto
+
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='live.proto',
+  package='live',
+  syntax='proto3',
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n\nlive.proto\x12\x04live\"\x88\x01\n\x04\x42ody\x12\n\n\x02\x62\x31\x18\x01 \x01(\x05\x12\n\n\x02\x62\x32\x18\x02 \x01(\x05\x12\n\n\x02\x62\x33\x18\x03 \x01(\x05\x12\n\n\x02\x62\x34\x18\x04 \x01(\x05\x12\n\n\x02\x62\x37\x18\x07 \x01(\t\x12\n\n\x02\x62\x38\x18\x08 \x01(\x05\x12\n\n\x02\x62\x39\x18\t \x01(\t\x12\x12\n\ntimestamp2\x18\n \x01(\x03\x12\x0b\n\x03\x62\x31\x31\x18\x0b \x01(\x05\x12\x0b\n\x03\x62\x31\x35\x18\x0f \x01(\x05\"\x19\n\x08\x43hatType\x12\r\n\x05value\x18\x01 \x01(\x05\"\x16\n\x05STR19\x12\r\n\x05value\x18\x01 \x01(\x05\"\x8a\x02\n\x12\x43ontinuationEntity\x12\x0e\n\x06header\x18\x03 \x01(\t\x12\x12\n\ntimestamp1\x18\x05 \x01(\x03\x12\n\n\x02s6\x18\x06 \x01(\x05\x12\n\n\x02s7\x18\x07 \x01(\x05\x12\n\n\x02s8\x18\x08 \x01(\x05\x12\x18\n\x04\x62ody\x18\t \x01(\x0b\x32\n.live.Body\x12\x12\n\ntimestamp3\x18\n \x01(\x03\x12\x12\n\ntimestamp4\x18\x0b \x01(\x03\x12\x0b\n\x03s13\x18\r \x01(\x05\x12 \n\x08\x63hattype\x18\x10 \x01(\x0b\x32\x0e.live.ChatType\x12\x0b\n\x03s17\x18\x11 \x01(\x05\x12\x1a\n\x05str19\x18\x13 \x01(\x0b\x32\x0b.live.STR19\x12\x12\n\ntimestamp5\x18\x14 \x01(\x03\";\n\x0c\x43ontinuation\x12+\n\x06\x65ntity\x18\xfa\xc0\x89\x39 \x01(\x0b\x32\x18.live.ContinuationEntityb\x06proto3'
+)
+
+
+
+
+_BODY = _descriptor.Descriptor(
+  name='Body',
+  full_name='live.Body',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='b1', full_name='live.Body.b1', index=0,
+      number=1, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='b2', full_name='live.Body.b2', index=1,
+      number=2, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='b3', full_name='live.Body.b3', index=2,
+      number=3, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='b4', full_name='live.Body.b4', index=3,
+      number=4, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='b7', full_name='live.Body.b7', index=4,
+      number=7, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='b8', full_name='live.Body.b8', index=5,
+      number=8, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='b9', full_name='live.Body.b9', index=6,
+      number=9, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='timestamp2', full_name='live.Body.timestamp2', index=7,
+      number=10, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='b11', full_name='live.Body.b11', index=8,
+      number=11, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='b15', full_name='live.Body.b15', index=9,
+      number=15, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=21,
+  serialized_end=157,
+)
+
+
+_CHATTYPE = _descriptor.Descriptor(
+  name='ChatType',
+  full_name='live.ChatType',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='value', full_name='live.ChatType.value', index=0,
+      number=1, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=159,
+  serialized_end=184,
+)
+
+
+_STR19 = _descriptor.Descriptor(
+  name='STR19',
+  full_name='live.STR19',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='value', full_name='live.STR19.value', index=0,
+      number=1, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=186,
+  serialized_end=208,
+)
+
+
+_CONTINUATIONENTITY = _descriptor.Descriptor(
+  name='ContinuationEntity',
+  full_name='live.ContinuationEntity',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='header', full_name='live.ContinuationEntity.header', index=0,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='timestamp1', full_name='live.ContinuationEntity.timestamp1', index=1,
+      number=5, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s6', full_name='live.ContinuationEntity.s6', index=2,
+      number=6, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s7', full_name='live.ContinuationEntity.s7', index=3,
+      number=7, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s8', full_name='live.ContinuationEntity.s8', index=4,
+      number=8, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='body', full_name='live.ContinuationEntity.body', index=5,
+      number=9, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='timestamp3', full_name='live.ContinuationEntity.timestamp3', index=6,
+      number=10, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='timestamp4', full_name='live.ContinuationEntity.timestamp4', index=7,
+      number=11, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s13', full_name='live.ContinuationEntity.s13', index=8,
+      number=13, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='chattype', full_name='live.ContinuationEntity.chattype', index=9,
+      number=16, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s17', full_name='live.ContinuationEntity.s17', index=10,
+      number=17, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='str19', full_name='live.ContinuationEntity.str19', index=11,
+      number=19, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='timestamp5', full_name='live.ContinuationEntity.timestamp5', index=12,
+      number=20, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=211,
+  serialized_end=477,
+)
+
+
+_CONTINUATION = _descriptor.Descriptor(
+  name='Continuation',
+  full_name='live.Continuation',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='entity', full_name='live.Continuation.entity', index=0,
+      number=119693434, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=479,
+  serialized_end=538,
+)
+
+_CONTINUATIONENTITY.fields_by_name['body'].message_type = _BODY
+_CONTINUATIONENTITY.fields_by_name['chattype'].message_type = _CHATTYPE
+_CONTINUATIONENTITY.fields_by_name['str19'].message_type = _STR19
+_CONTINUATION.fields_by_name['entity'].message_type = _CONTINUATIONENTITY
+DESCRIPTOR.message_types_by_name['Body'] = _BODY
+DESCRIPTOR.message_types_by_name['ChatType'] = _CHATTYPE
+DESCRIPTOR.message_types_by_name['STR19'] = _STR19
+DESCRIPTOR.message_types_by_name['ContinuationEntity'] = _CONTINUATIONENTITY
+DESCRIPTOR.message_types_by_name['Continuation'] = _CONTINUATION
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Body = _reflection.GeneratedProtocolMessageType('Body', (_message.Message,), {
+  'DESCRIPTOR' : _BODY,
+  '__module__' : 'live_pb2'
+  # @@protoc_insertion_point(class_scope:live.Body)
+  })
+_sym_db.RegisterMessage(Body)
+
+ChatType = _reflection.GeneratedProtocolMessageType('ChatType', (_message.Message,), {
+  'DESCRIPTOR' : _CHATTYPE,
+  '__module__' : 'live_pb2'
+  # @@protoc_insertion_point(class_scope:live.ChatType)
+  })
+_sym_db.RegisterMessage(ChatType)
+
+STR19 = _reflection.GeneratedProtocolMessageType('STR19', (_message.Message,), {
+  'DESCRIPTOR' : _STR19,
+  '__module__' : 'live_pb2'
+  # @@protoc_insertion_point(class_scope:live.STR19)
+  })
+_sym_db.RegisterMessage(STR19)
+
+ContinuationEntity = _reflection.GeneratedProtocolMessageType('ContinuationEntity', (_message.Message,), {
+  'DESCRIPTOR' : _CONTINUATIONENTITY,
+  '__module__' : 'live_pb2'
+  # @@protoc_insertion_point(class_scope:live.ContinuationEntity)
+  })
+_sym_db.RegisterMessage(ContinuationEntity)
+
+Continuation = _reflection.GeneratedProtocolMessageType('Continuation', (_message.Message,), {
+  'DESCRIPTOR' : _CONTINUATION,
+  '__module__' : 'live_pb2'
+  # @@protoc_insertion_point(class_scope:live.Continuation)
+  })
+_sym_db.RegisterMessage(Continuation)
+
+
+# @@protoc_insertion_point(module_scope)

pytchat/paramgen/pb/replay_pb2.py

@@ -0,0 +1,215 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: replay.proto
+
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='replay.proto',
+  package='replay',
+  syntax='proto3',
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n\x0creplay.proto\x12\x06replay\"\x19\n\x08\x43hatType\x12\r\n\x05value\x18\x01 \x01(\x05\"\xb2\x01\n\x12\x43ontinuationEntity\x12\x0e\n\x06header\x18\x03 \x01(\t\x12\x11\n\ttimestamp\x18\x05 \x01(\x03\x12\n\n\x02s6\x18\x06 \x01(\x05\x12\n\n\x02s7\x18\x07 \x01(\x05\x12\n\n\x02s8\x18\x08 \x01(\x05\x12\n\n\x02s9\x18\t \x01(\x05\x12\x0b\n\x03s10\x18\n \x01(\t\x12\x0b\n\x03s12\x18\x0c \x01(\x05\x12\"\n\x08\x63hattype\x18\x0e \x01(\x0b\x32\x10.replay.ChatType\x12\x0b\n\x03s15\x18\x0f \x01(\x05\"=\n\x0c\x43ontinuation\x12-\n\x06\x65ntity\x18\xd4\x83\xb6J \x01(\x0b\x32\x1a.replay.ContinuationEntityb\x06proto3'
+)
+
+
+
+
+_CHATTYPE = _descriptor.Descriptor(
+  name='ChatType',
+  full_name='replay.ChatType',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='value', full_name='replay.ChatType.value', index=0,
+      number=1, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=24,
+  serialized_end=49,
+)
+
+
+_CONTINUATIONENTITY = _descriptor.Descriptor(
+  name='ContinuationEntity',
+  full_name='replay.ContinuationEntity',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='header', full_name='replay.ContinuationEntity.header', index=0,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='timestamp', full_name='replay.ContinuationEntity.timestamp', index=1,
+      number=5, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s6', full_name='replay.ContinuationEntity.s6', index=2,
+      number=6, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s7', full_name='replay.ContinuationEntity.s7', index=3,
+      number=7, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s8', full_name='replay.ContinuationEntity.s8', index=4,
+      number=8, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s9', full_name='replay.ContinuationEntity.s9', index=5,
+      number=9, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s10', full_name='replay.ContinuationEntity.s10', index=6,
+      number=10, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s12', full_name='replay.ContinuationEntity.s12', index=7,
+      number=12, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='chattype', full_name='replay.ContinuationEntity.chattype', index=8,
+      number=14, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='s15', full_name='replay.ContinuationEntity.s15', index=9,
+      number=15, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=52,
+  serialized_end=230,
+)
+
+
+_CONTINUATION = _descriptor.Descriptor(
+  name='Continuation',
+  full_name='replay.Continuation',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='entity', full_name='replay.Continuation.entity', index=0,
+      number=156074452, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=232,
+  serialized_end=293,
+)
+
+_CONTINUATIONENTITY.fields_by_name['chattype'].message_type = _CHATTYPE
+_CONTINUATION.fields_by_name['entity'].message_type = _CONTINUATIONENTITY
+DESCRIPTOR.message_types_by_name['ChatType'] = _CHATTYPE
+DESCRIPTOR.message_types_by_name['ContinuationEntity'] = _CONTINUATIONENTITY
+DESCRIPTOR.message_types_by_name['Continuation'] = _CONTINUATION
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+ChatType = _reflection.GeneratedProtocolMessageType('ChatType', (_message.Message,), {
+  'DESCRIPTOR' : _CHATTYPE,
+  '__module__' : 'replay_pb2'
+  # @@protoc_insertion_point(class_scope:replay.ChatType)
+  })
+_sym_db.RegisterMessage(ChatType)
+
+ContinuationEntity = _reflection.GeneratedProtocolMessageType('ContinuationEntity', (_message.Message,), {
+  'DESCRIPTOR' : _CONTINUATIONENTITY,
+  '__module__' : 'replay_pb2'
+  # @@protoc_insertion_point(class_scope:replay.ContinuationEntity)
+  })
+_sym_db.RegisterMessage(ContinuationEntity)
+
+Continuation = _reflection.GeneratedProtocolMessageType('Continuation', (_message.Message,), {
+  'DESCRIPTOR' : _CONTINUATION,
+  '__module__' : 'replay_pb2'
+  # @@protoc_insertion_point(class_scope:replay.Continuation)
+  })
+_sym_db.RegisterMessage(Continuation)
+
+
+# @@protoc_insertion_point(module_scope)

pytchat/paramgen/proto/header.proto

@@ -0,0 +1,14 @@
+syntax = "proto3";
+
+message Video {
+    string id = 1;
+}
+
+message HeaderInfo {
+    Video video = 1;
+}
+
+message Header {
+    HeaderInfo info = 1;
+    int32 terminator = 4;
+}

pytchat/paramgen/proto/live.proto

@@ -0,0 +1,45 @@
+syntax = "proto3";
+
+package live;
+
+message Body {
+    int32 b1 = 1;
+    int32 b2 = 2;
+    int32 b3 = 3;
+    int32 b4 = 4;
+    string b7 = 7;
+    int32 b8 = 8;
+    string b9 = 9;
+    int64 timestamp2 = 10;
+    int32 b11 = 11;
+    int32 b15 = 15;
+}
+
+message ChatType {
+    int32 value = 1;
+}
+
+message STR19 {
+    int32 value = 1;
+}
+
+message ContinuationEntity {
+    string header = 3;
+    int64 timestamp1 = 5;
+    int32 s6 = 6;
+    int32 s7 = 7;
+    int32 s8 = 8;
+    Body body = 9;
+    int64 timestamp3 = 10;
+    int64 timestamp4 = 11;
+    int32 s13 = 13;
+    ChatType chattype = 16;
+    int32 s17 = 17;
+    STR19 str19 = 19;
+    int64 timestamp5 = 20;
+}
+
+message Continuation {
+    ContinuationEntity entity = 119693434;
+}
+

pytchat/paramgen/proto/replay.proto

@@ -0,0 +1,24 @@
+syntax = "proto3";
+
+package replay;
+
+message ChatType {
+    int32 value = 1;
+}
+
+message ContinuationEntity {
+    string header = 3;
+    int64 timestamp = 5;
+    int32 s6 = 6;
+    int32 s7 = 7;
+    int32 s8 = 8;
+    int32 s9 = 9;
+    string s10 = 10;
+    int32 s12 = 12;
+    ChatType chattype = 14;
+    int32 s15 = 15;
+}
+
+message Continuation {
+    ContinuationEntity entity = 156074452;
+}  

pytchat/parser/live.py

@@ -0,0 +1,121 @@
+"""
+pytchat.parser.live
+~~~~~~~~~~~~~~~~~~~
+Parser of live chat JSON.
+"""
+
+from .. import exceptions
+
+
+class Parser:
+    '''
+    Parser of chat json.
+    
+    Parameter
+    ----------
+    is_replay : bool
+
+    exception_holder : Object [default:Npne]
+        The object holding exceptions.
+        This is passed from the parent livechat object.
+    '''
+    __slots__ = ['is_replay', 'exception_holder']
+
+    def __init__(self, is_replay, exception_holder=None):
+        self.is_replay = is_replay
+        self.exception_holder = exception_holder
+
+    def get_contents(self, jsn):
+        if jsn is None:
+            self.raise_exception(exceptions.IllegalFunctionCall('Called with none JSON object.'))
+        if jsn['response']['responseContext'].get('errors'):
+            raise exceptions.ResponseContextError(
+                'The video_id would be wrong, or video is deleted or private.')
+        contents = jsn['response'].get('continuationContents')
+        return contents
+
+    def parse(self, contents):
+        """
+        Parameter
+        ----------
+        + contents : dict
+            + JSON of chat data from YouTube.
+
+        Returns
+        -------
+        tuple:
+        + metadata : dict
+         + timeout
+         + video_id
+         + continuation
+        + chatdata : List[dict]
+        """
+
+        if contents is None:
+            '''Broadcasting end or cannot fetch chat stream'''
+            self.raise_exception(exceptions.NoContents('Chat data stream is empty.'))
+
+        cont = contents['liveChatContinuation']['continuations'][0]
+        if cont is None:
+            self.raise_exception(exceptions.NoContinuation('No Continuation'))
+        metadata = (cont.get('invalidationContinuationData')
+                    or cont.get('timedContinuationData')
+                    or cont.get('reloadContinuationData')
+                    or cont.get('liveChatReplayContinuationData')
+                    )
+        if metadata is None:
+            if cont.get("playerSeekContinuationData"):
+                self.raise_exception(exceptions.ChatDataFinished('Finished chat data'))
+            unknown = list(cont.keys())[0]
+            if unknown:
+                self.raise_exception(exceptions.ReceivedUnknownContinuation(
+                    f"Received unknown continuation type:{unknown}"))
+            else:
+                self.raise_exception(exceptions.FailedExtractContinuation('Cannot extract continuation data'))
+        return self._create_data(metadata, contents)
+
+    def reload_continuation(self, contents):
+        """
+        When `seektime == 0` or seektime is abbreviated ,
+        check if fetched chat json has no chat data.
+        If so, try to fetch playerSeekContinuationData.
+        This function must be run only first fetching.
+        """
+        if contents is None:
+            '''Broadcasting end or cannot fetch chat stream'''
+            self.raise_exception(exceptions.NoContents('Chat data stream is empty.'))
+        cont = contents['liveChatContinuation']['continuations'][0]
+        if cont.get("liveChatReplayContinuationData"):
+            # chat data exist.
+            return None
+        # chat data do not exist, get playerSeekContinuationData.
+        init_cont = cont.get("playerSeekContinuationData")
+        if init_cont:
+            return init_cont.get("continuation")
+        self.raise_exception(exceptions.ChatDataFinished('Finished chat data'))
+
+    def _create_data(self, metadata, contents):
+        actions = contents['liveChatContinuation'].get('actions')
+        if self.is_replay:
+            interval = self._get_interval(actions)
+            metadata.setdefault("timeoutMs", interval)
+            """Archived chat has different structures than live chat,
+            so make it the same format."""
+            chatdata = [action["replayChatItemAction"]["actions"][0]
+                        for action in actions]
+        else:
+            metadata.setdefault('timeoutMs', 10000)
+            chatdata = actions
+        return metadata, chatdata
+
+    def _get_interval(self, actions: list):
+        if actions is None:
+            return 0
+        start = int(actions[0]["replayChatItemAction"]["videoOffsetTimeMsec"])
+        last = int(actions[-1]["replayChatItemAction"]["videoOffsetTimeMsec"])
+        return (last - start)
+
+    def raise_exception(self, exception):
+        if self.exception_holder is None:
+            raise exception
+        self.exception_holder = exception

pytchat/processors/chat_processor.py

@@ -1,27 +1,30 @@
 class ChatProcessor:
     '''
-     Listenerからチャットデータ（actions）を受け取り
-    チャットデータを加工するクラスの抽象クラス
+    Abstract class that processes chat data.
+    Receive chat data (actions) from Listener.
     '''
+
     def process(self, chat_components: list):
         '''
-        チャットデータの加工を表すインターフェース
-        Listenerから呼び出される。        
+        Interface that represents processing of chat data.
+        Called from LiveChat object.
+
         Parameter
         ----------
-        chat_components: list<component>
+        chat_components: List[component]
             component : dict {
                 "video_id" : str
-                    動画ID
                 "timeout"  : int
-                    次のチャットの再読み込みまでの時間(秒）
-                "chatdata" : list<object>
-                    チャットデータ（actions）のリスト
+                    Time to fetch next chat (seconds)
+                "chatdata" : List[dict]
+                    List of chat data.
             }
         '''
         pass
 
-
-
-
-
+    def finalize(self, *args, **kwargs):
+        '''
+        Interface for finalizing the process.
+        Called when chat fetching finished.
+        '''
+        pass

pytchat/processors/combinator.py

@@ -0,0 +1,42 @@
+from .chat_processor import ChatProcessor
+
+
+class Combinator(ChatProcessor):
+    '''
+    Combinator combines multiple chat processors.
+    Specify processors as tuple at `processor` params of LiveChat object.
+
+    For example:
+        [constructor]
+        chat = LiveChat("video_id", processor = ( Processor1(), Processor2(), Processor3() ) )
+
+        [receive return values]
+        ret1, ret2, ret3 = chat.get()
+
+    The return values are tuple of processed chat data,
+    the order of return depends on parameter order.
+
+    Parameter
+    ---------
+    processors : Tuple[ChatProcessor]
+        multiple processors for processing chat data
+    '''
+
+    def __init__(self, processors: tuple):
+        self.processors = processors
+
+    def process(self, chat_components: list):
+        '''
+        Called from LiveChat.get() function by user,
+        or LiveChat._listen() automatically.
+
+        Returns
+        -------
+            Tuple of chat data processed by each chat processor.
+        '''
+        return tuple(processor.process(chat_components)
+                     for processor in self.processors)
+
+    def finalize(self, *args, **kwargs):
+        [processor.finalize(*args, **kwargs)
+                     for processor in self.processors]

pytchat/processors/compatible/parser.py

@@ -1,43 +0,0 @@
-
-from .renderer.textmessage import LiveChatTextMessageRenderer
-from .renderer.paidmessage import LiveChatPaidMessageRenderer
-from .renderer.paidsticker import LiveChatPaidStickerRenderer
-from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
-
-def parse(sitem):
-
-    action = sitem.get("addChatItemAction")
-    if action:
-        item = action.get("item")
-    if item is None: return None
-    rd={}
-    try:
-        renderer = get_renderer(item)
-        if renderer == None:
-            return None
-
-        rd["kind"] = "youtube#liveChatMessage"
-        rd["etag"] = ""
-        rd["id"] = 'LCC.' + renderer.get_id()
-        rd["snippet"]       = renderer.get_snippet()
-        rd["authorDetails"] = renderer.get_authordetails()
-    except (KeyError,TypeError,AttributeError) as e:
-        print(f"------{str(type(e))}-{str(e)}----------")
-        print(sitem)
-        return None
-    
-    return rd        
-
-def get_renderer(item):
-    if item.get("liveChatTextMessageRenderer"):
-        renderer = LiveChatTextMessageRenderer(item)
-    elif item.get("liveChatPaidMessageRenderer"):
-        renderer = LiveChatPaidMessageRenderer(item)
-    elif item.get( "liveChatPaidStickerRenderer"):
-        renderer = LiveChatPaidStickerRenderer(item)
-    elif item.get("liveChatLegacyPaidMessageRenderer"):
-        renderer = LiveChatLegacyPaidMessageRenderer(item)
-    else:
-        renderer = None
-    return renderer
-

pytchat/processors/compatible/processor.py

@@ -1,16 +1,20 @@
-from . import parser
-import json
-import os
-import traceback
-import datetime
-import time
-class CompatibleProcessor():
-    
+from .renderer.textmessage import LiveChatTextMessageRenderer
+from .renderer.paidmessage import LiveChatPaidMessageRenderer
+from .renderer.paidsticker import LiveChatPaidStickerRenderer
+from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
+from .renderer.membership import LiveChatMembershipItemRenderer
+from .. chat_processor import ChatProcessor
+from ... import config
+logger = config.logger(__name__)
+
+
+class CompatibleProcessor(ChatProcessor):
+
     def process(self, chat_components: list):
 
         chatlist = []
         timeout = 0
-        ret={}
+        ret = {}
         ret["kind"] = "youtube#liveChatMessageListResponse"
         ret["etag"] = ""
         ret["nextPageToken"] = ""
@@ -19,21 +23,65 @@ class CompatibleProcessor():
             for chat_component in chat_components:
                 timeout += chat_component.get('timeout', 0)
                 chatdata = chat_component.get('chatdata')
-             
-                if chatdata is None: break
-                for action in chatdata:
-                    if action is None: continue
-                    if action.get('addChatItemAction') is None: continue
-                    if action['addChatItemAction'].get('item') is None: continue
 
-                    chat = parser.parse(action)
+                if chatdata is None:
+                    break
+                for action in chatdata:
+                    if action is None:
+                        continue
+                    if action.get('addChatItemAction') is None:
+                        continue
+                    if action['addChatItemAction'].get('item') is None:
+                        continue
+
+                    chat = self.parse(action)
                     if chat:
                         chatlist.append(chat)
-        ret["pollingIntervalMillis"] = int(timeout*1000)
-        ret["pageInfo"]={
-            "totalResults":len(chatlist),
-            "resultsPerPage":len(chatlist),
+        ret["pollingIntervalMillis"] = int(timeout * 1000)
+        ret["pageInfo"] = {
+            "totalResults": len(chatlist),
+            "resultsPerPage": len(chatlist),
         }
         ret["items"] = chatlist
 
-        return ret
+        return ret
+
+    def parse(self, sitem):
+
+        action = sitem.get("addChatItemAction")
+        if action:
+            item = action.get("item")
+        if item is None:
+            return None
+        rd = {}
+        try:
+            renderer = self.get_renderer(item)
+            if renderer is None:
+                return None
+
+            rd["kind"] = "youtube#liveChatMessage"
+            rd["etag"] = ""
+            rd["id"] = 'LCC.' + renderer.get_id()
+            rd["snippet"] = renderer.get_snippet()
+            rd["authorDetails"] = renderer.get_authordetails()
+        except (KeyError, TypeError, AttributeError) as e:
+            logger.error(f"Error: {str(type(e))}-{str(e)}")
+            logger.error(f"item: {sitem}")
+            return None
+
+        return rd
+
+    def get_renderer(self, item):
+        if item.get("liveChatTextMessageRenderer"):
+            renderer = LiveChatTextMessageRenderer(item)
+        elif item.get("liveChatPaidMessageRenderer"):
+            renderer = LiveChatPaidMessageRenderer(item)
+        elif item.get("liveChatPaidStickerRenderer"):
+            renderer = LiveChatPaidStickerRenderer(item)
+        elif item.get("liveChatLegacyPaidMessageRenderer"):
+            renderer = LiveChatLegacyPaidMessageRenderer(item)
+        elif item.get("liveChatMembershipItemRenderer"):
+            renderer = LiveChatMembershipItemRenderer(item)
+        else:
+            renderer = None
+        return renderer

pytchat/processors/compatible/renderer/base.py

@@ -1,68 +1,67 @@
-import datetime, pytz
+import datetime
+import pytz
+
 
 class BaseRenderer:
     def __init__(self, item, chattype):
         self.renderer = list(item.values())[0]
         self.chattype = chattype
 
-
     def get_snippet(self):
 
         message = self.get_message(self.renderer)
 
         return {
-            "type"              : self.chattype,
-            "liveChatId"        : "",
-            "authorChannelId"   : self.renderer.get("authorExternalChannelId"),
-            "publishedAt"       : self.get_publishedat(self.renderer.get("timestampUsec",0)),
-            "hasDisplayContent" : True,
-            "displayMessage"    : message,
+            "type": self.chattype,
+            "liveChatId": "",
+            "authorChannelId": self.renderer.get("authorExternalChannelId"),
+            "publishedAt": self.get_publishedat(self.renderer.get("timestampUsec", 0)),
+            "hasDisplayContent": True,
+            "displayMessage": message,
             "textMessageDetails": {
-                "messageText"   : message  
+                "messageText": message
             }
         }
 
-
     def get_authordetails(self):
         authorExternalChannelId = self.renderer.get("authorExternalChannelId")
-        #parse subscriber type
+        # parse subscriber type
         isVerified, isChatOwner, isChatSponsor, isChatModerator = (
             self.get_badges(self.renderer)
         )
-        return  {
-            "channelId"         : authorExternalChannelId,
-            "channelUrl"        : "http://www.youtube.com/channel/"+authorExternalChannelId,
-            "displayName"       : self.renderer["authorName"]["simpleText"],
-            "profileImageUrl"   : self.renderer["authorPhoto"]["thumbnails"][1]["url"] ,
-            "isVerified"        : isVerified,
-            "isChatOwner"       : isChatOwner,
-            "isChatSponsor"     : isChatSponsor,
-            "isChatModerator"   : isChatModerator
-        }    
+        return {
+            "channelId": authorExternalChannelId,
+            "channelUrl": "http://www.youtube.com/channel/" + authorExternalChannelId,
+            "displayName": self.renderer["authorName"]["simpleText"],
+            "profileImageUrl": self.renderer["authorPhoto"]["thumbnails"][1]["url"],
+            "isVerified": isVerified,
+            "isChatOwner": isChatOwner,
+            "isChatSponsor": isChatSponsor,
+            "isChatModerator": isChatModerator
+        }
 
-
-    def get_message(self,renderer):
+    def get_message(self, renderer):
         message = ''
         if renderer.get("message"):
-            runs=renderer["message"].get("runs")
+            runs = renderer["message"].get("runs")
             if runs:
                 for r in runs:
                     if r:
                         if r.get('emoji'):
-                            message += r['emoji'].get('shortcuts',[''])[0]
+                            message += r['emoji'].get('shortcuts', [''])[0]
                         else:
-                            message += r.get('text','')
+                            message += r.get('text', '')
         return message
 
-    def get_badges(self,renderer):
+    def get_badges(self, renderer):
         isVerified = False
         isChatOwner = False
         isChatSponsor = False
         isChatModerator = False
-        badges=renderer.get("authorBadges")
+        badges = renderer.get("authorBadges")
         if badges:
             for badge in badges:
-                author_type  = badge["liveChatAuthorBadgeRenderer"]["accessibility"]["accessibilityData"]["label"]
+                author_type = badge["liveChatAuthorBadgeRenderer"]["accessibility"]["accessibilityData"]["label"]
                 if author_type == '確認済み':
                     isVerified = True
                 if author_type == '所有者':
@@ -72,12 +71,11 @@ class BaseRenderer:
                 if author_type == 'モデレーター':
                     isChatModerator = True
         return isVerified, isChatOwner, isChatSponsor, isChatModerator
-    
+
     def get_id(self):
         return self.renderer.get('id')
-    
-    def get_publishedat(self,timestamp):
-        dt = datetime.datetime.fromtimestamp(int(timestamp)/1000000)
+
+    def get_publishedat(self, timestamp):
+        dt = datetime.datetime.fromtimestamp(int(timestamp) / 1000000)
         return dt.astimezone(pytz.utc).isoformat(
-            timespec='milliseconds').replace('+00:00','Z')
-        
+            timespec='milliseconds').replace('+00:00', 'Z')

pytchat/processors/compatible/renderer/currency.py

@@ -33,5 +33,6 @@ symbols = {
     "ARS\xa0": {"fxtext": "ARS", "jptext": "アルゼンチン・ペソ"},
     "CLP\xa0": {"fxtext": "CLP", "jptext": "チリ・ペソ"},
     "NOK\xa0": {"fxtext": "NOK", "jptext": "ノルウェー・クローネ"},
-    "BAM\xa0": {"fxtext": "BAM", "jptext": "ボスニア・兌換マルカ"}
-}
+    "BAM\xa0": {"fxtext": "BAM", "jptext": "ボスニア・兌換マルカ"},
+    "SGD\xa0": {"fxtext": "SGD", "jptext": "シンガポール・ドル"}
+}

pytchat/processors/compatible/renderer/legacypaid.py

@@ -1,4 +1,6 @@
 from .base import BaseRenderer
+
+
 class LiveChatLegacyPaidMessageRenderer(BaseRenderer):
     def __init__(self, item):
         super().__init__(item, "newSponsorEvent")
@@ -8,36 +10,33 @@ class LiveChatLegacyPaidMessageRenderer(BaseRenderer):
         message = self.get_message(self.renderer)
 
         return {
-            "type"              : self.chattype,
-            "liveChatId"        : "",
-            "authorChannelId"   : self.renderer.get("authorExternalChannelId"),
-            "publishedAt"       : self.get_publishedat(self.renderer.get("timestampUsec",0)),
-            "hasDisplayContent" : True,
-            "displayMessage"    : message,
-         
+            "type": self.chattype,
+            "liveChatId": "",
+            "authorChannelId": self.renderer.get("authorExternalChannelId"),
+            "publishedAt": self.get_publishedat(self.renderer.get("timestampUsec", 0)),
+            "hasDisplayContent": True,
+            "displayMessage": message,
+
         }
 
     def get_authordetails(self):
         authorExternalChannelId = self.renderer.get("authorExternalChannelId")
-        #parse subscriber type
+        # parse subscriber type
         isVerified, isChatOwner, _, isChatModerator = (
             self.get_badges(self.renderer)
         )
-        return  {
-            "channelId"         : authorExternalChannelId,
-            "channelUrl"        : "http://www.youtube.com/channel/"+authorExternalChannelId,
-            "displayName"       : self.renderer["authorName"]["simpleText"],
-            "profileImageUrl"   : self.renderer["authorPhoto"]["thumbnails"][1]["url"] ,
-            "isVerified"        : isVerified,
-            "isChatOwner"       : isChatOwner,
-            "isChatSponsor"     : True,
-            "isChatModerator"   : isChatModerator
-        }    
+        return {
+            "channelId": authorExternalChannelId,
+            "channelUrl": "http://www.youtube.com/channel/" + authorExternalChannelId,
+            "displayName": self.renderer["authorName"]["simpleText"],
+            "profileImageUrl": self.renderer["authorPhoto"]["thumbnails"][1]["url"],
+            "isVerified": isVerified,
+            "isChatOwner": isChatOwner,
+            "isChatSponsor": True,
+            "isChatModerator": isChatModerator
+        }
 
-
-    def get_message(self,renderer):
+    def get_message(self, renderer):
         message = (renderer["eventText"]["runs"][0]["text"]
-            )+' / '+(renderer["detailText"]["simpleText"])
+                   ) + ' / ' + (renderer["detailText"]["simpleText"])
         return message
-
-

pytchat/processors/compatible/renderer/membership.py

@@ -0,0 +1,40 @@
+from .base import BaseRenderer
+
+
+class LiveChatMembershipItemRenderer(BaseRenderer):
+    def __init__(self, item):
+        super().__init__(item, "newSponsorEvent")
+
+    def get_snippet(self):
+        message = self.get_message(self.renderer)
+        return {
+            "type": self.chattype,
+            "liveChatId": "",
+            "authorChannelId": self.renderer.get("authorExternalChannelId"),
+            "publishedAt": self.get_publishedat(self.renderer.get("timestampUsec", 0)),
+            "hasDisplayContent": True,
+            "displayMessage": message,
+
+        }
+
+    def get_authordetails(self):
+        authorExternalChannelId = self.renderer.get("authorExternalChannelId")
+        # parse subscriber type
+        isVerified, isChatOwner, _, isChatModerator = (
+            self.get_badges(self.renderer)
+        )
+        return {
+            "channelId": authorExternalChannelId,
+            "channelUrl": "http://www.youtube.com/channel/" + authorExternalChannelId,
+            "displayName": self.renderer["authorName"]["simpleText"],
+            "profileImageUrl": self.renderer["authorPhoto"]["thumbnails"][1]["url"],
+            "isVerified": isVerified,
+            "isChatOwner": isChatOwner,
+            "isChatSponsor": True,
+            "isChatModerator": isChatModerator
+        }
+
+    def get_message(self, renderer):
+        message = ''.join([mes.get("text", "")
+                           for mes in renderer["headerSubtext"]["runs"]])
+        return message, [message]

pytchat/processors/compatible/renderer/paidmessage.py

@@ -3,6 +3,7 @@ from . import currency
 from .base import BaseRenderer
 superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
 
+
 class LiveChatPaidMessageRenderer(BaseRenderer):
     def __init__(self, item):
         super().__init__(item, "superChatEvent")
@@ -10,32 +11,32 @@ class LiveChatPaidMessageRenderer(BaseRenderer):
     def get_snippet(self):
         authorName = self.renderer["authorName"]["simpleText"]
         message = self.get_message(self.renderer)
-        amountDisplayString, symbol, amountMicros =(
+        amountDisplayString, symbol, amountMicros = (
             self.get_amountdata(self.renderer)
         )
         return {
-            "type"              : self.chattype,
-            "liveChatId"        : "",
-            "authorChannelId"   : self.renderer.get("authorExternalChannelId"),
-            "publishedAt"       : self.get_publishedat(self.renderer.get("timestampUsec",0)),
-            "hasDisplayContent" : True,
-            "displayMessage"    : amountDisplayString+" from "+authorName+': \"'+ message+'\"',
-            "superChatDetails"  : {
-                "amountMicros"          : amountMicros,
-                "currency"              : currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol,
-                "amountDisplayString"   : amountDisplayString,
-                "tier"                  : 0,
-                "backgroundColor" : self.renderer.get("bodyBackgroundColor", 0)
+            "type": self.chattype,
+            "liveChatId": "",
+            "authorChannelId": self.renderer.get("authorExternalChannelId"),
+            "publishedAt": self.get_publishedat(self.renderer.get("timestampUsec", 0)),
+            "hasDisplayContent": True,
+            "displayMessage": amountDisplayString + " from " + authorName + ': \"' + message + '\"',
+            "superChatDetails": {
+                "amountMicros": amountMicros,
+                "currency": currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol,
+                "amountDisplayString": amountDisplayString,
+                "tier": 0,
+                "backgroundColor": self.renderer.get("bodyBackgroundColor", 0)
             }
         }
 
-    def get_amountdata(self,renderer):
+    def get_amountdata(self, renderer):
         amountDisplayString = renderer["purchaseAmountText"]["simpleText"]
         m = superchat_regex.search(amountDisplayString)
         if m:
             symbol = m.group(1)
-            amountMicros = int(float(m.group(2).replace(',',''))*1000000)
+            amountMicros = int(float(m.group(2).replace(',', '')) * 1000000)
         else:
             symbol = ""
             amountMicros = 0
-        return amountDisplayString, symbol, amountMicros
+        return amountDisplayString, symbol, amountMicros

pytchat/processors/compatible/renderer/paidsticker.py

@@ -3,46 +3,45 @@ from . import currency
 from .base import BaseRenderer
 superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
 
+
 class LiveChatPaidStickerRenderer(BaseRenderer):
     def __init__(self, item):
         super().__init__(item, "superStickerEvent")
 
     def get_snippet(self):
         authorName = self.renderer["authorName"]["simpleText"]
-        amountDisplayString, symbol, amountMicros =(
+        amountDisplayString, symbol, amountMicros = (
             self.get_amountdata(self.renderer)
         )
 
         return {
-            "type"              : self.chattype,
-            "liveChatId"        : "",
-            "authorChannelId"   : self.renderer.get("authorExternalChannelId"),
-            "publishedAt"       : self.get_publishedat(self.renderer.get("timestampUsec",0)),
-            "hasDisplayContent" : True,
-            "displayMessage"    : "Super Sticker " + amountDisplayString + " from "+authorName,
-            "superStickerDetails"  : {
-                "superStickerMetaData" : {
+            "type": self.chattype,
+            "liveChatId": "",
+            "authorChannelId": self.renderer.get("authorExternalChannelId"),
+            "publishedAt": self.get_publishedat(self.renderer.get("timestampUsec", 0)),
+            "hasDisplayContent": True,
+            "displayMessage": "Super Sticker " + amountDisplayString + " from " + authorName,
+            "superStickerDetails": {
+                "superStickerMetaData": {
                     "stickerId": "",
                     "altText": "",
-                    "language": ""                    
+                    "language": ""
                 },
-                "amountMicros"          : amountMicros,
-                "currency"              : currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol,
-                "amountDisplayString"   : amountDisplayString,
-                "tier"                  : 0,
-                "backgroundColor" : self.renderer.get("bodyBackgroundColor", 0)
+                "amountMicros": amountMicros,
+                "currency": currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol,
+                "amountDisplayString": amountDisplayString,
+                "tier": 0,
+                "backgroundColor": self.renderer.get("bodyBackgroundColor", 0)
             }
         }
 
-    def get_amountdata(self,renderer):
+    def get_amountdata(self, renderer):
         amountDisplayString = renderer["purchaseAmountText"]["simpleText"]
         m = superchat_regex.search(amountDisplayString)
         if m:
             symbol = m.group(1)
-            amountMicros = int(float(m.group(2).replace(',',''))*1000000)
+            amountMicros = int(float(m.group(2).replace(',', '')) * 1000000)
         else:
             symbol = ""
             amountMicros = 0
         return amountDisplayString, symbol, amountMicros
- 
-

pytchat/processors/compatible/renderer/textmessage.py

@@ -1,4 +1,6 @@
 from .base import BaseRenderer
+
+
 class LiveChatTextMessageRenderer(BaseRenderer):
     def __init__(self, item):
         super().__init__(item, "textMessageEvent")

pytchat/processors/default/custom_encoder.py

@@ -0,0 +1,11 @@
+import json
+from .renderer.base import Author
+from .renderer.paidmessage import Colors
+from .renderer.paidsticker import Colors2
+
+
+class CustomEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, Author) or isinstance(obj, Colors) or isinstance(obj, Colors2):
+            return vars(obj)
+        return json.JSONEncoder.default(self, obj)

pytchat/processors/default/parser.py

@@ -1,39 +0,0 @@
-
-from .renderer.textmessage import LiveChatTextMessageRenderer
-from .renderer.paidmessage import LiveChatPaidMessageRenderer
-from .renderer.paidsticker import LiveChatPaidStickerRenderer
-from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
-
-def parse(sitem):
-
-    action = sitem.get("addChatItemAction")
-    if action:
-        item = action.get("item")
-    if item is None: return None
-    try:
-        renderer = get_renderer(item)
-        if renderer == None:
-            return None
-
-        renderer.get_snippet()
-        renderer.get_authordetails()
-    except (KeyError,TypeError,AttributeError) as e:
-        print(f"------{str(type(e))}-{str(e)}----------")
-        print(sitem)
-        return None
-    
-    return renderer        
-
-def get_renderer(item):
-    if item.get("liveChatTextMessageRenderer"):
-        renderer = LiveChatTextMessageRenderer(item)
-    elif item.get("liveChatPaidMessageRenderer"):
-        renderer = LiveChatPaidMessageRenderer(item)
-    elif item.get( "liveChatPaidStickerRenderer"):
-        renderer = LiveChatPaidStickerRenderer(item)
-    elif item.get("liveChatLegacyPaidMessageRenderer"):
-        renderer = LiveChatLegacyPaidMessageRenderer(item)
-    else:
-        renderer = None
-    return renderer
-

pytchat/processors/default/processor.py

@@ -1,26 +1,132 @@
-from . import parser
 import asyncio
+import json
 import time
+from .custom_encoder import CustomEncoder
+from .renderer.textmessage import LiveChatTextMessageRenderer
+from .renderer.paidmessage import LiveChatPaidMessageRenderer
+from .renderer.paidsticker import LiveChatPaidStickerRenderer
+from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
+from .renderer.membership import LiveChatMembershipItemRenderer
+from .. chat_processor import ChatProcessor
+from ... import config
+
+logger = config.logger(__name__)
+
+
+class Chat:
+    def json(self) -> str:
+        return json.dumps(vars(self), ensure_ascii=False, cls=CustomEncoder)
 
 
 class Chatdata:
-    def __init__(self,chatlist:list, timeout:float):
+
+    def __init__(self, chatlist: list, timeout: float, abs_diff):
         self.items = chatlist
         self.interval = timeout
-    
+        self.abs_diff = abs_diff
+        self.itemcount = 0
+
     def tick(self):
-        if self.interval == 0:
-            time.sleep(3)
+        '''DEPRECATE
+            Use sync_items()
+        '''
+        if len(self.items) < 1:
+            time.sleep(1)
             return
-        time.sleep(self.interval/len(self.items))
+        if self.itemcount == 0:
+            self.starttime = time.time()
+        if len(self.items) == 1:
+            total_itemcount = 1
+        else:
+            total_itemcount = len(self.items) - 1
+        next_chattime = (self.items[0].timestamp + (self.items[-1].timestamp - self.items[0].timestamp) / total_itemcount * self.itemcount) / 1000
+        tobe_disptime = self.abs_diff + next_chattime
+        wait_sec = tobe_disptime - time.time()
+        self.itemcount += 1
+        
+        if wait_sec < 0:
+            wait_sec = 0
+       
+        time.sleep(wait_sec)
 
     async def tick_async(self):
-        if self.interval == 0:
-            await asyncio.sleep(0.5)
+        '''DEPRECATE
+            Use async_items()
+        '''
+        if len(self.items) < 1:
+            await asyncio.sleep(1)
             return
-        await asyncio.sleep(self.interval/len(self.items))
+        if self.itemcount == 0:
+            self.starttime = time.time()
+        if len(self.items) == 1:
+            total_itemcount = 1
+        else:
+            total_itemcount = len(self.items) - 1
+        next_chattime = (self.items[0].timestamp + (self.items[-1].timestamp - self.items[0].timestamp) / total_itemcount * self.itemcount) / 1000
+        tobe_disptime = self.abs_diff + next_chattime
+        wait_sec = tobe_disptime - time.time()
+        self.itemcount += 1
+        
+        if wait_sec < 0:
+            wait_sec = 0
+       
+        await asyncio.sleep(wait_sec)
+
+    def sync_items(self):
+        starttime = time.time()
+        if len(self.items) > 0:
+            last_chattime = self.items[-1].timestamp / 1000
+            tobe_disptime = self.abs_diff + last_chattime
+            wait_total_sec = max(tobe_disptime - time.time(), 0)
+            if len(self.items) > 1:
+                wait_sec = wait_total_sec / len(self.items)
+            elif len(self.items) == 1:
+                wait_sec = 0
+            for c in self.items:
+                if wait_sec < 0:
+                    wait_sec = 0
+                time.sleep(wait_sec)
+                yield c
+        stop_interval = time.time() - starttime
+        if stop_interval < 1:
+            time.sleep(1 - stop_interval)
+
+    async def async_items(self):
+        starttime = time.time()
+        if len(self.items) > 0:
+            last_chattime = self.items[-1].timestamp / 1000
+            tobe_disptime = self.abs_diff + last_chattime
+            wait_total_sec = max(tobe_disptime - time.time(), 0)
+            if len(self.items) > 1:
+                wait_sec = wait_total_sec / len(self.items)
+            elif len(self.items) == 1:
+                wait_sec = 0
+            for c in self.items:
+                if wait_sec < 0:
+                    wait_sec = 0
+                await asyncio.sleep(wait_sec)
+                yield c
+                
+        stop_interval = time.time() - starttime
+        if stop_interval < 1:
+            await asyncio.sleep(1 - stop_interval)
+
+    def json(self) -> str:
+        return json.dumps([vars(a) for a in self.items], ensure_ascii=False, cls=CustomEncoder)
+
+
+class DefaultProcessor(ChatProcessor):
+    def __init__(self):
+        self.first = True
+        self.abs_diff = 0
+        self.renderers = {
+            "liveChatTextMessageRenderer": LiveChatTextMessageRenderer(),
+            "liveChatPaidMessageRenderer": LiveChatPaidMessageRenderer(),
+            "liveChatPaidStickerRenderer": LiveChatPaidStickerRenderer(),
+            "liveChatLegacyPaidMessageRenderer": LiveChatLegacyPaidMessageRenderer(),
+            "liveChatMembershipItemRenderer": LiveChatMembershipItemRenderer()
+        }
 
-class DefaultProcessor:
     def process(self, chat_components: list):
 
         chatlist = []
@@ -28,17 +134,46 @@ class DefaultProcessor:
 
         if chat_components:
             for component in chat_components:
+                if component is None:
+                    continue
                 timeout += component.get('timeout', 0)
                 chatdata = component.get('chatdata')
-             
-                if chatdata is None: continue
+                if chatdata is None:
+                    continue
                 for action in chatdata:
-                    if action is None: continue
-                    if action.get('addChatItemAction') is None: continue
-                    if action['addChatItemAction'].get('item') is None: continue
-
-                    chat = parser.parse(action)
+                    if action is None:
+                        continue
+                    if action.get('addChatItemAction') is None:
+                        continue
+                    item = action['addChatItemAction'].get('item')
+                    if item is None:
+                        continue
+                    chat = self._parse(item)
                     if chat:
                         chatlist.append(chat)
-        return Chatdata(chatlist, float(timeout))
-  
+        
+        if self.first and chatlist:
+            self.abs_diff = time.time() - chatlist[0].timestamp / 1000 + 2
+            self.first = False
+
+        chatdata = Chatdata(chatlist, float(timeout), self.abs_diff)
+
+        return chatdata
+
+    def _parse(self, item):
+        try:
+            key = list(item.keys())[0]
+            renderer = self.renderers.get(key)
+            if renderer is None:
+                return None
+            renderer.setitem(item.get(key), Chat())
+            renderer.settype()
+            renderer.get_snippet()
+            renderer.get_authordetails()
+            rendered_chatobj = renderer.get_chatobj()
+            renderer.clear()
+        except (KeyError, TypeError) as e:
+            logger.error(f"{str(type(e))}-{str(e)} item:{str(item)}")
+            return None
+        
+        return rendered_chatobj

pytchat/processors/default/renderer/base.py

@@ -1,80 +1,101 @@
 from datetime import datetime
 
+
 class Author:
     pass
+
+
 class BaseRenderer:
-    def __init__(self, item, chattype):
-        self.renderer = list(item.values())[0]
-        self.chattype = chattype
-        self.author = Author()
+    def setitem(self, item, chat):
+        self.item = item
+        self.chat = chat
+        self.chat.author = Author()
+
+    def settype(self):
+        pass
 
     def get_snippet(self):
-        self.type = self.chattype
-        self.id = self.renderer.get('id')
-        timestampUsec = int(self.renderer.get("timestampUsec",0))
-        self.timestamp = int(timestampUsec/1000)
-        self.datetime = self.get_datetime(timestampUsec)
-        self.message = self.get_message(self.renderer)
-        self.id =  self.renderer.get('id')
-        self.amountValue= 0.0
-        self.amountString = ""
-        self.currency= ""
-        self.bgColor = 0
+        self.chat.id = self.item.get('id')
+        timestampUsec = int(self.item.get("timestampUsec", 0))
+        self.chat.timestamp = int(timestampUsec / 1000)
+        tst = self.item.get("timestampText")
+        if tst:
+            self.chat.elapsedTime = tst.get("simpleText")
+        else:
+            self.chat.elapsedTime = ""
+        self.chat.datetime = self.get_datetime(timestampUsec)
+        self.chat.message, self.chat.messageEx = self.get_message(self.item)
+        self.chat.id = self.item.get('id')
+        self.chat.amountValue = 0.0
+        self.chat.amountString = ""
+        self.chat.currency = ""
+        self.chat.bgColor = 0
 
     def get_authordetails(self):
-        self.author.badgeUrl = ""
-        (self.author.isVerified, 
-        self.author.isChatOwner, 
-        self.author.isChatSponsor, 
-        self.author.isChatModerator) = (
-            self.get_badges(self.renderer)
+        self.chat.author.badgeUrl = ""
+        (self.chat.author.isVerified,
+         self.chat.author.isChatOwner,
+         self.chat.author.isChatSponsor,
+         self.chat.author.isChatModerator) = (
+            self.get_badges(self.item)
         )
-        self.author.channelId = self.renderer.get("authorExternalChannelId")
-        self.author.channelUrl = "http://www.youtube.com/channel/"+self.author.channelId
-        self.author.name      = self.renderer["authorName"]["simpleText"]
-        self.author.imageUrl= self.renderer["authorPhoto"]["thumbnails"][1]["url"] 
-        
+        self.chat.author.channelId = self.item.get("authorExternalChannelId")
+        self.chat.author.channelUrl = "http://www.youtube.com/channel/" + self.chat.author.channelId
+        self.chat.author.name = self.item["authorName"]["simpleText"]
+        self.chat.author.imageUrl = self.item["authorPhoto"]["thumbnails"][1]["url"]
 
-
-    def get_message(self,renderer):
+    def get_message(self, item):
         message = ''
-        if renderer.get("message"):
-            runs=renderer["message"].get("runs")
-            if runs:
-                for r in runs:
-                    if r:
-                        if r.get('emoji'):
-                            message += r['emoji'].get('shortcuts',[''])[0]
-                        else:
-                            message += r.get('text','')
-        return message
+        message_ex = []
+        runs = item.get("message", {}).get("runs", {})
+        for r in runs:
+            if not hasattr(r, "get"):
+                continue
+            if r.get('emoji'):
+                message += r['emoji'].get('shortcuts', [''])[0]
+                message_ex.append({
+                    'id': r['emoji'].get('emojiId').split('/')[-1],
+                    'txt': r['emoji'].get('shortcuts', [''])[0],
+                    'url': r['emoji']['image']['thumbnails'][0].get('url')
+                })
+            else:
+                message += r.get('text', '')
+                message_ex.append(r.get('text', ''))
+        return message, message_ex
 
-    def get_badges(self,renderer):
+    def get_badges(self, renderer):
+        self.chat.author.type = ''
         isVerified = False
         isChatOwner = False
         isChatSponsor = False
         isChatModerator = False
-        badges=renderer.get("authorBadges")
-        if badges:
-            for badge in badges:
-                author_type  = badge["liveChatAuthorBadgeRenderer"]["accessibility"]["accessibilityData"]["label"]
-                if author_type == '確認済み':
+        badges = renderer.get("authorBadges", {})
+        for badge in badges:
+            if badge["liveChatAuthorBadgeRenderer"].get("icon"):
+                author_type = badge["liveChatAuthorBadgeRenderer"]["icon"]["iconType"]
+                self.chat.author.type = author_type
+                if author_type == 'VERIFIED':
                     isVerified = True
-                if author_type == '所有者':
+                if author_type == 'OWNER':
                     isChatOwner = True
-                if 'メンバー' in author_type:
-                    isChatSponsor = True
-                    self.get_badgeurl(badge)
-                if author_type == 'モデレーター':
+                if author_type == 'MODERATOR':
                     isChatModerator = True
+            if badge["liveChatAuthorBadgeRenderer"].get("customThumbnail"):
+                isChatSponsor = True
+                self.chat.author.type = 'MEMBER'
+                self.get_badgeurl(badge)
         return isVerified, isChatOwner, isChatSponsor, isChatModerator
-    
 
-    def get_badgeurl(self,badge):
-        self.author.badgeUrl = badge["liveChatAuthorBadgeRenderer"]["customThumbnail"]["thumbnails"][0]["url"]
+    def get_badgeurl(self, badge):
+        self.chat.author.badgeUrl = badge["liveChatAuthorBadgeRenderer"]["customThumbnail"]["thumbnails"][0]["url"]
 
+    def get_datetime(self, timestamp):
+        dt = datetime.fromtimestamp(timestamp / 1000000)
+        return dt.strftime('%Y-%m-%d %H:%M:%S')
 
+    def get_chatobj(self):
+        return self.chat
 
-    def get_datetime(self,timestamp):
-        dt = datetime.fromtimestamp(timestamp/1000000)
-        return dt.strftime('%Y-%m-%d %H:%M:%S')
+    def clear(self):
+        self.item = None
+        self.chat = None

pytchat/processors/default/renderer/currency.py

@@ -23,7 +23,7 @@ symbols = {
     "PLN\xa0": {"fxtext": "PLN", "jptext": "ポーランド・ズロチ"},
     "R$": {"fxtext": "BRL", "jptext": "ブラジル・レアル"},
     "RUB\xa0": {"fxtext": "RUB", "jptext": "ロシア・ルーブル"},
-    "SEK\xa0": {"fxtext": "SEK", "jptext": "スウェーデン・クローネ"},
+    "SEK\xa0": {"fxtext": "SEK", "jptext": "スウェーデン・クローナ"},
     "£": {"fxtext": "GBP", "jptext": "英・ポンド"},
     "₩": {"fxtext": "KRW", "jptext": "韓国・ウォン"},
     "€": {"fxtext": "EUR", "jptext": "欧・ユーロ"},
@@ -33,5 +33,6 @@ symbols = {
     "ARS\xa0": {"fxtext": "ARS", "jptext": "アルゼンチン・ペソ"},
     "CLP\xa0": {"fxtext": "CLP", "jptext": "チリ・ペソ"},
     "NOK\xa0": {"fxtext": "NOK", "jptext": "ノルウェー・クローネ"},
-    "BAM\xa0": {"fxtext": "BAM", "jptext": "ボスニア・兌換マルカ"}
-}
+    "BAM\xa0": {"fxtext": "BAM", "jptext": "ボスニア・兌換マルカ"},
+    "SGD\xa0": {"fxtext": "SGD", "jptext": "シンガポール・ドル"}
+}

pytchat/processors/default/renderer/legacypaid.py

@@ -1,18 +1,15 @@
 from .base import BaseRenderer
-class LiveChatLegacyPaidMessageRenderer(BaseRenderer):
-    def __init__(self, item):
-        super().__init__(item, "newSponsor")
 
-        
+
+class LiveChatLegacyPaidMessageRenderer(BaseRenderer):
+    def settype(self):
+        self.chat.type = "newSponsor"
 
     def get_authordetails(self):
         super().get_authordetails()
-        self.author.isChatSponsor =  True
-
-
-    def get_message(self,renderer):
-        message = (renderer["eventText"]["runs"][0]["text"]
-            )+' / '+(renderer["detailText"]["simpleText"])
-        return message
-
+        self.chat.author.isChatSponsor = True
 
+    def get_message(self, item):
+        message = (item["eventText"]["runs"][0]["text"]
+                   ) + ' / ' + (item["detailText"]["simpleText"])
+        return message, [message]

pytchat/processors/default/renderer/membership.py

@@ -0,0 +1,18 @@
+from .base import BaseRenderer
+
+
+class LiveChatMembershipItemRenderer(BaseRenderer):
+    def settype(self):
+        self.chat.type = "newSponsor"
+
+    def get_authordetails(self):
+        super().get_authordetails()
+        self.chat.author.isChatSponsor = True
+
+    def get_message(self, item):
+        try:
+            message = ''.join([mes.get("text", "")
+                           for mes in item["headerSubtext"]["runs"]])
+        except KeyError:
+            return "Welcome New Member!", ["Welcome New Member!"]
+        return message, [message]

pytchat/processors/default/renderer/paidmessage.py

@@ -3,34 +3,45 @@ from . import currency
 from .base import BaseRenderer
 superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
 
-class LiveChatPaidMessageRenderer(BaseRenderer):
-    def __init__(self, item):
-        super().__init__(item, "superChat")
 
+class Colors:
+    pass
+
+
+class LiveChatPaidMessageRenderer(BaseRenderer):
+    def settype(self):
+        self.chat.type = "superChat"
 
     def get_snippet(self):
         super().get_snippet()
-
-        self.author.name = self.renderer["authorName"]["simpleText"]
-
-        amountDisplayString, symbol, amount =(
-            self.get_amountdata(self.renderer)
+        amountDisplayString, symbol, amount = (
+            self.get_amountdata(self.item)
         )
-        self.message = self.get_message(self.renderer)
-        self.amountValue= amount
-        self.amountString = amountDisplayString
-        self.currency= currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol
-        self.bgColor= self.renderer.get("bodyBackgroundColor", 0)
-            
+        self.chat.amountValue = amount
+        self.chat.amountString = amountDisplayString
+        self.chat.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get(
+            symbol) else symbol
+        self.chat.bgColor = self.item.get("bodyBackgroundColor", 0)
+        self.chat.colors = self.get_colors()
 
-
-    def get_amountdata(self,renderer):
-        amountDisplayString = renderer["purchaseAmountText"]["simpleText"]
+    def get_amountdata(self, item):
+        amountDisplayString = item["purchaseAmountText"]["simpleText"]
         m = superchat_regex.search(amountDisplayString)
         if m:
             symbol = m.group(1)
-            amount = float(m.group(2).replace(',',''))
+            amount = float(m.group(2).replace(',', ''))
         else:
             symbol = ""
             amount = 0.0
-        return amountDisplayString, symbol, amount
+        return amountDisplayString, symbol, amount
+
+    def get_colors(self):
+        item = self.item
+        colors = Colors()
+        colors.headerBackgroundColor = item.get("headerBackgroundColor", 0)
+        colors.headerTextColor = item.get("headerTextColor", 0)
+        colors.bodyBackgroundColor = item.get("bodyBackgroundColor", 0)
+        colors.bodyTextColor = item.get("bodyTextColor", 0)
+        colors.timestampColor = item.get("timestampColor", 0)
+        colors.authorNameTextColor = item.get("authorNameTextColor", 0)
+        return colors

pytchat/processors/default/renderer/paidsticker.py

@@ -1,13 +1,47 @@
 import re
 from . import currency
-from .paidmessage import LiveChatPaidMessageRenderer
-
-class LiveChatPaidStickerRenderer(LiveChatPaidMessageRenderer):
-    def __init__(self, item):
-        super().__init__(item, "superSticker")
+from .base import BaseRenderer
+superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
 
 
- 
+class Colors2:
+    pass
 
- 
 
+class LiveChatPaidStickerRenderer(BaseRenderer):
+    def settype(self):
+        self.chat.type = "superSticker"
+
+    def get_snippet(self):
+        super().get_snippet()
+        amountDisplayString, symbol, amount = (
+            self.get_amountdata(self.item)
+        )
+        self.chat.amountValue = amount
+        self.chat.amountString = amountDisplayString
+        self.chat.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get(
+            symbol) else symbol
+        self.chat.bgColor = self.item.get("backgroundColor", 0)
+        self.chat.sticker = "".join(("https:",
+            self.item["sticker"]["thumbnails"][0]["url"]))
+        self.chat.colors = self.get_colors()
+
+    def get_amountdata(self, item):
+        amountDisplayString = item["purchaseAmountText"]["simpleText"]
+        m = superchat_regex.search(amountDisplayString)
+        if m:
+            symbol = m.group(1)
+            amount = float(m.group(2).replace(',', ''))
+        else:
+            symbol = ""
+            amount = 0.0
+        return amountDisplayString, symbol, amount
+
+    def get_colors(self):
+        item = self.item
+        colors = Colors2()
+        colors.moneyChipBackgroundColor = item.get("moneyChipBackgroundColor", 0)
+        colors.moneyChipTextColor = item.get("moneyChipTextColor", 0)
+        colors.backgroundColor = item.get("backgroundColor", 0)
+        colors.authorNameTextColor = item.get("authorNameTextColor", 0)
+        return colors

pytchat/processors/default/renderer/textmessage.py

@@ -1,4 +1,6 @@
 from .base import BaseRenderer
+
+
 class LiveChatTextMessageRenderer(BaseRenderer):
-    def __init__(self, item):
-        super().__init__(item, "textMessage")
+    def settype(self):
+        self.chat.type = "textMessage"

pytchat/processors/dummy_processor.py

@@ -0,0 +1,10 @@
+from .chat_processor import ChatProcessor
+
+
+class DummyProcessor(ChatProcessor):
+    '''
+    Dummy processor just returns received chat_components directly.
+    '''
+
+    def process(self, chat_components: list):
+        return chat_components

pytchat/processors/html_archiver.py

@@ -0,0 +1,157 @@
+import httpx
+import os
+import re
+import time
+from base64 import standard_b64encode
+from concurrent.futures import ThreadPoolExecutor
+from .chat_processor import ChatProcessor
+from .default.processor import DefaultProcessor
+from ..exceptions import UnknownConnectionError
+
+
+PATTERN = re.compile(r"(.*)\(([0-9]+)\)$")
+
+fmt_headers = ['datetime', 'elapsed', 'authorName',
+               'message', 'superchat', 'type', 'authorChannel']
+
+HEADER_HTML = '''
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html;charset=UTF-8">
+'''
+
+TABLE_CSS = '''
+table.css {
+ border-collapse: collapse;
+}
+ 
+table.css thead{
+ border-collapse: collapse;
+ border: 1px solid #000
+}
+ 
+table.css tr td{
+ padding: 0.3em;
+ border: 1px solid #000
+}
+
+table.css th{
+ padding: 0.3em;
+ border: 1px solid #000
+}
+'''
+
+
+class HTMLArchiver(ChatProcessor):
+    '''
+    HTMLArchiver saves chat data as HTML table format.
+    '''
+    def __init__(self, save_path, callback=None):
+        super().__init__()
+        self.client = httpx.Client(http2=True)
+        self.save_path = self._checkpath(save_path)
+        self.processor = DefaultProcessor()
+        self.emoji_table = {}  # dict for custom emojis. key: emoji_id, value: base64 encoded image binary.
+        self.header = [HEADER_HTML]
+        self.body = ['<body>\n', '<table class="css">\n', self._parse_table_header(fmt_headers)]
+        self.callback = callback
+        self.executor = ThreadPoolExecutor(max_workers=10)
+
+    def _checkpath(self, filepath):
+        splitter = os.path.splitext(os.path.basename(filepath))
+        body = splitter[0]
+        extention = splitter[1]
+        newpath = filepath
+        counter = 1
+        while os.path.exists(newpath):
+            match = re.search(PATTERN, body)
+            if match:
+                counter = int(match[2]) + 1
+                num_with_bracket = f'({str(counter)})'
+                body = f'{match[1]}{num_with_bracket}'
+            else:
+                body = f'{body}({str(counter)})'
+            newpath = os.path.join(os.path.dirname(filepath), body + extention)
+        return newpath
+
+    def process(self, chat_components: list):
+        """
+        Returns
+        ----------
+        dict :
+            save_path : str :
+                Actual save path of file.
+            total_lines : int :
+                Count of total lines written to the file.
+        """
+        if chat_components is None or len(chat_components) == 0:
+            return
+        for c in self.processor.process(chat_components).items:
+            self.body.extend(
+                self._parse_html_line((
+                    c.datetime,
+                    c.elapsedTime,
+                    c.author.name,
+                    self._parse_message(c.messageEx),
+                    c.amountString,
+                    c.author.type,
+                    c.author.channelId)
+                )
+            )
+            if self.callback:
+                self.callback(None, 1)
+
+    def _parse_html_line(self, raw_line):
+        return ''.join(('<tr>',
+                        ''.join(''.join(('<td>', cell, '</td>')) for cell in raw_line),
+                        '</tr>\n'))
+
+    def _parse_table_header(self, raw_line):
+        return ''.join(('<thead><tr>',
+                        ''.join(''.join(('<th>', cell, '</th>')) for cell in raw_line),
+                        '</tr></thead>\n'))
+        
+    def _parse_message(self, message_items: list) -> str:
+        return ''.join(''.join(('<span class="', self._set_emoji_table(item), '" title="', item['txt'], '"></span>'))
+                       if type(item) is dict else item
+                       for item in message_items)
+
+    def _encode_img(self, url):
+        err = None
+        for _ in range(5):
+            try:
+                resp = self.client.get(url, timeout=30)
+                break
+            except httpx.HTTPError as e:
+                err = e
+                time.sleep(3)
+        else:
+            raise UnknownConnectionError(str(err))
+
+        return standard_b64encode(resp.content).decode()
+
+    def _set_emoji_table(self, item: dict):
+        emoji_id = ''.join(('Z', item['id'])) if 48 <= ord(item['id'][0]) <= 57 else item['id']
+        if emoji_id not in self.emoji_table:
+            self.emoji_table.setdefault(emoji_id, self.executor.submit(self._encode_img, item['url']))
+        return emoji_id
+
+    def _stylecode(self, name, code, width, height):
+        return ''.join((".", name, " { display: inline-block; background-image: url(data:image/png;base64,",
+                        code, "); background-repeat: no-repeat; width: ",
+                        str(width), "; height: ", str(height), ";}"))
+    
+    def _create_styles(self):
+        return '\n'.join(('<style type="text/css">',
+                          TABLE_CSS,
+                          '\n'.join(self._stylecode(key, self.emoji_table[key].result(), 24, 24)
+                                for key in self.emoji_table.keys()),
+                          '</style>\n'))
+    
+    def finalize(self):
+        self.executor.shutdown()
+        self.header.extend([self._create_styles(), '</head>\n'])
+        self.body.extend(['</table>\n</body>\n</html>'])
+        with open(self.save_path, mode='a', encoding='utf-8') as f:
+            f.writelines(self.header)
+            f.writelines(self.body)

pytchat/processors/json_display_processor.py

@@ -1,13 +0,0 @@
-import json
-from .chat_processor import ChatProcessor
-
-class JsonDisplayProcessor(ChatProcessor):
-
-    def process(self,chat_components: list):
-        if chat_components:
-            for component in chat_components:
-                chatdata = component.get('chatdata')
-                if chatdata:
-                    for chat in chatdata:
-                        print(json.dumps(chat,ensure_ascii=False)[:200])
-

pytchat/processors/jsonfile_archive_processor.py

@@ -1,46 +0,0 @@
-import json
-import os
-import datetime
-from .chat_processor import ChatProcessor
-
-class JsonfileArchiveProcessor(ChatProcessor):
-    def __init__(self,filepath):
-        super().__init__()
-        if os.path.exists(filepath):
-            print('filepath is already exists!: ')
-            print('  '+filepath)
-            newpath=os.path.dirname(filepath) + \
-                '/'+datetime.datetime.now() \
-                .strftime('%Y-%m-%d %H-%M-%S')+'.data'
-
-            print('created alternate filename:')
-            print('   '+newpath)
-            self.filepath = newpath
-        else:
-            print('filepath: '+filepath)
-            self.filepath = filepath
-
-    def process(self,chat_components: list):
-        if chat_components:
-            with open(self.filepath, mode='a', encoding = 'utf-8') as f:
-                for component in chat_components:
-                    if component:
-                        chatdata = component.get('chatdata')
-                        for action in chatdata:
-                            if action:
-                                if action.get("addChatItemAction"):
-                                    if action["addChatItemAction"]["item"].get(
-                                    "liveChatViewerEngagementMessageRenderer"):
-                                        continue
-                                s = json.dumps(action,ensure_ascii = False)
-                                #print(s[:200])
-                                f.writelines(s+'\n')
-
-    def _parsedir(self,_dir):
-        if _dir[-1]=='\\' or _dir[-1]=='/':
-            separator =''
-        else:
-            separator ='/'
-        os.makedirs(_dir + separator, exist_ok=True)
-        return _dir  + separator
-

pytchat/processors/jsonfile_archiver.py

@@ -0,0 +1,69 @@
+import json
+import os
+import re
+from .chat_processor import ChatProcessor
+
+PATTERN = re.compile(r"(.*)\(([0-9]+)\)$")
+
+
+class JsonfileArchiver(ChatProcessor):
+    """
+    JsonfileArchiver saves chat data as text of JSON lines.
+
+    Parameter:
+    ----------
+    save_path : str :
+        save path of file.If a file with the same name exists,
+        it is automatically saved under a different name
+        with suffix '(number)'
+    """
+
+    def __init__(self, save_path):
+        super().__init__()
+        self.save_path = self._checkpath(save_path)
+        self.line_counter = 0
+
+    def process(self, chat_components: list):
+        """
+        Returns
+        ----------
+        dict :
+            save_path : str :
+                Actual save path of file.
+            total_lines : int :
+                count of total lines written to the file.
+        """
+        if chat_components is None:
+            return
+        with open(self.save_path, mode='a', encoding='utf-8') as f:
+            for component in chat_components:
+                if component is None:
+                    continue
+                chatdata = component.get('chatdata')
+                if chatdata is None:
+                    continue
+                for action in chatdata:
+                    if action is None:
+                        continue
+                    json_line = json.dumps(action, ensure_ascii=False)
+                    f.writelines(json_line + '\n')
+                    self.line_counter += 1
+            return {"save_path": self.save_path,
+                    "total_lines": self.line_counter}
+
+    def _checkpath(self, filepath):
+        splitter = os.path.splitext(os.path.basename(filepath))
+        body = splitter[0]
+        extention = splitter[1]
+        newpath = filepath
+        counter = 0
+        while os.path.exists(newpath):
+            match = re.search(PATTERN, body)
+            if match:
+                counter = int(match[2]) + 1
+                num_with_bracket = f'({str(counter)})'
+                body = f'{match[1]}{num_with_bracket}'
+            else:
+                body = f'{body}({str(counter)})'
+            newpath = os.path.join(os.path.dirname(filepath), body + extention)
+        return newpath

pytchat/processors/simple_display_processor.py

@@ -1,47 +1,49 @@
-import json
-import os
-import traceback
-import datetime
-import time
 from .chat_processor import ChatProcessor
-##version 2
+
+
 class SimpleDisplayProcessor(ChatProcessor):
-    
+
     def process(self, chat_components: list):
         chatlist = []
         timeout = 0
-        
+
         if chat_components is None:
-            return {"timeout":timeout, "chatlist":chatlist}
+            return {"timeout": timeout, "chatlist": chatlist}
         for component in chat_components:
             timeout += component.get('timeout', 0)
             chatdata = component.get('chatdata')
-            
-            if chatdata is None:break
-            for action in chatdata:
-                if action is None:continue
-                if action.get('addChatItemAction') is None:continue
-                if action['addChatItemAction'].get('item') is None:continue
 
-                root = action['addChatItemAction']['item'].get('liveChatTextMessageRenderer')
-                
+            if chatdata is None:
+                break
+            for action in chatdata:
+                if action is None:
+                    continue
+                if action.get('addChatItemAction') is None:
+                    continue
+                if action['addChatItemAction'].get('item') is None:
+                    continue
+
+                root = action['addChatItemAction']['item'].get(
+                    'liveChatTextMessageRenderer')
+
                 if root:
                     author_name = root['authorName']['simpleText']
                     message = self._parse_message(root.get('message'))
                     purchase_amount_text = ''
                 else:
-                    root = ( action['addChatItemAction']['item'].get('liveChatPaidMessageRenderer') or 
-                            action['addChatItemAction']['item'].get('liveChatPaidMessageRenderer') )
+                    root = (action['addChatItemAction']['item'].get('liveChatPaidMessageRenderer')
+                            or action['addChatItemAction']['item'].get('liveChatPaidStickerRenderer'))
                     if root:
                         author_name = root['authorName']['simpleText']
                         message = self._parse_message(root.get('message'))
                         purchase_amount_text = root['purchaseAmountText']['simpleText']
                     else:
                         continue
-                chatlist.append(f'[{author_name}]:  {message}  {purchase_amount_text}')
-        return {"timeout":timeout, "chatlist":chatlist}
-    
-    def _parse_message(self,message):
+                chatlist.append(
+                    f'[{author_name}]:  {message}  {purchase_amount_text}')
+        return {"timeout": timeout, "chatlist": chatlist}
+
+    def _parse_message(self, message):
         if message is None:
             return ''
         if message.get('simpleText'):
@@ -51,11 +53,9 @@ class SimpleDisplayProcessor(ChatProcessor):
             tmp = ''
             for run in runs:
                 if run.get('emoji'):
-                    tmp+=(run['emoji']['shortcuts'][0])
+                    tmp += (run['emoji']['shortcuts'][0])
                 elif run.get('text'):
-                    tmp+=(run['text'])
+                    tmp += (run['text'])
             return tmp
         else:
             return ''
-
-

pytchat/processors/speed/calculator.py

@@ -0,0 +1,203 @@
+"""
+speed_calculator.py
+チャットの勢いを算出するChatProcessor
+Calculate speed of chat.
+"""
+import time
+from .. chat_processor import ChatProcessor
+
+
+class RingQueue:
+    """
+    リング型キュー
+
+    Attributes
+    ----------
+    items : list
+        格納されているアイテムのリスト。
+    first_pos : int
+        キュー内の一番古いアイテムを示すリストのインデックス。
+    last_pos : int
+        キュー内の一番新しいアイテムを示すリストのインデックス。
+    mergin : boolean
+        キュー内に余裕があるか。キュー内のアイテム個数が、キューの最大個数未満であればTrue。
+    """
+
+    def __init__(self, capacity):
+        """
+        コンストラクタ
+
+        Parameter
+        ----------
+            capacity:このキューに格納するアイテムの最大個数。
+                     格納時に最大個数を超える場合は一番古いアイテムから
+                     上書きする。
+        """
+        if capacity <= 0:
+            raise ValueError
+        self.items = list()
+        self.capacity = capacity
+        self.first_pos = 0
+        self.last_pos = 0
+        self.mergin = True
+
+    def put(self, item):
+        """
+        引数itemに指定されたアイテムをこのキューに格納する。
+        キューの最大個数を超える場合は、一番古いアイテムの位置に上書きする。
+
+        Parameter
+        ----------
+            item:格納するアイテム
+        """
+        if self.mergin:
+            self.items.append(item)
+            self.last_pos = len(self.items) - 1
+            if self.last_pos == self.capacity - 1:
+                self.mergin = False
+            return
+        self.last_pos += 1
+        if self.last_pos > self.capacity - 1:
+            self.last_pos = 0
+        self.items[self.last_pos] = item
+
+        self.first_pos += 1
+        if self.first_pos > self.capacity - 1:
+            self.first_pos = 0
+
+    def get(self):
+        """
+        キュー内の一番古いアイテムへの参照を返す
+        （アイテムは削除しない）
+
+        Return
+        ----------
+            キュー内の一番古いアイテムへの参照
+        """
+        return self.items[self.first_pos]
+
+    def item_count(self):
+        return len(self.items)
+
+
+class SpeedCalculator(ChatProcessor, RingQueue):
+    """
+    チャットの勢いを計算する。
+
+    一定期間のチャットデータのうち、最初のチャットの投稿時刻と
+    最後のチャットの投稿時刻の差を、チャット数で割り返し
+    1分あたりの速度に換算する。
+
+    Parameter
+    ----------
+    capacity : int
+        RingQueueに格納するチャット勢い算出用データの最大数
+    """
+
+    def __init__(self, capacity=10):
+        super().__init__(capacity)
+        self.speed = 0
+
+    def process(self, chat_components: list):
+        chatdata = []
+        if chat_components:
+            for component in chat_components:
+                if component.get("chatdata"):
+                    chatdata.extend(component.get("chatdata"))
+
+            self._put_chatdata(chatdata)
+            self.speed = self._calc_speed()
+        return self.speed
+
+    def _calc_speed(self):
+        """
+        RingQueue内のチャット勢い算出用データリストを元に、
+        チャット速度を計算して返す
+
+        Return
+        ---------------------------
+            チャット速度（１分間で換算したチャット数）
+        """
+        try:
+            # キュー内の総チャット数
+            total = sum(item['chat_count'] for item in self.items)
+            # キュー内の最初と最後のチャットの時間差
+            duration = (self.items[self.last_pos]['endtime'] - self.items[self.first_pos]['starttime'])
+            if duration != 0:
+                return int(total * 60 / duration)
+            return 0
+        except IndexError:
+            return 0
+
+    def _put_chatdata(self, actions):
+        """
+        チャットデータからタイムスタンプを読み取り、勢い測定用のデータを組み立て、
+        RingQueueに投入する。
+        200円以上のスパチャはtickerとmessageの2つのデータが生成されるが、
+        tickerの方は時刻データの場所が異なることを利用し、勢いの集計から除外している。
+        Parameter
+        ---------
+        actions : List[dict]
+            チャットデータ(addChatItemAction) のリスト
+        """
+        def _put_emptydata():
+            '''
+            チャットデータがない場合に空のデータをキューに投入する。
+            '''
+            timestamp_now = int(time.time())
+            self.put({
+                'chat_count': 0,
+                'starttime': int(timestamp_now),
+                'endtime': int(timestamp_now)
+            })
+
+        def _get_timestamp(action: dict):
+            """
+            チャットデータから時刻データを取り出す。
+            """
+            try:
+                item = action['addChatItemAction']['item']
+                timestamp = int(item[list(item.keys())[0]]['timestampUsec'])
+            except (KeyError, TypeError):
+                return None
+            return timestamp
+
+        if actions is None or len(actions) == 0:
+            _put_emptydata()
+            return
+
+        # actions内の時刻データを持つチャットデータの数
+        counter = 0
+        # actions内の最初のチャットデータの時刻
+        starttime = None
+        # actions内の最後のチャットデータの時刻
+        endtime = None
+
+        for action in actions:
+            # チャットデータからtimestampUsecを読み取る
+            gettime = _get_timestamp(action)
+
+            # 時刻のないデータだった場合は次の行のデータで読み取り試行
+            if gettime is None:
+                continue
+
+            # 最初に有効な時刻を持つデータのtimestampをstarttimeに設定
+            if starttime is None:
+                starttime = gettime
+
+            # 最後のtimestampを設定(途中で時刻のないデータの場合もあるので上書きしていく)
+            endtime = gettime
+
+            # チャットの数をインクリメント
+            counter += 1
+
+        # チャット速度用のデータをRingQueueに送る
+        if starttime is None or endtime is None:
+            _put_emptydata()
+            return
+
+        self.put({
+            'chat_count': counter,
+            'starttime': int(starttime / 1000000),
+            'endtime': int(endtime / 1000000)
+        })

pytchat/processors/superchat/calculator.py

@@ -0,0 +1,75 @@
+import re
+from pytchat.processors.chat_processor import ChatProcessor
+
+superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
+
+items_paid = [
+    'addChatItemAction',
+    'item',
+    'liveChatPaidMessageRenderer'
+]
+
+items_sticker = [
+    'addChatItemAction',
+    'item',
+    'liveChatPaidStickerRenderer'
+]
+
+
+class SuperchatCalculator(ChatProcessor):
+    """
+    Calculate the amount of SuperChat by currency.
+    """
+
+    def __init__(self):
+        self.results = {}
+
+    def process(self, chat_components: list):
+        """
+        Return
+        ------------
+        results : dict :
+            List of amount by currency.
+            key: currency symbol, value: total amount.
+        """
+        if chat_components is None:
+            return self.results
+        for component in chat_components:
+            chatdata = component.get('chatdata')
+            if chatdata is None:
+                continue
+            for action in chatdata:
+                renderer = self._get_item(action, items_paid) or \
+                    self._get_item(action, items_sticker)
+                if renderer is None:
+                    continue
+                symbol, amount = self._parse(renderer)
+                self.results.setdefault(symbol, 0)
+                self.results[symbol] += amount
+        return self.results
+
+    def _parse(self, renderer):
+        purchase_amount_text = renderer["purchaseAmountText"]["simpleText"]
+        m = superchat_regex.search(purchase_amount_text)
+        if m:
+            symbol = m.group(1)
+            amount = float(m.group(2).replace(',', ''))
+        else:
+            symbol = ""
+            amount = 0.0
+        return symbol, amount
+
+    def _get_item(self, dict_body, items: list):
+        for item in items:
+            if dict_body is None:
+                break
+            if isinstance(dict_body, dict):
+                dict_body = dict_body.get(item)
+                continue
+            if isinstance(item, int) and \
+               isinstance(dict_body, list) and \
+               len(dict_body) > item:
+                dict_body = dict_body[item]
+                continue
+            return None
+        return dict_body

pytchat/processors/tsv_archiver.py

@@ -0,0 +1,67 @@
+import csv
+import os
+import re
+from .chat_processor import ChatProcessor
+from .default.processor import DefaultProcessor
+
+PATTERN = re.compile(r"(.*)\(([0-9]+)\)$")
+fmt_headers = ['datetime', 'elapsed', 'authorName', 'message',
+               'superchatAmount', 'authorType', 'authorChannel']
+
+
+class TSVArchiver(ChatProcessor):
+    '''
+    TsvArchiver saves chat data as Tab Separated Values format text.
+    '''
+
+    def __init__(self, save_path):
+        super().__init__()
+        self.save_path = self._checkpath(save_path)
+        with open(self.save_path, mode='a', encoding='utf-8') as f:
+            writer = csv.writer(f, delimiter='\t')
+            writer.writerow(fmt_headers)
+        self.processor = DefaultProcessor()
+
+    def _checkpath(self, filepath):
+        splitter = os.path.splitext(os.path.basename(filepath))
+        body = splitter[0]
+        extention = splitter[1]
+        newpath = filepath
+        counter = 0
+        while os.path.exists(newpath):
+            match = re.search(PATTERN, body)
+            if match:
+                counter = int(match[2]) + 1
+                num_with_bracket = f'({str(counter)})'
+                body = f'{match[1]}{num_with_bracket}'
+            else:
+                body = f'{body}({str(counter)})'
+            newpath = os.path.join(os.path.dirname(filepath), body + extention)
+        return newpath
+
+    def process(self, chat_components: list):
+        """
+        Returns
+        ----------
+        dict :
+            save_path : str :
+                Actual save path of file.
+            total_lines : int :
+                count of total lines written to the file.
+        """
+        if chat_components is None or len(chat_components) == 0:
+            return
+
+        with open(self.save_path, mode='a', encoding='utf-8') as f:
+            writer = csv.writer(f, delimiter='\t')
+            chats = self.processor.process(chat_components).items
+            for c in chats:
+                writer.writerow([
+                    c.datetime,
+                    c.elapsedTime,
+                    c.author.name,
+                    c.message,
+                    c.amountString,
+                    c.author.type,
+                    c.author.channelId
+                ])

pytchat/tool/extract/asyncdl.py

@@ -0,0 +1,176 @@
+import asyncio
+import httpx
+import socket
+from . import parser
+from . block import Block
+from . worker import ExtractWorker
+from . patch import Patch
+from ... import config
+from ... paramgen import arcparam
+from ... exceptions import UnknownConnectionError
+from concurrent.futures import CancelledError
+from json import JSONDecodeError
+from urllib.parse import quote
+
+
+headers = config.headers
+REPLAY_URL = "https://www.youtube.com/live_chat_replay/" \
+             "get_live_chat_replay?continuation="
+MAX_RETRY_COUNT = 3
+
+# Set to avoid duplicate parameters
+param_set = set()
+
+
+def _split(start, end, count, min_interval_sec=120):
+    """
+    Split section from `start` to `end` into `count` pieces,
+    and returns the beginning of each piece.
+    The `count` is adjusted so that the length of each piece
+    is no smaller than `min_interval`.
+
+    Returns:
+    --------
+        List of the offset of each block's first chat data.
+    """
+    if not (isinstance(start, int) or isinstance(start, float)) or \
+       not (isinstance(end, int) or isinstance(end, float)):
+        raise ValueError("start/end must be int or float")
+    if not isinstance(count, int):
+        raise ValueError("count must be int")
+    if start > end:
+        raise ValueError("end must be equal to or greater than start.")
+    if count < 1:
+        raise ValueError("count must be equal to or greater than 1.")
+    if (end - start) / count < min_interval_sec:
+        count = int((end - start) / min_interval_sec)
+        if count == 0:
+            count = 1
+    interval = (end - start) / count
+
+    if count == 1:
+        return [start]
+    return sorted(list(set([int(start + interval * j)
+                            for j in range(count)])))
+
+
+def ready_blocks(video_id, duration, div, callback):
+    param_set.clear()
+    if div <= 0:
+        raise ValueError
+
+    async def _get_blocks(video_id, duration, div, callback):
+        async with httpx.AsyncClient(http2=True) as session:
+            tasks = [_create_block(session, video_id, seektime, callback)
+                     for seektime in _split(-1, duration, div)]
+            return await asyncio.gather(*tasks)
+
+    async def _create_block(session, video_id, seektime, callback):
+        continuation = arcparam.getparam(video_id, seektime=seektime)
+        url = f"{REPLAY_URL}{quote(continuation)}&pbj=1"
+        err = None
+        for _ in range(MAX_RETRY_COUNT):
+            try:
+                if continuation in param_set:
+                    next_continuation, actions = None, []
+                    break
+                param_set.add(continuation)
+                resp = await session.get(url, headers=headers, timeout=10)
+                next_continuation, actions = parser.parse(resp.json())
+                break
+            except JSONDecodeError:
+                await asyncio.sleep(3)
+            except httpx.HTTPError as e:
+                err = e
+                await asyncio.sleep(3)
+        else:
+            cancel()
+            raise UnknownConnectionError("Abort:" + str(err))
+
+        if actions:
+            first = parser.get_offset(actions[0])
+            last = parser.get_offset(actions[-1])
+            if callback:
+                callback(actions, last - first)
+            return Block(
+                continuation=next_continuation,
+                chat_data=actions,
+                first=first,
+                last=last
+            )
+
+    """
+    fetch initial blocks.
+    """
+    loop = asyncio.get_event_loop()
+    blocks = loop.run_until_complete(
+        _get_blocks(video_id, duration, div, callback))
+    return blocks
+
+
+def fetch_patch(callback, blocks, video_id):
+
+    async def _allocate_workers():
+        workers = [
+            ExtractWorker(
+                fetch=_fetch, block=block,
+                blocks=blocks, video_id=video_id
+            )
+            for block in blocks
+        ]
+        async with httpx.AsyncClient() as session:
+            tasks = [worker.run(session) for worker in workers]
+            return await asyncio.gather(*tasks)
+
+    async def _fetch(continuation, session) -> Patch:
+        url = f"{REPLAY_URL}{quote(continuation)}&pbj=1"
+        err = None
+        for _ in range(MAX_RETRY_COUNT):
+            try:
+                if continuation in param_set:
+                    continuation, actions = None, []
+                    break
+                param_set.add(continuation)
+                resp = await session.get(url, headers=config.headers)
+                continuation, actions = parser.parse(resp.json())
+                break
+            except JSONDecodeError:
+                await asyncio.sleep(3)
+            except httpx.HTTPError as e:
+                err = e
+                await asyncio.sleep(3)
+            except socket.error as error:
+                print("socket error", error.errno)
+                await asyncio.sleep(3)
+        else:
+            cancel()
+            raise UnknownConnectionError("Abort:" + str(err))
+
+        if actions:
+            last = parser.get_offset(actions[-1])
+            first = parser.get_offset(actions[0])
+            if callback:
+                callback(actions, last - first)
+            return Patch(actions, continuation, first, last)
+        return Patch(continuation=continuation)
+
+    """
+    allocate workers and assign blocks.
+    """
+    loop = asyncio.get_event_loop()
+    try:
+        loop.run_until_complete(_allocate_workers())
+    except CancelledError:
+        pass
+
+
+async def _shutdown():
+    tasks = [t for t in asyncio.all_tasks()
+             if t is not asyncio.current_task()]
+    for task in tasks:
+        task.cancel()
+
+
+def cancel():
+    loop = asyncio.get_event_loop()
+    loop.create_task(_shutdown())

pytchat/tool/extract/block.py

@@ -0,0 +1,56 @@
+class Block:
+    """Block object represents something like a box
+    to join chunk of chatdata.
+
+    Parameter:
+    ---------
+    first : int :
+        videoOffsetTimeMs of the first chat_data
+        (chat_data[0])
+
+    last : int :
+        videoOffsetTimeMs of the last chat_data.
+        (chat_data[-1])
+
+        this value increases as fetching chatdata progresses.
+
+    end : int :
+        target videoOffsetTimeMs of last chat data for extract,
+        equals to first videoOffsetTimeMs of next block.
+        when extract worker reaches this offset, stop fetching.
+
+    continuation : str :
+        continuation param of last chat data.
+
+    chat_data : list
+
+    done : bool :
+        whether this block has been fetched.
+
+    remaining : int :
+        remaining data to extract.
+        equals end - last.
+
+    is_last : bool :
+        whether this block is the last one in blocklist.
+
+    during_split : bool :
+        whether this block is in the process of during_split.
+        while True, this block is excluded from duplicate split procedure.
+    """
+
+    __slots__ = ['first', 'last', 'end', 'continuation', 'chat_data', 'remaining',
+                 'done', 'is_last', 'during_split']
+
+    def __init__(self, first=0, last=0, end=0,
+                 continuation='', chat_data=[], is_last=False,
+                 during_split=False):
+        self.first = first
+        self.last = last
+        self.end = end
+        self.continuation = continuation
+        self.chat_data = chat_data
+        self.done = False
+        self.remaining = self.end - self.last
+        self.is_last = is_last
+        self.during_split = during_split

pytchat/tool/extract/duplcheck.py

@@ -0,0 +1,153 @@
+from . import parser
+
+
+def check_duplicate(chatdata):
+    max_range = len(chatdata) - 1
+    tbl_offset = [None] * max_range
+    tbl_id = [None] * max_range
+    tbl_type = [None] * max_range
+
+    def create_table(chatdata, max_range):
+        for i in range(max_range):
+            tbl_offset[i] = parser.get_offset(chatdata[i])
+            tbl_id[i] = parser.get_id(chatdata[i])
+            tbl_type[i] = parser.get_type(chatdata[i])
+
+    def is_duplicate(i, j):
+        return (
+            tbl_offset[i] == tbl_offset[j]
+            and tbl_id[i] == tbl_id[j]
+            and tbl_type[i] == tbl_type[j]
+        )
+    print("creating table...")
+    create_table(chatdata, max_range)
+    print("searching duplicate data...")
+    return [{"i": {
+        "index": i, "id": parser.get_id(chatdata[i]),
+        "offsetTime": parser.get_offset(chatdata[i]),
+        "type": parser.get_type(chatdata[i])
+    },
+        "j":{
+        "index": j, "id": parser.get_id(chatdata[j]),
+        "offsetTime": parser.get_offset(chatdata[j]),
+        "type": parser.get_type(chatdata[j])
+    }
+    }
+        for i in range(max_range) for j in range(i + 1, max_range)
+        if is_duplicate(i, j)]
+
+
+def check_duplicate_offset(chatdata):
+    max_range = len(chatdata)
+    tbl_offset = [None] * max_range
+    tbl_id = [None] * max_range
+    tbl_type = [None] * max_range
+
+    def create_table(chatdata, max_range):
+        for i in range(max_range):
+            tbl_offset[i] = parser.get_offset(chatdata[i])
+            tbl_id[i] = parser.get_id(chatdata[i])
+            tbl_type[i] = parser.get_type(chatdata[i])
+
+    def is_duplicate(i, j):
+        return (
+            tbl_offset[i] == tbl_offset[j]
+            and tbl_id[i] == tbl_id[j]
+        )
+
+    print("creating table...")
+    create_table(chatdata, max_range)
+    print("searching duplicate data...")
+
+    return [{
+        "index": i, "id": tbl_id[i],
+        "offsetTime": tbl_offset[i],
+        "type:": tbl_type[i]
+    }
+        for i in range(max_range - 1)
+        if is_duplicate(i, i + 1)]
+
+
+def remove_duplicate_head(blocks):
+    if len(blocks) == 0 or len(blocks) == 1:
+        return blocks
+
+    def is_duplicate_head(index):
+
+        if len(blocks[index].chat_data) == 0:
+            return True
+        elif len(blocks[index + 1].chat_data) == 0:
+            return False
+
+        id_0 = parser.get_id(blocks[index].chat_data[0])
+        id_1 = parser.get_id(blocks[index + 1].chat_data[0])
+        type_0 = parser.get_type(blocks[index].chat_data[0])
+        type_1 = parser.get_type(blocks[index + 1].chat_data[0])
+        return (
+            blocks[index].first == blocks[index + 1].first
+            and id_0 == id_1
+            and type_0 == type_1
+        )
+    ret = [blocks[i] for i in range(len(blocks) - 1)
+           if (len(blocks[i].chat_data) > 0
+               and not is_duplicate_head(i))]
+    ret.append(blocks[-1])
+    return ret
+
+
+def remove_duplicate_tail(blocks):
+    if len(blocks) == 0 or len(blocks) == 1:
+        return blocks
+
+    def is_duplicate_tail(index):
+        if len(blocks[index].chat_data) == 0:
+            return True
+        elif len(blocks[index - 1].chat_data) == 0:
+            return False
+        id_0 = parser.get_id(blocks[index - 1].chat_data[-1])
+        id_1 = parser.get_id(blocks[index].chat_data[-1])
+        type_0 = parser.get_type(blocks[index - 1].chat_data[-1])
+        type_1 = parser.get_type(blocks[index].chat_data[-1])
+        return (
+            blocks[index - 1].last == blocks[index].last
+            and id_0 == id_1
+            and type_0 == type_1
+        )
+
+    ret = [blocks[i] for i in range(0, len(blocks))
+           if i == 0 or not is_duplicate_tail(i)]
+    return ret
+
+
+def remove_overlap(blocks):
+    """
+    Fix overlapped blocks after ready_blocks().
+    Align the last offset of each block to the first offset
+    of next block (equals `end` offset of each block).
+    """
+    if len(blocks) == 0 or len(blocks) == 1:
+        return blocks
+
+    for block in blocks:
+        if block.is_last:
+            break
+        if len(block.chat_data) == 0:
+            continue
+        block_end = block.end
+        if block.last >= block_end:
+            for line in reversed(block.chat_data):
+                if parser.get_offset(line) < block_end:
+                    break
+                block.chat_data.pop()
+            block.last = parser.get_offset(line)
+            block.remaining = 0
+            block.done = True
+            block.continuation = None
+    return blocks
+
+
+def _dump(blocks):
+    print("----------        first         last         end---")
+    for i, block in enumerate(blocks):
+        print(
+            f"block[{i:3}]   {block.first:>10}   {block.last:>10}  {block.end:>10}")

pytchat/tool/extract/extractor.py

@@ -0,0 +1,96 @@
+from . import asyncdl
+from . import duplcheck
+from .. videoinfo import VideoInfo
+from ... import config
+from ... exceptions import InvalidVideoIdException
+from ... util.extract_video_id import extract_video_id
+
+logger = config.logger(__name__)
+headers = config.headers
+
+
+class Extractor:
+    def __init__(self, video_id, div=1, callback=None, processor=None):
+        if not isinstance(div, int) or div < 1:
+            raise ValueError('div must be positive integer.')
+        elif div > 10:
+            div = 10
+        self.video_id = extract_video_id(video_id)
+        self.div = div
+        self.callback = callback
+        self.processor = processor
+        self.duration = self._get_duration_of_video(video_id)
+        self.blocks = []
+
+    def _get_duration_of_video(self, video_id):
+        duration = 0
+        try:
+            duration = VideoInfo(video_id).get_duration()
+        except InvalidVideoIdException:
+            raise
+        return duration
+
+    def _ready_blocks(self):
+        blocks = asyncdl.ready_blocks(
+            self.video_id, self.duration, self.div, self.callback)
+        self.blocks = [block for block in blocks if block]
+        return self
+
+    def _remove_duplicate_head(self):
+        self.blocks = duplcheck.remove_duplicate_head(self.blocks)
+        return self
+
+    def _set_block_end(self):
+        if len(self.blocks) > 0:
+            for i in range(len(self.blocks) - 1):
+                self.blocks[i].end = self.blocks[i + 1].first
+            self.blocks[-1].end = self.duration * 1000
+            self.blocks[-1].is_last = True
+        return self
+
+    def _remove_overlap(self):
+        self.blocks = duplcheck.remove_overlap(self.blocks)
+        return self
+
+    def _download_blocks(self):
+        asyncdl.fetch_patch(self.callback, self.blocks, self.video_id)
+        return self
+
+    def _remove_duplicate_tail(self):
+        self.blocks = duplcheck.remove_duplicate_tail(self.blocks)
+        return self
+
+    def _combine(self):
+        ret = []
+        for block in self.blocks:
+            ret.extend(block.chat_data)
+        return ret
+
+    def _execute_extract_operations(self):
+        return (
+            self._ready_blocks()
+                ._remove_duplicate_head()
+                ._set_block_end()
+                ._remove_overlap()
+                ._download_blocks()
+                ._remove_duplicate_tail()
+                ._combine()
+        )
+
+    def extract(self):
+        if self.duration == 0:
+            print("\nCannot extract chat data:\n The specified video has not yet been archived.")
+            return []
+        data = self._execute_extract_operations()
+        if self.processor is None:
+            return data
+        ret = self.processor.process(
+            [{'video_id': None,
+              'timeout': 1,
+              'chatdata': (action["replayChatItemAction"]["actions"][0] for action in data)}]
+        )
+        self.processor.finalize()
+        return ret
+
+    def cancel(self):
+        asyncdl.cancel()

pytchat/tool/extract/parser.py

@@ -0,0 +1,55 @@
+from ... import config
+from ... import exceptions
+
+logger = config.logger(__name__)
+
+
+def parse(jsn):
+    """
+    Parse replay chat data.
+    Parameter:
+    ----------
+    jsn : dict
+        JSON of replay chat data.
+    Returns:
+    ------
+        continuation : str
+        actions : list
+
+    """
+    if jsn is None:
+        raise ValueError("parameter JSON is None")
+    if jsn['response']['responseContext'].get('errors'):
+        raise exceptions.ResponseContextError(
+            'video_id is invalid or private/deleted.')
+    contents = jsn['response'].get('continuationContents')
+    if contents is None:
+        raise exceptions.NoContents('No chat data.')
+
+    cont = contents['liveChatContinuation']['continuations'][0]
+    if cont is None:
+        raise exceptions.NoContinuation('No Continuation')
+    metadata = cont.get('liveChatReplayContinuationData')
+    if metadata:
+        continuation = metadata.get("continuation")
+        actions = contents['liveChatContinuation'].get('actions')
+        return continuation, actions
+    return None, []
+
+
+def get_offset(item):
+    return int(item['replayChatItemAction']["videoOffsetTimeMsec"])
+
+
+def get_id(item):
+    a = list(item['replayChatItemAction']["actions"][0].values())[0].get('item')
+    if a:
+        return list(a.values())[0].get('id')
+    return None
+
+
+def get_type(item):
+    a = list(item['replayChatItemAction']["actions"][0].values())[0].get('item')
+    if a:
+        return list(a.keys())[0]
+    return None

pytchat/tool/extract/patch.py

@@ -0,0 +1,55 @@
+from . import parser
+from . block import Block
+from typing import NamedTuple
+
+
+class Patch(NamedTuple):
+    """
+    Patch represents chunk of chat data
+    which is fetched by asyncdl.fetch_patch._fetch().
+    """
+    chats: list = []
+    continuation: str = None
+    first: int = None
+    last: int = None
+
+
+def fill(block: Block, patch: Patch):
+    block_end = block.end
+    if patch.last < block_end or block.is_last:
+        set_patch(block, patch)
+        return
+    for line in reversed(patch.chats):
+        line_offset = parser.get_offset(line)
+        if line_offset < block_end:
+            break
+        patch.chats.pop()
+    set_patch(block, patch._replace(
+        continuation=None,
+        last=line_offset
+    )
+    )
+    block.remaining = 0
+    block.done = True
+
+
+def split(parent_block: Block, child_block: Block, patch: Patch):
+    parent_block.during_split = False
+    if patch.first <= parent_block.last:
+        ''' When patch overlaps with parent_block,
+        discard this block. '''
+        child_block.continuation = None
+        ''' Leave child_block.during_split == True
+         to exclude from during_split sequence. '''
+        return
+    child_block.during_split = False
+    child_block.first = patch.first
+    parent_block.end = patch.first
+    fill(child_block, patch)
+
+
+def set_patch(block: Block, patch: Patch):
+    block.continuation = patch.continuation
+    block.chat_data.extend(patch.chats)
+    block.last = patch.last
+    block.remaining = block.end - block.last

pytchat/tool/extract/worker.py

@@ -0,0 +1,92 @@
+from . block import Block
+from . patch import fill, split
+from ... paramgen import arcparam
+from typing import Tuple
+
+
+class ExtractWorker:
+    """
+    ExtractWorker associates a download session with a block.
+    When the worker finishes fetching, the block
+    being fetched is splitted and assigned the free worker.
+
+    Parameter
+    ----------
+    fetch : func :
+        extract function of asyncdl
+
+    block : Block :
+        Block object that includes chat_data
+
+    blocks : list :
+        List of Block(s)
+
+    video_id : str :
+
+    parent_block : Block :
+        the block from which current block is splitted
+    """
+    __slots__ = ['block', 'fetch', 'blocks', 'video_id', 'parent_block']
+
+    def __init__(self, fetch, block, blocks, video_id):
+        self.block = block
+        self.fetch = fetch
+        self.blocks = blocks
+        self.video_id = video_id
+        self.parent_block = None
+
+    async def run(self, session):
+        while self.block.continuation:
+            patch = await self.fetch(
+                self.block.continuation, session)
+            if patch.continuation is None:
+                """TODO : make the worker assigned to the last block
+                to work more than twice as possible.
+                """
+                break
+            if self.parent_block:
+                split(self.parent_block, self.block, patch)
+                self.parent_block = None
+            else:
+                fill(self.block, patch)
+            if self.block.continuation is None:
+                """finished fetching this block """
+                self.block.done = True
+                self.block = _search_new_block(self)
+
+
+def _search_new_block(worker) -> Block:
+    index, undone_block = _get_undone_block(worker.blocks)
+    if undone_block is None:
+        return Block(continuation=None)
+    mean = (undone_block.last + undone_block.end) / 2
+    continuation = arcparam.getparam(worker.video_id, seektime=mean / 1000)
+    worker.parent_block = undone_block
+    worker.parent_block.during_split = True
+    new_block = Block(
+        end=undone_block.end,
+        chat_data=[],
+        continuation=continuation,
+        during_split=True,
+        is_last=worker.parent_block.is_last)
+    '''swap last block'''
+    if worker.parent_block.is_last:
+        worker.parent_block.is_last = False
+    worker.blocks.insert(index + 1, new_block)
+    return new_block
+
+
+def _get_undone_block(blocks) -> Tuple[int, Block]:
+    min_interval_ms = 120000
+    max_remaining = 0
+    undone_block = None
+    index_undone_block = 0
+    for index, block in enumerate(blocks):
+        if block.done or block.during_split:
+            continue
+        remaining = block.remaining
+        if remaining > max_remaining and remaining > min_interval_ms:
+            index_undone_block = index
+            undone_block = block
+            max_remaining = remaining
+    return index_undone_block, undone_block

pytchat/tool/videoinfo.py

@@ -0,0 +1,201 @@
+import httpx
+import json
+import re
+import time
+from .. import config
+from ..exceptions import InvalidVideoIdException, PatternUnmatchError, UnknownConnectionError
+from ..util.extract_video_id import extract_video_id
+
+
+headers = config.headers
+pattern = re.compile(r"['\"]PLAYER_CONFIG['\"]:\s*({.*})")
+pattern2 = re.compile(r"yt\.setConfig\((\{[\s\S]*?\})\);")
+
+item_channel_id = [
+    "videoDetails",
+    "embeddedPlayerOverlayVideoDetailsRenderer",
+    "channelThumbnailEndpoint",
+    "channelThumbnailEndpoint",
+    "urlEndpoint",
+    "urlEndpoint",
+    "url"
+]
+
+item_renderer = [
+    "embedPreview",
+    "thumbnailPreviewRenderer"
+]
+
+item_response = [
+    "args",
+    "embedded_player_response"
+]
+
+item_response2 = [
+    "PLAYER_VARS",
+    "embedded_player_response"
+]
+item_author_image = [
+    "videoDetails",
+    "embeddedPlayerOverlayVideoDetailsRenderer",
+    "channelThumbnail",
+    "thumbnails",
+    0,
+    "url"
+]
+
+item_thumbnail = [
+    "defaultThumbnail",
+    "thumbnails",
+    2,
+    "url"
+]
+
+item_channel_name = [
+    "videoDetails",
+    "embeddedPlayerOverlayVideoDetailsRenderer",
+    "expandedRenderer",
+    "embeddedPlayerOverlayVideoDetailsExpandedRenderer",
+    "title",
+    "runs",
+    0,
+    "text"
+]
+
+item_moving_thumbnail = [
+    "movingThumbnail",
+    "thumbnails",
+    0,
+    "url"
+]
+
+
+class VideoInfo:
+    '''
+    VideoInfo object retrieves YouTube video information.
+
+    Parameter
+    ---------
+    video_id : str
+
+    Exception
+    ---------
+    InvalidVideoIdException :
+        Occurs when video_id does not exist on YouTube.
+    '''
+
+    def __init__(self, video_id):
+        self.video_id = extract_video_id(video_id)
+        self.client = httpx.Client(http2=True)
+        self.new_pattern_text = False
+        err = None
+        for _ in range(3):
+            try:
+                text = self._get_page_text(self.video_id)
+                self._parse(text)
+                break
+            except (InvalidVideoIdException, UnknownConnectionError) as e:
+                raise e
+            except Exception as e:
+                err = e
+                time.sleep(2)
+                pass
+        else:
+            raise err
+
+    def _get_page_text(self, video_id):
+        url = f"https://www.youtube.com/embed/{video_id}"
+        err = None
+        for _ in range(3):
+            try:
+                resp = self.client.get(url, headers=headers)
+                resp.raise_for_status()
+                break
+            except httpx.HTTPError as e:
+                err = e
+                time.sleep(3)
+        else:
+            raise UnknownConnectionError(str(err))
+
+        return resp.text
+
+    def _parse(self, text):
+        result = re.search(pattern, text)
+        if result is None:
+            result = re.search(pattern2, text)
+            if result is None:
+                raise PatternUnmatchError(doc=text)
+            else:
+                self.new_pattern_text = True
+        decoder = json.JSONDecoder()
+        if self.new_pattern_text:
+            res = decoder.raw_decode(result.group(1))[0]
+        else:
+            res = decoder.raw_decode(result.group(1)[:-1])[0]
+        if self.new_pattern_text:
+            response = self._get_item(res, item_response2)
+        else:
+            response = self._get_item(res, item_response)
+        if response is None:
+            if self.new_pattern_text:
+                self._check_video_is_private(res.get("PLAYER_VARS"))
+            else:
+                self._check_video_is_private(res.get("args"))
+        self._renderer = self._get_item(json.loads(response), item_renderer)
+        if self._renderer is None:
+            raise InvalidVideoIdException(
+                f"No renderer found in video_id: [{self.video_id}].")
+
+    def _check_video_is_private(self, args):
+        if args and args.get("video_id"):
+            raise InvalidVideoIdException(
+                f"video_id [{self.video_id}] is private or deleted.")
+        raise InvalidVideoIdException(
+            f"video_id [{self.video_id}] is invalid.")
+
+    def _get_item(self, dict_body, items: list):
+        for item in items:
+            if dict_body is None:
+                break
+            if isinstance(dict_body, dict):
+                dict_body = dict_body.get(item)
+                continue
+            if isinstance(item, int) and \
+               isinstance(dict_body, list) and \
+               len(dict_body) > item:
+                dict_body = dict_body[item]
+                continue
+            return None
+        return dict_body
+
+    def get_duration(self):
+        duration_seconds = self._renderer.get("videoDurationSeconds")
+        if duration_seconds:
+            '''Fetched value is string, so cast to integer.'''
+            return int(duration_seconds)
+        '''When key is not found, explicitly returns None.'''
+        return None
+
+    def get_title(self):
+        if self._renderer.get("title"):
+            return [''.join(run["text"])
+                    for run in self._renderer["title"]["runs"]][0]
+        return None
+
+    def get_channel_id(self):
+        channel_url = self._get_item(self._renderer, item_channel_id)
+        if channel_url:
+            return channel_url[9:]
+        return None
+
+    def get_author_image(self):
+        return self._get_item(self._renderer, item_author_image)
+
+    def get_thumbnail(self):
+        return self._get_item(self._renderer, item_thumbnail)
+
+    def get_channel_name(self):
+        return self._get_item(self._renderer, item_channel_name)
+
+    def get_moving_thumbnail(self):
+        return self._get_item(self._renderer, item_moving_thumbnail)

pytchat/util/__init__.py

@@ -1,15 +1,40 @@
-import requests,json,datetime
+import datetime
+import httpx
+import json
+import os
+import re
 from .. import config
 
-def download(cls,url):
-    _session = requests.Session()
+PATTERN = re.compile(r"(.*)\(([0-9]+)\)$")
+
+
+def extract(url):
+    _session = httpx.Client(http2=True)
     html = _session.get(url, headers=config.headers)
     with open(str(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')
-                        )+'test.json',mode ='w',encoding='utf-8') as f:
-        json.dump(html.json(),f,ensure_ascii=False)
+                  ) + 'test.json', mode='w', encoding='utf-8') as f:
+        json.dump(html.json(), f, ensure_ascii=False)
 
 
-def save(cls,data,filename):
-    with open(str(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')
-                            )+filename,mode ='w',encoding='utf-8') as f:
+def save(data, filename, extention):
+    with open(filename + "_" + (datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')) + extention,
+              mode='w', encoding='utf-8') as f:
         f.writelines(data)
+
+
+def checkpath(filepath):
+    splitter = os.path.splitext(os.path.basename(filepath))
+    body = splitter[0]
+    extention = splitter[1]
+    newpath = filepath
+    counter = 1
+    while os.path.exists(newpath):
+        match = re.search(PATTERN, body)
+        if match:
+            counter = int(match[2]) + 1
+            num_with_bracket = f'({str(counter)})'
+            body = f'{match[1]}{num_with_bracket}'
+        else:
+            body = f'{body}({str(counter)})'
+        newpath = os.path.join(os.path.dirname(filepath), body + extention)
+    return newpath

pytchat/util/extract_video_id.py

@@ -0,0 +1,28 @@
+import re
+from .. exceptions import InvalidVideoIdException
+
+
+PATTERN = re.compile(r"((?<=(v|V)/)|(?<=be/)|(?<=(\?|\&)v=)|(?<=embed/))([\w-]+)")
+YT_VIDEO_ID_LENGTH = 11
+
+
+def extract_video_id(url_or_id: str) -> str:
+    ret = ''
+    if '[' in url_or_id:
+        url_or_id = url_or_id.replace('[', '').replace(']', '')
+
+    if type(url_or_id) != str:
+        raise TypeError(f"{url_or_id}: URL or VideoID must be str, but {type(url_or_id)} is passed.")
+    if len(url_or_id) == YT_VIDEO_ID_LENGTH:
+        return url_or_id
+    match = re.search(PATTERN, url_or_id)
+    if match is None:
+        raise InvalidVideoIdException(f"Invalid video id: {url_or_id}")
+    try:
+        ret = match.group(4)
+    except IndexError:
+        raise InvalidVideoIdException(f"Invalid video id: {url_or_id}")
+
+    if ret is None or len(ret) != YT_VIDEO_ID_LENGTH:
+        raise InvalidVideoIdException(f"Invalid video id: {url_or_id}")
+    return ret

requirements.txt

@@ -1,4 +1,4 @@
-aiohttp
+httpx[http2]==0.16.1
+protobuf==3.14.0
 pytz
-requests
 urllib3

requirements_test.txt

@@ -1,5 +1,2 @@
-aioresponses
-mock
-mocker
-pytest
-pytest-mock
+pytest-mock==3.3.1
+pytest-httpx==0.10.0

setup.py

@@ -1,6 +1,5 @@
-from setuptools import setup, find_packages
-from codecs import open
-from os import path
+from setuptools import setup, find_packages, Command
+from os import path, system, remove, rename, removedirs
 import re
 
 package_name = "pytchat"
@@ -8,20 +7,27 @@ package_name = "pytchat"
 root_dir = path.abspath(path.dirname(__file__))
 
 def _requirements():
-    return [name.rstrip() for name in open(path.join(root_dir, 'requirements.txt')).readlines()]
+    return [name.rstrip() 
+        for name in open(path.join(
+            root_dir, 'requirements.txt')).readlines()]
 
 def _test_requirements():
-    return [name.rstrip() for name in open(path.join(root_dir, 'requirements_test.txt')).readlines()]
-
-
+    return [name.rstrip() 
+        for name in open(path.join(
+            root_dir, 'requirements_test.txt')).readlines()]
 
 with open(path.join(root_dir, package_name, '__init__.py')) as f:
     init_text = f.read()
-    version = re.search(r'__version__\s*=\s*[\'\"](.+?)[\'\"]', init_text).group(1)
-    license = re.search(r'__license__\s*=\s*[\'\"](.+?)[\'\"]', init_text).group(1)
-    author = re.search(r'__author__\s*=\s*[\'\"](.+?)[\'\"]', init_text).group(1)
-    author_email = re.search(r'__author_email__\s*=\s*[\'\"](.+?)[\'\"]', init_text).group(1)
-    url = re.search(r'__url__\s*=\s*[\'\"](.+?)[\'\"]', init_text).group(1)
+    version = re.search(
+        r'__version__\s*=\s*[\'\"](.+?)[\'\"]', init_text).group(1)
+    license = re.search(
+        r'__license__\s*=\s*[\'\"](.+?)[\'\"]', init_text).group(1)
+    author = re.search(
+        r'__author__\s*=\s*[\'\"](.+?)[\'\"]', init_text).group(1)
+    author_email = re.search(
+        r'__author_email__\s*=\s*[\'\"](.+?)[\'\"]', init_text).group(1)
+    url = re.search(
+        r'__url__\s*=\s*[\'\"](.+?)[\'\"]', init_text).group(1)
 
 assert version
 assert license
@@ -29,33 +35,38 @@ assert author
 assert author_email
 assert url
 
+
+
+
 with open('README.md', encoding='utf-8') as f:
     long_description = f.read()
 
+
+
 setup(
-    name=package_name,
-    packages=find_packages(),
-    version=version,
-    url=url,
     author=author,
     author_email=author_email,
-    long_description=long_description,
-    long_description_content_type='text/markdown',
-    license=license,
-    install_requires=_requirements(),
-    tests_require=_test_requirements(),
-    description="a python library for fetching youtube live chat.",
     classifiers=[
         'Natural Language :: Japanese',
         'Development Status :: 4 - Beta',
         'Programming Language :: Python',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.4',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',        
+        'Programming Language :: Python :: 3.8',        
         'License :: OSI Approved :: MIT License',
     ],
+    description="a python library for fetching youtube live chat.",
+    entry_points=
+    '''
+        [console_scripts]
+        pytchat=pytchat.cli:main
+    ''',
+    install_requires=_requirements(),
     keywords='youtube livechat asyncio',
-
+    license=license,
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    name=package_name,
+    packages=find_packages(exclude=['*log.txt','*tests','*testrun']),
+    url=url,
+    version=version,
 )

tests/test_arcparam.py

@@ -0,0 +1,32 @@
+import json
+import httpx
+import pytchat.config as config
+from pytchat.paramgen import arcparam
+from pytchat.parser.live import Parser
+
+
+def test_arcparam_0(mocker):
+    param = arcparam.getparam("01234567890", -1)
+    assert param == "op2w0wQmGhxDZzhLRFFvTE1ERXlNelExTmpjNE9UQWdBUT09SARgAXICCAE%3D" 
+
+
+def test_arcparam_1(mocker):
+    param = arcparam.getparam("01234567890", seektime=100000)
+    assert param == "op2w0wQtGhxDZzhLRFFvTE1ERXlNelExTmpjNE9UQWdBUT09KIDQ28P0AkgDYAFyAggB"
+
+
+def test_arcparam_2(mocker):
+    param = arcparam.getparam("SsjCnHOk-Sk", seektime=100)
+    url = f"https://www.youtube.com/live_chat_replay/get_live_chat_replay?continuation={param}&pbj=1"
+    resp = httpx.Client(http2=True).get(url, headers=config.headers)
+    jsn = json.loads(resp.text)
+    parser = Parser(is_replay=True)
+    contents = parser.get_contents(jsn)
+    _, chatdata = parser.parse(contents)
+    test_id = chatdata[0]["addChatItemAction"]["item"]["liveChatTextMessageRenderer"]["id"]
+    assert test_id == "CjoKGkNMYXBzZTdudHVVQ0Zjc0IxZ0FkTnFnQjVREhxDSnlBNHV2bnR1VUNGV0dnd2dvZDd3NE5aZy0w"
+
+
+def test_arcparam_3(mocker):
+    param = arcparam.getparam("01234567890")
+    assert param == "op2w0wQmGhxDZzhLRFFvTE1ERXlNelExTmpjNE9UQWdBUT09SARgAXICCAE%3D"

tests/test_calculator_get_item.py

@@ -0,0 +1,140 @@
+from pytchat.processors.superchat.calculator import SuperchatCalculator
+
+get_item = SuperchatCalculator()._get_item
+
+dict_test = {
+    'root':{
+        'node0' : 'value0',
+        'node1' : 'value1',
+        'node2' : {
+            'node2-0' : 'value2-0'
+        },
+    
+        'node3' : [
+            {'node3-0' : 'value3-0'},
+            {'node3-1' : 
+                {'node3-1-0' : 'value3-1-0'}
+            }
+        ],
+        'node4' : [],
+        'node5' : [
+            [
+                {'node5-1-0' : 'value5-1-0'},
+                {'node5-1-1' : 'value5-1-1'},
+            ],
+                {'node5-0' : 'value5-0'},
+            
+        ]
+    }
+}
+
+items_test0 = [
+    'root',
+    'node1'
+]
+
+
+items_test_not_found0 = [
+    'root',
+    'other_data'
+]
+
+
+items_test_nest = [
+    'root',
+    'node2',
+    'node2-0'
+]
+
+items_test_list0 = [
+    'root',
+    'node3',
+    1,
+    'node3-1'
+]
+
+items_test_list1 = [
+    'root',
+    'node3',
+    1,
+    'node3-1',
+    'node3-1-0'
+]
+
+items_test_list2 = [
+    'root',
+    'node4',
+    None
+]
+
+items_test_list3 = [
+    'root',
+    'node4'
+]
+
+items_test_list_nest = [
+    'root',
+    'node5',
+    0,
+    1,
+    'node5-1-1'
+]
+
+items_test_list_nest_not_found1 = [
+    'root',
+    'node5',
+    0,
+    1,
+    'node5-1-1',
+    'nodez'
+]
+
+items_test_not_found1 = [
+    'root',
+    'node3',
+    2,
+    'node3-1',
+    'node3-1-0'
+]
+
+items_test_not_found2 = [
+    'root',
+    'node3',
+    2,
+    'node3-1',
+    'node3-1-0',
+    'nodex'
+]
+def test_get_items_0():
+    assert get_item(dict_test, items_test0) == 'value1'
+
+def test_get_items_1():
+    assert get_item(dict_test, items_test_not_found0) is None
+
+def test_get_items_2():
+    assert get_item(dict_test, items_test_nest) == 'value2-0'
+
+def test_get_items_3():
+    assert get_item(
+        dict_test, items_test_list0) ==  {'node3-1-0' : 'value3-1-0'}
+
+def test_get_items_4():
+    assert get_item(dict_test, items_test_list1) ==  'value3-1-0'
+
+def test_get_items_5():
+    assert get_item(dict_test, items_test_not_found1) ==  None
+
+def test_get_items_6():
+    assert get_item(dict_test, items_test_not_found2) ==  None
+
+def test_get_items_7():
+    assert get_item(dict_test, items_test_list2) ==  None
+
+def test_get_items_8():
+    assert get_item(dict_test, items_test_list_nest) ==  'value5-1-1'
+
+def test_get_items_9():
+    assert get_item(dict_test, items_test_list_nest_not_found1) ==  None
+
+def test_get_items_10():
+    assert get_item(dict_test, items_test_list3) ==  []

tests/test_calculator_parse.py

@@ -0,0 +1,68 @@
+import json
+from pytchat.parser.live import Parser
+from pytchat.processors.superchat.calculator import SuperchatCalculator
+from pytchat.exceptions import ChatParseException
+parse = SuperchatCalculator()._parse
+
+
+def _open_file(path):
+    with open(path,mode ='r',encoding = 'utf-8') as f:
+        return f.read()
+
+def load_chatdata(filepath):
+    parser = Parser(is_replay=True)
+    #print(json.loads(_open_file(filepath)))
+    contents = parser.get_contents( json.loads(_open_file(filepath)))
+    return parser.parse(contents)[1]
+
+
+
+def test_parse_1():
+    renderer ={"purchaseAmountText":{"simpleText":"￥2,000"}}
+    symbol ,amount = parse(renderer)
+    assert symbol == '￥'
+    assert amount == 2000.0
+
+def test_parse_2():
+    renderer ={"purchaseAmountText":{"simpleText":"ABC\x0a200"}}
+    symbol ,amount = parse(renderer)
+    assert symbol == 'ABC\x0a'
+    assert amount == 200.0
+
+def test_process_0():
+    """
+    parse superchat data
+    """    
+    chat_component = {
+        'video_id':'',
+        'timeout':10,
+        'chatdata':load_chatdata(r"tests/testdata/calculator/superchat_0.json")
+    }
+    assert SuperchatCalculator().process([chat_component])=={'￥': 6800.0, '€': 2.0}
+
+def test_process_1():
+    """
+    parse no superchat data
+    """
+    chat_component = {
+        'video_id':'',
+        'timeout':10,
+        'chatdata':load_chatdata(r"tests/testdata/calculator/text_only.json")
+    }
+    assert SuperchatCalculator().process([chat_component])=={}
+
+def test_process_2():
+    """
+    try to parse after replay end
+    """
+    try:
+        chat_component = {
+            'video_id':'',
+            'timeout':10,
+            'chatdata':load_chatdata(r"tests/testdata/calculator/replay_end.json")
+        }
+        assert False
+        SuperchatCalculator().process([chat_component])
+    except ChatParseException:
+        assert True
+    

tests/test_compatible_processor.py

@@ -1,16 +1,9 @@
 import json
-import pytest
-import asyncio,aiohttp
-from pytchat.core_async.parser import Parser
+from pytchat.parser.live import Parser
 from pytchat.processors.compatible.processor import CompatibleProcessor
-from pytchat.exceptions import (
-    NoLivechatRendererException,NoYtinitialdataException,
-    ResponseContextError, NoContentsException)
 
-from pytchat.processors.compatible.renderer.textmessage import LiveChatTextMessageRenderer
-from pytchat.processors.compatible.renderer.paidmessage import LiveChatPaidMessageRenderer
-from pytchat.processors.compatible.renderer.paidsticker import LiveChatPaidStickerRenderer
-from pytchat.processors.compatible.renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
+parser = Parser(is_replay=False)
+
 
 def test_textmessage(mocker):
     '''api互換processorのテスト：通常テキストメッセージ'''
@@ -18,36 +11,39 @@ def test_textmessage(mocker):
 
     _json = _open_file("tests/testdata/compatible/textmessage.json")
 
-    _, chatdata = Parser.parse(json.loads(_json))
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
     data = {
-        "video_id" : "",
-        "timeout" : 7,
-        "chatdata" : chatdata
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
     }
     ret = processor.process([data])
 
-    assert ret["kind"]== "youtube#liveChatMessageListResponse"
-    assert ret["pollingIntervalMillis"]==data["timeout"]*1000
+    assert ret["kind"] == "youtube#liveChatMessageListResponse"
+    assert ret["pollingIntervalMillis"] == data["timeout"] * 1000
     assert ret.keys() == {
-        "kind",   "etag",   "pageInfo",   "nextPageToken","pollingIntervalMillis","items"
+        "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
     }
     assert ret["pageInfo"].keys() == {
-        "totalResults",   "resultsPerPage"
+        "totalResults", "resultsPerPage"
     }
     assert ret["items"][0].keys() == {
-        "kind",   "etag",   "id",   "snippet", "authorDetails"
+        "kind", "etag", "id", "snippet", "authorDetails"
     }
     assert ret["items"][0]["snippet"].keys() == {
-        'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage', 'textMessageDetails'
+        'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage',
+        'textMessageDetails'
     }
     assert ret["items"][0]["authorDetails"].keys() == {
-        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator'
+        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor',
+        'isChatModerator'
     }
     assert ret["items"][0]["snippet"]["textMessageDetails"].keys() == {
         'messageText'
     }
-    assert "LCC." in ret["items"][0]["id"] 
-    assert ret["items"][0]["snippet"]["type"]=="textMessageEvent"
+    assert "LCC." in ret["items"][0]["id"]
+    assert ret["items"][0]["snippet"]["type"] == "textMessageEvent"
+
 
 def test_newsponcer(mocker):
     '''api互換processorのテスト：メンバ新規登録'''
@@ -55,34 +51,72 @@ def test_newsponcer(mocker):
 
     _json = _open_file("tests/testdata/compatible/newSponsor.json")
 
-    _, chatdata = Parser.parse(json.loads(_json))
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
     data = {
-        "video_id" : "",
-        "timeout" : 7,
-        "chatdata" : chatdata
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
     }
     ret = processor.process([data])
 
-    assert ret["kind"]== "youtube#liveChatMessageListResponse"
-    assert ret["pollingIntervalMillis"]==data["timeout"]*1000
+    assert ret["kind"] == "youtube#liveChatMessageListResponse"
+    assert ret["pollingIntervalMillis"] == data["timeout"] * 1000
     assert ret.keys() == {
-        "kind",   "etag",   "pageInfo",   "nextPageToken","pollingIntervalMillis","items"
+        "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
     }
     assert ret["pageInfo"].keys() == {
-        "totalResults",   "resultsPerPage"
+        "totalResults", "resultsPerPage"
     }
     assert ret["items"][0].keys() == {
-        "kind",   "etag",   "id",   "snippet","authorDetails"
+        "kind", "etag", "id", "snippet", "authorDetails"
     }
     assert ret["items"][0]["snippet"].keys() == {
         'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage'
 
     }
     assert ret["items"][0]["authorDetails"].keys() == {
-        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator'
+        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor',
+        'isChatModerator'
     }
-    assert "LCC." in ret["items"][0]["id"] 
-    assert ret["items"][0]["snippet"]["type"]=="newSponsorEvent"
+    assert "LCC." in ret["items"][0]["id"]
+    assert ret["items"][0]["snippet"]["type"] == "newSponsorEvent"
+
+
+def test_newsponcer_rev(mocker):
+    '''api互換processorのテスト：メンバ新規登録'''
+    processor = CompatibleProcessor()
+
+    _json = _open_file("tests/testdata/compatible/newSponsor_rev.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+    ret = processor.process([data])
+
+    assert ret["kind"] == "youtube#liveChatMessageListResponse"
+    assert ret["pollingIntervalMillis"] == data["timeout"] * 1000
+    assert ret.keys() == {
+        "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
+    }
+    assert ret["pageInfo"].keys() == {
+        "totalResults", "resultsPerPage"
+    }
+    assert ret["items"][0].keys() == {
+        "kind", "etag", "id", "snippet", "authorDetails"
+    }
+    assert ret["items"][0]["snippet"].keys() == {
+        'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage'
+
+    }
+    assert ret["items"][0]["authorDetails"].keys() == {
+        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor',
+        'isChatModerator'
+    }
+    assert "LCC." in ret["items"][0]["id"]
+    assert ret["items"][0]["snippet"]["type"] == "newSponsorEvent"
 
 
 def test_superchat(mocker):
@@ -91,38 +125,56 @@ def test_superchat(mocker):
 
     _json = _open_file("tests/testdata/compatible/superchat.json")
 
-    _, chatdata = Parser.parse(json.loads(_json))
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
     data = {
-        "video_id" : "",
-        "timeout" : 7,
-        "chatdata" : chatdata
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
     }
     ret = processor.process([data])
 
-    assert ret["kind"]== "youtube#liveChatMessageListResponse"
-    assert ret["pollingIntervalMillis"]==data["timeout"]*1000
+    assert ret["kind"] == "youtube#liveChatMessageListResponse"
+    assert ret["pollingIntervalMillis"] == data["timeout"] * 1000
     assert ret.keys() == {
-        "kind",   "etag",   "pageInfo",   "nextPageToken","pollingIntervalMillis","items"
+        "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
     }
     assert ret["pageInfo"].keys() == {
-        "totalResults",   "resultsPerPage"
+        "totalResults", "resultsPerPage"
     }
     assert ret["items"][0].keys() == {
-        "kind",   "etag",   "id",   "snippet", "authorDetails"
+        "kind", "etag", "id", "snippet", "authorDetails"
     }
     assert ret["items"][0]["snippet"].keys() == {
-        'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage', 'superChatDetails'
+        'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage',
+        'superChatDetails'
     }
     assert ret["items"][0]["authorDetails"].keys() == {
-        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator'
+        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor',
+        'isChatModerator'
     }
     assert ret["items"][0]["snippet"]["superChatDetails"].keys() == {
         'amountMicros', 'currency', 'amountDisplayString', 'tier', 'backgroundColor'
     }
-    assert "LCC." in ret["items"][0]["id"] 
-    assert ret["items"][0]["snippet"]["type"]=="superChatEvent"
+    assert "LCC." in ret["items"][0]["id"]
+    assert ret["items"][0]["snippet"]["type"] == "superChatEvent"
+
+
+def test_unregistered_currency(mocker):
+    processor = CompatibleProcessor()
+
+    _json = _open_file("tests/testdata/unregistered_currency.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+    ret = processor.process([data])
+    assert ret["items"][0]["snippet"]["superChatDetails"]["currency"] == "[UNREGISTERD]"
 
 
 def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
+    with open(path, mode='r', encoding='utf-8') as f:
         return f.read()

tests/test_default_processor.py

@@ -0,0 +1,218 @@
+import json
+from pytchat.parser.live import Parser
+from pytchat.processors.default.processor import DefaultProcessor
+
+
+def test_textmessage(mocker):
+    '''text message'''
+    processor = DefaultProcessor()
+    parser = Parser(is_replay=False)
+    _json = _open_file("tests/testdata/default/textmessage.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+
+    ret = processor.process([data]).items[0]
+    assert ret.id == "dummy_id"
+    assert ret.message == "dummy_message"
+    assert ret.timestamp == 1570678496000
+    assert ret.datetime == "2019-10-10 12:34:56"
+    assert ret.author.name == "author_name"
+    assert ret.author.channelId == "author_channel_id"
+    assert ret.author.channelUrl == "http://www.youtube.com/channel/author_channel_id"
+    assert ret.author.imageUrl == "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg"
+    assert ret.author.badgeUrl == ""
+    assert ret.author.isVerified is False
+    assert ret.author.isChatOwner is False
+    assert ret.author.isChatSponsor is False
+    assert ret.author.isChatModerator is False
+
+
+def test_textmessage_replay_member(mocker):
+    '''text message replay member'''
+    processor = DefaultProcessor()
+    parser = Parser(is_replay=True)
+    _json = _open_file("tests/testdata/default/replay_member_text.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+    
+    ret = processor.process([data]).items[0]
+    assert ret.type == "textMessage"
+    assert ret.id == "dummy_id"
+    assert ret.message == "dummy_message"
+    assert ret.messageEx == ["dummy_message"]
+    assert ret.timestamp == 1570678496000
+    assert ret.datetime == "2019-10-10 12:34:56"
+    assert ret.elapsedTime == "1:23:45"
+    assert ret.author.name == "author_name"
+    assert ret.author.channelId == "author_channel_id"
+    assert ret.author.channelUrl == "http://www.youtube.com/channel/author_channel_id"
+    assert ret.author.imageUrl == "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg"
+    assert ret.author.badgeUrl == "https://yt3.ggpht.com/X=s16-c-k"
+    assert ret.author.isVerified is False
+    assert ret.author.isChatOwner is False
+    assert ret.author.isChatSponsor is True
+    assert ret.author.isChatModerator is False
+
+
+def test_superchat(mocker):
+    '''superchat'''
+    processor = DefaultProcessor()
+    parser = Parser(is_replay=False)
+    _json = _open_file("tests/testdata/default/superchat.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+    
+    ret = processor.process([data]).items[0]
+    assert ret.type == "superChat"
+    assert ret.id == "dummy_id"
+    assert ret.message == "dummy_message"
+    assert ret.messageEx == ["dummy_message"]
+    assert ret.timestamp == 1570678496000
+    assert ret.datetime == "2019-10-10 12:34:56"
+    assert ret.elapsedTime == ""
+    assert ret.amountValue == 800
+    assert ret.amountString == "￥800"
+    assert ret.currency == "JPY"
+    assert ret.bgColor == 4280150454
+    assert ret.author.name == "author_name"
+    assert ret.author.channelId == "author_channel_id"
+    assert ret.author.channelUrl == "http://www.youtube.com/channel/author_channel_id"
+    assert ret.author.imageUrl == "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg"
+    assert ret.author.badgeUrl == ""
+    assert ret.author.isVerified is False
+    assert ret.author.isChatOwner is False
+    assert ret.author.isChatSponsor is False
+    assert ret.author.isChatModerator is False
+    assert ret.colors.headerBackgroundColor == 4278239141
+    assert ret.colors.headerTextColor == 4278190080
+    assert ret.colors.bodyBackgroundColor == 4280150454
+    assert ret.colors.bodyTextColor == 4278190080
+    assert ret.colors.authorNameTextColor == 2315255808
+    assert ret.colors.timestampColor == 2147483648
+
+
+def test_supersticker(mocker):
+    '''supersticker'''
+    processor = DefaultProcessor()
+    parser = Parser(is_replay=False)
+    _json = _open_file("tests/testdata/default/supersticker.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+   
+    ret = processor.process([data]).items[0]
+    assert ret.type == "superSticker"
+    assert ret.id == "dummy_id"
+    assert ret.message == ""
+    assert ret.messageEx == []
+    assert ret.timestamp == 1570678496000
+    assert ret.datetime == "2019-10-10 12:34:56"
+    assert ret.elapsedTime == ""
+    assert ret.amountValue == 200
+    assert ret.amountString == "￥200"
+    assert ret.currency == "JPY"
+    assert ret.bgColor == 4278237396
+    assert ret.sticker == "https://lh3.googleusercontent.com/param_s=s72-rp"
+    assert ret.author.name == "author_name"
+    assert ret.author.channelId == "author_channel_id"
+    assert ret.author.channelUrl == "http://www.youtube.com/channel/author_channel_id"
+    assert ret.author.imageUrl == "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg"
+    assert ret.author.badgeUrl == ""
+    assert ret.author.isVerified is False
+    assert ret.author.isChatOwner is False
+    assert ret.author.isChatSponsor is False
+    assert ret.author.isChatModerator is False
+    assert ret.colors.backgroundColor == 4278237396
+    assert ret.colors.moneyChipBackgroundColor == 4278248959
+    assert ret.colors.moneyChipTextColor == 4278190080
+    assert ret.colors.authorNameTextColor == 3003121664
+
+
+def test_sponsor(mocker):
+    '''sponsor(membership)'''
+    processor = DefaultProcessor()
+    parser = Parser(is_replay=False)
+    _json = _open_file("tests/testdata/default/newSponsor_current.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+    
+    ret = processor.process([data]).items[0]
+    assert ret.type == "newSponsor"
+    assert ret.id == "dummy_id"
+    assert ret.message == "新規メンバー"
+    assert ret.messageEx == ["新規メンバー"]
+    assert ret.timestamp == 1570678496000
+    assert ret.datetime == "2019-10-10 12:34:56"
+    assert ret.elapsedTime == ""
+    assert ret.bgColor == 0
+    assert ret.author.name == "author_name"
+    assert ret.author.channelId == "author_channel_id"
+    assert ret.author.channelUrl == "http://www.youtube.com/channel/author_channel_id"
+    assert ret.author.imageUrl == "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg"
+    assert ret.author.badgeUrl == "https://yt3.ggpht.com/X=s32-c-k"
+    assert ret.author.isVerified is False
+    assert ret.author.isChatOwner is False
+    assert ret.author.isChatSponsor is True
+    assert ret.author.isChatModerator is False
+
+
+def test_sponsor_legacy(mocker):
+    '''lagacy sponsor(membership)'''
+    processor = DefaultProcessor()
+    parser = Parser(is_replay=False)
+    _json = _open_file("tests/testdata/default/newSponsor_lagacy.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+    
+    ret = processor.process([data]).items[0]
+    assert ret.type == "newSponsor"
+    assert ret.id == "dummy_id"
+    assert ret.message == "新規メンバー / ようこそ、author_name！"
+    assert ret.messageEx == ["新規メンバー / ようこそ、author_name！"]
+    assert ret.timestamp == 1570678496000
+    assert ret.datetime == "2019-10-10 12:34:56"
+    assert ret.elapsedTime == ""
+    assert ret.bgColor == 0
+    assert ret.author.name == "author_name"
+    assert ret.author.channelId == "author_channel_id"
+    assert ret.author.channelUrl == "http://www.youtube.com/channel/author_channel_id"
+    assert ret.author.imageUrl == "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg"
+    assert ret.author.badgeUrl == ""
+    assert ret.author.isVerified is False
+    assert ret.author.isChatOwner is False
+    assert ret.author.isChatSponsor is True
+    assert ret.author.isChatModerator is False
+
+
+def _open_file(path):
+    with open(path, mode='r', encoding='utf-8') as f:
+        return f.read()

tests/test_extract_duplcheck.py

@@ -0,0 +1,134 @@
+import json
+from pytchat.tool.extract import duplcheck
+from pytchat.tool.extract import parser
+from pytchat.tool.extract.block import Block
+from pytchat.tool.extract.duplcheck import _dump
+
+
+def _open_file(path):
+    with open(path, mode='r', encoding='utf-8') as f:
+        return f.read()
+
+
+def test_overlap():
+    """
+    test overlap data
+        operation : [0]  [2] [3]  [4] -> last :align to end
+                    [1] , [5] -> no change
+
+    """
+
+    def load_chatdata(filename):
+        return parser.parse(
+            json.loads(_open_file(
+                "tests/testdata/extract_duplcheck/overlap/" + filename))
+        )[1]
+
+    blocks = (
+        Block(first=0, last=12771, end=9890,
+              chat_data=load_chatdata("dp0-0.json")),
+        Block(first=9890, last=15800, end=20244,
+              chat_data=load_chatdata("dp0-1.json")),
+        Block(first=20244, last=45146, end=32476,
+              chat_data=load_chatdata("dp0-2.json")),
+        Block(first=32476, last=50520, end=41380,
+              chat_data=load_chatdata("dp0-3.json")),
+        Block(first=41380, last=62875, end=52568,
+              chat_data=load_chatdata("dp0-4.json")),
+        Block(first=52568, last=62875, end=54000,
+              chat_data=load_chatdata("dp0-5.json"), is_last=True)
+    )
+    result = duplcheck.remove_overlap(blocks)
+    # dp0-0.json has item offset time is 9890 (equals block[0].end = block[1].first),
+    # but must be aligne to the most close and smaller value:9779.
+    assert result[0].last == 9779
+
+    assert result[1].last == 15800
+
+    assert result[2].last == 32196
+
+    assert result[3].last == 41116
+
+    assert result[4].last == 52384
+
+    # the last block must be always added to result.
+    assert result[5].last == 62875
+
+
+def test_duplicate_head():
+
+    def load_chatdata(filename):
+        return parser.parse(
+            json.loads(_open_file(
+                "tests/testdata/extract_duplcheck/head/" + filename))
+        )[1]
+
+    """
+    test duplicate head data 
+        operation : [0] , [1]  -> discard [0]
+                    [1] , [2]  -> discard [1]
+                    [2] , [3]  -> append  [2]
+                    [3] , [4]  -> discard [3]
+                    [4] , [5]  -> append  [4]
+                    append [5]
+
+        result    : [2] , [4] , [5] 
+    """
+
+    # chat data offsets are ignored.
+    blocks = (
+        Block(first=0, last=2500, chat_data=load_chatdata("dp0-0.json")),
+        Block(first=0, last=38771, chat_data=load_chatdata("dp0-1.json")),
+        Block(first=0, last=45146, chat_data=load_chatdata("dp0-2.json")),
+        Block(first=20244, last=60520, chat_data=load_chatdata("dp0-3.json")),
+        Block(first=20244, last=62875, chat_data=load_chatdata("dp0-4.json")),
+        Block(first=52568, last=62875, chat_data=load_chatdata("dp0-5.json"))
+    )
+    _dump(blocks)
+    result = duplcheck.remove_duplicate_head(blocks)
+
+    assert len(result) == 3
+    assert result[0].first == blocks[2].first
+    assert result[0].last == blocks[2].last
+    assert result[1].first == blocks[4].first
+    assert result[1].last == blocks[4].last
+    assert result[2].first == blocks[5].first
+    assert result[2].last == blocks[5].last
+
+
+def test_duplicate_tail():
+    """
+    test duplicate tail data 
+        operation : append [0]
+                    [0] , [1]  -> discard [1]
+                    [1] , [2]  -> append  [2]
+                    [2] , [3]  -> discard [3]
+                    [3] , [4]  -> append  [4]
+                    [4] , [5]  -> discard [5]
+
+        result    : [0] , [2] , [4] 
+    """
+    def load_chatdata(filename):
+        return parser.parse(
+            json.loads(_open_file(
+                "tests/testdata/extract_duplcheck/head/" + filename))
+        )[1]
+    # chat data offsets are ignored.
+    blocks = (
+        Block(first=0, last=2500, chat_data=load_chatdata("dp0-0.json")),
+        Block(first=1500, last=2500, chat_data=load_chatdata("dp0-1.json")),
+        Block(first=10000, last=45146, chat_data=load_chatdata("dp0-2.json")),
+        Block(first=20244, last=45146, chat_data=load_chatdata("dp0-3.json")),
+        Block(first=20244, last=62875, chat_data=load_chatdata("dp0-4.json")),
+        Block(first=52568, last=62875, chat_data=load_chatdata("dp0-5.json"))
+    )
+
+    result = duplcheck.remove_duplicate_tail(blocks)
+    _dump(result)
+    assert len(result) == 3
+    assert result[0].first == blocks[0].first
+    assert result[0].last == blocks[0].last
+    assert result[1].first == blocks[2].first
+    assert result[1].last == blocks[2].last
+    assert result[2].first == blocks[4].first
+    assert result[2].last == blocks[4].last

tests/test_extract_patch.py

@@ -0,0 +1,239 @@
+import json
+
+from pytchat.tool.extract import parser
+from pytchat.tool.extract.block import Block
+from pytchat.tool.extract.patch import Patch, split
+
+
+def _open_file(path):
+    with open(path, mode='r', encoding='utf-8') as f:
+        return f.read()
+
+
+def load_chatdata(filename):
+    return parser.parse(
+        json.loads(_open_file("tests/testdata/fetch_patch/" + filename))
+    )[1]
+
+
+def test_split_0():
+    """
+    Normal case
+
+    ~~~~~~ before ~~~~~~
+
+     @parent_block  (# = already fetched)
+    
+     first    last                                     end
+       |########----------------------------------------|
+    
+
+     @child_block
+    
+     first = last = 0                                  end (=parent_end)
+       |                                                |
+    
+
+     @fetched patch
+                            |-- patch --|
+    
+     
+                            |
+                            |
+                            V 
+
+    ~~~~~~ after ~~~~~~
+    
+
+     @parent_block
+    
+     first    last         end (after split)   
+       |########------------|
+    
+     @child_block
+                          first       last            end            
+                            |###########---------------|
+    
+     @fetched patch
+                            |-- patch --|
+    """
+    parent = Block(first=0, last=4000, end=60000,
+                   continuation='parent', during_split=True)
+    child = Block(first=0, last=0, end=60000,
+                  continuation='mean', during_split=True)
+    patch = Patch(chats=load_chatdata('pt0-5.json'),
+                  first=32500, last=34000, continuation='patch')
+
+    split(parent, child, patch)
+
+    assert child.continuation == 'patch'
+    assert parent.last < child.first
+    assert parent.end == child.first
+    assert child.first < child.last
+    assert child.last < child.end
+    assert parent.during_split is False
+    assert child.during_split is False
+
+
+def test_split_1():
+    """patch.first <= parent_block.last
+
+    While awaiting at run()->asyncdl._fetch()
+    fetching parent_block proceeds, 
+    and parent.block.last exceeds patch.first.
+
+    In this case, fetched patch is all discarded,
+    and worker searches other processing block again. 
+
+    ~~~~~~ before ~~~~~~
+
+                          patch.first
+      first                 |    last                  end
+       |####################|#####|---------------------|
+                            ^
+     @child_block
+     first = last = 0                                  end (=parent_end)
+       |                                                |
+     
+     @fetched patch
+                            |-- patch --|
+    
+     
+                            |
+                            |
+                            V 
+    
+    ~~~~~~ after ~~~~~~
+
+     @parent_block
+     first                       last                  end
+       |###########################|--------------------|
+    
+     @child_block
+                                
+                            ..............　-> 　discard all data
+                   
+    """
+    parent = Block(first=0, last=33000, end=60000, continuation='parent', during_split=True)
+    child = Block(first=0, last=0, end=60000, continuation='mean', during_split=True)
+    patch = Patch(chats=load_chatdata('pt0-5.json'),
+        first=32500, last=34000, continuation='patch')
+
+    split(parent, child, patch)
+
+    assert parent.last == 33000  # no change
+    assert parent.end == 60000  # no change
+    assert child.continuation is None
+    assert parent.during_split is False
+    assert child.during_split is True  # exclude during_split sequence
+
+
+def test_split_2():
+    """child_block.end < patch.last:
+
+    Case the last offset of patch exceeds child_block.end.
+    In this case, remove overlapped data of patch.
+
+    ~~~~~~ before ~~~~~~
+
+     @parent_block  (# = already fetched)
+     first    last                           end (before split)
+       |########------------------------------|
+    
+     @child_block
+     first = last = 0                        end (=parent_end)
+       |                                      |
+    
+    continuation:succeed from patch
+    
+     @fetched patch
+                            |-------- patch --------|
+    
+     
+                            |
+                            |
+                            V 
+
+    ~~~~~~ after ~~~~~~
+
+     @parent_block
+     first    last         end (after split)   
+       |########------------|
+
+     @child_block                                  old patch.end            
+                          first            last=end |
+                            |#################|......   cut extra data.
+                                                    ^
+    continuation : None (extract complete)
+
+     @fetched patch                                 
+                            |-------- patch --------|
+    """
+    parent = Block(first=0, last=4000, end=33500, continuation='parent', during_split=True)
+    child = Block(first=0, last=0, end=33500, continuation='mean', during_split=True)
+    patch = Patch(chats=load_chatdata('pt0-5.json'),
+        first=32500, last=34000, continuation='patch')
+     
+    split(parent, child, patch)
+
+    assert child.continuation is None
+    assert parent.last < child.first
+    assert parent.end == child.first
+    assert child.first < child.last
+    assert child.last < child.end
+    assert child.continuation is None
+    assert parent.during_split is False
+    assert child.during_split is False
+
+
+def test_split_none():
+    """patch.last <= parent_block.last
+
+    While awaiting at run()->asyncdl._fetch()
+    fetching parent_block proceeds, 
+    and parent.block.last exceeds patch.first.
+
+    In this case, fetched patch is all discarded,
+    and worker searches other processing block again.
+    
+    ~~~~~~ before ~~~~~~
+
+                          patch.first
+     first                  |   last                   end
+       |####################|###################|-------|
+                            ^
+     @child_block
+     first = last = 0                                  end (=parent_end)
+       |                                                |
+     
+     @fetched patch
+                            |-- patch --|
+                                      patch.last < parent_block.last                       .
+     
+                            |
+                            |
+                            V 
+    
+    ~~~~~~ after ~~~~~~
+
+     @parent_block
+     first                       last           end (before split)
+       |########################################|-------|
+
+     @child_block
+                                           
+                            ............    -> discard all data.
+
+    """
+    parent = Block(first=0, last=40000, end=60000, continuation='parent', during_split=True)
+    child = Block(first=0, last=0, end=60000, continuation='mean', during_split=True)
+    patch = Patch(chats=load_chatdata('pt0-5.json'),
+        first=32500, last=34000, continuation='patch')
+    
+    split(parent, child, patch)
+
+    assert parent.last == 40000  # no change
+    assert parent.end == 60000  # no change
+    assert child.continuation is None
+    assert parent.during_split is False
+    assert child.during_split is True  # exclude during_split sequence

tests/test_extract_video_id.py

@@ -0,0 +1,55 @@
+from pytchat.util.extract_video_id import extract_video_id
+from pytchat.exceptions import InvalidVideoIdException
+
+VALID_TEST_PATTERNS = (
+    ("ABC_EFG_IJK", "ABC_EFG_IJK"),
+    ("vid_test_be", "vid_test_be"),
+    ("https://www.youtube.com/watch?v=123_456_789", "123_456_789"),
+    ("https://www.youtube.com/watch?v=123_456_789&t=123s", "123_456_789"),
+    ("www.youtube.com/watch?v=123_456_789", "123_456_789"),
+    ("watch?v=123_456_789", "123_456_789"),
+    ("youtube.com/watch?v=123_456_789", "123_456_789"),
+    ("http://youtu.be/ABC_EFG_IJK", "ABC_EFG_IJK"),
+    ("youtu.be/ABC_EFG_IJK", "ABC_EFG_IJK"),
+    ("https://www.youtube.com/watch?v=ABC_EFG_IJK&list=XYZ_ABC_12345&start_radio=1&t=1", "ABC_EFG_IJK"),
+    ("https://www.youtube.com/embed/ABC_EFG_IJK", "ABC_EFG_IJK"),
+    ("www.youtube.com/embed/ABC_EFG_IJK", "ABC_EFG_IJK"),
+    ("youtube.com/embed/ABC_EFG_IJK", "ABC_EFG_IJK")
+)
+
+INVALID_TEST_PATTERNS = (
+    ("", ""),
+    ("0123456789", "0123456789"),  # less than 11 letters id
+    ("more_than_11_letter_string", "more_than_11_letter_string"),
+    ("https://www.youtube.com/watch?v=more_than_11_letter_string", "more_than_11_letter_string"),
+    ("https://www.youtube.com/channel/123_456_789", "123_456_789"),
+)
+
+TYPEERROR_TEST_PATTERNS = (
+    (100, 100),  # not string
+    (["123_456_789"], "123_456_789"),  # not string
+)
+
+
+def test_extract_valid_pattern():
+    for pattern in VALID_TEST_PATTERNS:
+        ret = extract_video_id(pattern[0])
+        assert ret == pattern[1]
+
+
+def test_extract_invalid_pattern():
+    for pattern in INVALID_TEST_PATTERNS:
+        try:
+            extract_video_id(pattern[0])
+            assert False
+        except InvalidVideoIdException:
+            assert True
+
+
+def test_extract_typeerror_pattern():
+    for pattern in TYPEERROR_TEST_PATTERNS:
+        try:
+            extract_video_id(pattern[0])
+            assert False
+        except TypeError:
+            assert True

tests/test_jsonfile_archiver.py

@@ -0,0 +1,48 @@
+import json
+from pytchat.processors.jsonfile_archiver import JsonfileArchiver
+from unittest.mock import patch, mock_open
+from tests.testdata.jsonfile_archiver.chat_component import chat_component
+
+def _open_file(path):
+    with open(path,mode ='r',encoding = 'utf-8') as f:
+        return f.read()
+
+def test_checkpath(mocker):
+    processor = JsonfileArchiver("path")
+    mocker.patch('os.path.exists').side_effect = exists_file
+    '''Test no duplicate file.'''
+    assert processor._checkpath("z:/other.txt") == "z:/other.txt"
+    
+    '''Test duplicate filename. 
+    The case the name first renamed ('test.txt -> test(0).txt')
+    is also duplicated.
+    '''
+    assert processor._checkpath("z:/test.txt") == "z:/test(1).txt"
+    
+    '''Test no extention file (duplicate).'''
+    assert processor._checkpath("z:/test") == "z:/test(0)"
+
+
+def test_read_write():
+    '''Test read and write chatdata'''
+    mock = mock_open(read_data = "")
+    with patch('builtins.open',mock):
+        processor = JsonfileArchiver("path")
+        save_path = processor.process([chat_component])
+    fh = mock()
+    actuals = [args[0] for (args, kwargs) in fh.writelines.call_args_list]
+    '''write format is json dump string  with 0x0A'''
+    to_be_written = [json.dumps(action, ensure_ascii=False)+'\n' 
+        for action in chat_component["chatdata"]]
+    for i in range(len(actuals)):
+        assert actuals[i] == to_be_written[i]
+    assert save_path == {'save_path': 'path', 'total_lines': 7}    
+
+
+def exists_file(path):
+    if path == "z:/test.txt":
+        return True
+    if path == "z:/test(0).txt":
+        return True
+    if path == "z:/test":
+        return True

tests/test_livechat.py

@@ -1,53 +1,48 @@
-import pytest
-from pytchat.core_async.parser import Parser as AsyncParser
-from pytchat.core_multithread.parser import Parser as ThreadParser
+import asyncio
 import json
-import asyncio,aiohttp
-
-from aioresponses import aioresponses
-from pytchat.core_async.livechat import LiveChatAsync
-from pytchat.exceptions import (
-    NoLivechatRendererException,NoYtinitialdataException,
-    ResponseContextError,NoContentsException)
-
-
+from pytest_httpx import HTTPXMock
+from concurrent.futures import CancelledError
 from pytchat.core_multithread.livechat import LiveChat
-import unittest
-from unittest import TestCase
+from pytchat.core_async.livechat import LiveChatAsync
+from pytchat.exceptions import ResponseContextError
+
 
 def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
+    with open(path, mode='r', encoding='utf-8') as f:
         return f.read()
 
-@aioresponses()
-def test_Async(*mock):
-    vid=''
-    _text = _open_file('tests/testdata/paramgen_firstread.json')
-    _text = json.loads(_text)
-    mock[0].get(f"https://www.youtube.com/live_chat?v={vid}&is_popout=1", status=200, body=_text)
+
+def add_response_file(httpx_mock: HTTPXMock, jsonfile_path: str):
+    testdata = json.loads(_open_file(jsonfile_path))
+    httpx_mock.add_response(json=testdata)
+
+
+def test_async(httpx_mock: HTTPXMock):
+    add_response_file(httpx_mock, 'tests/testdata/paramgen_firstread.json')
+
+    async def test_loop():
+        try:
+            chat = LiveChatAsync(video_id='__test_id__')
+            _ = await chat.get()
+            assert chat.is_alive()
+            chat.terminate()
+            assert not chat.is_alive()
+        except ResponseContextError:
+            assert False
+    loop = asyncio.get_event_loop()
     try:
-        chat = LiveChatAsync(video_id='')
+        loop.run_until_complete(test_loop())
+    except CancelledError:
+        assert True
+
+
+def test_multithread(httpx_mock: HTTPXMock):
+    add_response_file(httpx_mock, 'tests/testdata/paramgen_firstread.json')
+    try:
+        chat = LiveChat(video_id='__test_id__')
+        _ = chat.get()
         assert chat.is_alive()
         chat.terminate()
         assert not chat.is_alive()
     except ResponseContextError:
-        assert not chat.is_alive()
-
-def test_MultiThread(mocker):
-    _text = _open_file('tests/testdata/paramgen_firstread.json')
-    _text = json.loads(_text)
-    responseMock = mocker.Mock()
-    responseMock.status_code = 200
-    responseMock.text = _text
-    mocker.patch('requests.Session.get').return_value = responseMock
-    try:
-        chat = LiveChatAsync(video_id='')
-        assert chat.is_alive()
-        chat.terminate()
-        assert not chat.is_alive()
-    except ResponseContextError:
-        chat.terminate()
-        assert not chat.is_alive()
-
-
-
+        assert False

tests/test_livechat_2.py

@@ -0,0 +1,113 @@
+import asyncio
+import json
+from pytest_httpx import HTTPXMock
+from concurrent.futures import CancelledError
+from pytchat.core_multithread.livechat import LiveChat
+from pytchat.core_async.livechat import LiveChatAsync
+from pytchat.processors.dummy_processor import DummyProcessor
+
+
+def _open_file(path):
+    with open(path, mode='r', encoding='utf-8') as f:
+        return f.read()
+
+
+def add_response_file(httpx_mock: HTTPXMock, jsonfile_path: str):
+    testdata = json.loads(_open_file(jsonfile_path))
+    httpx_mock.add_response(json=testdata)
+
+
+def test_async_live_stream(httpx_mock: HTTPXMock):
+    add_response_file(httpx_mock, 'tests/testdata/test_stream.json')
+
+    async def test_loop():
+        chat = LiveChatAsync(video_id='__test_id__', processor=DummyProcessor())
+        chats = await chat.get()
+        rawdata = chats[0]["chatdata"]
+        assert list(rawdata[0]["addChatItemAction"]["item"].keys())[
+            0] == "liveChatTextMessageRenderer"
+        assert list(rawdata[1]["addChatItemAction"]["item"].keys())[
+            0] == "liveChatTextMessageRenderer"
+        assert list(rawdata[2]["addChatItemAction"]["item"].keys())[
+            0] == "liveChatPlaceholderItemRenderer"
+        assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[
+            0] == "liveChatTickerPaidMessageItemRenderer"
+        assert list(rawdata[4]["addChatItemAction"]["item"].keys())[
+            0] == "liveChatPaidMessageRenderer"
+        assert list(rawdata[5]["addChatItemAction"]["item"].keys())[
+            0] == "liveChatPaidStickerRenderer"
+        assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[
+            0] == "liveChatTickerSponsorItemRenderer"
+
+    loop = asyncio.get_event_loop()
+    try:
+        loop.run_until_complete(test_loop())
+    except CancelledError:
+        assert True
+
+
+def test_async_replay_stream(httpx_mock: HTTPXMock):
+    add_response_file(httpx_mock, 'tests/testdata/finished_live.json')
+    add_response_file(httpx_mock, 'tests/testdata/chatreplay.json')
+
+    async def test_loop():
+        chat = LiveChatAsync(video_id='__test_id__', processor=DummyProcessor())
+        chats = await chat.get()
+        rawdata = chats[0]["chatdata"]
+        # assert fetching replaychat data
+        assert list(rawdata[0]["addChatItemAction"]["item"].keys())[
+            0] == "liveChatTextMessageRenderer"
+        assert list(rawdata[14]["addChatItemAction"]["item"].keys())[
+            0] == "liveChatPaidMessageRenderer"
+
+    loop = asyncio.get_event_loop()
+    try:
+        loop.run_until_complete(test_loop())
+    except CancelledError:
+        assert True
+
+
+def test_async_force_replay(httpx_mock: HTTPXMock):
+    add_response_file(httpx_mock, 'tests/testdata/test_stream.json')
+    add_response_file(httpx_mock, 'tests/testdata/chatreplay.json')
+
+    async def test_loop():
+        chat = LiveChatAsync(
+            video_id='__test_id__', processor=DummyProcessor(), force_replay=True)
+        chats = await chat.get()
+        rawdata = chats[0]["chatdata"]
+        # assert fetching replaychat data
+        assert list(rawdata[14]["addChatItemAction"]["item"].keys())[
+            0] == "liveChatPaidMessageRenderer"
+        # assert not mix livechat data
+        assert list(rawdata[2]["addChatItemAction"]["item"].keys())[
+            0] != "liveChatPlaceholderItemRenderer"
+
+    loop = asyncio.get_event_loop()
+    try:
+        loop.run_until_complete(test_loop())
+    except CancelledError:
+        assert True
+
+
+def test_multithread_live_stream(httpx_mock: HTTPXMock):
+    add_response_file(httpx_mock, 'tests/testdata/test_stream.json')
+    chat = LiveChat(video_id='__test_id__', processor=DummyProcessor())
+    chats = chat.get()
+    rawdata = chats[0]["chatdata"]
+    # assert fetching livachat data
+    assert list(rawdata[0]["addChatItemAction"]["item"].keys())[
+        0] == "liveChatTextMessageRenderer"
+    assert list(rawdata[1]["addChatItemAction"]["item"].keys())[
+        0] == "liveChatTextMessageRenderer"
+    assert list(rawdata[2]["addChatItemAction"]["item"].keys())[
+        0] == "liveChatPlaceholderItemRenderer"
+    assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[
+        0] == "liveChatTickerPaidMessageItemRenderer"
+    assert list(rawdata[4]["addChatItemAction"]["item"].keys())[
+        0] == "liveChatPaidMessageRenderer"
+    assert list(rawdata[5]["addChatItemAction"]["item"].keys())[
+        0] == "liveChatPaidStickerRenderer"
+    assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[
+        0] == "liveChatTickerSponsorItemRenderer"
+    chat.terminate()

tests/test_liveparam.py

@@ -0,0 +1,9 @@
+import pytest
+from pytchat.paramgen import liveparam
+
+def test_liveparam_0(mocker):
+    _ts1= 1546268400
+    param = liveparam._build("01234567890",
+    *([_ts1*1000000 for i in range(5)]), topchat_only=False)
+    test_param="0ofMyANcGhxDZzhLRFFvTE1ERXlNelExTmpjNE9UQWdBUT09KIC41tWqyt8CQAFKC1CAuNbVqsrfAlgDUIC41tWqyt8CWIC41tWqyt8CaAGCAQIIAZoBAKABgLjW1arK3wI%3D"
+    assert test_param == param

tests/test_parser.py

@@ -1,44 +1,40 @@
-import pytest
-from pytchat.core_async.parser import Parser
+from pytchat.parser.live import Parser
 import json
-import asyncio,aiohttp
-from aioresponses import aioresponses
-from pytchat.exceptions import (
-    NoLivechatRendererException,NoYtinitialdataException,
-    ResponseContextError, NoContentsException)
+from pytchat.exceptions import NoContents
+
+
+parser = Parser(is_replay=False)
 
 
 def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
+    with open(path, mode='r', encoding='utf-8') as f:
         return f.read()
 
 
-@aioresponses()
 def test_finishedlive(*mock):
     '''配信が終了した動画を正しく処理できるか'''
-
     _text = _open_file('tests/testdata/finished_live.json')
     _text = json.loads(_text)
 
-    try:    
-        Parser.parse(_text)
+    try:
+        parser.parse(parser.get_contents(_text))
         assert False
-    except NoContentsException:
+    except NoContents:
         assert True
 
-@aioresponses()
+
 def test_parsejson(*mock):
     '''jsonを正常にパースできるか'''
-
     _text = _open_file('tests/testdata/paramgen_firstread.json')
     _text = json.loads(_text)
 
-    try:    
-        Parser.parse(_text)
+    try:
+        parser.parse(parser.get_contents(_text))
         jsn = _text
         timeout = jsn["response"]["continuationContents"]["liveChatContinuation"]["continuations"][0]["timedContinuationData"]["timeoutMs"]
-        continuation = jsn["response"]["continuationContents"]["liveChatContinuation"]["continuations"][0]["timedContinuationData"]["continuation"]
-        assert 5035 == timeout
-        assert "0ofMyAPiARp8Q2c4S0RRb0xhelJMZDBsWFQwdERkalFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5YXpSTGQwbFhUMHREZGpRbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiPz5-Os-PkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgJqXjrPj5AJYA1CRwciOs-PkAli3pNq1k-PkAmgBggEECAEQAIgBAKABjbfnjrPj5AI%3D" == continuation
-    except:
-        assert False
+        continuation = jsn["response"]["continuationContents"]["liveChatContinuation"][
+            "continuations"][0]["timedContinuationData"]["continuation"]
+        assert timeout == 5035
+        assert continuation == "0ofMyAPiARp8Q2c4S0RRb0xhelJMZDBsWFQwdERkalFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5YXpSTGQwbFhUMHREZGpRbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiPz5-Os-PkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgJqXjrPj5AJYA1CRwciOs-PkAli3pNq1k-PkAmgBggEECAEQAIgBAKABjbfnjrPj5AI%3D"
+    except Exception:
+        assert False

tests/test_speed_calculator.py

@@ -0,0 +1,64 @@
+import json
+from pytchat.parser.live import Parser
+from pytchat.processors.speed.calculator import SpeedCalculator
+
+parser = Parser(is_replay=False)
+
+
+def test_speed_1(mocker):
+    '''test speed calculation with normal json.
+    test json has 15 chatdata, duration is 30 seconds,
+    so the speed of chatdata is 30 chats/minute.
+    '''
+
+    processor = SpeedCalculator(capacity=30)
+
+    _json = _open_file("tests/testdata/speed/speedtest_normal.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    data = {
+        "video_id": "",
+        "timeout": 10,
+        "chatdata": chatdata
+    }
+    ret = processor.process([data])
+    assert 30 == ret
+
+
+def test_speed_2(mocker):
+    '''test speed calculation with no valid chat data.
+    '''
+    processor = SpeedCalculator(capacity=30)
+
+    _json = _open_file("tests/testdata/speed/speedtest_undefined.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    data = {
+        "video_id": "",
+        "timeout": 10,
+        "chatdata": chatdata
+    }
+    ret = processor.process([data])
+    assert ret == 0
+
+
+def test_speed_3(mocker):
+    '''test speed calculation with empty data.
+    '''
+    processor = SpeedCalculator(capacity=30)
+
+    _json = _open_file("tests/testdata/speed/speedtest_empty.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    data = {
+        "video_id": "",
+        "timeout": 10,
+        "chatdata": chatdata
+    }
+    ret = processor.process([data])
+    assert ret == 0
+
+
+def _open_file(path):
+    with open(path, mode='r', encoding='utf-8') as f:
+        return f.read()

tests/test_videoinfo.py

@@ -0,0 +1,101 @@
+from json.decoder import JSONDecodeError
+from pytchat.tool.videoinfo import VideoInfo
+from pytchat.exceptions import InvalidVideoIdException
+
+
+def _open_file(path):
+    with open(path, mode='r', encoding='utf-8') as f:
+        return f.read()
+
+
+def _set_test_data(filepath, mocker):
+    _text = _open_file(filepath)
+    response_mock = mocker.Mock()
+    response_mock.status_code = 200
+    response_mock.text = _text
+    mocker.patch('httpx.Client.get').return_value = response_mock
+
+
+def test_archived_page(mocker):
+    _set_test_data('tests/testdata/videoinfo/archived_page.txt', mocker)
+    info = VideoInfo('__test_id__')
+    actual_thumbnail_url = 'https://i.ytimg.com/vi/fzI9FNjXQ0o/hqdefault.jpg'
+    assert info.video_id == '__test_id__'
+    assert info.get_channel_name() == 'GitHub'
+    assert info.get_thumbnail() == actual_thumbnail_url
+    assert info.get_title() == 'GitHub Arctic Code Vault'
+    assert info.get_channel_id() == 'UC7c3Kb6jYCRj4JOHHZTxKsQ'
+    assert info.get_duration() == 148
+
+
+def test_live_page(mocker):
+    _set_test_data('tests/testdata/videoinfo/live_page.txt', mocker)
+    info = VideoInfo('__test_id__')
+    '''live page: duration==0'''
+    assert info.get_duration() == 0
+    assert info.video_id == '__test_id__'
+    assert info.get_channel_name() == 'BGM channel'
+    assert info.get_thumbnail() == \
+        'https://i.ytimg.com/vi/fEvM-OUbaKs/hqdefault_live.jpg'
+    assert info.get_title() == (
+        'Coffee Jazz Music - Chill Out Lounge Jazz Music Radio'
+        ' - 24/7 Live Stream - Slow Jazz')
+    assert info.get_channel_id() == 'UCQINXHZqCU5i06HzxRkujfg'
+
+
+def test_invalid_video_id(mocker):
+    '''Test case invalid video_id is specified.'''
+    _set_test_data(
+        'tests/testdata/videoinfo/invalid_video_id_page.txt', mocker)
+    try:
+        _ = VideoInfo('__test_id__')
+        assert False
+    except InvalidVideoIdException:
+        assert True
+
+
+def test_no_info(mocker):
+    '''Test case the video page has renderer, but no info.'''
+    _set_test_data(
+        'tests/testdata/videoinfo/no_info_page.txt', mocker)
+    info = VideoInfo('__test_id__')
+    assert info.video_id == '__test_id__'
+    assert info.get_channel_name() is None
+    assert info.get_thumbnail() is None
+    assert info.get_title() is None
+    assert info.get_channel_id() is None
+    assert info.get_duration() is None
+
+
+def test_collapsed_data(mocker):
+    '''Test case the video page's info is collapsed.'''
+    _set_test_data(
+        'tests/testdata/videoinfo/collapsed_page.txt', mocker)
+    try:
+        _ = VideoInfo('__test_id__')
+        assert False
+    except JSONDecodeError:
+        assert True
+
+
+def test_pattern_unmatch(mocker):
+    '''Test case the pattern for extraction is unmatched.'''
+    _set_test_data(
+        'tests/testdata/videoinfo/pattern_unmatch.txt', mocker)
+    try:
+        _ = VideoInfo('__test_id__')
+        assert False
+    except JSONDecodeError:
+        assert True
+
+
+def test_extradata_handling(mocker):
+    '''Test case the extracted data are JSON lines.'''
+    _set_test_data(
+        'tests/testdata/videoinfo/extradata_page.txt', mocker)
+    try:
+        _ = VideoInfo('__test_id__')
+        assert True
+    except JSONDecodeError as e:
+        print(e.doc)
+        assert False

tests/testdata/calculator/replay_end.json

@@ -0,0 +1,18 @@
+{
+ "response": {
+  "responseContext": {
+   "webResponseContextExtensionData": ""
+  },
+  "continuationContents": {
+   "liveChatContinuation": {
+    "continuations": [
+     {
+      "playerSeekContinuationData": {
+       "continuation": "___reload_continuation___"
+      }
+     }
+    ]
+   }
+  }
+ }
+}

tests/testdata/calculator/text_only.json

@@ -0,0 +1,89 @@
+{
+ "response": {
+  "responseContext": {
+   "webResponseContextExtensionData": ""
+  },
+  "continuationContents": {
+   "liveChatContinuation": {
+    "continuations": [
+     {
+      "invalidationContinuationData": {
+       "invalidationId": {
+        "objectSource": 1000,
+        "objectId": "___objectId___",
+        "topic": "chat~00000000000~0000000",
+        "subscribeToGcmTopics": true,
+        "protoCreationTimestampMs": "1577804400000"
+       },
+       "timeoutMs": 10000,
+       "continuation": "___continuation___"
+      }
+     }
+    ],
+    "actions": [
+     {
+      "replayChatItemAction": {
+       "actions": [
+        {
+         "addChatItemAction": {
+          "item": {
+           "liveChatTextMessageRenderer": {
+            "message": {
+             "runs": [
+              {
+               "text": "dummy_message"
+              }
+             ]
+            },
+            "authorName": {
+             "simpleText": "author_name"
+            },
+            "authorPhoto": {
+             "thumbnails": [
+              {
+               "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+               "width": 32,
+               "height": 32
+              },
+              {
+               "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+               "width": 64,
+               "height": 64
+              }
+             ]
+            },
+            "contextMenuEndpoint": {
+             "commandMetadata": {
+              "webCommandMetadata": {
+               "ignoreNavigation": true
+              }
+             },
+             "liveChatItemContextMenuEndpoint": {
+              "params": "___params___"
+             }
+            },
+            "id": "dummy_id",
+            "timestampUsec": 0,
+            "authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url",
+            "contextMenuAccessibility": {
+             "accessibilityData": {
+              "label": "コメントの操作"
+             }
+            },
+            "timestampText": {
+             "simpleText": "0:00"
+            }
+           }
+          },
+          "clientId": "dummy_client_id"
+         }
+        }
+       ],
+       "videoOffsetTimeMsec": "10000"
+      }
+     }
+    ]
+   }
+  }
+ }
+}

tests/testdata/chat.json

@@ -0,0 +1,164 @@
+{
+  "timing": {
+    "info": {
+      "st": 164
+    }
+  },
+  "csn": "",
+  "response": {
+    "responseContext": {
+      "serviceTrackingParams": [{
+        "service": "CSI",
+        "params": [{
+          "key": "GetLiveChat_rid",
+          "value": ""
+        }, {
+          "key": "c",
+          "value": "WEB"
+        }, {
+          "key": "cver",
+          "value": "2.20191219.03.01"
+        }, {
+          "key": "yt_li",
+          "value": "0"
+        }]
+      }, {
+        "service": "GFEEDBACK",
+        "params": [{
+          "key": "e",
+          "value": ""
+        }, {
+          "key": "logged_in",
+          "value": "0"
+        }]
+      }, {
+        "service": "GUIDED_HELP",
+        "params": [{
+          "key": "logged_in",
+          "value": "0"
+        }]
+      }, {
+        "service": "ECATCHER",
+        "params": [{
+          "key": "client.name",
+          "value": "WEB"
+        }, {
+          "key": "client.version",
+          "value": "2.2"
+        }, {
+          "key": "innertube.build.changelist",
+          "value": "228"
+        }, {
+          "key": "innertube.build.experiments.source_version",
+          "value": "2858"
+        }, {
+          "key": "innertube.build.label",
+          "value": "youtube.ytfe.innertube_"
+        }, {
+          "key": "innertube.build.timestamp",
+          "value": "154"
+        }, {
+          "key": "innertube.build.variants.checksum",
+          "value": "e"
+        }, {
+          "key": "innertube.run.job",
+          "value": "ytfe-innertube-replica-only.ytfe"
+        }]
+      }],
+      "webResponseContextExtensionData": {
+        "ytConfigData": {
+          "csn": "ADw",
+          "visitorData": "%3D%3D"
+        }
+      }
+    },
+    "continuationContents": {
+      "liveChatContinuation": {
+        "continuations": [{
+          "timedContinuationData": {
+            "timeoutMs": 10000,
+            "continuation": "continuation"
+          }
+        }],
+        "actions": [{
+          "addChatItemAction": {
+            "item": {
+              "liveChatTextMessageRenderer": {
+                "message": {
+                  "runs": [{
+                    "text": "message"
+                  }]
+                },
+                "authorName": {
+                  "simpleText": "authorName"
+                },
+                "authorPhoto": {
+                  "thumbnails": [{
+                    "url": "https://yt3.ggpht.com/photo.jpg",
+                    "width": 32,
+                    "height": 32
+                  }, {
+                    "url": "https://yt3.ggpht.com/photo.jpg",
+                    "width": 64,
+                    "height": 64
+                  }]
+                },
+                "contextMenuEndpoint": {
+                  "commandMetadata": {
+                    "webCommandMetadata": {
+                      "ignoreNavigation": true
+                    }
+                  },
+                  "liveChatItemContextMenuEndpoint": {
+                    "params": "params"
+                  }
+                },
+                "id": "id",
+                "timestampUsec": "1576851922945411",
+                "authorBadges": [{
+                  "liveChatAuthorBadgeRenderer": {
+                    "customThumbnail": {
+                      "thumbnails": [{
+                        "url": "https://yt3.ggpht.com/photo.jpg"
+                      }, {
+                        "url": "https://yt3.ggpht.com/photo.jpg"
+                      }]
+                    },
+                    "tooltip": "メンバー（6 か月）",
+                    "accessibility": {
+                      "accessibilityData": {
+                        "label": "メンバー（6 か月）"
+                      }
+                    }
+                  }
+                }],
+                "authorExternalChannelId": "UC",
+                "contextMenuAccessibility": {
+                  "accessibilityData": {
+                    "label": "コメントの操作"
+                  }
+                }
+              }
+            },
+            "clientId": "00000000000000000000"
+          }
+        }
+      ]}
+    },
+        
+    "xsrf_token": "xsrf_token",
+    "url": "/live_chat/get_live_chat?continuation=0",
+    "endpoint": {
+      "commandMetadata": {
+        "webCommandMetadata": {
+          "url": "/live_chat/get_live_chat?continuation=0",
+          "rootVe": 0
+        }
+      },
+      "urlEndpoint": {
+        "url": "/live_chat/get_live_chat?continuation=0"
+      }
+    }
+  }
+}
+