From 9751289ecae91c8dedbf9bf44b1423b7a844875f Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Thu, 2 Jan 2020 12:47:40 +0900 Subject: [PATCH 01/31] Integrate _get_initial_continuation --- pytchat/core_async/replaychat.py | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/pytchat/core_async/replaychat.py b/pytchat/core_async/replaychat.py index 95499fe..9479458 100644 --- a/pytchat/core_async/replaychat.py +++ b/pytchat/core_async/replaychat.py @@ -135,28 +135,9 @@ class ReplayChatAsync: """最初のcontinuationパラメータを取得し、 _listenループのタスクを作成し開始する """ - initial_continuation = await self._get_initial_continuation() - if initial_continuation is None: - self.terminate() - logger.debug(f"[{self.video_id}]No initial continuation.") - return + initial_continuation = arcparam.get(self.video_id,self.seektime) await self._listen(initial_continuation) - async def _get_initial_continuation(self): - ''' チャットデータ取得に必要な最初のcontinuationを取得する。''' - try: - initial_continuation = arcparam.get(self.video_id,self.seektime) - except ChatParseException as e: - self.terminate() - logger.debug(f"[{self.video_id}]Error:{str(e)}") - return - except KeyError: - logger.debug(f"[{self.video_id}]KeyError:" - f"{traceback.format_exc(limit = -1)}") - self.terminate() - return - return initial_continuation - async def _listen(self, continuation): ''' continuationに紐付いたチャットデータを取得し Bufferにチャットデータを格納、 From 7308a87a611bf77fab113f7d2bc9b79a42a23199 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Thu, 2 Jan 2020 13:15:41 +0900 Subject: [PATCH 02/31] Implement pause/pauser to livechat --- pytchat/core_async/livechat.py | 43 ++++++++++++++++---------------- pytchat/core_async/replaychat.py | 6 +++-- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index c671d1a..09f1c69 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -8,6 +8,7 @@ import traceback import urllib.parse from aiohttp.client_exceptions import ClientConnectorError from concurrent.futures import CancelledError +from queue import Queue from .buffer import Buffer from ..parser.live import Parser from .. import config @@ -63,6 +64,7 @@ class LiveChatAsync: _setup_finished = False def __init__(self, video_id, + seektime = 0, processor = DefaultProcessor(), buffer = None, interruptable = True, @@ -71,6 +73,7 @@ class LiveChatAsync: exception_handler = None, direct_mode = False): self.video_id = video_id + self.seektime = seektime if isinstance(processor, tuple): self.processor = Combinator(processor) else: @@ -82,6 +85,8 @@ class LiveChatAsync: self._direct_mode = direct_mode self._is_alive = True self._parser = Parser() + self._pauser = Queue() + self._pauser.put_nowait(None) self._setup() if not LiveChatAsync._setup_finished: @@ -126,28 +131,9 @@ class LiveChatAsync: """最初のcontinuationパラメータを取得し、 _listenループのタスクを作成し開始する """ - initial_continuation = await self._get_initial_continuation() - if initial_continuation is None: - self.terminate() - logger.debug(f"[{self.video_id}]No initial continuation.") - return + initial_continuation = liveparam.getparam(self.video_id) await self._listen(initial_continuation) - async def _get_initial_continuation(self): - ''' チャットデータ取得に必要な最初のcontinuationを取得する。''' - try: - initial_continuation = liveparam.getparam(self.video_id) - except ChatParseException as e: - self.terminate() - logger.debug(f"[{self.video_id}]Error:{str(e)}") - return - except KeyError: - logger.debug(f"[{self.video_id}]KeyError:" - f"{traceback.format_exc(limit = -1)}") - self.terminate() - return - return initial_continuation - async def _listen(self, continuation): ''' continuationに紐付いたチャットデータを取得し Bufferにチャットデータを格納、 @@ -161,6 +147,12 @@ class LiveChatAsync: try: async with aiohttp.ClientSession() as session: while(continuation and self._is_alive): + if self._pauser.empty(): + #pause + await self._pauser.get() + #resume + #prohibit from blocking by putting None into _pauser. + self._pauser.put_nowait(None) livechat_json = (await self._get_livechat_json(continuation, session, headers) ) @@ -246,6 +238,15 @@ class LiveChatAsync: raise IllegalFunctionCall( "既にcallbackを登録済みのため、get()は実行できません。") + def pause(self): + if not self._pauser.empty(): + self._pauser.get() + + def resume(self): + if self._pauser.empty(): + self._pauser.put_nowait(None) + + def is_alive(self): return self._is_alive @@ -264,7 +265,7 @@ class LiveChatAsync: if self._direct_mode == False: #bufferにダミーオブジェクトを入れてis_alive()を判定させる self._buffer.put_nowait({'chatdata':'','timeout':1}) - logger.info(f'終了しました:[{self.video_id}]') + logger.info(f'[{self.video_id}]終了しました') @classmethod def _set_exception_handler(cls, handler): diff --git a/pytchat/core_async/replaychat.py b/pytchat/core_async/replaychat.py index 9479458..d66a2f6 100644 --- a/pytchat/core_async/replaychat.py +++ b/pytchat/core_async/replaychat.py @@ -18,8 +18,9 @@ from ..processors.default.processor import DefaultProcessor from ..processors.combinator import Combinator logger = config.logger(__name__) -MAX_RETRY = 10 headers = config.headers +MAX_RETRY = 10 + @@ -178,11 +179,12 @@ class ReplayChatAsync: await asyncio.sleep(diff_time) continuation = metadata.get('continuation') except ChatParseException as e: + self.terminate() logger.error(f"{str(e)}(video_id:\"{self.video_id}\")") return except (TypeError , json.JSONDecodeError) : - logger.error(f"{traceback.format_exc(limit = -1)}") self.terminate() + logger.error(f"{traceback.format_exc(limit = -1)}") return logger.debug(f"[{self.video_id}]チャット取得を終了しました。") From 2bb481a228d5856c4c2f9579d51184d1e13a40ee Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Thu, 2 Jan 2020 14:27:36 +0900 Subject: [PATCH 03/31] Disable _pauser when callback is unset --- pytchat/config/__init__.py | 2 +- pytchat/core_async/livechat.py | 19 +++++++++++-------- pytchat/paramgen/liveparam.py | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/pytchat/config/__init__.py b/pytchat/config/__init__.py index 1a84b59..542551e 100644 --- a/pytchat/config/__init__.py +++ b/pytchat/config/__init__.py @@ -1,7 +1,7 @@ import logging from . import mylogger -LOGGER_MODE = None +LOGGER_MODE = logging.DEBUG headers = { 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'} diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index 09f1c69..172a96d 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -8,7 +8,7 @@ import traceback import urllib.parse from aiohttp.client_exceptions import ClientConnectorError from concurrent.futures import CancelledError -from queue import Queue +from asyncio import Queue from .buffer import Buffer from ..parser.live import Parser from .. import config @@ -148,11 +148,13 @@ class LiveChatAsync: async with aiohttp.ClientSession() as session: while(continuation and self._is_alive): if self._pauser.empty(): - #pause + '''pause''' await self._pauser.get() - #resume - #prohibit from blocking by putting None into _pauser. + '''resume: + prohibit from blocking by putting None into _pauser. + ''' self._pauser.put_nowait(None) + continuation= liveparam.getparam(self.video_id) livechat_json = (await self._get_livechat_json(continuation, session, headers) ) @@ -239,14 +241,17 @@ class LiveChatAsync: "既にcallbackを登録済みのため、get()は実行できません。") def pause(self): + if self._callback is None: + return if not self._pauser.empty(): - self._pauser.get() + self._pauser.get_nowait() def resume(self): + if self._callback is None: + return if self._pauser.empty(): self._pauser.put_nowait(None) - def is_alive(self): return self._is_alive @@ -270,12 +275,10 @@ class LiveChatAsync: @classmethod def _set_exception_handler(cls, handler): loop = asyncio.get_event_loop() - #default handler: cls._handle_exception loop.set_exception_handler(handler) @classmethod def _handle_exception(cls, loop, context): - #msg = context.get("exception", context["message"]) if not isinstance(context["exception"],CancelledError): logger.error(f"Caught exception: {context}") loop= asyncio.get_event_loop() diff --git a/pytchat/paramgen/liveparam.py b/pytchat/paramgen/liveparam.py index f53fdda..f3bb8b7 100644 --- a/pytchat/paramgen/liveparam.py +++ b/pytchat/paramgen/liveparam.py @@ -155,7 +155,7 @@ def _times(past_sec): return list(map(lambda x:int(x*1000000),[_ts1,_ts2,_ts3,_ts4,_ts5])) -def getparam(video_id,past_sec = 60): +def getparam(video_id,past_sec = 0): ''' Parameter --------- From d6ea673f98b4523d224d326095193db569138121 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Thu, 2 Jan 2020 15:12:48 +0900 Subject: [PATCH 04/31] Fix getting arcparam when resume --- pytchat/core_async/livechat.py | 4 ++-- pytchat/core_async/replaychat.py | 19 +++++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index 172a96d..19aadd3 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -154,7 +154,7 @@ class LiveChatAsync: prohibit from blocking by putting None into _pauser. ''' self._pauser.put_nowait(None) - continuation= liveparam.getparam(self.video_id) + continuation= liveparam.getparam(self.video_id,3) livechat_json = (await self._get_livechat_json(continuation, session, headers) ) @@ -185,6 +185,7 @@ class LiveChatAsync: return logger.debug(f"[{self.video_id}]チャット取得を終了しました。") + self.terminate() async def _get_livechat_json(self, continuation, session, headers): ''' @@ -209,7 +210,6 @@ class LiveChatAsync: else: logger.error(f"[{self.video_id}]" f"Exceeded retry count. status_code={status_code}") - self.terminate() return None return livechat_json diff --git a/pytchat/core_async/replaychat.py b/pytchat/core_async/replaychat.py index d66a2f6..ab5fd5b 100644 --- a/pytchat/core_async/replaychat.py +++ b/pytchat/core_async/replaychat.py @@ -8,7 +8,7 @@ import traceback import urllib.parse from aiohttp.client_exceptions import ClientConnectorError from concurrent.futures import CancelledError -from queue import Queue +from asyncio import Queue from .buffer import Buffer from ..parser.replay import Parser from .. import config @@ -153,11 +153,13 @@ class ReplayChatAsync: async with aiohttp.ClientSession() as session: while(continuation and self._is_alive): if self._pauser.empty(): - #pause + '''pause''' await self._pauser.get() - #resume - #prohibit from blocking by putting None into _pauser. + '''resume: + prohibit from blocking by putting None into _pauser. + ''' self._pauser.put_nowait(None) + #continuation= arcparam.get(self.video_id) livechat_json = (await self._get_livechat_json(continuation, session, headers) ) @@ -244,14 +246,17 @@ class ReplayChatAsync: "既にcallbackを登録済みのため、get()は実行できません。") def pause(self): + if self._callback is None: + return if not self._pauser.empty(): - self._pauser.get() + self._pauser.get_nowait() def resume(self): + if self._callback is None: + return if self._pauser.empty(): self._pauser.put_nowait(None) - def is_alive(self): return self._is_alive @@ -275,12 +280,10 @@ class ReplayChatAsync: @classmethod def _set_exception_handler(cls, handler): loop = asyncio.get_event_loop() - #default handler: cls._handle_exception loop.set_exception_handler(handler) @classmethod def _handle_exception(cls, loop, context): - #msg = context.get("exception", context["message"]) if not isinstance(context["exception"],CancelledError): logger.error(f"Caught exception: {context}") loop= asyncio.get_event_loop() From 2616e4c4b53a947e8d398b56841cb3b1660d0cf3 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Thu, 2 Jan 2020 15:20:34 +0900 Subject: [PATCH 05/31] Adjust amount of first fetching chat --- pytchat/core_async/livechat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index 19aadd3..82b3bdc 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -131,7 +131,7 @@ class LiveChatAsync: """最初のcontinuationパラメータを取得し、 _listenループのタスクを作成し開始する """ - initial_continuation = liveparam.getparam(self.video_id) + initial_continuation = liveparam.getparam(self.video_id,3) await self._listen(initial_continuation) async def _listen(self, continuation): From 907f8aba0b37051d7237bda915f551932a1a09d8 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Thu, 2 Jan 2020 15:42:32 +0900 Subject: [PATCH 06/31] Rename function name --- pytchat/core_async/replaychat.py | 4 ++-- pytchat/core_multithread/replaychat.py | 2 +- pytchat/paramgen/arcparam.py | 12 +++++++++--- tests/test_arcparam.py | 6 +++--- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/pytchat/core_async/replaychat.py b/pytchat/core_async/replaychat.py index ab5fd5b..45f0c6e 100644 --- a/pytchat/core_async/replaychat.py +++ b/pytchat/core_async/replaychat.py @@ -136,7 +136,7 @@ class ReplayChatAsync: """最初のcontinuationパラメータを取得し、 _listenループのタスクを作成し開始する """ - initial_continuation = arcparam.get(self.video_id,self.seektime) + initial_continuation = arcparam.getparam(self.video_id, self.seektime) await self._listen(initial_continuation) async def _listen(self, continuation): @@ -159,7 +159,7 @@ class ReplayChatAsync: prohibit from blocking by putting None into _pauser. ''' self._pauser.put_nowait(None) - #continuation= arcparam.get(self.video_id) + #when replay, not reacquire continuation param livechat_json = (await self._get_livechat_json(continuation, session, headers) ) diff --git a/pytchat/core_multithread/replaychat.py b/pytchat/core_multithread/replaychat.py index 43b5e2e..10106ca 100644 --- a/pytchat/core_multithread/replaychat.py +++ b/pytchat/core_multithread/replaychat.py @@ -139,7 +139,7 @@ class ReplayChat: def _get_initial_continuation(self): ''' チャットデータ取得に必要な最初のcontinuationを取得する。''' try: - initial_continuation = arcparam.get(self.video_id,self.seektime) + initial_continuation = arcparam.getparam(self.video_id,self.seektime) except ChatParseException as e: self.terminate() logger.debug(f"[{self.video_id}]Error:{str(e)}") diff --git a/pytchat/paramgen/arcparam.py b/pytchat/paramgen/arcparam.py index 6cc7650..45da2c7 100644 --- a/pytchat/paramgen/arcparam.py +++ b/pytchat/paramgen/arcparam.py @@ -65,7 +65,7 @@ def _tzparity(video_id,times): return ((times^t) % 2).to_bytes(1,'big') -def get(video_id, seektime = 0, topchatonly = False): +def _build(video_id, seektime, topchatonly = False): switch_01 = b'\x04' if topchatonly else b'\x01' @@ -116,5 +116,11 @@ def get(video_id, seektime = 0, topchatonly = False): ).decode() ) - - +def getparam(video_id, seektime = 0): + ''' + Parameter + --------- + seektime : int + seconds to load past chat data + ''' + return _build(video_id, seektime) diff --git a/tests/test_arcparam.py b/tests/test_arcparam.py index 926dd59..e53eda3 100644 --- a/tests/test_arcparam.py +++ b/tests/test_arcparam.py @@ -5,16 +5,16 @@ import requests, json from pytchat.paramgen import arcparam def test_arcparam_0(mocker): - param = arcparam.get("01234567890") + param = arcparam.getparam("01234567890") assert "op2w0wRyGjxDZzhhRFFvTE1ERXlNelExTmpjNE9UQWFFLXFvM2JrQkRRb0xNREV5TXpRMU5qYzRPVEFnQVElM0QlM0QoATAAOABAAEgEUhwIABAAGAAgACoOc3RhdGljY2hlY2tzdW1AAFgDYAFoAXIECAEQAXgA" == param def test_arcparam_1(mocker): - param = arcparam.get("01234567890", seektime = 100000) + param = arcparam.getparam("01234567890", seektime = 100000) assert "op2w0wR3GjxDZzhhRFFvTE1ERXlNelExTmpjNE9UQWFFLXFvM2JrQkRRb0xNREV5TXpRMU5qYzRPVEFnQVElM0QlM0QogNDbw_QCMAA4AEAASANSHAgAEAAYACAAKg5zdGF0aWNjaGVja3N1bUAAWANgAWgBcgQIARABeAA%3D" == param def test_arcparam_2(mocker): - param = arcparam.get("SsjCnHOk-Sk") + param = arcparam.getparam("SsjCnHOk-Sk") url=f"https://www.youtube.com/live_chat_replay/get_live_chat_replay?continuation={param}&pbj=1" resp = requests.Session().get(url,headers = config.headers) jsn = json.loads(resp.text) From 48b6f2c24e5b4c9672320a06be811148522691ae Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Thu, 2 Jan 2020 15:46:45 +0900 Subject: [PATCH 07/31] Add comment --- pytchat/paramgen/arcparam.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytchat/paramgen/arcparam.py b/pytchat/paramgen/arcparam.py index 45da2c7..99154d7 100644 --- a/pytchat/paramgen/arcparam.py +++ b/pytchat/paramgen/arcparam.py @@ -121,6 +121,7 @@ def getparam(video_id, seektime = 0): Parameter --------- seektime : int - seconds to load past chat data + unit:seconds + start position of fetching chat data. ''' return _build(video_id, seektime) From 7766a39c9c0d8236114adb665247ee5d34ad6f20 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Thu, 2 Jan 2020 19:35:58 +0900 Subject: [PATCH 08/31] Integrate replaychat into livechat --- pytchat/core_async/livechat copy.py | 297 ++++++++++++++++++++++++++++ pytchat/core_async/livechat.py | 29 ++- pytchat/parser/live.py | 56 ++++-- tests/test_compatible_processor.py | 6 +- tests/test_parser.py | 4 +- tests/test_speed_calculator.py | 6 +- 6 files changed, 372 insertions(+), 26 deletions(-) create mode 100644 pytchat/core_async/livechat copy.py diff --git a/pytchat/core_async/livechat copy.py b/pytchat/core_async/livechat copy.py new file mode 100644 index 0000000..82b3bdc --- /dev/null +++ b/pytchat/core_async/livechat copy.py @@ -0,0 +1,297 @@ +import aiohttp, asyncio +import datetime +import json +import random +import signal +import time +import traceback +import urllib.parse +from aiohttp.client_exceptions import ClientConnectorError +from concurrent.futures import CancelledError +from asyncio import Queue +from .buffer import Buffer +from ..parser.live import Parser +from .. import config +from ..exceptions import ChatParseException,IllegalFunctionCall +from ..paramgen import liveparam +from ..processors.default.processor import DefaultProcessor +from ..processors.combinator import Combinator + +logger = config.logger(__name__) +headers = config.headers +MAX_RETRY = 10 + + +class LiveChatAsync: + '''asyncio(aiohttp)を利用してYouTubeのライブ配信のチャットデータを取得する。 + + Parameter + --------- + video_id : str + 動画ID + + processor : ChatProcessor + チャットデータを加工するオブジェクト + + buffer : Buffer(maxsize:20[default]) + チャットデータchat_componentを格納するバッファ。 + maxsize : 格納できるchat_componentの個数 + default値20個。1個で約5~10秒分。 + + interruptable : bool + Ctrl+Cによる処理中断を行うかどうか。 + + callback : func + _listen()関数から一定間隔で自動的に呼びだす関数。 + + done_callback : func + listener終了時に呼び出すコールバック。 + + exception_handler : func + 例外を処理する関数 + + direct_mode : bool + Trueの場合、bufferを使わずにcallbackを呼ぶ。 + Trueの場合、callbackの設定が必須 + (設定していない場合IllegalFunctionCall例外を発生させる) + + Attributes + --------- + _is_alive : bool + チャット取得を停止するためのフラグ + ''' + + _setup_finished = False + + def __init__(self, video_id, + seektime = 0, + processor = DefaultProcessor(), + buffer = None, + interruptable = True, + callback = None, + done_callback = None, + exception_handler = None, + direct_mode = False): + self.video_id = video_id + self.seektime = seektime + if isinstance(processor, tuple): + self.processor = Combinator(processor) + else: + self.processor = processor + self._buffer = buffer + self._callback = callback + self._done_callback = done_callback + self._exception_handler = exception_handler + self._direct_mode = direct_mode + self._is_alive = True + self._parser = Parser() + self._pauser = Queue() + self._pauser.put_nowait(None) + self._setup() + + if not LiveChatAsync._setup_finished: + LiveChatAsync._setup_finished = True + if exception_handler == None: + self._set_exception_handler(self._handle_exception) + else: + self._set_exception_handler(exception_handler) + if interruptable: + signal.signal(signal.SIGINT, + (lambda a, b:asyncio.create_task( + LiveChatAsync.shutdown(None,signal.SIGINT,b)) + )) + + def _setup(self): + #direct modeがTrueでcallback未設定の場合例外発生。 + if self._direct_mode: + if self._callback is None: + raise IllegalFunctionCall( + "direct_mode=Trueの場合callbackの設定が必須です。") + else: + #direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成 + if self._buffer is None: + self._buffer = Buffer(maxsize = 20) + #callbackが指定されている場合はcallbackを呼ぶループタスクを作成 + if self._callback is None: + pass + else: + #callbackを呼ぶループタスクの開始 + loop = asyncio.get_event_loop() + loop.create_task(self._callback_loop(self._callback)) + #_listenループタスクの開始 + loop = asyncio.get_event_loop() + listen_task = loop.create_task(self._startlisten()) + #add_done_callbackの登録 + if self._done_callback is None: + listen_task.add_done_callback(self.finish) + else: + listen_task.add_done_callback(self._done_callback) + + async def _startlisten(self): + """最初のcontinuationパラメータを取得し、 + _listenループのタスクを作成し開始する + """ + initial_continuation = liveparam.getparam(self.video_id,3) + await self._listen(initial_continuation) + + async def _listen(self, continuation): + ''' continuationに紐付いたチャットデータを取得し + Bufferにチャットデータを格納、 + 次のcontinuaitonを取得してループする。 + + Parameter + --------- + continuation : str + 次のチャットデータ取得に必要なパラメータ + ''' + try: + async with aiohttp.ClientSession() as session: + while(continuation and self._is_alive): + if self._pauser.empty(): + '''pause''' + await self._pauser.get() + '''resume: + prohibit from blocking by putting None into _pauser. + ''' + self._pauser.put_nowait(None) + continuation= liveparam.getparam(self.video_id,3) + livechat_json = (await + self._get_livechat_json(continuation, session, headers) + ) + metadata, chatdata = self._parser.parse( livechat_json ) + timeout = metadata['timeoutMs']/1000 + chat_component = { + "video_id" : self.video_id, + "timeout" : timeout, + "chatdata" : chatdata + } + time_mark =time.time() + if self._direct_mode: + await self._callback( + self.processor.process([chat_component]) + ) + else: + await self._buffer.put(chat_component) + diff_time = timeout - (time.time()-time_mark) + await asyncio.sleep(diff_time) + continuation = metadata.get('continuation') + except ChatParseException as e: + self.terminate() + logger.error(f"{str(e)}(video_id:\"{self.video_id}\")") + return + except (TypeError , json.JSONDecodeError) : + self.terminate() + logger.error(f"{traceback.format_exc(limit = -1)}") + return + + logger.debug(f"[{self.video_id}]チャット取得を終了しました。") + self.terminate() + + async def _get_livechat_json(self, continuation, session, headers): + ''' + チャットデータが格納されたjsonデータを取得する。 + ''' + continuation = urllib.parse.quote(continuation) + livechat_json = None + status_code = 0 + url =( + f"https://www.youtube.com/live_chat/get_live_chat?" + f"continuation={continuation}&pbj=1") + for _ in range(MAX_RETRY + 1): + async with session.get(url ,headers = headers) as resp: + try: + text = await resp.text() + status_code = resp.status + livechat_json = json.loads(text) + break + except (ClientConnectorError,json.JSONDecodeError) : + await asyncio.sleep(1) + continue + else: + logger.error(f"[{self.video_id}]" + f"Exceeded retry count. status_code={status_code}") + return None + return livechat_json + + async def _callback_loop(self,callback): + """ コンストラクタでcallbackを指定している場合、バックグラウンドで + callbackに指定された関数に一定間隔でチャットデータを投げる。 + + Parameter + --------- + callback : func + 加工済みのチャットデータを渡す先の関数。 + """ + while self.is_alive(): + items = await self._buffer.get() + data = self.processor.process(items) + await callback(data) + + async def get(self): + """ bufferからデータを取り出し、processorに投げ、 + 加工済みのチャットデータを返す。 + + Returns + : Processorによって加工されたチャットデータ + """ + if self._callback is None: + items = await self._buffer.get() + return self.processor.process(items) + raise IllegalFunctionCall( + "既にcallbackを登録済みのため、get()は実行できません。") + + def pause(self): + if self._callback is None: + return + if not self._pauser.empty(): + self._pauser.get_nowait() + + def resume(self): + if self._callback is None: + return + if self._pauser.empty(): + self._pauser.put_nowait(None) + + def is_alive(self): + return self._is_alive + + def finish(self,sender): + '''Listener終了時のコールバック''' + try: + self.terminate() + except CancelledError: + logger.debug(f'[{self.video_id}]cancelled:{sender}') + + def terminate(self): + ''' + Listenerを終了する。 + ''' + self._is_alive = False + if self._direct_mode == False: + #bufferにダミーオブジェクトを入れてis_alive()を判定させる + self._buffer.put_nowait({'chatdata':'','timeout':1}) + logger.info(f'[{self.video_id}]終了しました') + + @classmethod + def _set_exception_handler(cls, handler): + loop = asyncio.get_event_loop() + loop.set_exception_handler(handler) + + @classmethod + def _handle_exception(cls, loop, context): + if not isinstance(context["exception"],CancelledError): + logger.error(f"Caught exception: {context}") + loop= asyncio.get_event_loop() + loop.create_task(cls.shutdown(None,None,None)) + + @classmethod + async def shutdown(cls, event, sig = None, handler=None): + logger.debug("シャットダウンしています") + tasks = [t for t in asyncio.all_tasks() if t is not + asyncio.current_task()] + [task.cancel() for task in tasks] + + logger.debug(f"残っているタスクを終了しています") + await asyncio.gather(*tasks,return_exceptions=True) + loop = asyncio.get_event_loop() + loop.stop() \ No newline at end of file diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index 82b3bdc..1eb7269 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -13,7 +13,7 @@ from .buffer import Buffer from ..parser.live import Parser from .. import config from ..exceptions import ChatParseException,IllegalFunctionCall -from ..paramgen import liveparam +from ..paramgen import liveparam, arcparam from ..processors.default.processor import DefaultProcessor from ..processors.combinator import Combinator @@ -88,7 +88,9 @@ class LiveChatAsync: self._pauser = Queue() self._pauser.put_nowait(None) self._setup() - + #self._paramgen = liveparam + self._first_fetch = True + self._fetch_url = "live_chat/get_live_chat?continuation=" if not LiveChatAsync._setup_finished: LiveChatAsync._setup_finished = True if exception_handler == None: @@ -154,11 +156,26 @@ class LiveChatAsync: prohibit from blocking by putting None into _pauser. ''' self._pauser.put_nowait(None) - continuation= liveparam.getparam(self.video_id,3) + if self._parser.mode == 'LIVE': + continuation = liveparam.getparam(self.video_id,3) livechat_json = (await self._get_livechat_json(continuation, session, headers) ) - metadata, chatdata = self._parser.parse( livechat_json ) + contents = self._parser.get_contents(livechat_json) + '''switch live or replay''' + if self._first_fetch: + if contents is None: + self._parser.mode = 'REPLAY' + self._fetch_url = ("live_chat_replay/" + "get_live_chat_replay?continuation=") + continuation = arcparam.getparam(self.video_id) + livechat_json = (await self._get_livechat_json( + continuation, session, headers)) + contents = self._parser.get_contents(livechat_json) + self._first_fetch = False + + metadata, chatdata = self._parser.parse( contents ) + timeout = metadata['timeoutMs']/1000 chat_component = { "video_id" : self.video_id, @@ -191,12 +208,12 @@ class LiveChatAsync: ''' チャットデータが格納されたjsonデータを取得する。 ''' + continuation = urllib.parse.quote(continuation) livechat_json = None status_code = 0 url =( - f"https://www.youtube.com/live_chat/get_live_chat?" - f"continuation={continuation}&pbj=1") + f"https://www.youtube.com/{self._fetch_url}{continuation}&pbj=1") for _ in range(MAX_RETRY + 1): async with session.get(url ,headers = headers) as resp: try: diff --git a/pytchat/parser/live.py b/pytchat/parser/live.py index e32177f..0207ac8 100644 --- a/pytchat/parser/live.py +++ b/pytchat/parser/live.py @@ -14,9 +14,24 @@ from .. exceptions import ( logger = config.logger(__name__) - +from .. import util class Parser: - def parse(self, jsn): + URL_LIVE = "live_chat/get_live_chat?continuation=" + URL_REPLAY = "live_chat_replay/get_live_chat_replay?continuation=" + + def __init__(self): + self.mode = 'LIVE' + + def get_contents(self, jsn): + if jsn is None: + return {'timeoutMs':0,'continuation':None},[] + if jsn['response']['responseContext'].get('errors'): + raise ResponseContextError('The video_id would be wrong, or video is deleted or private.') + contents=jsn['response'].get('continuationContents') + return contents + + def parse(self, contents): + util.save(json.dumps(contents,ensure_ascii=False,indent=2),"v:\\~\\test_",".json") """ このparse関数はLiveChat._listen() 関数から定期的に呼び出される。 引数jsnはYoutubeから取得したチャットデータの生JSONであり、 @@ -27,7 +42,7 @@ class Parser: Parameter ---------- - + jsn : dict + + contents : dict + Youtubeから取得したチャットデータのJSONオブジェクト。 (pythonの辞書形式に変換済みの状態で渡される) @@ -38,19 +53,19 @@ class Parser: + chatdata : list[dict] + チャットデータ本体のリスト。 """ - if jsn is None: - return {'timeoutMs':0,'continuation':None},[] - if jsn['response']['responseContext'].get('errors'): - raise ResponseContextError('動画に接続できません。' - '動画IDが間違っているか、動画が削除/非公開の可能性があります。') - contents=jsn['response'].get('continuationContents') - #配信が終了した場合、もしくはチャットデータが取得できない場合 + # if jsn is None: + # return {'timeoutMs':0,'continuation':None},[] + # if jsn['response']['responseContext'].get('errors'): + # raise ResponseContextError('動画に接続できません。' + # '動画IDが間違っているか、動画が削除/非公開の可能性があります。') + # contents=jsn['response'].get('continuationContents') + '''配信が終了した場合、もしくはチャットデータが取得できない場合''' if contents is None: raise NoContentsException('チャットデータを取得できませんでした。') cont = contents['liveChatContinuation']['continuations'][0] if cont is None: - raise NoContinuationsException('Continuationがありません。') + raise NoContinuationsException('No Continuation') metadata = (cont.get('invalidationContinuationData') or cont.get('timedContinuationData') or cont.get('reloadContinuationData') @@ -60,6 +75,23 @@ class Parser: if unknown: logger.debug(f"Received unknown continuation type:{unknown}") metadata = cont.get(unknown) - metadata.setdefault('timeoutMs', 10000) + return self._create_data(metadata, contents) + + def _create_data(self, metadata, contents): chatdata = contents['liveChatContinuation'].get('actions') + if self.mode == 'LIVE': + metadata.setdefault('timeoutMs', 10000) + else: + interval = self._get_interval(chatdata) + metadata.setdefault("timeoutMs",interval) + """アーカイブ済みチャットはライブチャットと構造が異なっているため、以下の行により + ライブチャットと同じ形式にそろえる""" + chatdata = [action["replayChatItemAction"]["actions"][0] for action in chatdata] return metadata, chatdata + + def _get_interval(self, actions: list): + if actions is None: + return 0 + start = int(actions[0]["replayChatItemAction"]["videoOffsetTimeMsec"]) + last = int(actions[-1]["replayChatItemAction"]["videoOffsetTimeMsec"]) + return (last - start) \ No newline at end of file diff --git a/tests/test_compatible_processor.py b/tests/test_compatible_processor.py index fc3051a..efc81b9 100644 --- a/tests/test_compatible_processor.py +++ b/tests/test_compatible_processor.py @@ -20,7 +20,7 @@ def test_textmessage(mocker): _json = _open_file("tests/testdata/compatible/textmessage.json") - _, chatdata = parser.parse(json.loads(_json)) + _, chatdata = parser.parse(parser.get_contents(json.loads(_json))) data = { "video_id" : "", "timeout" : 7, @@ -57,7 +57,7 @@ def test_newsponcer(mocker): _json = _open_file("tests/testdata/compatible/newSponsor.json") - _, chatdata = parser.parse(json.loads(_json)) + _, chatdata = parser.parse(parser.get_contents(json.loads(_json))) data = { "video_id" : "", "timeout" : 7, @@ -93,7 +93,7 @@ def test_superchat(mocker): _json = _open_file("tests/testdata/compatible/superchat.json") - _, chatdata = parser.parse(json.loads(_json)) + _, chatdata = parser.parse(parser.get_contents(json.loads(_json))) data = { "video_id" : "", "timeout" : 7, diff --git a/tests/test_parser.py b/tests/test_parser.py index b338908..b40bee8 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -21,7 +21,7 @@ def test_finishedlive(*mock): _text = json.loads(_text) try: - parser.parse(_text) + parser.parse(parser.get_contents(_text)) assert False except NoContentsException: assert True @@ -34,7 +34,7 @@ def test_parsejson(*mock): _text = json.loads(_text) try: - parser.parse(_text) + parser.parse(parser.get_contents(_text)) jsn = _text timeout = jsn["response"]["continuationContents"]["liveChatContinuation"]["continuations"][0]["timedContinuationData"]["timeoutMs"] continuation = jsn["response"]["continuationContents"]["liveChatContinuation"]["continuations"][0]["timedContinuationData"]["continuation"] diff --git a/tests/test_speed_calculator.py b/tests/test_speed_calculator.py index 8a096c8..f137c52 100644 --- a/tests/test_speed_calculator.py +++ b/tests/test_speed_calculator.py @@ -21,7 +21,7 @@ def test_speed_1(mocker): _json = _open_file("tests/testdata/speed/speedtest_normal.json") - _, chatdata = parser.parse(json.loads(_json)) + _, chatdata = parser.parse(parser.get_contents(json.loads(_json))) data = { "video_id" : "", "timeout" : 10, @@ -37,7 +37,7 @@ def test_speed_2(mocker): _json = _open_file("tests/testdata/speed/speedtest_undefined.json") - _, chatdata = parser.parse(json.loads(_json)) + _, chatdata = parser.parse(parser.get_contents(json.loads(_json))) data = { "video_id" : "", "timeout" : 10, @@ -53,7 +53,7 @@ def test_speed_3(mocker): _json = _open_file("tests/testdata/speed/speedtest_empty.json") - _, chatdata = parser.parse(json.loads(_json)) + _, chatdata = parser.parse(parser.get_contents(json.loads(_json))) data = { "video_id" : "", "timeout" : 10, From 347707a5144e990f540d69fd11f1138074dbb42d Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Thu, 2 Jan 2020 20:08:45 +0900 Subject: [PATCH 09/31] Delete unnecessary lines --- pytchat/core_async/livechat.py | 1 - pytchat/parser/live.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index 1eb7269..09e3c5f 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -218,7 +218,6 @@ class LiveChatAsync: async with session.get(url ,headers = headers) as resp: try: text = await resp.text() - status_code = resp.status livechat_json = json.loads(text) break except (ClientConnectorError,json.JSONDecodeError) : diff --git a/pytchat/parser/live.py b/pytchat/parser/live.py index 0207ac8..f362be9 100644 --- a/pytchat/parser/live.py +++ b/pytchat/parser/live.py @@ -16,8 +16,6 @@ logger = config.logger(__name__) from .. import util class Parser: - URL_LIVE = "live_chat/get_live_chat?continuation=" - URL_REPLAY = "live_chat_replay/get_live_chat_replay?continuation=" def __init__(self): self.mode = 'LIVE' From f4dc5e9d4a5f56d0b94bf5994dd651f2ae36d099 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Thu, 2 Jan 2020 20:31:51 +0900 Subject: [PATCH 10/31] Delete unnecessary lines --- pytchat/core_async/livechat.py | 37 ++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index 09e3c5f..16e3367 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -161,19 +161,7 @@ class LiveChatAsync: livechat_json = (await self._get_livechat_json(continuation, session, headers) ) - contents = self._parser.get_contents(livechat_json) - '''switch live or replay''' - if self._first_fetch: - if contents is None: - self._parser.mode = 'REPLAY' - self._fetch_url = ("live_chat_replay/" - "get_live_chat_replay?continuation=") - continuation = arcparam.getparam(self.video_id) - livechat_json = (await self._get_livechat_json( - continuation, session, headers)) - contents = self._parser.get_contents(livechat_json) - self._first_fetch = False - + contents = await self._get_contents(livechat_json, session, headers) metadata, chatdata = self._parser.parse( contents ) timeout = metadata['timeoutMs']/1000 @@ -204,6 +192,29 @@ class LiveChatAsync: logger.debug(f"[{self.video_id}]チャット取得を終了しました。") self.terminate() + async def _get_contents(self, livechat_json, session, headers): + '''Get 'contents' dict from livechat json. + If contents is None at first fetching, + try to fetch archive chat data. + + Return: + ------- + 'contents' dict which includes metadata & chatdata. + ''' + contents = self._parser.get_contents(livechat_json) + if self._first_fetch: + if contents is None: + '''Try to fetch archive chat data.''' + self._parser.mode = 'REPLAY' + self._fetch_url = ("live_chat_replay/" + "get_live_chat_replay?continuation=") + continuation = arcparam.getparam(self.video_id) + livechat_json = (await self._get_livechat_json( + continuation, session, headers)) + contents = self._parser.get_contents(livechat_json) + self._first_fetch = False + return contents + async def _get_livechat_json(self, continuation, session, headers): ''' チャットデータが格納されたjsonデータを取得する。 From fc5979c025a79511027cccb784546b780828e13c Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Thu, 2 Jan 2020 20:51:59 +0900 Subject: [PATCH 11/31] Moved livechat_json part --- pytchat/core_async/livechat.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index 16e3367..e178b46 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -158,11 +158,9 @@ class LiveChatAsync: self._pauser.put_nowait(None) if self._parser.mode == 'LIVE': continuation = liveparam.getparam(self.video_id,3) - livechat_json = (await - self._get_livechat_json(continuation, session, headers) - ) - contents = await self._get_contents(livechat_json, session, headers) - metadata, chatdata = self._parser.parse( contents ) + contents = await self._get_contents( + continuation, session, headers) + metadata, chatdata = self._parser.parse(contents) timeout = metadata['timeoutMs']/1000 chat_component = { @@ -192,7 +190,7 @@ class LiveChatAsync: logger.debug(f"[{self.video_id}]チャット取得を終了しました。") self.terminate() - async def _get_contents(self, livechat_json, session, headers): + async def _get_contents(self, continuation, session, headers): '''Get 'contents' dict from livechat json. If contents is None at first fetching, try to fetch archive chat data. @@ -201,6 +199,10 @@ class LiveChatAsync: ------- 'contents' dict which includes metadata & chatdata. ''' + + livechat_json = (await + self._get_livechat_json(continuation, session, headers) + ) contents = self._parser.get_contents(livechat_json) if self._first_fetch: if contents is None: From 7b7323abf846732967880d680e6c793758900611 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Thu, 2 Jan 2020 20:52:36 +0900 Subject: [PATCH 12/31] Delete debug line --- pytchat/parser/live.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pytchat/parser/live.py b/pytchat/parser/live.py index f362be9..c445780 100644 --- a/pytchat/parser/live.py +++ b/pytchat/parser/live.py @@ -29,7 +29,6 @@ class Parser: return contents def parse(self, contents): - util.save(json.dumps(contents,ensure_ascii=False,indent=2),"v:\\~\\test_",".json") """ このparse関数はLiveChat._listen() 関数から定期的に呼び出される。 引数jsnはYoutubeから取得したチャットデータの生JSONであり、 From 18400724b1d8b790953902d5a50a53a3eb0d44ed Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Thu, 2 Jan 2020 21:08:53 +0900 Subject: [PATCH 13/31] Modify metadata selection --- pytchat/core_async/livechat.py | 1 - pytchat/parser/live.py | 28 ++++++++++++---------------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index e178b46..cc1ab46 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -199,7 +199,6 @@ class LiveChatAsync: ------- 'contents' dict which includes metadata & chatdata. ''' - livechat_json = (await self._get_livechat_json(continuation, session, headers) ) diff --git a/pytchat/parser/live.py b/pytchat/parser/live.py index c445780..031c031 100644 --- a/pytchat/parser/live.py +++ b/pytchat/parser/live.py @@ -31,11 +31,8 @@ class Parser: def parse(self, contents): """ このparse関数はLiveChat._listen() 関数から定期的に呼び出される。 - 引数jsnはYoutubeから取得したチャットデータの生JSONであり、 - このparse関数によって与えられたJSONを以下に分割して返す。 - + timeout (次のチャットデータ取得までのインターバル) - + chat data(チャットデータ本体) - + continuation (次のチャットデータ取得に必要となるパラメータ). + 引数contentsはYoutubeから取得したチャットデータの生JSONであり、 + 与えられたJSONをチャットデータとメタデータに分割して返す。 Parameter ---------- @@ -45,19 +42,17 @@ class Parser: Returns ------- - + metadata : dict - + チャットデータに付随するメタデータ。timeout、 動画ID、continuationパラメータで構成される。 + tuple: + + metadata : dict  チャットデータに付随するメタデータ + + timeout + + video_id + + continuation + chatdata : list[dict] - + チャットデータ本体のリスト。 +     チャットデータ本体のリスト。 """ - # if jsn is None: - # return {'timeoutMs':0,'continuation':None},[] - # if jsn['response']['responseContext'].get('errors'): - # raise ResponseContextError('動画に接続できません。' - # '動画IDが間違っているか、動画が削除/非公開の可能性があります。') - # contents=jsn['response'].get('continuationContents') - '''配信が終了した場合、もしくはチャットデータが取得できない場合''' + if contents is None: + '''配信が終了した場合、もしくはチャットデータが取得できない場合''' raise NoContentsException('チャットデータを取得できませんでした。') cont = contents['liveChatContinuation']['continuations'][0] @@ -65,7 +60,8 @@ class Parser: raise NoContinuationsException('No Continuation') metadata = (cont.get('invalidationContinuationData') or cont.get('timedContinuationData') or - cont.get('reloadContinuationData') + cont.get('reloadContinuationData') or + cont.get('liveChatReplayContinuationData') ) if metadata is None: unknown = list(cont.keys())[0] From 0fc9d14780ebe567de880d8ba5b4ea50d0714455 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Thu, 2 Jan 2020 21:14:22 +0900 Subject: [PATCH 14/31] Fix handling exception --- pytchat/parser/live.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pytchat/parser/live.py b/pytchat/parser/live.py index 031c031..f0fd176 100644 --- a/pytchat/parser/live.py +++ b/pytchat/parser/live.py @@ -9,7 +9,8 @@ from .. import config from .. exceptions import ( ResponseContextError, NoContentsException, - NoContinuationsException ) + NoContinuationsException, + ChatParseException ) logger = config.logger(__name__) @@ -22,7 +23,7 @@ class Parser: def get_contents(self, jsn): if jsn is None: - return {'timeoutMs':0,'continuation':None},[] + raise ChatParseException('Called with none JSON object.') if jsn['response']['responseContext'].get('errors'): raise ResponseContextError('The video_id would be wrong, or video is deleted or private.') contents=jsn['response'].get('continuationContents') From 4c558491a39ad2f3e82f21729bf1c5af0663ceab Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Thu, 2 Jan 2020 21:17:37 +0900 Subject: [PATCH 15/31] Change exception message --- pytchat/parser/live.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytchat/parser/live.py b/pytchat/parser/live.py index f0fd176..1cc1125 100644 --- a/pytchat/parser/live.py +++ b/pytchat/parser/live.py @@ -54,7 +54,7 @@ class Parser: if contents is None: '''配信が終了した場合、もしくはチャットデータが取得できない場合''' - raise NoContentsException('チャットデータを取得できませんでした。') + raise NoContentsException('Chat data stream is empty.') cont = contents['liveChatContinuation']['continuations'][0] if cont is None: From 2fdd834caf6fbb6e1885cf043c9cfdbdeae88ed1 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Thu, 2 Jan 2020 22:08:26 +0900 Subject: [PATCH 16/31] Extract method _check_pause() --- pytchat/core_async/livechat.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index cc1ab46..7b69b4d 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -149,15 +149,7 @@ class LiveChatAsync: try: async with aiohttp.ClientSession() as session: while(continuation and self._is_alive): - if self._pauser.empty(): - '''pause''' - await self._pauser.get() - '''resume: - prohibit from blocking by putting None into _pauser. - ''' - self._pauser.put_nowait(None) - if self._parser.mode == 'LIVE': - continuation = liveparam.getparam(self.video_id,3) + continuation = await self._check_pause(continuation) contents = await self._get_contents( continuation, session, headers) metadata, chatdata = self._parser.parse(contents) @@ -190,6 +182,18 @@ class LiveChatAsync: logger.debug(f"[{self.video_id}]チャット取得を終了しました。") self.terminate() + async def _check_pause(self, continuation): + if self._pauser.empty(): + '''pause''' + await self._pauser.get() + '''resume: + prohibit from blocking by putting None into _pauser. + ''' + self._pauser.put_nowait(None) + if self._parser.mode == 'LIVE': + continuation = liveparam.getparam(self.video_id,3) + return continuation + async def _get_contents(self, continuation, session, headers): '''Get 'contents' dict from livechat json. If contents is None at first fetching, From d742a9fdf35fc409b33c999e10e02534841da014 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Thu, 2 Jan 2020 22:27:21 +0900 Subject: [PATCH 17/31] Fix seek time param --- pytchat/core_async/livechat.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index 7b69b4d..72e7d10 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -88,7 +88,6 @@ class LiveChatAsync: self._pauser = Queue() self._pauser.put_nowait(None) self._setup() - #self._paramgen = liveparam self._first_fetch = True self._fetch_url = "live_chat/get_live_chat?continuation=" if not LiveChatAsync._setup_finished: @@ -213,7 +212,7 @@ class LiveChatAsync: self._parser.mode = 'REPLAY' self._fetch_url = ("live_chat_replay/" "get_live_chat_replay?continuation=") - continuation = arcparam.getparam(self.video_id) + continuation = arcparam.getparam(self.video_id, self.seektime) livechat_json = (await self._get_livechat_json( continuation, session, headers)) contents = self._parser.get_contents(livechat_json) @@ -222,9 +221,8 @@ class LiveChatAsync: async def _get_livechat_json(self, continuation, session, headers): ''' - チャットデータが格納されたjsonデータを取得する。 + Get json which includes chat data. ''' - continuation = urllib.parse.quote(continuation) livechat_json = None status_code = 0 From 30708470f26b4c2ba69d9e7eb59acc1449751ca7 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Thu, 2 Jan 2020 22:43:23 +0900 Subject: [PATCH 18/31] Change comments --- pytchat/core_async/livechat.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index 72e7d10..e0d1f8b 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -107,7 +107,7 @@ class LiveChatAsync: if self._direct_mode: if self._callback is None: raise IllegalFunctionCall( - "direct_mode=Trueの場合callbackの設定が必須です。") + "When direct_mode=True, callback parameter is required.") else: #direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成 if self._buffer is None: @@ -129,21 +129,20 @@ class LiveChatAsync: listen_task.add_done_callback(self._done_callback) async def _startlisten(self): - """最初のcontinuationパラメータを取得し、 - _listenループのタスクを作成し開始する + """Fetch first continuation parameter, + create and start _listen loop. """ initial_continuation = liveparam.getparam(self.video_id,3) await self._listen(initial_continuation) async def _listen(self, continuation): - ''' continuationに紐付いたチャットデータを取得し - Bufferにチャットデータを格納、 - 次のcontinuaitonを取得してループする。 + ''' Fetch chat data and store them into buffer, + get next continuaiton parameter and loop. Parameter --------- continuation : str - 次のチャットデータ取得に必要なパラメータ + parameter for next chat data ''' try: async with aiohttp.ClientSession() as session: @@ -178,7 +177,7 @@ class LiveChatAsync: logger.error(f"{traceback.format_exc(limit = -1)}") return - logger.debug(f"[{self.video_id}]チャット取得を終了しました。") + logger.debug(f"[{self.video_id}]finished fetching chat.") self.terminate() async def _check_pause(self, continuation): @@ -300,7 +299,7 @@ class LiveChatAsync: if self._direct_mode == False: #bufferにダミーオブジェクトを入れてis_alive()を判定させる self._buffer.put_nowait({'chatdata':'','timeout':1}) - logger.info(f'[{self.video_id}]終了しました') + logger.info(f'[{self.video_id}]finished.') @classmethod def _set_exception_handler(cls, handler): @@ -316,12 +315,12 @@ class LiveChatAsync: @classmethod async def shutdown(cls, event, sig = None, handler=None): - logger.debug("シャットダウンしています") + logger.debug("shutdown...") tasks = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()] [task.cancel() for task in tasks] - logger.debug(f"残っているタスクを終了しています") + logger.debug(f"complete remaining tasks...") await asyncio.gather(*tasks,return_exceptions=True) loop = asyncio.get_event_loop() loop.stop() \ No newline at end of file From 2c684d04b50fe1d5a5d48bf649a160fb64ce99db Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Fri, 3 Jan 2020 02:09:39 +0900 Subject: [PATCH 19/31] Intgegrate replaychat into livechat (multithread) --- pytchat/core_async/livechat.py | 10 ++- pytchat/core_multithread/livechat.py | 130 +++++++++++++++++---------- pytchat/parser/live.py | 4 + 3 files changed, 95 insertions(+), 49 deletions(-) diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index e0d1f8b..a78dc87 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -169,16 +169,15 @@ class LiveChatAsync: await asyncio.sleep(diff_time) continuation = metadata.get('continuation') except ChatParseException as e: - self.terminate() - logger.error(f"{str(e)}(video_id:\"{self.video_id}\")") + #self.terminate() + logger.debug(f"[{self.video_id}]{str(e)}") return except (TypeError , json.JSONDecodeError) : - self.terminate() + #self.terminate() logger.error(f"{traceback.format_exc(limit = -1)}") return logger.debug(f"[{self.video_id}]finished fetching chat.") - self.terminate() async def _check_pause(self, continuation): if self._pauser.empty(): @@ -269,6 +268,9 @@ class LiveChatAsync: raise IllegalFunctionCall( "既にcallbackを登録済みのため、get()は実行できません。") + def get_mode(self): + return self._parser.mode + def pause(self): if self._callback is None: return diff --git a/pytchat/core_multithread/livechat.py b/pytchat/core_multithread/livechat.py index 30b9249..a07a439 100644 --- a/pytchat/core_multithread/livechat.py +++ b/pytchat/core_multithread/livechat.py @@ -7,11 +7,12 @@ import time import traceback import urllib.parse from concurrent.futures import CancelledError, ThreadPoolExecutor +from queue import Queue from .buffer import Buffer from ..parser.live import Parser from .. import config from ..exceptions import ChatParseException,IllegalFunctionCall -from ..paramgen import liveparam +from ..paramgen import liveparam, arcparam from ..processors.default.processor import DefaultProcessor from ..processors.combinator import Combinator @@ -63,6 +64,7 @@ class LiveChat: #チャット監視中のListenerのリスト _listeners= [] def __init__(self, video_id, + seektime = 0, processor = DefaultProcessor(), buffer = None, interruptable = True, @@ -71,6 +73,7 @@ class LiveChat: direct_mode = False ): self.video_id = video_id + self.seektime = seektime if isinstance(processor, tuple): self.processor = Combinator(processor) else: @@ -82,7 +85,11 @@ class LiveChat: self._direct_mode = direct_mode self._is_alive = True self._parser = Parser() + self._pauser = Queue() + self._pauser.put_nowait(None) self._setup() + self._first_fetch = True + self._fetch_url = "live_chat/get_live_chat?continuation=" if not LiveChat._setup_finished: LiveChat._setup_finished = True @@ -93,11 +100,12 @@ class LiveChat: LiveChat._listeners.append(self) def _setup(self): + #logger.debug("setup") #direct modeがTrueでcallback未設定の場合例外発生。 if self._direct_mode: if self._callback is None: raise IllegalFunctionCall( - "direct_mode=Trueの場合callbackの設定が必須です。") + "When direct_mode=True, callback parameter is required.") else: #direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成 if self._buffer is None: @@ -117,48 +125,30 @@ class LiveChat: listen_task.add_done_callback(self._done_callback) def _startlisten(self): - """最初のcontinuationパラメータを取得し、 - _listenループのタスクを作成し開始する + time.sleep(0.1) #sleep shortly to prohibit skipping fetching data + """Fetch first continuation parameter, + create and start _listen loop. """ - initial_continuation = self._get_initial_continuation() - if initial_continuation is None: - self.terminate() - logger.debug(f"[{self.video_id}]No initial continuation.") - return + initial_continuation = liveparam.getparam(self.video_id,3) self._listen(initial_continuation) - def _get_initial_continuation(self): - ''' チャットデータ取得に必要な最初のcontinuationを取得する。''' - try: - initial_continuation = liveparam.getparam(self.video_id) - except ChatParseException as e: - self.terminate() - logger.debug(f"[{self.video_id}]Error:{str(e)}") - return - except KeyError: - logger.debug(f"[{self.video_id}]KeyError:" - f"{traceback.format_exc(limit = -1)}") - self.terminate() - return - return initial_continuation - def _listen(self, continuation): - ''' continuationに紐付いたチャットデータを取得し - Bufferにチャットデータを格納、 - 次のcontinuaitonを取得してループする。 + ''' Fetch chat data and store them into buffer, + get next continuaiton parameter and loop. Parameter --------- continuation : str - 次のチャットデータ取得に必要なパラメータ + parameter for next chat data ''' try: with requests.Session() as session: while(continuation and self._is_alive): - livechat_json = ( - self._get_livechat_json(continuation, session, headers) - ) - metadata, chatdata = self._parser.parse( livechat_json ) + continuation = self._check_pause(continuation) + contents = self._get_contents( + continuation, session, headers) + metadata, chatdata = self._parser.parse(contents) + timeout = metadata['timeoutMs']/1000 chat_component = { "video_id" : self.video_id, @@ -173,35 +163,70 @@ class LiveChat: else: self._buffer.put(chat_component) diff_time = timeout - (time.time()-time_mark) - if diff_time < 0 : diff_time=0 - time.sleep(diff_time) + time.sleep(diff_time if diff_time > 0 else 0) continuation = metadata.get('continuation') except ChatParseException as e: - self.terminate() - logger.error(f"{str(e)}(video_id:\"{self.video_id}\")") + #self.terminate() + logger.debug(f"[{self.video_id}]{str(e)}") return except (TypeError , json.JSONDecodeError) : - self.terminate() + #self.terminate() logger.error(f"{traceback.format_exc(limit = -1)}") return - logger.debug(f"[{self.video_id}]チャット取得を終了しました。") + logger.debug(f"[{self.video_id}]finished fetching chat.") + + def _check_pause(self, continuation): + if self._pauser.empty(): + '''pause''' + self._pauser.get() + '''resume: + prohibit from blocking by putting None into _pauser. + ''' + self._pauser.put_nowait(None) + if self._parser.mode == 'LIVE': + continuation = liveparam.getparam(self.video_id,3) + return continuation + + def _get_contents(self, continuation, session, headers): + '''Get 'contents' dict from livechat json. + If contents is None at first fetching, + try to fetch archive chat data. + + Return: + ------- + 'contents' dict which includes metadata & chatdata. + ''' + livechat_json = ( + self._get_livechat_json(continuation, session, headers) + ) + contents = self._parser.get_contents(livechat_json) + if self._first_fetch: + if contents is None: + '''Try to fetch archive chat data.''' + self._parser.mode = 'REPLAY' + self._fetch_url = ("live_chat_replay/" + "get_live_chat_replay?continuation=") + continuation = arcparam.getparam(self.video_id, self.seektime) + livechat_json = ( self._get_livechat_json( + continuation, session, headers)) + contents = self._parser.get_contents(livechat_json) + self._first_fetch = False + return contents def _get_livechat_json(self, continuation, session, headers): ''' - チャットデータが格納されたjsonデータを取得する。 + Get json which includes chat data. ''' continuation = urllib.parse.quote(continuation) livechat_json = None status_code = 0 url =( - f"https://www.youtube.com/live_chat/get_live_chat?" - f"continuation={continuation}&pbj=1") + f"https://www.youtube.com/{self._fetch_url}{continuation}&pbj=1") for _ in range(MAX_RETRY + 1): with session.get(url ,headers = headers) as resp: try: text = resp.text - status_code = resp.status_code livechat_json = json.loads(text) break except json.JSONDecodeError : @@ -210,7 +235,7 @@ class LiveChat: else: logger.error(f"[{self.video_id}]" f"Exceeded retry count. status_code={status_code}") - self.terminate() + #self.terminate() return None return livechat_json @@ -241,6 +266,21 @@ class LiveChat: raise IllegalFunctionCall( "既にcallbackを登録済みのため、get()は実行できません。") + def get_mode(self): + return self._parser.mode + + def pause(self): + if self._callback is None: + return + if not self._pauser.empty(): + self._pauser.get() + + def resume(self): + if self._callback is None: + return + if self._pauser.empty(): + self._pauser.put_nowait(None) + def is_alive(self): return self._is_alive @@ -259,10 +299,10 @@ class LiveChat: if self._direct_mode == False: #bufferにダミーオブジェクトを入れてis_alive()を判定させる self._buffer.put({'chatdata':'','timeout':1}) - logger.info(f'[{self.video_id}]終了しました') + logger.info(f'[{self.video_id}]finished.') @classmethod def shutdown(cls, event, sig = None, handler=None): - logger.debug("シャットダウンしています") + logger.debug("shutdown...") for t in LiveChat._listeners: t._is_alive = False \ No newline at end of file diff --git a/pytchat/parser/live.py b/pytchat/parser/live.py index 1cc1125..b58ab3d 100644 --- a/pytchat/parser/live.py +++ b/pytchat/parser/live.py @@ -65,10 +65,14 @@ class Parser: cont.get('liveChatReplayContinuationData') ) if metadata is None: + if cont.get("playerSeekContinuationData"): + raise ChatParseException('Finished chat data') unknown = list(cont.keys())[0] if unknown: logger.debug(f"Received unknown continuation type:{unknown}") metadata = cont.get(unknown) + else: + raise ChatParseException('Cannot extract continuation data') return self._create_data(metadata, contents) def _create_data(self, metadata, contents): From a1e48b56e6594ad0164ef7446fda9e75b320b2b2 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Fri, 3 Jan 2020 23:45:08 +0900 Subject: [PATCH 20/31] Add warning for deprecating replaychat --- pytchat/core_async/replaychat.py | 17 ++++++++++++++++- pytchat/core_multithread/replaychat.py | 19 ++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/pytchat/core_async/replaychat.py b/pytchat/core_async/replaychat.py index 45f0c6e..adfa811 100644 --- a/pytchat/core_async/replaychat.py +++ b/pytchat/core_async/replaychat.py @@ -6,6 +6,7 @@ import signal import time import traceback import urllib.parse +import warnings from aiohttp.client_exceptions import ClientConnectorError from concurrent.futures import CancelledError from asyncio import Queue @@ -25,7 +26,15 @@ MAX_RETRY = 10 class ReplayChatAsync: - '''asyncio(aiohttp)を利用してYouTubeのチャットデータを取得する。 + ''' + ### ----------------------------------------------------------- + ### [Warning] ReplayChatAsync is integrated into LiveChatAsync. + ### This class is deprecated and will be removed at v0.0.5.0. + ### ReplayChatAsyncはLiveChatAsyncに統合しました。 + ### このクラスはv0.0.5.0で廃止予定です。 + ### ----------------------------------------------------------- + + asyncio(aiohttp)を利用してYouTubeのチャットデータを取得する。 Parameter --------- @@ -77,6 +86,12 @@ class ReplayChatAsync: done_callback = None, exception_handler = None, direct_mode = False): + + warnings.warn("" + f"\n{'-'*60}\n[WARNING] ReplayChatAsync is integrated " + f"into LiveChatAsync.\n{' '*5} This is deprecated and will" + f" be removed at v0.0.5.0.\n{'-'*60}\n" + ) self.video_id = video_id self.seektime = seektime if isinstance(processor, tuple): diff --git a/pytchat/core_multithread/replaychat.py b/pytchat/core_multithread/replaychat.py index 10106ca..2256828 100644 --- a/pytchat/core_multithread/replaychat.py +++ b/pytchat/core_multithread/replaychat.py @@ -6,6 +6,7 @@ import signal import time import traceback import urllib.parse +import warnings from concurrent.futures import CancelledError, ThreadPoolExecutor from queue import Queue from .buffer import Buffer @@ -22,7 +23,15 @@ MAX_RETRY = 10 class ReplayChat: - ''' スレッドプールを利用してYouTubeのライブ配信のチャットデータを取得する + ''' + ### ----------------------------------------------------------- + ### [Warning] ReplayChat is integrated into LiveChat. + ### This class is deprecated and will be removed at v0.0.5.0. + ### ReplayChatはLiveChatに統合しました。 + ### このクラスはv0.0.5.0で廃止予定です。 + ### ----------------------------------------------------------- + + スレッドプールを利用してYouTubeのライブ配信のチャットデータを取得する Parameter --------- @@ -64,8 +73,10 @@ class ReplayChat: ''' _setup_finished = False + #チャット監視中のListenerのリスト _listeners= [] + def __init__(self, video_id, seektime = 0, processor = DefaultProcessor(), @@ -75,6 +86,12 @@ class ReplayChat: done_callback = None, direct_mode = False ): + + warnings.warn("" + f"\n{'-'*60}\n[WARNING] ReplayChat is integrated into LiveChat.\n" + f"{' '*5}This is deprecated and will be removed at v0.0.5.0.\n" + f"{'-'*60}\n" + ) self.video_id = video_id self.seektime = seektime if isinstance(processor, tuple): From 5d228589f1066773afddb98c95b701e06ccc05d1 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Sat, 4 Jan 2020 00:25:22 +0900 Subject: [PATCH 21/31] Delete unnecessary file --- pytchat/core_async/livechat copy.py | 297 ---------------------------- 1 file changed, 297 deletions(-) delete mode 100644 pytchat/core_async/livechat copy.py diff --git a/pytchat/core_async/livechat copy.py b/pytchat/core_async/livechat copy.py deleted file mode 100644 index 82b3bdc..0000000 --- a/pytchat/core_async/livechat copy.py +++ /dev/null @@ -1,297 +0,0 @@ -import aiohttp, asyncio -import datetime -import json -import random -import signal -import time -import traceback -import urllib.parse -from aiohttp.client_exceptions import ClientConnectorError -from concurrent.futures import CancelledError -from asyncio import Queue -from .buffer import Buffer -from ..parser.live import Parser -from .. import config -from ..exceptions import ChatParseException,IllegalFunctionCall -from ..paramgen import liveparam -from ..processors.default.processor import DefaultProcessor -from ..processors.combinator import Combinator - -logger = config.logger(__name__) -headers = config.headers -MAX_RETRY = 10 - - -class LiveChatAsync: - '''asyncio(aiohttp)を利用してYouTubeのライブ配信のチャットデータを取得する。 - - Parameter - --------- - video_id : str - 動画ID - - processor : ChatProcessor - チャットデータを加工するオブジェクト - - buffer : Buffer(maxsize:20[default]) - チャットデータchat_componentを格納するバッファ。 - maxsize : 格納できるchat_componentの個数 - default値20個。1個で約5~10秒分。 - - interruptable : bool - Ctrl+Cによる処理中断を行うかどうか。 - - callback : func - _listen()関数から一定間隔で自動的に呼びだす関数。 - - done_callback : func - listener終了時に呼び出すコールバック。 - - exception_handler : func - 例外を処理する関数 - - direct_mode : bool - Trueの場合、bufferを使わずにcallbackを呼ぶ。 - Trueの場合、callbackの設定が必須 - (設定していない場合IllegalFunctionCall例外を発生させる) - - Attributes - --------- - _is_alive : bool - チャット取得を停止するためのフラグ - ''' - - _setup_finished = False - - def __init__(self, video_id, - seektime = 0, - processor = DefaultProcessor(), - buffer = None, - interruptable = True, - callback = None, - done_callback = None, - exception_handler = None, - direct_mode = False): - self.video_id = video_id - self.seektime = seektime - if isinstance(processor, tuple): - self.processor = Combinator(processor) - else: - self.processor = processor - self._buffer = buffer - self._callback = callback - self._done_callback = done_callback - self._exception_handler = exception_handler - self._direct_mode = direct_mode - self._is_alive = True - self._parser = Parser() - self._pauser = Queue() - self._pauser.put_nowait(None) - self._setup() - - if not LiveChatAsync._setup_finished: - LiveChatAsync._setup_finished = True - if exception_handler == None: - self._set_exception_handler(self._handle_exception) - else: - self._set_exception_handler(exception_handler) - if interruptable: - signal.signal(signal.SIGINT, - (lambda a, b:asyncio.create_task( - LiveChatAsync.shutdown(None,signal.SIGINT,b)) - )) - - def _setup(self): - #direct modeがTrueでcallback未設定の場合例外発生。 - if self._direct_mode: - if self._callback is None: - raise IllegalFunctionCall( - "direct_mode=Trueの場合callbackの設定が必須です。") - else: - #direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成 - if self._buffer is None: - self._buffer = Buffer(maxsize = 20) - #callbackが指定されている場合はcallbackを呼ぶループタスクを作成 - if self._callback is None: - pass - else: - #callbackを呼ぶループタスクの開始 - loop = asyncio.get_event_loop() - loop.create_task(self._callback_loop(self._callback)) - #_listenループタスクの開始 - loop = asyncio.get_event_loop() - listen_task = loop.create_task(self._startlisten()) - #add_done_callbackの登録 - if self._done_callback is None: - listen_task.add_done_callback(self.finish) - else: - listen_task.add_done_callback(self._done_callback) - - async def _startlisten(self): - """最初のcontinuationパラメータを取得し、 - _listenループのタスクを作成し開始する - """ - initial_continuation = liveparam.getparam(self.video_id,3) - await self._listen(initial_continuation) - - async def _listen(self, continuation): - ''' continuationに紐付いたチャットデータを取得し - Bufferにチャットデータを格納、 - 次のcontinuaitonを取得してループする。 - - Parameter - --------- - continuation : str - 次のチャットデータ取得に必要なパラメータ - ''' - try: - async with aiohttp.ClientSession() as session: - while(continuation and self._is_alive): - if self._pauser.empty(): - '''pause''' - await self._pauser.get() - '''resume: - prohibit from blocking by putting None into _pauser. - ''' - self._pauser.put_nowait(None) - continuation= liveparam.getparam(self.video_id,3) - livechat_json = (await - self._get_livechat_json(continuation, session, headers) - ) - metadata, chatdata = self._parser.parse( livechat_json ) - timeout = metadata['timeoutMs']/1000 - chat_component = { - "video_id" : self.video_id, - "timeout" : timeout, - "chatdata" : chatdata - } - time_mark =time.time() - if self._direct_mode: - await self._callback( - self.processor.process([chat_component]) - ) - else: - await self._buffer.put(chat_component) - diff_time = timeout - (time.time()-time_mark) - await asyncio.sleep(diff_time) - continuation = metadata.get('continuation') - except ChatParseException as e: - self.terminate() - logger.error(f"{str(e)}(video_id:\"{self.video_id}\")") - return - except (TypeError , json.JSONDecodeError) : - self.terminate() - logger.error(f"{traceback.format_exc(limit = -1)}") - return - - logger.debug(f"[{self.video_id}]チャット取得を終了しました。") - self.terminate() - - async def _get_livechat_json(self, continuation, session, headers): - ''' - チャットデータが格納されたjsonデータを取得する。 - ''' - continuation = urllib.parse.quote(continuation) - livechat_json = None - status_code = 0 - url =( - f"https://www.youtube.com/live_chat/get_live_chat?" - f"continuation={continuation}&pbj=1") - for _ in range(MAX_RETRY + 1): - async with session.get(url ,headers = headers) as resp: - try: - text = await resp.text() - status_code = resp.status - livechat_json = json.loads(text) - break - except (ClientConnectorError,json.JSONDecodeError) : - await asyncio.sleep(1) - continue - else: - logger.error(f"[{self.video_id}]" - f"Exceeded retry count. status_code={status_code}") - return None - return livechat_json - - async def _callback_loop(self,callback): - """ コンストラクタでcallbackを指定している場合、バックグラウンドで - callbackに指定された関数に一定間隔でチャットデータを投げる。 - - Parameter - --------- - callback : func - 加工済みのチャットデータを渡す先の関数。 - """ - while self.is_alive(): - items = await self._buffer.get() - data = self.processor.process(items) - await callback(data) - - async def get(self): - """ bufferからデータを取り出し、processorに投げ、 - 加工済みのチャットデータを返す。 - - Returns - : Processorによって加工されたチャットデータ - """ - if self._callback is None: - items = await self._buffer.get() - return self.processor.process(items) - raise IllegalFunctionCall( - "既にcallbackを登録済みのため、get()は実行できません。") - - def pause(self): - if self._callback is None: - return - if not self._pauser.empty(): - self._pauser.get_nowait() - - def resume(self): - if self._callback is None: - return - if self._pauser.empty(): - self._pauser.put_nowait(None) - - def is_alive(self): - return self._is_alive - - def finish(self,sender): - '''Listener終了時のコールバック''' - try: - self.terminate() - except CancelledError: - logger.debug(f'[{self.video_id}]cancelled:{sender}') - - def terminate(self): - ''' - Listenerを終了する。 - ''' - self._is_alive = False - if self._direct_mode == False: - #bufferにダミーオブジェクトを入れてis_alive()を判定させる - self._buffer.put_nowait({'chatdata':'','timeout':1}) - logger.info(f'[{self.video_id}]終了しました') - - @classmethod - def _set_exception_handler(cls, handler): - loop = asyncio.get_event_loop() - loop.set_exception_handler(handler) - - @classmethod - def _handle_exception(cls, loop, context): - if not isinstance(context["exception"],CancelledError): - logger.error(f"Caught exception: {context}") - loop= asyncio.get_event_loop() - loop.create_task(cls.shutdown(None,None,None)) - - @classmethod - async def shutdown(cls, event, sig = None, handler=None): - logger.debug("シャットダウンしています") - tasks = [t for t in asyncio.all_tasks() if t is not - asyncio.current_task()] - [task.cancel() for task in tasks] - - logger.debug(f"残っているタスクを終了しています") - await asyncio.gather(*tasks,return_exceptions=True) - loop = asyncio.get_event_loop() - loop.stop() \ No newline at end of file From b5e302cdf312dd895d9770fcaa2fd4f0df8afddd Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Sat, 4 Jan 2020 00:41:58 +0900 Subject: [PATCH 22/31] Make it possible to retrieve chat before broadcast by specifying negative number in seektime --- pytchat/paramgen/arcparam.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pytchat/paramgen/arcparam.py b/pytchat/paramgen/arcparam.py index 99154d7..ba725b0 100644 --- a/pytchat/paramgen/arcparam.py +++ b/pytchat/paramgen/arcparam.py @@ -70,10 +70,11 @@ def _build(video_id, seektime, topchatonly = False): if seektime < 0: - raise ValueError('seektime is 0 or positive number.') + times =_nval(0) + switch = b'\x04' if seektime == 0: times =_nval(1) - switch = b'\x04' + switch = b'\x03' else: times =_nval(int(seektime*1000000)) switch = b'\x03' @@ -88,8 +89,8 @@ def _build(video_id, seektime, topchatonly = False): sep_2 = b'\x52\x1C\x08\x00\x10\x00\x18\x00\x20\x00' chkstr = b'\x2A\x0E\x73\x74\x61\x74\x69\x63\x63\x68\x65\x63\x6B\x73\x75\x6D\x40' sep_3 = b'\x00\x58\x03\x60' - sep_4 = b'\x68'+parity+b'\x72\x04\x08' - sep_5 = b'\x10'+parity+b'\x78\x00' + sep_4 = b'\x68' + parity + b'\x72\x04\x08' + sep_5 = b'\x10' + parity + b'\x78\x00' body = [ sep_0, _nval(len(vid)), From 5d86fb4b71d3e00ebed4287a8119da4e8e6fe750 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Sat, 4 Jan 2020 09:28:44 +0900 Subject: [PATCH 23/31] Fix parameter switching and tests --- pytchat/paramgen/arcparam.py | 19 ++----------------- tests/test_arcparam.py | 11 +++++++---- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/pytchat/paramgen/arcparam.py b/pytchat/paramgen/arcparam.py index ba725b0..1d80ae1 100644 --- a/pytchat/paramgen/arcparam.py +++ b/pytchat/paramgen/arcparam.py @@ -52,33 +52,18 @@ def _nval(val): buf += val.to_bytes(1,'big') return buf - -def _tzparity(video_id,times): - t=0 - for i,s in enumerate(video_id): - ss = ord(s) - if(ss % 2 == 0): - t += ss*(12-i) - else: - t ^= ss*i - - return ((times^t) % 2).to_bytes(1,'big') - - def _build(video_id, seektime, topchatonly = False): switch_01 = b'\x04' if topchatonly else b'\x01' - - if seektime < 0: times =_nval(0) switch = b'\x04' - if seektime == 0: + elif seektime == 0: times =_nval(1) switch = b'\x03' else: times =_nval(int(seektime*1000000)) switch = b'\x03' - parity = _tzparity(video_id, seektime) + parity = b'\x00' header_magic= b'\xA2\x9D\xB0\xD3\x04' sep_0 = b'\x1A' diff --git a/tests/test_arcparam.py b/tests/test_arcparam.py index e53eda3..e30466e 100644 --- a/tests/test_arcparam.py +++ b/tests/test_arcparam.py @@ -5,13 +5,13 @@ import requests, json from pytchat.paramgen import arcparam def test_arcparam_0(mocker): - param = arcparam.getparam("01234567890") - assert "op2w0wRyGjxDZzhhRFFvTE1ERXlNelExTmpjNE9UQWFFLXFvM2JrQkRRb0xNREV5TXpRMU5qYzRPVEFnQVElM0QlM0QoATAAOABAAEgEUhwIABAAGAAgACoOc3RhdGljY2hlY2tzdW1AAFgDYAFoAXIECAEQAXgA" == param + param = arcparam.getparam("01234567890",-1) + assert "op2w0wRyGjxDZzhhRFFvTE1ERXlNelExTmpjNE9UQWFFLXFvM2JrQkRRb0xNREV5TXpRMU5qYzRPVEFnQVElM0QlM0QoADAAOABAAEgEUhwIABAAGAAgACoOc3RhdGljY2hlY2tzdW1AAFgDYAFoAHIECAEQAHgA" == param def test_arcparam_1(mocker): param = arcparam.getparam("01234567890", seektime = 100000) - assert "op2w0wR3GjxDZzhhRFFvTE1ERXlNelExTmpjNE9UQWFFLXFvM2JrQkRRb0xNREV5TXpRMU5qYzRPVEFnQVElM0QlM0QogNDbw_QCMAA4AEAASANSHAgAEAAYACAAKg5zdGF0aWNjaGVja3N1bUAAWANgAWgBcgQIARABeAA%3D" == param + assert "op2w0wR3GjxDZzhhRFFvTE1ERXlNelExTmpjNE9UQWFFLXFvM2JrQkRRb0xNREV5TXpRMU5qYzRPVEFnQVElM0QlM0QogNDbw_QCMAA4AEAASANSHAgAEAAYACAAKg5zdGF0aWNjaGVja3N1bUAAWANgAWgAcgQIARAAeAA%3D" == param def test_arcparam_2(mocker): param = arcparam.getparam("SsjCnHOk-Sk") @@ -23,4 +23,7 @@ def test_arcparam_2(mocker): test_id = chatdata[0]["addChatItemAction"]["item"]["liveChatTextMessageRenderer"]["id"] print(test_id) assert "CjoKGkNMYXBzZTdudHVVQ0Zjc0IxZ0FkTnFnQjVREhxDSnlBNHV2bnR1VUNGV0dnd2dvZDd3NE5aZy0w" == test_id - \ No newline at end of file + +def test_arcparam_3(mocker): + param = arcparam.getparam("01234567890") + assert "op2w0wRyGjxDZzhhRFFvTE1ERXlNelExTmpjNE9UQWFFLXFvM2JrQkRRb0xNREV5TXpRMU5qYzRPVEFnQVElM0QlM0QoATAAOABAAEgDUhwIABAAGAAgACoOc3RhdGljY2hlY2tzdW1AAFgDYAFoAHIECAEQAHgA" == param From 26fefddddfa34ed7d4e7c8301ff6cf563d3dcea1 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Sat, 4 Jan 2020 13:23:32 +0900 Subject: [PATCH 24/31] Implement force_replay_mode --- pytchat/core_async/livechat.py | 30 ++++++++++++++------- pytchat/core_multithread/livechat.py | 28 ++++++++++++------- pytchat/parser/live.py | 40 +++++++++++++--------------- 3 files changed, 58 insertions(+), 40 deletions(-) diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index a78dc87..27341ee 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -30,6 +30,11 @@ class LiveChatAsync: video_id : str 動画ID + seektime : int + (ライブチャット取得時は無視) + 取得開始するアーカイブ済みチャットの経過時間(秒) + マイナス値を指定した場合は、配信開始前のチャットも取得する。 + processor : ChatProcessor チャットデータを加工するオブジェクト @@ -53,7 +58,11 @@ class LiveChatAsync: direct_mode : bool Trueの場合、bufferを使わずにcallbackを呼ぶ。 Trueの場合、callbackの設定が必須 - (設定していない場合IllegalFunctionCall例外を発生させる) + (設定していない場合IllegalFunctionCall例外を発生させる) + + force_replay : bool + Trueの場合、ライブチャットが取得できる場合であっても + 強制的にアーカイブ済みチャットを取得する。 Attributes --------- @@ -71,7 +80,9 @@ class LiveChatAsync: callback = None, done_callback = None, exception_handler = None, - direct_mode = False): + direct_mode = False, + force_replay = False + ): self.video_id = video_id self.seektime = seektime if isinstance(processor, tuple): @@ -84,7 +95,8 @@ class LiveChatAsync: self._exception_handler = exception_handler self._direct_mode = direct_mode self._is_alive = True - self._parser = Parser() + self._is_replay = force_replay + self._parser = Parser(is_replay = self._is_replay) self._pauser = Queue() self._pauser.put_nowait(None) self._setup() @@ -187,7 +199,7 @@ class LiveChatAsync: prohibit from blocking by putting None into _pauser. ''' self._pauser.put_nowait(None) - if self._parser.mode == 'LIVE': + if not self._is_replay: continuation = liveparam.getparam(self.video_id,3) return continuation @@ -205,9 +217,9 @@ class LiveChatAsync: ) contents = self._parser.get_contents(livechat_json) if self._first_fetch: - if contents is None: + if contents is None or self._is_replay: '''Try to fetch archive chat data.''' - self._parser.mode = 'REPLAY' + self._parser.is_replay = True self._fetch_url = ("live_chat_replay/" "get_live_chat_replay?continuation=") continuation = arcparam.getparam(self.video_id, self.seektime) @@ -268,8 +280,8 @@ class LiveChatAsync: raise IllegalFunctionCall( "既にcallbackを登録済みのため、get()は実行できません。") - def get_mode(self): - return self._parser.mode + def is_replay(self): + return self._is_replay def pause(self): if self._callback is None: @@ -300,7 +312,7 @@ class LiveChatAsync: self._is_alive = False if self._direct_mode == False: #bufferにダミーオブジェクトを入れてis_alive()を判定させる - self._buffer.put_nowait({'chatdata':'','timeout':1}) + self._buffer.put_nowait({'chatdata':'','timeout':0}) logger.info(f'[{self.video_id}]finished.') @classmethod diff --git a/pytchat/core_multithread/livechat.py b/pytchat/core_multithread/livechat.py index a07a439..9687289 100644 --- a/pytchat/core_multithread/livechat.py +++ b/pytchat/core_multithread/livechat.py @@ -28,6 +28,11 @@ class LiveChat: --------- video_id : str 動画ID + + seektime : int + (ライブチャット取得時は無視) + 取得開始するアーカイブ済みチャットの経過時間(秒) + マイナス値を指定した場合は、配信開始前のチャットも取得する。 processor : ChatProcessor チャットデータを加工するオブジェクト @@ -51,6 +56,10 @@ class LiveChat: Trueの場合、callbackの設定が必須 (設定していない場合IllegalFunctionCall例外を発生させる) + force_replay : bool + Trueの場合、ライブチャットが取得できる場合であっても + 強制的にアーカイブ済みチャットを取得する。 + Attributes --------- _executor : ThreadPoolExecutor @@ -70,7 +79,8 @@ class LiveChat: interruptable = True, callback = None, done_callback = None, - direct_mode = False + direct_mode = False, + force_replay = False ): self.video_id = video_id self.seektime = seektime @@ -84,7 +94,8 @@ class LiveChat: self._executor = ThreadPoolExecutor(max_workers=2) self._direct_mode = direct_mode self._is_alive = True - self._parser = Parser() + self._is_replay = force_replay + self._parser = Parser(is_replay = self._is_replay) self._pauser = Queue() self._pauser.put_nowait(None) self._setup() @@ -184,7 +195,7 @@ class LiveChat: prohibit from blocking by putting None into _pauser. ''' self._pauser.put_nowait(None) - if self._parser.mode == 'LIVE': + if not self._is_replay: continuation = liveparam.getparam(self.video_id,3) return continuation @@ -202,9 +213,9 @@ class LiveChat: ) contents = self._parser.get_contents(livechat_json) if self._first_fetch: - if contents is None: + if contents is None or self._is_replay: '''Try to fetch archive chat data.''' - self._parser.mode = 'REPLAY' + self._parser.is_replay = True self._fetch_url = ("live_chat_replay/" "get_live_chat_replay?continuation=") continuation = arcparam.getparam(self.video_id, self.seektime) @@ -235,7 +246,6 @@ class LiveChat: else: logger.error(f"[{self.video_id}]" f"Exceeded retry count. status_code={status_code}") - #self.terminate() return None return livechat_json @@ -266,8 +276,8 @@ class LiveChat: raise IllegalFunctionCall( "既にcallbackを登録済みのため、get()は実行できません。") - def get_mode(self): - return self._parser.mode + def is_replay(self): + return self._is_replay def pause(self): if self._callback is None: @@ -298,7 +308,7 @@ class LiveChat: self._is_alive = False if self._direct_mode == False: #bufferにダミーオブジェクトを入れてis_alive()を判定させる - self._buffer.put({'chatdata':'','timeout':1}) + self._buffer.put({'chatdata':'','timeout':0}) logger.info(f'[{self.video_id}]finished.') @classmethod diff --git a/pytchat/parser/live.py b/pytchat/parser/live.py index b58ab3d..90b03fe 100644 --- a/pytchat/parser/live.py +++ b/pytchat/parser/live.py @@ -1,7 +1,7 @@ """ pytchat.parser.live ~~~~~~~~~~~~~~~~~~~ -This module is parser of live chat JSON. +Parser of live chat JSON. """ import json @@ -15,11 +15,12 @@ from .. exceptions import ( logger = config.logger(__name__) -from .. import util class Parser: - def __init__(self): - self.mode = 'LIVE' + __slots__ = ['is_replay'] + + def __init__(self, is_replay): + self.is_replay = is_replay def get_contents(self, jsn): if jsn is None: @@ -31,29 +32,23 @@ class Parser: def parse(self, contents): """ - このparse関数はLiveChat._listen() 関数から定期的に呼び出される。 - 引数contentsはYoutubeから取得したチャットデータの生JSONであり、 - 与えられたJSONをチャットデータとメタデータに分割して返す。 - Parameter ---------- + contents : dict - + Youtubeから取得したチャットデータのJSONオブジェクト。 - (pythonの辞書形式に変換済みの状態で渡される) + + JSON of chat data from YouTube. Returns ------- tuple: - + metadata : dict  チャットデータに付随するメタデータ + + metadata : dict + timeout + video_id + continuation - + chatdata : list[dict] -     チャットデータ本体のリスト。 + + chatdata : List[dict] """ if contents is None: - '''配信が終了した場合、もしくはチャットデータが取得できない場合''' + '''Broadcasting end or cannot fetch chat stream''' raise NoContentsException('Chat data stream is empty.') cont = contents['liveChatContinuation']['continuations'][0] @@ -76,15 +71,16 @@ class Parser: return self._create_data(metadata, contents) def _create_data(self, metadata, contents): - chatdata = contents['liveChatContinuation'].get('actions') - if self.mode == 'LIVE': - metadata.setdefault('timeoutMs', 10000) - else: - interval = self._get_interval(chatdata) + actions = contents['liveChatContinuation'].get('actions') + if self.is_replay: + interval = self._get_interval(actions) metadata.setdefault("timeoutMs",interval) - """アーカイブ済みチャットはライブチャットと構造が異なっているため、以下の行により - ライブチャットと同じ形式にそろえる""" - chatdata = [action["replayChatItemAction"]["actions"][0] for action in chatdata] + """Archived chat has different structures than live chat, + so make it the same format.""" + chatdata = [action["replayChatItemAction"]["actions"][0] for action in actions] + else: + metadata.setdefault('timeoutMs', 10000) + chatdata = actions return metadata, chatdata def _get_interval(self, actions: list): From 3b27c81166c7bae7167d3cbf97f65c3b56b97f8b Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Wed, 8 Jan 2020 00:44:50 +0900 Subject: [PATCH 25/31] Add tests --- tests/test_compatible_processor.py | 2 +- tests/test_livechat_2.py | 125 +++++++ tests/test_parser.py | 2 +- tests/test_speed_calculator.py | 2 +- tests/testdata/finished_live.json | 113 ++++++- tests/testdata/test_stream.json | 509 +++++++++++++++++++++++++++++ 6 files changed, 749 insertions(+), 4 deletions(-) create mode 100644 tests/test_livechat_2.py create mode 100644 tests/testdata/test_stream.json diff --git a/tests/test_compatible_processor.py b/tests/test_compatible_processor.py index efc81b9..39cf9fe 100644 --- a/tests/test_compatible_processor.py +++ b/tests/test_compatible_processor.py @@ -12,7 +12,7 @@ from pytchat.processors.compatible.renderer.paidmessage import LiveChatPaidMessa from pytchat.processors.compatible.renderer.paidsticker import LiveChatPaidStickerRenderer from pytchat.processors.compatible.renderer.legacypaid import LiveChatLegacyPaidMessageRenderer -parser = Parser() +parser = Parser(is_replay=False) def test_textmessage(mocker): '''api互換processorのテスト:通常テキストメッセージ''' diff --git a/tests/test_livechat_2.py b/tests/test_livechat_2.py new file mode 100644 index 0000000..f582ae0 --- /dev/null +++ b/tests/test_livechat_2.py @@ -0,0 +1,125 @@ +import asyncio, aiohttp +import json +import pytest +import re +import requests +import sys +import time +from aioresponses import aioresponses +from concurrent.futures import CancelledError +from unittest import TestCase +from pytchat.core_multithread.livechat import LiveChat +from pytchat.core_async.livechat import LiveChatAsync +from pytchat.exceptions import ( + NoLivechatRendererException,NoYtinitialdataException, + ResponseContextError,NoContentsException) +from pytchat.parser.live import Parser +from pytchat.processors.dummy_processor import DummyProcessor + +def _open_file(path): + with open(path,mode ='r',encoding = 'utf-8') as f: + return f.read() + +@aioresponses() +def test_async_live_stream(*mock): + + async def test_loop(*mock): + pattern = re.compile(r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$') + _text = _open_file('tests/testdata/test_stream.json') + mock[0].get(pattern, status=200, body=_text) + chat = LiveChatAsync(video_id='', processor = DummyProcessor()) + chats = await chat.get() + rawdata = chats[0]["chatdata"] + #assert fetching livachat data + assert list(rawdata[0]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer" + assert list(rawdata[1]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer" + assert list(rawdata[2]["addChatItemAction"]["item"].keys())[0] == "liveChatPlaceholderItemRenderer" + assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerPaidMessageItemRenderer" + assert list(rawdata[4]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer" + assert list(rawdata[5]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidStickerRenderer" + assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerSponsorItemRenderer" + + loop = asyncio.get_event_loop() + try: + loop.run_until_complete(test_loop(*mock)) + except CancelledError: + assert True + +@aioresponses() +def test_async_replay_stream(*mock): + + async def test_loop(*mock): + pattern_live = re.compile(r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$') + pattern_replay = re.compile(r'^https://www.youtube.com/live_chat_replay/get_live_chat_replay\?continuation=.*$') + #empty livechat -> switch to fetch replaychat + _text_live = _open_file('tests/testdata/finished_live.json') + _text_replay = _open_file('tests/testdata/chatreplay.json') + mock[0].get(pattern_live, status=200, body=_text_live) + mock[0].get(pattern_replay, status=200, body=_text_replay) + + chat = LiveChatAsync(video_id='', processor = DummyProcessor()) + chats = await chat.get() + rawdata = chats[0]["chatdata"] + #assert fetching replaychat data + assert list(rawdata[0]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer" + assert list(rawdata[14]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer" + + loop = asyncio.get_event_loop() + try: + loop.run_until_complete(test_loop(*mock)) + except CancelledError: + assert True + +@aioresponses() +def test_async_force_replay(*mock): + + async def test_loop(*mock): + pattern_live = re.compile(r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$') + pattern_replay = re.compile(r'^https://www.youtube.com/live_chat_replay/get_live_chat_replay\?continuation=.*$') + #valid live data, but force_replay = True + _text_live = _open_file('tests/testdata/test_stream.json') + #valid replay data + _text_replay = _open_file('tests/testdata/chatreplay.json') + + mock[0].get(pattern_live, status=200, body=_text_live) + mock[0].get(pattern_replay, status=200, body=_text_replay) + #force replay + chat = LiveChatAsync(video_id='', processor = DummyProcessor(), force_replay = True) + chats = await chat.get() + rawdata = chats[0]["chatdata"] + # assert fetching replaychat data + assert list(rawdata[14]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer" + # assert not mix livechat data + assert list(rawdata[2]["addChatItemAction"]["item"].keys())[0] != "liveChatPlaceholderItemRenderer" + + loop = asyncio.get_event_loop() + try: + loop.run_until_complete(test_loop(*mock)) + except CancelledError: + assert True + +def test_multithread_live_stream(mocker): + + _text = _open_file('tests/testdata/test_stream.json') + responseMock = mocker.Mock() + responseMock.status_code = 200 + responseMock.text = _text + mocker.patch('requests.Session.get').return_value.__enter__.return_value = responseMock + + chat = LiveChat(video_id='test_id', processor = DummyProcessor()) + chats = chat.get() + rawdata = chats[0]["chatdata"] + #assert fetching livachat data + assert list(rawdata[0]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer" + assert list(rawdata[1]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer" + assert list(rawdata[2]["addChatItemAction"]["item"].keys())[0] == "liveChatPlaceholderItemRenderer" + assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerPaidMessageItemRenderer" + assert list(rawdata[4]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer" + assert list(rawdata[5]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidStickerRenderer" + assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerSponsorItemRenderer" + chat.terminate() + + + + + diff --git a/tests/test_parser.py b/tests/test_parser.py index b40bee8..9832f7a 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -11,7 +11,7 @@ from pytchat.exceptions import ( def _open_file(path): with open(path,mode ='r',encoding = 'utf-8') as f: return f.read() -parser = Parser() +parser = Parser(is_replay = False) @aioresponses() def test_finishedlive(*mock): diff --git a/tests/test_speed_calculator.py b/tests/test_speed_calculator.py index f137c52..96c6761 100644 --- a/tests/test_speed_calculator.py +++ b/tests/test_speed_calculator.py @@ -9,7 +9,7 @@ from pytchat.exceptions import ( from pytchat.processors.speed_calculator import SpeedCalculator -parser = Parser() +parser = Parser(is_replay =False) def test_speed_1(mocker): '''test speed calculation with normal json. diff --git a/tests/testdata/finished_live.json b/tests/testdata/finished_live.json index 0300acc..39a897a 100644 --- a/tests/testdata/finished_live.json +++ b/tests/testdata/finished_live.json @@ -1 +1,112 @@ -{"csn":"zeiIXfXHJYOA1d8Pyuaw4A4","response":{"responseContext":{"serviceTrackingParams":[{"service":"CSI","params":[{"key":"GetLiveChat_rid","value":"0x96761cd683987638"},{"key":"c","value":"WEB"},{"key":"cver","value":"2.20190920.05.01"},{"key":"yt_li","value":"0"}]},{"service":"GFEEDBACK","params":[{"key":"e","value":"23744176,23757412,23788838,23788875,23793834,23804281,23808952,23818920,23828084,23828243,23829335,23832543,23835014,23836965,23837741,23837772,23837957,23837993,23838272,23838302,23838823,23838823,23839284,23839362,23840216,23840243,23841118,23842662,23842986,23843283,23843289,23843534,23844042,24630096,9449243,9471235"},{"key":"logged_in","value":"0"}]},{"service":"GUIDED_HELP","params":[{"key":"logged_in","value":"0"}]},{"service":"ECATCHER","params":[{"key":"client.name","value":"WEB"},{"key":"client.version","value":"2.20190920"},{"key":"innertube.build.changelist","value":"270293990"},{"key":"innertube.build.experiments.source_version","value":"270377311"},{"key":"innertube.build.label","value":"youtube.ytfe.innertube_20190920_5_RC0"},{"key":"innertube.build.timestamp","value":"1568999515"},{"key":"innertube.build.variants.checksum","value":"669625af1d321c1e95dffac8db989afa"},{"key":"innertube.run.job","value":"ytfe-innertube-replica-only.ytfe"}]}],"webResponseContextExtensionData":{"ytConfigData":{"csn":"zeiIXfXHJYOA1d8Pyuaw4A4","visitorData":"CgtLWW1kYjAxZTBaRSjN0aPsBQ%3D%3D"}}}},"xsrf_token":"QUFFLUhqbnhXaGhpblNhWmEzdjJJR2JNeW02M01PQ0p6Z3xBQ3Jtc0ttekpfU1dhZlA4ZWJhSGNrOFN5ZGFFSmNSMjBWRERWYUtOSS03RG5sbDRaa01KWmZFd2pPZzNEdW10WThmUXRiQjRKQ1ZPUkd1b09nT0k5dEZJTGdFYWxEVGNOWkUzcGNEQjdTNnN2OTRjN1Qtc0haZlpSWGlxd1k4LUdnVEhVb1FtMW8yZHJfankzN1JhUFo3aFZvS0s4NkIzTGc=","url":"\/live_chat\/get_live_chat?continuation=0ofMyAORAhqsAUNqZ0tEUW9MWjAwdGEwMWFaMmRxY2xrcUp3b1lWVU53VGtneVdtc3laM2N6U2tKcVYwRkxVM2xhWTFGUkVndG5UUzFyVFZwbloycHlXUnBEcXJuQnZRRTlDanRvZEhSd2N6b3ZMM2QzZHk1NWIzVjBkV0psTG1OdmJTOXNhWFpsWDJOb1lYUV9kajFuVFMxclRWcG5aMnB5V1NacGMxOXdiM0J2ZFhROU1TQUMo5-aA_KPn5AIwADgAQAJKKwgAEAAYACAAKg5zdGF0aWNjaGVja3N1bToAQABKAggBUMKMlt2k5-QCWANQ5KzA_KPn5AJYt7rZo9Tm5AJoAYIBAggBiAEAoAHorZb_pOfkAg%253D%253D","endpoint":{"commandMetadata":{"webCommandMetadata":{"url":"/live_chat/get_live_chat?continuation=0ofMyAORAhqsAUNqZ0tEUW9MWjAwdGEwMWFaMmRxY2xrcUp3b1lWVU53VGtneVdtc3laM2N6U2tKcVYwRkxVM2xhWTFGUkVndG5UUzFyVFZwbloycHlXUnBEcXJuQnZRRTlDanRvZEhSd2N6b3ZMM2QzZHk1NWIzVjBkV0psTG1OdmJTOXNhWFpsWDJOb1lYUV9kajFuVFMxclRWcG5aMnB5V1NacGMxOXdiM0J2ZFhROU1TQUMo5-aA_KPn5AIwADgAQAJKKwgAEAAYACAAKg5zdGF0aWNjaGVja3N1bToAQABKAggBUMKMlt2k5-QCWANQ5KzA_KPn5AJYt7rZo9Tm5AJoAYIBAggBiAEAoAHorZb_pOfkAg%253D%253D"}},"urlEndpoint":{"url":"/live_chat/get_live_chat?continuation=0ofMyAORAhqsAUNqZ0tEUW9MWjAwdGEwMWFaMmRxY2xrcUp3b1lWVU53VGtneVdtc3laM2N6U2tKcVYwRkxVM2xhWTFGUkVndG5UUzFyVFZwbloycHlXUnBEcXJuQnZRRTlDanRvZEhSd2N6b3ZMM2QzZHk1NWIzVjBkV0psTG1OdmJTOXNhWFpsWDJOb1lYUV9kajFuVFMxclRWcG5aMnB5V1NacGMxOXdiM0J2ZFhROU1TQUMo5-aA_KPn5AIwADgAQAJKKwgAEAAYACAAKg5zdGF0aWNjaGVja3N1bToAQABKAggBUMKMlt2k5-QCWANQ5KzA_KPn5AJYt7rZo9Tm5AJoAYIBAggBiAEAoAHorZb_pOfkAg%253D%253D"}},"timing":{"info":{"st":64}}} \ No newline at end of file +{ + "csn": "zeiIXfXHJYOA1d8Pyuaw4A4", + "response": { + "responseContext": { + "serviceTrackingParams": [ + { + "service": "CSI", + "params": [ + { + "key": "GetLiveChat_rid", + "value": "0x96761cd683987638" + }, + { + "key": "c", + "value": "WEB" + }, + { + "key": "cver", + "value": "2.20190920.05.01" + }, + { + "key": "yt_li", + "value": "0" + } + ] + }, + { + "service": "GFEEDBACK", + "params": [ + { + "key": "e", + "value": "23744176,23757412,23788838,23788875,23793834,23804281,23808952,23818920,23828084,23828243,23829335,23832543,23835014,23836965,23837741,23837772,23837957,23837993,23838272,23838302,23838823,23838823,23839284,23839362,23840216,23840243,23841118,23842662,23842986,23843283,23843289,23843534,23844042,24630096,9449243,9471235" + }, + { + "key": "logged_in", + "value": "0" + } + ] + }, + { + "service": "GUIDED_HELP", + "params": [ + { + "key": "logged_in", + "value": "0" + } + ] + }, + { + "service": "ECATCHER", + "params": [ + { + "key": "client.name", + "value": "WEB" + }, + { + "key": "client.version", + "value": "2.20190920" + }, + { + "key": "innertube.build.changelist", + "value": "270293990" + }, + { + "key": "innertube.build.experiments.source_version", + "value": "270377311" + }, + { + "key": "innertube.build.label", + "value": "youtube.ytfe.innertube_20190920_5_RC0" + }, + { + "key": "innertube.build.timestamp", + "value": "1568999515" + }, + { + "key": "innertube.build.variants.checksum", + "value": "669625af1d321c1e95dffac8db989afa" + }, + { + "key": "innertube.run.job", + "value": "ytfe-innertube-replica-only.ytfe" + } + ] + } + ], + "webResponseContextExtensionData": { + "ytConfigData": { + "csn": "zeiIXfXHJYOA1d8Pyuaw4A4", + "visitorData": "CgtLWW1kYjAxZTBaRSjN0aPsBQ%3D%3D" + } + } + } + }, + "xsrf_token": "QUFFLUhqbnhXaGhpblNhWmEzdjJJR2JNeW02M01PQ0p6Z3xBQ3Jtc0ttekpfU1dhZlA4ZWJhSGNrOFN5ZGFFSmNSMjBWRERWYUtOSS03RG5sbDRaa01KWmZFd2pPZzNEdW10WThmUXRiQjRKQ1ZPUkd1b09nT0k5dEZJTGdFYWxEVGNOWkUzcGNEQjdTNnN2OTRjN1Qtc0haZlpSWGlxd1k4LUdnVEhVb1FtMW8yZHJfankzN1JhUFo3aFZvS0s4NkIzTGc=", + "url": "\/live_chat\/get_live_chat?continuation=0ofMyAORAhqsAUNqZ0tEUW9MWjAwdGEwMWFaMmRxY2xrcUp3b1lWVU53VGtneVdtc3laM2N6U2tKcVYwRkxVM2xhWTFGUkVndG5UUzFyVFZwbloycHlXUnBEcXJuQnZRRTlDanRvZEhSd2N6b3ZMM2QzZHk1NWIzVjBkV0psTG1OdmJTOXNhWFpsWDJOb1lYUV9kajFuVFMxclRWcG5aMnB5V1NacGMxOXdiM0J2ZFhROU1TQUMo5-aA_KPn5AIwADgAQAJKKwgAEAAYACAAKg5zdGF0aWNjaGVja3N1bToAQABKAggBUMKMlt2k5-QCWANQ5KzA_KPn5AJYt7rZo9Tm5AJoAYIBAggBiAEAoAHorZb_pOfkAg%253D%253D", + "endpoint": { + "commandMetadata": { + "webCommandMetadata": { + "url": "/live_chat/get_live_chat?continuation=0ofMyAORAhqsAUNqZ0tEUW9MWjAwdGEwMWFaMmRxY2xrcUp3b1lWVU53VGtneVdtc3laM2N6U2tKcVYwRkxVM2xhWTFGUkVndG5UUzFyVFZwbloycHlXUnBEcXJuQnZRRTlDanRvZEhSd2N6b3ZMM2QzZHk1NWIzVjBkV0psTG1OdmJTOXNhWFpsWDJOb1lYUV9kajFuVFMxclRWcG5aMnB5V1NacGMxOXdiM0J2ZFhROU1TQUMo5-aA_KPn5AIwADgAQAJKKwgAEAAYACAAKg5zdGF0aWNjaGVja3N1bToAQABKAggBUMKMlt2k5-QCWANQ5KzA_KPn5AJYt7rZo9Tm5AJoAYIBAggBiAEAoAHorZb_pOfkAg%253D%253D" + } + }, + "urlEndpoint": { + "url": "/live_chat/get_live_chat?continuation=0ofMyAORAhqsAUNqZ0tEUW9MWjAwdGEwMWFaMmRxY2xrcUp3b1lWVU53VGtneVdtc3laM2N6U2tKcVYwRkxVM2xhWTFGUkVndG5UUzFyVFZwbloycHlXUnBEcXJuQnZRRTlDanRvZEhSd2N6b3ZMM2QzZHk1NWIzVjBkV0psTG1OdmJTOXNhWFpsWDJOb1lYUV9kajFuVFMxclRWcG5aMnB5V1NacGMxOXdiM0J2ZFhROU1TQUMo5-aA_KPn5AIwADgAQAJKKwgAEAAYACAAKg5zdGF0aWNjaGVja3N1bToAQABKAggBUMKMlt2k5-QCWANQ5KzA_KPn5AJYt7rZo9Tm5AJoAYIBAggBiAEAoAHorZb_pOfkAg%253D%253D" + } + }, + "timing": { + "info": { + "st": 64 + } + } +} \ No newline at end of file diff --git a/tests/testdata/test_stream.json b/tests/testdata/test_stream.json new file mode 100644 index 0000000..46f85d6 --- /dev/null +++ b/tests/testdata/test_stream.json @@ -0,0 +1,509 @@ +{ + "response": { + "responseContext": { + "webResponseContextExtensionData": "" + }, + "continuationContents": { + "liveChatContinuation": { + "continuations": [ + { + "invalidationContinuationData": { + "invalidationId": { + "objectSource": 1000, + "objectId": "___objectId___", + "topic": "chat~00000000000~0000000", + "subscribeToGcmTopics": true, + "protoCreationTimestampMs": "1577804400000" + }, + "timeoutMs": 5000, + "continuation": "___continuation___" + } + } + ], + "actions": [ + { + "addChatItemAction": { + "item": { + "liveChatTextMessageRenderer": { + "message": { + "runs": [ + { + "text": "This is normal message." + } + ] + }, + "authorName": { + "simpleText": "author_name" + }, + "authorPhoto": { + "thumbnails": [ + { + "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg", + "width": 32, + "height": 32 + }, + { + "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg", + "width": 64, + "height": 64 + } + ] + }, + "contextMenuEndpoint": { + "commandMetadata": { + "webCommandMetadata": { + "ignoreNavigation": true + } + }, + "liveChatItemContextMenuEndpoint": { + "params": "___params___" + } + }, + "id": "dummy_id", + "timestampUsec": 0, + "authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url", + "contextMenuAccessibility": { + "accessibilityData": { + "label": "コメントの操作" + } + } + } + }, + "clientId": "dummy_client_id" + } + }, + { + "addChatItemAction": { + "item": { + "liveChatTextMessageRenderer": { + "message": { + "runs": [ + { + "text": "This is members's message" + } + ] + }, + "authorName": { + "simpleText": "author_name" + }, + "authorPhoto": { + "thumbnails": [ + { + "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg", + "width": 32, + "height": 32 + }, + { + "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg", + "width": 64, + "height": 64 + } + ] + }, + "contextMenuEndpoint": { + "commandMetadata": { + "webCommandMetadata": { + "ignoreNavigation": true + } + }, + "liveChatItemContextMenuEndpoint": { + "params": "___params___" + } + }, + "id": "dummy_id", + "timestampUsec": 0, + "authorBadges": [ + { + "liveChatAuthorBadgeRenderer": { + "customThumbnail": { + "thumbnails": [ + { + "url": "https://yt3.ggpht.com/X=s32-c-k" + }, + { + "url": "https://yt3.ggpht.com/X=s32-c-k" + } + ] + }, + "tooltip": "メンバー(2 か月)", + "accessibility": { + "accessibilityData": { + "label": "メンバー(2 か月)" + } + } + } + } + ], + "authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url", + "contextMenuAccessibility": { + "accessibilityData": { + "label": "コメントの操作" + } + } + } + }, + "clientId": "dummy_client_id" + } + }, + { + "addChatItemAction": { + "item": { + "liveChatPlaceholderItemRenderer": { + "id": "dummy_id", + "timestampUsec": 0 + } + }, + "clientId": "dummy_client_id" + } + }, + { + "addLiveChatTickerItemAction": { + "item": { + "liveChatTickerPaidMessageItemRenderer": { + "id": "dummy_id", + "amount": { + "simpleText": "¥10,000" + }, + "amountTextColor": 4294967295, + "startBackgroundColor": 4293271831, + "endBackgroundColor": 4291821568, + "authorPhoto": { + "thumbnails": [ + { + "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg", + "width": 32, + "height": 32 + }, + { + "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg", + "width": 64, + "height": 64 + } + ] + }, + "durationSec": 3600, + "showItemEndpoint": { + "commandMetadata": { + "webCommandMetadata": { + "ignoreNavigation": true + } + }, + "showLiveChatItemEndpoint": { + "renderer": { + "liveChatPaidMessageRenderer": { + "id": "dummy_id", + "timestampUsec": 0, + "authorName": { + "simpleText": "author_name" + }, + "authorPhoto": { + "thumbnails": [ + { + "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg", + "width": 32, + "height": 32 + }, + { + "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg", + "width": 64, + "height": 64 + } + ] + }, + "purchaseAmountText": { + "simpleText": "¥10,000" + }, + "message": { + "runs": [ + { + "text": "This is superchat message." + } + ] + }, + "headerBackgroundColor": 4291821568, + "headerTextColor": 4294967295, + "bodyBackgroundColor": 4293271831, + "bodyTextColor": 4294967295, + "authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url", + "authorNameTextColor": 3019898879, + "contextMenuEndpoint": { + "commandMetadata": { + "webCommandMetadata": { + "ignoreNavigation": true + } + }, + "liveChatItemContextMenuEndpoint": { + "params": "___params___" + } + }, + "timestampColor": 2164260863, + "contextMenuAccessibility": { + "accessibilityData": { + "label": "コメントの操作" + } + } + } + } + } + }, + "authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url", + "fullDurationSec": 3600 + } + }, + "durationSec": "3600" + } + }, + { + "addChatItemAction": { + "item": { + "liveChatPaidMessageRenderer": { + "id": "dummy_id", + "timestampUsec": 0, + "authorName": { + "simpleText": "author_name" + }, + "authorPhoto": { + "thumbnails": [ + { + "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg", + "width": 32, + "height": 32 + }, + { + "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg", + "width": 64, + "height": 64 + } + ] + }, + "purchaseAmountText": { + "simpleText": "¥10,800" + }, + "message": { + "runs": [ + { + "text": "This is superchat message." + } + ] + }, + "headerBackgroundColor": 4291821568, + "headerTextColor": 4294967295, + "bodyBackgroundColor": 4293271831, + "bodyTextColor": 4294967295, + "authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url", + "authorNameTextColor": 3019898879, + "contextMenuEndpoint": { + "commandMetadata": { + "webCommandMetadata": { + "ignoreNavigation": true + } + }, + "liveChatItemContextMenuEndpoint": { + "params": "___params___" + } + }, + "timestampColor": 2164260863, + "contextMenuAccessibility": { + "accessibilityData": { + "label": "コメントの操作" + } + } + } + } + } + }, + { + "addChatItemAction": { + "item": { + "liveChatPaidStickerRenderer": { + "id": "dummy_id", + "contextMenuEndpoint": { + "clickTrackingParams": "___clickTrackingParams___", + "commandMetadata": { + "webCommandMetadata": { + "ignoreNavigation": true + } + }, + "liveChatItemContextMenuEndpoint": { + "params": "___params___" + } + }, + "contextMenuAccessibility": { + "accessibilityData": { + "label": "コメントの操作" + } + }, + "timestampUsec": 0, + "authorPhoto": { + "thumbnails": [ + { + "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg", + "width": 32, + "height": 32 + }, + { + "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg", + "width": 64, + "height": 64 + } + ] + }, + "authorName": { + "simpleText": "author_name" + }, + "authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url", + "sticker": { + "thumbnails": [ + { + "url": "//lh3.googleusercontent.com/param_s=s40-rp", + "width": 40, + "height": 40 + }, + { + "url": "//lh3.googleusercontent.com/param_s=s80-rp", + "width": 80, + "height": 80 + } + ], + "accessibility": { + "accessibilityData": { + "label": "___sticker_label___" + } + } + }, + "moneyChipBackgroundColor": 4280191205, + "moneyChipTextColor": 4294967295, + "purchaseAmountText": { + "simpleText": "¥150" + }, + "stickerDisplayWidth": 40, + "stickerDisplayHeight": 40, + "backgroundColor": 4279592384, + "authorNameTextColor": 3019898879, + "trackingParams": "___trackingParams___" + } + } + } + }, + { + "addLiveChatTickerItemAction": { + "item": { + "liveChatTickerSponsorItemRenderer": { + "id": "dummy_id", + "detailText": { + "runs": [ + { + "text": "メンバー" + } + ] + }, + "detailTextColor": 4294967295, + "startBackgroundColor": 4279213400, + "endBackgroundColor": 4278943811, + "sponsorPhoto": { + "thumbnails": [ + { + "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg", + "width": 32, + "height": 32 + }, + { + "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg", + "width": 64, + "height": 64 + } + ] + }, + "durationSec": 300, + "showItemEndpoint": { + "commandMetadata": { + "webCommandMetadata": { + "ignoreNavigation": true + } + }, + "showLiveChatItemEndpoint": { + "renderer": { + "liveChatMembershipItemRenderer": { + "id": "dummy_id", + "timestampUsec": 0, + "authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url", + "headerSubtext": { + "runs": [ + { + "text": "メンバーシップ" + }, + { + "text": " へようこそ!" + } + ] + }, + "authorName": { + "simpleText": "author_name" + }, + "authorPhoto": { + "thumbnails": [ + { + "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg", + "width": 32, + "height": 32 + }, + { + "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg", + "width": 64, + "height": 64 + } + ] + }, + "authorBadges": [ + { + "liveChatAuthorBadgeRenderer": { + "customThumbnail": { + "thumbnails": [ + { + "url": "https://yt3.ggpht.com/X=s32-c-k" + }, + { + "url": "https://yt3.ggpht.com/X=s32-c-k" + } + ] + }, + "tooltip": "新規メンバー", + "accessibility": { + "accessibilityData": { + "label": "新規メンバー" + } + } + } + } + ], + "contextMenuEndpoint": { + "commandMetadata": { + "webCommandMetadata": { + "ignoreNavigation": true + } + }, + "liveChatItemContextMenuEndpoint": { + "params": "___params___" + } + }, + "contextMenuAccessibility": { + "accessibilityData": { + "label": "コメントの操作" + } + } + } + } + } + }, + "authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url", + "fullDurationSec": 300 + } + }, + "durationSec": "300" + } + } + ] + } + } + } +} \ No newline at end of file From a0c5ea035a061c6223a562025adf4b617cf99202 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Wed, 8 Jan 2020 00:46:23 +0900 Subject: [PATCH 26/31] Fix comment --- pytchat/processors/speed_calculator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytchat/processors/speed_calculator.py b/pytchat/processors/speed_calculator.py index fb01d24..504762a 100644 --- a/pytchat/processors/speed_calculator.py +++ b/pytchat/processors/speed_calculator.py @@ -1,5 +1,5 @@ """ -speedmeter.py +speed_calculator.py チャットの勢いを算出するChatProcessor Calculate speed of chat. """ From 89d2f8978fdb0685c7480d830c519b037205b665 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Wed, 8 Jan 2020 01:02:05 +0900 Subject: [PATCH 27/31] Modify README --- README.md | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index e60adc3..4616174 100644 --- a/README.md +++ b/README.md @@ -46,11 +46,12 @@ def display(data): print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}") data.tick() -#entry point -chat = LiveChat("rsHWP7IjMiw", callback = display) -while chat.is_alive(): - time.sleep(3) - #other background operation. +if __name__ == '__main__': + chat = LiveChat("rsHWP7IjMiw", callback = display) + while chat.is_alive(): + #other background operation. + time.sleep(3) + ``` ### asyncio context: @@ -62,8 +63,8 @@ import asyncio async def main(): chat = LiveChatAsync("rsHWP7IjMiw", callback = func) while chat.is_alive(): - await asyncio.sleep(3) #other background operation. + await asyncio.sleep(3) #callback function is automatically called. async def func(data): @@ -71,11 +72,12 @@ async def func(data): print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}") await data.tick_async() -try: - loop = asyncio.get_event_loop() - loop.run_until_complete(main()) -except CancelledError: - pass +if __name__ == '__main__': + try: + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) + except CancelledError: + pass ``` @@ -97,9 +99,12 @@ while chat.is_alive(): time.sleep(polling/len(data['items'])) ``` -### replay: +### replay: +If specified video is not live, +automatically try to fetch archived chat data. + ```python -from pytchat import ReplayChat +from pytchat import LiveChat def main(): #seektime (seconds): start position of chat. @@ -110,7 +115,8 @@ def main(): print(f"{c.elapsedTime} [{c.author.name}]-{c.message} {c.amountString}") data.tick() -main() +if __name__ == '__main__': + main() ``` ## Structure of Default Processor From 0f7a0218b62ad0b336b050c0604b390f622d4418 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Wed, 8 Jan 2020 01:12:58 +0900 Subject: [PATCH 28/31] Delete unnecessary lines --- pytchat/core_multithread/livechat.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pytchat/core_multithread/livechat.py b/pytchat/core_multithread/livechat.py index 9687289..766cfcc 100644 --- a/pytchat/core_multithread/livechat.py +++ b/pytchat/core_multithread/livechat.py @@ -177,11 +177,9 @@ class LiveChat: time.sleep(diff_time if diff_time > 0 else 0) continuation = metadata.get('continuation') except ChatParseException as e: - #self.terminate() logger.debug(f"[{self.video_id}]{str(e)}") return except (TypeError , json.JSONDecodeError) : - #self.terminate() logger.error(f"{traceback.format_exc(limit = -1)}") return From 705bfe0bed70bc3881bf1987307e6d4f67b7511a Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Wed, 8 Jan 2020 01:23:19 +0900 Subject: [PATCH 29/31] Modify MANIFEST.in --- MANIFEST.in | 6 +++++- pytchat/__init__.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index da4d961..0125d4d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,7 @@ include requirements.txt include requirements_test.txt - +prune testrun*.py +prune log.txt +prune quote.txt +prune .gitignore +prun tests diff --git a/pytchat/__init__.py b/pytchat/__init__.py index c9d3131..0b571f5 100644 --- a/pytchat/__init__.py +++ b/pytchat/__init__.py @@ -2,7 +2,7 @@ pytchat is a python library for fetching youtube live chat without using yt api, Selenium, or BeautifulSoup. """ __copyright__ = 'Copyright (C) 2019 taizan-hokuto' -__version__ = '0.0.4.2' +__version__ = '0.0.4.3' __license__ = 'MIT' __author__ = 'taizan-hokuto' __author_email__ = '55448286+taizan-hokuto@users.noreply.github.com' From 60c389f3f726ce06f8ad9076cb06c1026162756f Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Wed, 8 Jan 2020 01:25:19 +0900 Subject: [PATCH 30/31] Change debug mode --- pytchat/__init__.py | 2 +- pytchat/config/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pytchat/__init__.py b/pytchat/__init__.py index 0b571f5..c9d3131 100644 --- a/pytchat/__init__.py +++ b/pytchat/__init__.py @@ -2,7 +2,7 @@ pytchat is a python library for fetching youtube live chat without using yt api, Selenium, or BeautifulSoup. """ __copyright__ = 'Copyright (C) 2019 taizan-hokuto' -__version__ = '0.0.4.3' +__version__ = '0.0.4.2' __license__ = 'MIT' __author__ = 'taizan-hokuto' __author_email__ = '55448286+taizan-hokuto@users.noreply.github.com' diff --git a/pytchat/config/__init__.py b/pytchat/config/__init__.py index 542551e..1a84b59 100644 --- a/pytchat/config/__init__.py +++ b/pytchat/config/__init__.py @@ -1,7 +1,7 @@ import logging from . import mylogger -LOGGER_MODE = logging.DEBUG +LOGGER_MODE = None headers = { 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'} From d4a1d00e282b458eefa516c83f55b158ca12931d Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Wed, 8 Jan 2020 01:25:34 +0900 Subject: [PATCH 31/31] Increment version --- pytchat/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytchat/__init__.py b/pytchat/__init__.py index c9d3131..0b571f5 100644 --- a/pytchat/__init__.py +++ b/pytchat/__init__.py @@ -2,7 +2,7 @@ pytchat is a python library for fetching youtube live chat without using yt api, Selenium, or BeautifulSoup. """ __copyright__ = 'Copyright (C) 2019 taizan-hokuto' -__version__ = '0.0.4.2' +__version__ = '0.0.4.3' __license__ = 'MIT' __author__ = 'taizan-hokuto' __author_email__ = '55448286+taizan-hokuto@users.noreply.github.com'