diff --git a/pytchat/cli/__init__.py b/pytchat/cli/__init__.py index 060b4eb..696af45 100644 --- a/pytchat/cli/__init__.py +++ b/pytchat/cli/__init__.py @@ -1,7 +1,7 @@ import argparse from pathlib import Path from .arguments import Arguments -from .. exceptions import InvalidVideoIdException, NoContentsException +from .. exceptions import InvalidVideoIdException, NoContents from .. processors.html_archiver import HTMLArchiver from .. tool.extract.extractor import Extractor from .. tool.videoinfo import VideoInfo @@ -50,7 +50,7 @@ def main(): callback=_disp_progress ).extract() print("\nExtraction end.\n") - except (InvalidVideoIdException, NoContentsException) as e: + except (InvalidVideoIdException, NoContents) as e: print(e) return parser.print_help() diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index 1853025..715064e 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -11,7 +11,7 @@ from asyncio import Queue from .buffer import Buffer from ..parser.live import Parser from .. import config -from ..exceptions import ChatParseException, IllegalFunctionCall +from .. import exceptions from ..paramgen import liveparam, arcparam from ..processors.default.processor import DefaultProcessor from ..processors.combinator import Combinator @@ -86,7 +86,7 @@ class LiveChatAsync: topchat_only=False, logger=config.logger(__name__), ): - self.video_id = video_id + self._video_id = video_id self.seektime = seektime if isinstance(processor, tuple): self.processor = Combinator(processor) @@ -102,28 +102,26 @@ class LiveChatAsync: self._parser = Parser(is_replay=self._is_replay) self._pauser = Queue() self._pauser.put_nowait(None) - self._setup() self._first_fetch = True self._fetch_url = "live_chat/get_live_chat?continuation=" self._topchat_only = topchat_only self._logger = logger + self.exception = None LiveChatAsync._logger = logger - if not LiveChatAsync._setup_finished: - LiveChatAsync._setup_finished = True - if exception_handler: - self._set_exception_handler(exception_handler) - if interruptable: - signal.signal(signal.SIGINT, - (lambda a, b: asyncio.create_task( - LiveChatAsync.shutdown(None, signal.SIGINT, b)) - )) + if exception_handler: + self._set_exception_handler(exception_handler) + if interruptable: + signal.signal(signal.SIGINT, + (lambda a, b: asyncio.create_task( + LiveChatAsync.shutdown(None, signal.SIGINT, b)))) + self._setup() def _setup(self): # direct modeがTrueでcallback未設定の場合例外発生。 if self._direct_mode: if self._callback is None: - raise IllegalFunctionCall( + raise exceptions.IllegalFunctionCall( "When direct_mode=True, callback parameter is required.") else: # direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成 @@ -138,18 +136,18 @@ class LiveChatAsync: loop.create_task(self._callback_loop(self._callback)) # _listenループタスクの開始 loop = asyncio.get_event_loop() - listen_task = loop.create_task(self._startlisten()) + self.listen_task = loop.create_task(self._startlisten()) # add_done_callbackの登録 if self._done_callback is None: - listen_task.add_done_callback(self.finish) + self.listen_task.add_done_callback(self._finish) else: - listen_task.add_done_callback(self._done_callback) + self.listen_task.add_done_callback(self._done_callback) async def _startlisten(self): """Fetch first continuation parameter, create and start _listen loop. """ - initial_continuation = liveparam.getparam(self.video_id, 3) + initial_continuation = liveparam.getparam(self._video_id, 3) await self._listen(initial_continuation) async def _listen(self, continuation): @@ -171,7 +169,7 @@ class LiveChatAsync: timeout = metadata['timeoutMs'] / 1000 chat_component = { - "video_id": self.video_id, + "video_id": self._video_id, "timeout": timeout, "chatdata": chatdata } @@ -188,14 +186,15 @@ class LiveChatAsync: diff_time = timeout - (time.time() - time_mark) await asyncio.sleep(diff_time) continuation = metadata.get('continuation') - except ChatParseException as e: - self._logger.debug(f"[{self.video_id}]{str(e)}") - return + except exceptions.ChatParseException as e: + self._logger.debug(f"[{self._video_id}]{str(e)}") + raise except (TypeError, json.JSONDecodeError): self._logger.error(f"{traceback.format_exc(limit = -1)}") - return + raise - self._logger.debug(f"[{self.video_id}]finished fetching chat.") + self._logger.debug(f"[{self._video_id}]finished fetching chat.") + raise exceptions.ChatDataFinished async def _check_pause(self, continuation): if self._pauser.empty(): @@ -207,7 +206,7 @@ class LiveChatAsync: self._pauser.put_nowait(None) if not self._is_replay: continuation = liveparam.getparam( - self.video_id, 3, self._topchat_only) + self._video_id, 3, self._topchat_only) return continuation async def _get_contents(self, continuation, session, headers): @@ -227,7 +226,7 @@ class LiveChatAsync: self._parser.is_replay = True self._fetch_url = "live_chat_replay/get_live_chat_replay?continuation=" continuation = arcparam.getparam( - self.video_id, self.seektime, self._topchat_only) + self._video_id, self.seektime, self._topchat_only) livechat_json = (await self._get_livechat_json( continuation, session, headers)) reload_continuation = self._parser.reload_continuation( @@ -258,7 +257,7 @@ class LiveChatAsync: await asyncio.sleep(1) continue else: - self._logger.error(f"[{self.video_id}]" + self._logger.error(f"[{self._video_id}]" f"Exceeded retry count. status_code={status_code}") return None return livechat_json @@ -288,9 +287,12 @@ class LiveChatAsync: : Processorによって加工されたチャットデータ """ if self._callback is None: - items = await self._buffer.get() - return self.processor.process(items) - raise IllegalFunctionCall( + if self.is_alive(): + items = await self._buffer.get() + return self.processor.process(items) + else: + return [] + raise exceptions.IllegalFunctionCall( "既にcallbackを登録済みのため、get()は実行できません。") def is_replay(self): @@ -311,22 +313,36 @@ class LiveChatAsync: def is_alive(self): return self._is_alive - def finish(self, sender): + def _finish(self, sender): '''Listener終了時のコールバック''' try: - self.terminate() + self._task_finished() except CancelledError: - self._logger.debug(f'[{self.video_id}]cancelled:{sender}') + self._logger.debug(f'[{self._video_id}]cancelled:{sender}') def terminate(self): + if self._pauser.empty(): + self._pauser.put_nowait(None) + self._is_alive = False + self._buffer.put_nowait({}) + + def _task_finished(self): ''' Listenerを終了する。 ''' - self._is_alive = False - if self._direct_mode is False: - # bufferにダミーオブジェクトを入れてis_alive()を判定させる - self._buffer.put_nowait({'chatdata': '', 'timeout': 0}) - self._logger.info(f'[{self.video_id}]finished.') + if self.is_alive(): + self.terminate() + try: + self.listen_task.result() + except Exception as e: + self.exception = e + if not isinstance(e, exceptions.ChatParseException): + self._logger.error(f'Internal exception - {type(e)}{str(e)}') + self._logger.info(f'[{self._video_id}]終了しました') + + def raise_for_status(self): + if self.exception is not None: + raise self.exception @classmethod def _set_exception_handler(cls, handler): diff --git a/pytchat/core_multithread/livechat.py b/pytchat/core_multithread/livechat.py index c1aa0ad..e06bc8a 100644 --- a/pytchat/core_multithread/livechat.py +++ b/pytchat/core_multithread/livechat.py @@ -6,10 +6,11 @@ import traceback import urllib.parse from concurrent.futures import CancelledError, ThreadPoolExecutor from queue import Queue +from threading import Event from .buffer import Buffer from ..parser.live import Parser from .. import config -from ..exceptions import ChatParseException, IllegalFunctionCall +from .. import exceptions from ..paramgen import liveparam, arcparam from ..processors.default.processor import DefaultProcessor from ..processors.combinator import Combinator @@ -83,7 +84,7 @@ class LiveChat: topchat_only=False, logger=config.logger(__name__) ): - self.video_id = video_id + self._video_id = video_id self.seektime = seektime if isinstance(processor, tuple): self.processor = Combinator(processor) @@ -102,7 +103,9 @@ class LiveChat: self._first_fetch = True self._fetch_url = "live_chat/get_live_chat?continuation=" self._topchat_only = topchat_only + self._event = Event() self._logger = logger + self.exception = None if interruptable: signal.signal(signal.SIGINT, lambda a, b: self.terminate()) self._setup() @@ -111,7 +114,7 @@ class LiveChat: # direct modeがTrueでcallback未設定の場合例外発生。 if self._direct_mode: if self._callback is None: - raise IllegalFunctionCall( + raise exceptions.IllegalFunctionCall( "When direct_mode=True, callback parameter is required.") else: # direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成 @@ -124,19 +127,19 @@ class LiveChat: # callbackを呼ぶループタスクの開始 self._executor.submit(self._callback_loop, self._callback) # _listenループタスクの開始 - listen_task = self._executor.submit(self._startlisten) + self.listen_task = self._executor.submit(self._startlisten) # add_done_callbackの登録 if self._done_callback is None: - listen_task.add_done_callback(self.finish) + self.listen_task.add_done_callback(self._finish) else: - listen_task.add_done_callback(self._done_callback) + self.listen_task.add_done_callback(self._done_callback) def _startlisten(self): time.sleep(0.1) # sleep shortly to prohibit skipping fetching data """Fetch first continuation parameter, create and start _listen loop. """ - initial_continuation = liveparam.getparam(self.video_id, 3) + initial_continuation = liveparam.getparam(self._video_id, 3) self._listen(initial_continuation) def _listen(self, continuation): @@ -152,13 +155,11 @@ class LiveChat: with requests.Session() as session: while(continuation and self._is_alive): continuation = self._check_pause(continuation) - contents = self._get_contents( - continuation, session, headers) + contents = self._get_contents(continuation, session, headers) metadata, chatdata = self._parser.parse(contents) - timeout = metadata['timeoutMs'] / 1000 chat_component = { - "video_id": self.video_id, + "video_id": self._video_id, "timeout": timeout, "chatdata": chatdata } @@ -173,16 +174,17 @@ class LiveChat: else: self._buffer.put(chat_component) diff_time = timeout - (time.time() - time_mark) - time.sleep(diff_time if diff_time > 0 else 0) + self._event.wait(diff_time if diff_time > 0 else 0) continuation = metadata.get('continuation') - except ChatParseException as e: - self._logger.debug(f"[{self.video_id}]{str(e)}") - return + except exceptions.ChatParseException as e: + self._logger.debug(f"[{self._video_id}]{str(e)}") + raise except (TypeError, json.JSONDecodeError): self._logger.error(f"{traceback.format_exc(limit=-1)}") - return + raise - self._logger.debug(f"[{self.video_id}]finished fetching chat.") + self._logger.debug(f"[{self._video_id}]finished fetching chat.") + raise exceptions.ChatDataFinished def _check_pause(self, continuation): if self._pauser.empty(): @@ -193,7 +195,7 @@ class LiveChat: ''' self._pauser.put_nowait(None) if not self._is_replay: - continuation = liveparam.getparam(self.video_id, 3) + continuation = liveparam.getparam(self._video_id, 3) return continuation def _get_contents(self, continuation, session, headers): @@ -215,9 +217,8 @@ class LiveChat: self._parser.is_replay = True self._fetch_url = "live_chat_replay/get_live_chat_replay?continuation=" continuation = arcparam.getparam( - self.video_id, self.seektime, self._topchat_only) - livechat_json = (self._get_livechat_json( - continuation, session, headers)) + self._video_id, self.seektime, self._topchat_only) + livechat_json = (self._get_livechat_json(continuation, session, headers)) reload_continuation = self._parser.reload_continuation( self._parser.get_contents(livechat_json)) if reload_continuation: @@ -246,9 +247,9 @@ class LiveChat: time.sleep(1) continue else: - self._logger.error(f"[{self.video_id}]" + self._logger.error(f"[{self._video_id}]" f"Exceeded retry count. status_code={status_code}") - return None + raise exceptions.RetryExceedMaxCount() return livechat_json def _callback_loop(self, callback): @@ -276,9 +277,12 @@ class LiveChat: : Processorによって加工されたチャットデータ """ if self._callback is None: - items = self._buffer.get() - return self.processor.process(items) - raise IllegalFunctionCall( + if self.is_alive(): + items = self._buffer.get() + return self.processor.process(items) + else: + return [] + raise exceptions.IllegalFunctionCall( "既にcallbackを登録済みのため、get()は実行できません。") def is_replay(self): @@ -299,18 +303,34 @@ class LiveChat: def is_alive(self): return self._is_alive - def finish(self, sender): + def _finish(self, sender): '''Listener終了時のコールバック''' try: - self.terminate() + self._task_finished() except CancelledError: - self._logger.debug(f'[{self.video_id}]cancelled:{sender}') + self._logger.debug(f'[{self._video_id}]cancelled:{sender}') def terminate(self): + if self._pauser.empty(): + self._pauser.put_nowait(None) + self._is_alive = False + self._buffer.put({}) + self._event.set() + + def _task_finished(self): ''' Listenerを終了する。 ''' if self.is_alive(): - self._is_alive = False - self._buffer.put({}) - self._logger.info(f'[{self.video_id}]終了しました') + self.terminate() + try: + self.listen_task.result() + except Exception as e: + self.exception = e + if not isinstance(e, exceptions.ChatParseException): + self._logger.error(f'Internal exception - {type(e)}{str(e)}') + self._logger.info(f'[{self._video_id}]終了しました') + + def raise_for_status(self): + if self.exception is not None: + raise self.exception diff --git a/pytchat/exceptions.py b/pytchat/exceptions.py index 1f45829..d8de40e 100644 --- a/pytchat/exceptions.py +++ b/pytchat/exceptions.py @@ -5,13 +5,6 @@ class ChatParseException(Exception): pass -class NoYtinitialdataException(ChatParseException): - ''' - Thrown when the video is not found. - ''' - pass - - class ResponseContextError(ChatParseException): ''' Thrown when chat data is invalid. @@ -19,21 +12,14 @@ class ResponseContextError(ChatParseException): pass -class NoLivechatRendererException(ChatParseException): - ''' - Thrown when livechatRenderer is missing in JSON. - ''' - pass - - -class NoContentsException(ChatParseException): +class NoContents(ChatParseException): ''' Thrown when ContinuationContents is missing in JSON. ''' pass -class NoContinuationsException(ChatParseException): +class NoContinuation(ChatParseException): ''' Thrown when continuation is missing in ContinuationContents. ''' @@ -42,8 +28,8 @@ class NoContinuationsException(ChatParseException): class IllegalFunctionCall(Exception): ''' - Thrown when get () is called even though - set_callback () has been executed. + Thrown when get() is called even though + set_callback() has been executed. ''' pass @@ -57,3 +43,22 @@ class InvalidVideoIdException(Exception): class UnknownConnectionError(Exception): pass + + +class RetryExceedMaxCount(Exception): + ''' + thrown when the number of retries exceeds the maximum value. + ''' + pass + + +class ChatDataFinished(ChatParseException): + pass + + +class ReceivedUnknownContinuation(ChatParseException): + pass + + +class FailedExtractContinuation(ChatDataFinished): + pass diff --git a/pytchat/parser/live.py b/pytchat/parser/live.py index 5fd6bdb..13540cb 100644 --- a/pytchat/parser/live.py +++ b/pytchat/parser/live.py @@ -4,11 +4,7 @@ pytchat.parser.live Parser of live chat JSON. """ -from .. exceptions import ( - ResponseContextError, - NoContentsException, - NoContinuationsException, - ChatParseException) +from .. import exceptions class Parser: @@ -20,9 +16,9 @@ class Parser: def get_contents(self, jsn): if jsn is None: - raise ChatParseException('Called with none JSON object.') + raise exceptions.IllegalFunctionCall('Called with none JSON object.') if jsn['response']['responseContext'].get('errors'): - raise ResponseContextError( + raise exceptions.ResponseContextError( 'The video_id would be wrong, or video is deleted or private.') contents = jsn['response'].get('continuationContents') return contents @@ -46,11 +42,11 @@ class Parser: if contents is None: '''Broadcasting end or cannot fetch chat stream''' - raise NoContentsException('Chat data stream is empty.') + raise exceptions.NoContents('Chat data stream is empty.') cont = contents['liveChatContinuation']['continuations'][0] if cont is None: - raise NoContinuationsException('No Continuation') + raise exceptions.NoContinuation('No Continuation') metadata = (cont.get('invalidationContinuationData') or cont.get('timedContinuationData') or cont.get('reloadContinuationData') @@ -58,22 +54,25 @@ class Parser: ) if metadata is None: if cont.get("playerSeekContinuationData"): - raise ChatParseException('Finished chat data') + raise exceptions.ChatDataFinished('Finished chat data') unknown = list(cont.keys())[0] if unknown: - raise ChatParseException( + raise exceptions.ReceivedUnknownContinuation( f"Received unknown continuation type:{unknown}") else: - raise ChatParseException('Cannot extract continuation data') + raise exceptions.FailedExtractContinuation('Cannot extract continuation data') return self._create_data(metadata, contents) def reload_continuation(self, contents): """ - When `seektime = 0` or seektime is abbreviated , + When `seektime == 0` or seektime is abbreviated , check if fetched chat json has no chat data. If so, try to fetch playerSeekContinuationData. This function must be run only first fetching. """ + if contents is None: + '''Broadcasting end or cannot fetch chat stream''' + raise exceptions.NoContents('Chat data stream is empty.') cont = contents['liveChatContinuation']['continuations'][0] if cont.get("liveChatReplayContinuationData"): # chat data exist. @@ -82,7 +81,7 @@ class Parser: init_cont = cont.get("playerSeekContinuationData") if init_cont: return init_cont.get("continuation") - raise ChatParseException('Finished chat data') + raise exceptions.ChatDataFinished('Finished chat data') def _create_data(self, metadata, contents): actions = contents['liveChatContinuation'].get('actions') diff --git a/pytchat/tool/extract/parser.py b/pytchat/tool/extract/parser.py index 9dc6989..a2568a4 100644 --- a/pytchat/tool/extract/parser.py +++ b/pytchat/tool/extract/parser.py @@ -1,8 +1,5 @@ from ... import config -from ... exceptions import ( - ResponseContextError, - NoContentsException, - NoContinuationsException) +from ... import exceptions logger = config.logger(__name__) @@ -23,15 +20,15 @@ def parse(jsn): if jsn is None: raise ValueError("parameter JSON is None") if jsn['response']['responseContext'].get('errors'): - raise ResponseContextError( + raise exceptions.ResponseContextError( 'video_id is invalid or private/deleted.') contents = jsn['response'].get('continuationContents') if contents is None: - raise NoContentsException('No chat data.') + raise exceptions.NoContents('No chat data.') cont = contents['liveChatContinuation']['continuations'][0] if cont is None: - raise NoContinuationsException('No Continuation') + raise exceptions.NoContinuation('No Continuation') metadata = cont.get('liveChatReplayContinuationData') if metadata: continuation = metadata.get("continuation") diff --git a/pytchat/tool/mining/parser.py b/pytchat/tool/mining/parser.py index 5dfd9dc..f9a692f 100644 --- a/pytchat/tool/mining/parser.py +++ b/pytchat/tool/mining/parser.py @@ -1,12 +1,12 @@ -import json +import re from ... import config -from ... exceptions import ( - ResponseContextError, - NoContentsException, - NoContinuationsException ) +from ... exceptions import ( + ResponseContextError, + NoContents, NoContinuation) logger = config.logger(__name__) + def parse(jsn): """ Parse replay chat data. @@ -20,45 +20,51 @@ def parse(jsn): actions : list """ - if jsn is None: + if jsn is None: raise ValueError("parameter JSON is None") if jsn['response']['responseContext'].get('errors'): raise ResponseContextError( - 'video_id is invalid or private/deleted.') - contents=jsn["response"].get('continuationContents') + 'video_id is invalid or private/deleted.') + contents = jsn["response"].get('continuationContents') if contents is None: - raise NoContentsException('No chat data.') + raise NoContents('No chat data.') cont = contents['liveChatContinuation']['continuations'][0] if cont is None: - raise NoContinuationsException('No Continuation') + raise NoContinuation('No Continuation') metadata = cont.get('liveChatReplayContinuationData') if metadata: continuation = metadata.get("continuation") actions = contents['liveChatContinuation'].get('actions') if continuation: - return continuation, [action["replayChatItemAction"]["actions"][0] - for action in actions - if list(action['replayChatItemAction']["actions"][0].values() - )[0]['item'].get("liveChatPaidMessageRenderer") - or list(action['replayChatItemAction']["actions"][0].values() - )[0]['item'].get("liveChatPaidStickerRenderer") - ] + return continuation, [action["replayChatItemAction"]["actions"][0] + for action in actions + if list(action['replayChatItemAction']["actions"][0].values() + )[0]['item'].get("liveChatPaidMessageRenderer") + or list(action['replayChatItemAction']["actions"][0].values() + )[0]['item'].get("liveChatPaidStickerRenderer") + ] return None, [] def get_offset(item): return int(item['replayChatItemAction']["videoOffsetTimeMsec"]) + def get_id(item): return list((list(item['replayChatItemAction']["actions"][0].values() - )[0])['item'].values())[0].get('id') + )[0])['item'].values())[0].get('id') + def get_type(item): return list((list(item['replayChatItemAction']["actions"][0].values() - )[0])['item'].keys())[0] -import re -_REGEX_YTINIT = re.compile("window\\[\"ytInitialData\"\\]\\s*=\\s*({.+?});\\s+") + )[0])['item'].keys())[0] + + +_REGEX_YTINIT = re.compile( + "window\\[\"ytInitialData\"\\]\\s*=\\s*({.+?});\\s+") + + def extract(text): match = re.findall(_REGEX_YTINIT, str(text)) diff --git a/tests/test_arcparam_mining.py b/tests/test_arcparam_mining.py index 7cfcee4..556df6e 100644 --- a/tests/test_arcparam_mining.py +++ b/tests/test_arcparam_mining.py @@ -1,40 +1,41 @@ -import pytest from pytchat.tool.mining import parser import pytchat.config as config -import requests, json +import requests +import json from pytchat.paramgen import arcparam_mining as arcparam + def test_arcparam_e(mocker): try: - arcparam.getparam("01234567890",-1) + arcparam.getparam("01234567890", -1) assert False except ValueError: assert True - - def test_arcparam_0(mocker): - param = arcparam.getparam("01234567890",0) + param = arcparam.getparam("01234567890", 0) - assert param =="op2w0wQsGiBDZzhhRFFvTE1ERXlNelExTmpjNE9UQWdBUSUzRCUzREABYARyAggBeAE%3D" + assert param == "op2w0wQsGiBDZzhhRFFvTE1ERXlNelExTmpjNE9UQWdBUSUzRCUzREABYARyAggBeAE%3D" def test_arcparam_1(mocker): - param = arcparam.getparam("01234567890", seektime = 100000) + param = arcparam.getparam("01234567890", seektime=100000) print(param) assert param == "op2w0wQzGiBDZzhhRFFvTE1ERXlNelExTmpjNE9UQWdBUSUzRCUzREABWgUQgMLXL2AEcgIIAXgB" + def test_arcparam_2(mocker): - param = arcparam.getparam("PZz9NB0-Z64",1) - url=f"https://www.youtube.com/live_chat_replay?continuation={param}&playerOffsetMs=1000&pbj=1" - resp = requests.Session().get(url,headers = config.headers) + param = arcparam.getparam("PZz9NB0-Z64", 1) + url = f"https://www.youtube.com/live_chat_replay?continuation={param}&playerOffsetMs=1000&pbj=1" + resp = requests.Session().get(url, headers=config.headers) jsn = json.loads(resp.text) - _ , chatdata = parser.parse(jsn[1]) + _, chatdata = parser.parse(jsn[1]) test_id = chatdata[0]["addChatItemAction"]["item"]["liveChatPaidMessageRenderer"]["id"] print(test_id) assert test_id == "ChwKGkNKSGE0YnFJeWVBQ0ZWcUF3Z0VkdGIwRm9R" + def test_arcparam_3(mocker): param = arcparam.getparam("01234567890") assert param == "op2w0wQsGiBDZzhhRFFvTE1ERXlNelExTmpjNE9UQWdBUSUzRCUzREABYARyAggBeAE%3D" diff --git a/tests/test_compatible_processor.py b/tests/test_compatible_processor.py index 45aec94..a06fe88 100644 --- a/tests/test_compatible_processor.py +++ b/tests/test_compatible_processor.py @@ -1,17 +1,6 @@ import json -import pytest -import asyncio -import aiohttp from pytchat.parser.live import Parser from pytchat.processors.compatible.processor import CompatibleProcessor -from pytchat.exceptions import ( - NoLivechatRendererException, NoYtinitialdataException, - ResponseContextError, NoContentsException) - -from pytchat.processors.compatible.renderer.textmessage import LiveChatTextMessageRenderer -from pytchat.processors.compatible.renderer.paidmessage import LiveChatPaidMessageRenderer -from pytchat.processors.compatible.renderer.paidsticker import LiveChatPaidStickerRenderer -from pytchat.processors.compatible.renderer.legacypaid import LiveChatLegacyPaidMessageRenderer parser = Parser(is_replay=False) @@ -31,21 +20,23 @@ def test_textmessage(mocker): ret = processor.process([data]) assert ret["kind"] == "youtube#liveChatMessageListResponse" - assert ret["pollingIntervalMillis"] == data["timeout"]*1000 + assert ret["pollingIntervalMillis"] == data["timeout"] * 1000 assert ret.keys() == { - "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items" + "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items" } assert ret["pageInfo"].keys() == { - "totalResults", "resultsPerPage" + "totalResults", "resultsPerPage" } assert ret["items"][0].keys() == { - "kind", "etag", "id", "snippet", "authorDetails" + "kind", "etag", "id", "snippet", "authorDetails" } assert ret["items"][0]["snippet"].keys() == { - 'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage', 'textMessageDetails' + 'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage', + 'textMessageDetails' } assert ret["items"][0]["authorDetails"].keys() == { - 'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator' + 'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', + 'isChatModerator' } assert ret["items"][0]["snippet"]["textMessageDetails"].keys() == { 'messageText' @@ -69,22 +60,23 @@ def test_newsponcer(mocker): ret = processor.process([data]) assert ret["kind"] == "youtube#liveChatMessageListResponse" - assert ret["pollingIntervalMillis"] == data["timeout"]*1000 + assert ret["pollingIntervalMillis"] == data["timeout"] * 1000 assert ret.keys() == { - "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items" + "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items" } assert ret["pageInfo"].keys() == { - "totalResults", "resultsPerPage" + "totalResults", "resultsPerPage" } assert ret["items"][0].keys() == { - "kind", "etag", "id", "snippet", "authorDetails" + "kind", "etag", "id", "snippet", "authorDetails" } assert ret["items"][0]["snippet"].keys() == { 'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage' } assert ret["items"][0]["authorDetails"].keys() == { - 'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator' + 'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', + 'isChatModerator' } assert "LCC." in ret["items"][0]["id"] assert ret["items"][0]["snippet"]["type"] == "newSponsorEvent" @@ -105,22 +97,23 @@ def test_newsponcer_rev(mocker): ret = processor.process([data]) assert ret["kind"] == "youtube#liveChatMessageListResponse" - assert ret["pollingIntervalMillis"] == data["timeout"]*1000 + assert ret["pollingIntervalMillis"] == data["timeout"] * 1000 assert ret.keys() == { - "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items" + "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items" } assert ret["pageInfo"].keys() == { - "totalResults", "resultsPerPage" + "totalResults", "resultsPerPage" } assert ret["items"][0].keys() == { - "kind", "etag", "id", "snippet", "authorDetails" + "kind", "etag", "id", "snippet", "authorDetails" } assert ret["items"][0]["snippet"].keys() == { 'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage' } assert ret["items"][0]["authorDetails"].keys() == { - 'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator' + 'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', + 'isChatModerator' } assert "LCC." in ret["items"][0]["id"] assert ret["items"][0]["snippet"]["type"] == "newSponsorEvent" @@ -141,21 +134,23 @@ def test_superchat(mocker): ret = processor.process([data]) assert ret["kind"] == "youtube#liveChatMessageListResponse" - assert ret["pollingIntervalMillis"] == data["timeout"]*1000 + assert ret["pollingIntervalMillis"] == data["timeout"] * 1000 assert ret.keys() == { - "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items" + "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items" } assert ret["pageInfo"].keys() == { - "totalResults", "resultsPerPage" + "totalResults", "resultsPerPage" } assert ret["items"][0].keys() == { - "kind", "etag", "id", "snippet", "authorDetails" + "kind", "etag", "id", "snippet", "authorDetails" } assert ret["items"][0]["snippet"].keys() == { - 'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage', 'superChatDetails' + 'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage', + 'superChatDetails' } assert ret["items"][0]["authorDetails"].keys() == { - 'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator' + 'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', + 'isChatModerator' } assert ret["items"][0]["snippet"]["superChatDetails"].keys() == { 'amountMicros', 'currency', 'amountDisplayString', 'tier', 'backgroundColor' diff --git a/tests/test_livechat.py b/tests/test_livechat.py index 475f26c..6c0d38f 100644 --- a/tests/test_livechat.py +++ b/tests/test_livechat.py @@ -1,30 +1,21 @@ -import pytest -from pytchat.parser.live import Parser - import json -import asyncio,aiohttp - from aioresponses import aioresponses from pytchat.core_async.livechat import LiveChatAsync -from pytchat.exceptions import ( - NoLivechatRendererException,NoYtinitialdataException, - ResponseContextError,NoContentsException) +from pytchat.exceptions import ResponseContextError -from pytchat.core_multithread.livechat import LiveChat -import unittest -from unittest import TestCase - def _open_file(path): - with open(path,mode ='r',encoding = 'utf-8') as f: + with open(path, mode='r', encoding='utf-8') as f: return f.read() + @aioresponses() def test_Async(*mock): - vid='' + vid = '' _text = _open_file('tests/testdata/paramgen_firstread.json') _text = json.loads(_text) - mock[0].get(f"https://www.youtube.com/live_chat?v={vid}&is_popout=1", status=200, body=_text) + mock[0].get( + f"https://www.youtube.com/live_chat?v={vid}&is_popout=1", status=200, body=_text) try: chat = LiveChatAsync(video_id='') assert chat.is_alive() @@ -33,6 +24,7 @@ def test_Async(*mock): except ResponseContextError: assert not chat.is_alive() + def test_MultiThread(mocker): _text = _open_file('tests/testdata/paramgen_firstread.json') _text = json.loads(_text) @@ -48,6 +40,3 @@ def test_MultiThread(mocker): except ResponseContextError: chat.terminate() assert not chat.is_alive() - - - diff --git a/tests/test_livechat_2.py b/tests/test_livechat_2.py index f582ae0..0fbe42a 100644 --- a/tests/test_livechat_2.py +++ b/tests/test_livechat_2.py @@ -1,43 +1,43 @@ -import asyncio, aiohttp -import json -import pytest +import asyncio import re -import requests -import sys -import time from aioresponses import aioresponses from concurrent.futures import CancelledError -from unittest import TestCase from pytchat.core_multithread.livechat import LiveChat from pytchat.core_async.livechat import LiveChatAsync -from pytchat.exceptions import ( - NoLivechatRendererException,NoYtinitialdataException, - ResponseContextError,NoContentsException) -from pytchat.parser.live import Parser from pytchat.processors.dummy_processor import DummyProcessor + def _open_file(path): - with open(path,mode ='r',encoding = 'utf-8') as f: + with open(path, mode='r', encoding='utf-8') as f: return f.read() + @aioresponses() def test_async_live_stream(*mock): async def test_loop(*mock): - pattern = re.compile(r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$') + pattern = re.compile( + r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$') _text = _open_file('tests/testdata/test_stream.json') mock[0].get(pattern, status=200, body=_text) - chat = LiveChatAsync(video_id='', processor = DummyProcessor()) + chat = LiveChatAsync(video_id='', processor=DummyProcessor()) chats = await chat.get() rawdata = chats[0]["chatdata"] - #assert fetching livachat data - assert list(rawdata[0]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer" - assert list(rawdata[1]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer" - assert list(rawdata[2]["addChatItemAction"]["item"].keys())[0] == "liveChatPlaceholderItemRenderer" - assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerPaidMessageItemRenderer" - assert list(rawdata[4]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer" - assert list(rawdata[5]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidStickerRenderer" - assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerSponsorItemRenderer" + # assert fetching livachat data + assert list(rawdata[0]["addChatItemAction"]["item"].keys())[ + 0] == "liveChatTextMessageRenderer" + assert list(rawdata[1]["addChatItemAction"]["item"].keys())[ + 0] == "liveChatTextMessageRenderer" + assert list(rawdata[2]["addChatItemAction"]["item"].keys())[ + 0] == "liveChatPlaceholderItemRenderer" + assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[ + 0] == "liveChatTickerPaidMessageItemRenderer" + assert list(rawdata[4]["addChatItemAction"]["item"].keys())[ + 0] == "liveChatPaidMessageRenderer" + assert list(rawdata[5]["addChatItemAction"]["item"].keys())[ + 0] == "liveChatPaidStickerRenderer" + assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[ + 0] == "liveChatTickerSponsorItemRenderer" loop = asyncio.get_event_loop() try: @@ -45,24 +45,29 @@ def test_async_live_stream(*mock): except CancelledError: assert True -@aioresponses() + +@aioresponses() def test_async_replay_stream(*mock): async def test_loop(*mock): - pattern_live = re.compile(r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$') - pattern_replay = re.compile(r'^https://www.youtube.com/live_chat_replay/get_live_chat_replay\?continuation=.*$') - #empty livechat -> switch to fetch replaychat + pattern_live = re.compile( + r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$') + pattern_replay = re.compile( + r'^https://www.youtube.com/live_chat_replay/get_live_chat_replay\?continuation=.*$') + # empty livechat -> switch to fetch replaychat _text_live = _open_file('tests/testdata/finished_live.json') _text_replay = _open_file('tests/testdata/chatreplay.json') mock[0].get(pattern_live, status=200, body=_text_live) mock[0].get(pattern_replay, status=200, body=_text_replay) - chat = LiveChatAsync(video_id='', processor = DummyProcessor()) + chat = LiveChatAsync(video_id='', processor=DummyProcessor()) chats = await chat.get() rawdata = chats[0]["chatdata"] - #assert fetching replaychat data - assert list(rawdata[0]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer" - assert list(rawdata[14]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer" + # assert fetching replaychat data + assert list(rawdata[0]["addChatItemAction"]["item"].keys())[ + 0] == "liveChatTextMessageRenderer" + assert list(rawdata[14]["addChatItemAction"]["item"].keys())[ + 0] == "liveChatPaidMessageRenderer" loop = asyncio.get_event_loop() try: @@ -70,56 +75,66 @@ def test_async_replay_stream(*mock): except CancelledError: assert True + @aioresponses() def test_async_force_replay(*mock): async def test_loop(*mock): - pattern_live = re.compile(r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$') - pattern_replay = re.compile(r'^https://www.youtube.com/live_chat_replay/get_live_chat_replay\?continuation=.*$') - #valid live data, but force_replay = True + pattern_live = re.compile( + r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$') + pattern_replay = re.compile( + r'^https://www.youtube.com/live_chat_replay/get_live_chat_replay\?continuation=.*$') + # valid live data, but force_replay = True _text_live = _open_file('tests/testdata/test_stream.json') - #valid replay data + # valid replay data _text_replay = _open_file('tests/testdata/chatreplay.json') - + mock[0].get(pattern_live, status=200, body=_text_live) mock[0].get(pattern_replay, status=200, body=_text_replay) - #force replay - chat = LiveChatAsync(video_id='', processor = DummyProcessor(), force_replay = True) + # force replay + chat = LiveChatAsync( + video_id='', processor=DummyProcessor(), force_replay=True) chats = await chat.get() rawdata = chats[0]["chatdata"] # assert fetching replaychat data - assert list(rawdata[14]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer" + assert list(rawdata[14]["addChatItemAction"]["item"].keys())[ + 0] == "liveChatPaidMessageRenderer" # assert not mix livechat data - assert list(rawdata[2]["addChatItemAction"]["item"].keys())[0] != "liveChatPlaceholderItemRenderer" - + assert list(rawdata[2]["addChatItemAction"]["item"].keys())[ + 0] != "liveChatPlaceholderItemRenderer" + loop = asyncio.get_event_loop() try: loop.run_until_complete(test_loop(*mock)) except CancelledError: assert True + def test_multithread_live_stream(mocker): _text = _open_file('tests/testdata/test_stream.json') responseMock = mocker.Mock() responseMock.status_code = 200 responseMock.text = _text - mocker.patch('requests.Session.get').return_value.__enter__.return_value = responseMock + mocker.patch( + 'requests.Session.get').return_value.__enter__.return_value = responseMock - chat = LiveChat(video_id='test_id', processor = DummyProcessor()) + chat = LiveChat(video_id='test_id', processor=DummyProcessor()) chats = chat.get() rawdata = chats[0]["chatdata"] - #assert fetching livachat data - assert list(rawdata[0]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer" - assert list(rawdata[1]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer" - assert list(rawdata[2]["addChatItemAction"]["item"].keys())[0] == "liveChatPlaceholderItemRenderer" - assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerPaidMessageItemRenderer" - assert list(rawdata[4]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer" - assert list(rawdata[5]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidStickerRenderer" - assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerSponsorItemRenderer" + # assert fetching livachat data + assert list(rawdata[0]["addChatItemAction"]["item"].keys())[ + 0] == "liveChatTextMessageRenderer" + assert list(rawdata[1]["addChatItemAction"]["item"].keys())[ + 0] == "liveChatTextMessageRenderer" + assert list(rawdata[2]["addChatItemAction"]["item"].keys())[ + 0] == "liveChatPlaceholderItemRenderer" + assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[ + 0] == "liveChatTickerPaidMessageItemRenderer" + assert list(rawdata[4]["addChatItemAction"]["item"].keys())[ + 0] == "liveChatPaidMessageRenderer" + assert list(rawdata[5]["addChatItemAction"]["item"].keys())[ + 0] == "liveChatPaidStickerRenderer" + assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[ + 0] == "liveChatTickerSponsorItemRenderer" chat.terminate() - - - - - diff --git a/tests/test_parser.py b/tests/test_parser.py index 9832f7a..140d07b 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,17 +1,16 @@ -import pytest from pytchat.parser.live import Parser import json -import asyncio,aiohttp from aioresponses import aioresponses -from pytchat.exceptions import ( - NoLivechatRendererException,NoYtinitialdataException, - ResponseContextError, NoContentsException) +from pytchat.exceptions import NoContents def _open_file(path): - with open(path,mode ='r',encoding = 'utf-8') as f: + with open(path, mode='r', encoding='utf-8') as f: return f.read() -parser = Parser(is_replay = False) + + +parser = Parser(is_replay=False) + @aioresponses() def test_finishedlive(*mock): @@ -20,12 +19,13 @@ def test_finishedlive(*mock): _text = _open_file('tests/testdata/finished_live.json') _text = json.loads(_text) - try: + try: parser.parse(parser.get_contents(_text)) assert False - except NoContentsException: + except NoContents: assert True + @aioresponses() def test_parsejson(*mock): '''jsonを正常にパースできるか''' @@ -33,12 +33,13 @@ def test_parsejson(*mock): _text = _open_file('tests/testdata/paramgen_firstread.json') _text = json.loads(_text) - try: + try: parser.parse(parser.get_contents(_text)) jsn = _text timeout = jsn["response"]["continuationContents"]["liveChatContinuation"]["continuations"][0]["timedContinuationData"]["timeoutMs"] - continuation = jsn["response"]["continuationContents"]["liveChatContinuation"]["continuations"][0]["timedContinuationData"]["continuation"] - assert 5035 == timeout - assert "0ofMyAPiARp8Q2c4S0RRb0xhelJMZDBsWFQwdERkalFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5YXpSTGQwbFhUMHREZGpRbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiPz5-Os-PkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgJqXjrPj5AJYA1CRwciOs-PkAli3pNq1k-PkAmgBggEECAEQAIgBAKABjbfnjrPj5AI%3D" == continuation - except: - assert False \ No newline at end of file + continuation = jsn["response"]["continuationContents"]["liveChatContinuation"][ + "continuations"][0]["timedContinuationData"]["continuation"] + assert timeout == 5035 + assert continuation == "0ofMyAPiARp8Q2c4S0RRb0xhelJMZDBsWFQwdERkalFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5YXpSTGQwbFhUMHREZGpRbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiPz5-Os-PkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgJqXjrPj5AJYA1CRwciOs-PkAli3pNq1k-PkAmgBggEECAEQAIgBAKABjbfnjrPj5AI%3D" + except Exception: + assert False diff --git a/tests/test_speed_calculator.py b/tests/test_speed_calculator.py index 171ea1e..86496cc 100644 --- a/tests/test_speed_calculator.py +++ b/tests/test_speed_calculator.py @@ -1,15 +1,9 @@ import json -import pytest -import asyncio,aiohttp from pytchat.parser.live import Parser -from pytchat.processors.compatible.processor import CompatibleProcessor -from pytchat.exceptions import ( - NoLivechatRendererException,NoYtinitialdataException, - ResponseContextError, NoContentsException) - from pytchat.processors.speed.calculator import SpeedCalculator -parser = Parser(is_replay =False) +parser = Parser(is_replay=False) + def test_speed_1(mocker): '''test speed calculation with normal json. @@ -23,13 +17,14 @@ def test_speed_1(mocker): _, chatdata = parser.parse(parser.get_contents(json.loads(_json))) data = { - "video_id" : "", - "timeout" : 10, - "chatdata" : chatdata + "video_id": "", + "timeout": 10, + "chatdata": chatdata } ret = processor.process([data]) assert 30 == ret + def test_speed_2(mocker): '''test speed calculation with no valid chat data. ''' @@ -39,13 +34,14 @@ def test_speed_2(mocker): _, chatdata = parser.parse(parser.get_contents(json.loads(_json))) data = { - "video_id" : "", - "timeout" : 10, - "chatdata" : chatdata + "video_id": "", + "timeout": 10, + "chatdata": chatdata } ret = processor.process([data]) - assert 0 == ret - + assert ret == 0 + + def test_speed_3(mocker): '''test speed calculation with empty data. ''' @@ -55,14 +51,14 @@ def test_speed_3(mocker): _, chatdata = parser.parse(parser.get_contents(json.loads(_json))) data = { - "video_id" : "", - "timeout" : 10, - "chatdata" : chatdata + "video_id": "", + "timeout": 10, + "chatdata": chatdata } ret = processor.process([data]) - assert 0 == ret - + assert ret == 0 + def _open_file(path): - with open(path,mode ='r',encoding = 'utf-8') as f: + with open(path, mode='r', encoding='utf-8') as f: return f.read()