ramforth/pytchat-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement raise_for_status()

Browse Source
This commit is contained in:
taizan-hokuto
2020-06-17 23:56:07 +09:00
parent 2474207691
commit 94d4eebd0f
13 changed files with 326 additions and 286 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pytchat/cli/__init__.py
View File

@@ -1,7 +1,7 @@
import argparse import argparse
from pathlib import Path from pathlib import Path
from .arguments import Arguments from .arguments import Arguments
from .. exceptions import InvalidVideoIdException, NoContentsException from .. exceptions import InvalidVideoIdException, NoContents
from .. processors.html_archiver import HTMLArchiver from .. processors.html_archiver import HTMLArchiver
from .. tool.extract.extractor import Extractor from .. tool.extract.extractor import Extractor
from .. tool.videoinfo import VideoInfo from .. tool.videoinfo import VideoInfo
@@ -50,7 +50,7 @@ def main():
callback=_disp_progress callback=_disp_progress
).extract() ).extract()
print("\nExtraction end.\n") print("\nExtraction end.\n")
except (InvalidVideoIdException, NoContentsException) as e: except (InvalidVideoIdException, NoContents) as e:
print(e) print(e)
return return
parser.print_help() parser.print_help()

90
pytchat/core_async/livechat.py
View File

@@ -11,7 +11,7 @@ from asyncio import Queue
from .buffer import Buffer from .buffer import Buffer
from ..parser.live import Parser from ..parser.live import Parser
from .. import config from .. import config
from ..exceptions import ChatParseException, IllegalFunctionCall from .. import exceptions
from ..paramgen import liveparam, arcparam from ..paramgen import liveparam, arcparam
from ..processors.default.processor import DefaultProcessor from ..processors.default.processor import DefaultProcessor
from ..processors.combinator import Combinator from ..processors.combinator import Combinator
@@ -86,7 +86,7 @@ class LiveChatAsync:
topchat_only=False, topchat_only=False,
logger=config.logger(__name__), logger=config.logger(__name__),
): ):
self.video_id = video_id self._video_id = video_id
self.seektime = seektime self.seektime = seektime
if isinstance(processor, tuple): if isinstance(processor, tuple):
self.processor = Combinator(processor) self.processor = Combinator(processor)
@@ -102,28 +102,26 @@ class LiveChatAsync:
self._parser = Parser(is_replay=self._is_replay) self._parser = Parser(is_replay=self._is_replay)
self._pauser = Queue() self._pauser = Queue()
self._pauser.put_nowait(None) self._pauser.put_nowait(None)
self._setup()
self._first_fetch = True self._first_fetch = True
self._fetch_url = "live_chat/get_live_chat?continuation=" self._fetch_url = "live_chat/get_live_chat?continuation="
self._topchat_only = topchat_only self._topchat_only = topchat_only
self._logger = logger self._logger = logger
self.exception = None
LiveChatAsync._logger = logger LiveChatAsync._logger = logger
if not LiveChatAsync._setup_finished: if exception_handler:
LiveChatAsync._setup_finished = True self._set_exception_handler(exception_handler)
if exception_handler: if interruptable:
self._set_exception_handler(exception_handler) signal.signal(signal.SIGINT,
if interruptable: (lambda a, b: asyncio.create_task(
signal.signal(signal.SIGINT, LiveChatAsync.shutdown(None, signal.SIGINT, b))))
(lambda a, b: asyncio.create_task( self._setup()
LiveChatAsync.shutdown(None, signal.SIGINT, b))
))
def _setup(self): def _setup(self):
# direct modeがTrueでcallback未設定の場合例外発生。 # direct modeがTrueでcallback未設定の場合例外発生。
if self._direct_mode: if self._direct_mode:
if self._callback is None: if self._callback is None:
raise IllegalFunctionCall( raise exceptions.IllegalFunctionCall(
"When direct_mode=True, callback parameter is required.") "When direct_mode=True, callback parameter is required.")
else: else:
# direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成 # direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成
@@ -138,18 +136,18 @@ class LiveChatAsync:
loop.create_task(self._callback_loop(self._callback)) loop.create_task(self._callback_loop(self._callback))
# _listenループタスクの開始 # _listenループタスクの開始
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
listen_task = loop.create_task(self._startlisten()) self.listen_task = loop.create_task(self._startlisten())
# add_done_callbackの登録 # add_done_callbackの登録
if self._done_callback is None: if self._done_callback is None:
listen_task.add_done_callback(self.finish) self.listen_task.add_done_callback(self._finish)
else: else:
listen_task.add_done_callback(self._done_callback) self.listen_task.add_done_callback(self._done_callback)
async def _startlisten(self): async def _startlisten(self):
"""Fetch first continuation parameter, """Fetch first continuation parameter,
create and start _listen loop. create and start _listen loop.
""" """
initial_continuation = liveparam.getparam(self.video_id, 3) initial_continuation = liveparam.getparam(self._video_id, 3)
await self._listen(initial_continuation) await self._listen(initial_continuation)
async def _listen(self, continuation): async def _listen(self, continuation):
@@ -171,7 +169,7 @@ class LiveChatAsync:
timeout = metadata['timeoutMs'] / 1000 timeout = metadata['timeoutMs'] / 1000
chat_component = { chat_component = {
"video_id": self.video_id, "video_id": self._video_id,
"timeout": timeout, "timeout": timeout,
"chatdata": chatdata "chatdata": chatdata
} }
@@ -188,14 +186,15 @@ class LiveChatAsync:
diff_time = timeout - (time.time() - time_mark) diff_time = timeout - (time.time() - time_mark)
await asyncio.sleep(diff_time) await asyncio.sleep(diff_time)
continuation = metadata.get('continuation') continuation = metadata.get('continuation')
except ChatParseException as e: except exceptions.ChatParseException as e:
self._logger.debug(f"[{self.video_id}]{str(e)}") self._logger.debug(f"[{self._video_id}]{str(e)}")
return raise
except (TypeError, json.JSONDecodeError): except (TypeError, json.JSONDecodeError):
self._logger.error(f"{traceback.format_exc(limit = -1)}") self._logger.error(f"{traceback.format_exc(limit = -1)}")
return raise
self._logger.debug(f"[{self.video_id}]finished fetching chat.") self._logger.debug(f"[{self._video_id}]finished fetching chat.")
raise exceptions.ChatDataFinished
async def _check_pause(self, continuation): async def _check_pause(self, continuation):
if self._pauser.empty(): if self._pauser.empty():
@@ -207,7 +206,7 @@ class LiveChatAsync:
self._pauser.put_nowait(None) self._pauser.put_nowait(None)
if not self._is_replay: if not self._is_replay:
continuation = liveparam.getparam( continuation = liveparam.getparam(
self.video_id, 3, self._topchat_only) self._video_id, 3, self._topchat_only)
return continuation return continuation
async def _get_contents(self, continuation, session, headers): async def _get_contents(self, continuation, session, headers):
@@ -227,7 +226,7 @@ class LiveChatAsync:
self._parser.is_replay = True self._parser.is_replay = True
self._fetch_url = "live_chat_replay/get_live_chat_replay?continuation=" self._fetch_url = "live_chat_replay/get_live_chat_replay?continuation="
continuation = arcparam.getparam( continuation = arcparam.getparam(
self.video_id, self.seektime, self._topchat_only) self._video_id, self.seektime, self._topchat_only)
livechat_json = (await self._get_livechat_json( livechat_json = (await self._get_livechat_json(
continuation, session, headers)) continuation, session, headers))
reload_continuation = self._parser.reload_continuation( reload_continuation = self._parser.reload_continuation(
@@ -258,7 +257,7 @@ class LiveChatAsync:
await asyncio.sleep(1) await asyncio.sleep(1)
continue continue
else: else:
self._logger.error(f"[{self.video_id}]" self._logger.error(f"[{self._video_id}]"
f"Exceeded retry count. status_code={status_code}") f"Exceeded retry count. status_code={status_code}")
return None return None
return livechat_json return livechat_json
@@ -288,9 +287,12 @@ class LiveChatAsync:
: Processorによって加工されたチャットデータ : Processorによって加工されたチャットデータ
""" """
if self._callback is None: if self._callback is None:
items = await self._buffer.get() if self.is_alive():
return self.processor.process(items) items = await self._buffer.get()
raise IllegalFunctionCall( return self.processor.process(items)
else:
return []
raise exceptions.IllegalFunctionCall(
"既にcallbackを登録済みのため、get()は実行できません。") "既にcallbackを登録済みのため、get()は実行できません。")
def is_replay(self): def is_replay(self):
@@ -311,22 +313,36 @@ class LiveChatAsync:
def is_alive(self): def is_alive(self):
return self._is_alive return self._is_alive
def finish(self, sender): def _finish(self, sender):
'''Listener終了時のコールバック''' '''Listener終了時のコールバック'''
try: try:
self.terminate() self._task_finished()
except CancelledError: except CancelledError:
self._logger.debug(f'[{self.video_id}]cancelled:{sender}') self._logger.debug(f'[{self._video_id}]cancelled:{sender}')
def terminate(self): def terminate(self):
if self._pauser.empty():
self._pauser.put_nowait(None)
self._is_alive = False
self._buffer.put_nowait({})
def _task_finished(self):
''' '''
Listenerを終了する。 Listenerを終了する。
''' '''
self._is_alive = False if self.is_alive():
if self._direct_mode is False: self.terminate()
# bufferにダミーオブジェクトを入れてis_alive()を判定させる try:
self._buffer.put_nowait({'chatdata': '', 'timeout': 0}) self.listen_task.result()
self._logger.info(f'[{self.video_id}]finished.') except Exception as e:
self.exception = e
if not isinstance(e, exceptions.ChatParseException):
self._logger.error(f'Internal exception - {type(e)}{str(e)}')
self._logger.info(f'[{self._video_id}]終了しました')
def raise_for_status(self):
if self.exception is not None:
raise self.exception
@classmethod @classmethod
def _set_exception_handler(cls, handler): def _set_exception_handler(cls, handler):

84
pytchat/core_multithread/livechat.py
View File

@@ -6,10 +6,11 @@ import traceback
import urllib.parse import urllib.parse
from concurrent.futures import CancelledError, ThreadPoolExecutor from concurrent.futures import CancelledError, ThreadPoolExecutor
from queue import Queue from queue import Queue
from threading import Event
from .buffer import Buffer from .buffer import Buffer
from ..parser.live import Parser from ..parser.live import Parser
from .. import config from .. import config
from ..exceptions import ChatParseException, IllegalFunctionCall from .. import exceptions
from ..paramgen import liveparam, arcparam from ..paramgen import liveparam, arcparam
from ..processors.default.processor import DefaultProcessor from ..processors.default.processor import DefaultProcessor
from ..processors.combinator import Combinator from ..processors.combinator import Combinator
@@ -83,7 +84,7 @@ class LiveChat:
topchat_only=False, topchat_only=False,
logger=config.logger(__name__) logger=config.logger(__name__)
): ):
self.video_id = video_id self._video_id = video_id
self.seektime = seektime self.seektime = seektime
if isinstance(processor, tuple): if isinstance(processor, tuple):
self.processor = Combinator(processor) self.processor = Combinator(processor)
@@ -102,7 +103,9 @@ class LiveChat:
self._first_fetch = True self._first_fetch = True
self._fetch_url = "live_chat/get_live_chat?continuation=" self._fetch_url = "live_chat/get_live_chat?continuation="
self._topchat_only = topchat_only self._topchat_only = topchat_only
self._event = Event()
self._logger = logger self._logger = logger
self.exception = None
if interruptable: if interruptable:
signal.signal(signal.SIGINT, lambda a, b: self.terminate()) signal.signal(signal.SIGINT, lambda a, b: self.terminate())
self._setup() self._setup()
@@ -111,7 +114,7 @@ class LiveChat:
# direct modeがTrueでcallback未設定の場合例外発生。 # direct modeがTrueでcallback未設定の場合例外発生。
if self._direct_mode: if self._direct_mode:
if self._callback is None: if self._callback is None:
raise IllegalFunctionCall( raise exceptions.IllegalFunctionCall(
"When direct_mode=True, callback parameter is required.") "When direct_mode=True, callback parameter is required.")
else: else:
# direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成 # direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成
@@ -124,19 +127,19 @@ class LiveChat:
# callbackを呼ぶループタスクの開始 # callbackを呼ぶループタスクの開始
self._executor.submit(self._callback_loop, self._callback) self._executor.submit(self._callback_loop, self._callback)
# _listenループタスクの開始 # _listenループタスクの開始
listen_task = self._executor.submit(self._startlisten) self.listen_task = self._executor.submit(self._startlisten)
# add_done_callbackの登録 # add_done_callbackの登録
if self._done_callback is None: if self._done_callback is None:
listen_task.add_done_callback(self.finish) self.listen_task.add_done_callback(self._finish)
else: else:
listen_task.add_done_callback(self._done_callback) self.listen_task.add_done_callback(self._done_callback)
def _startlisten(self): def _startlisten(self):
time.sleep(0.1) # sleep shortly to prohibit skipping fetching data time.sleep(0.1) # sleep shortly to prohibit skipping fetching data
"""Fetch first continuation parameter, """Fetch first continuation parameter,
create and start _listen loop. create and start _listen loop.
""" """
initial_continuation = liveparam.getparam(self.video_id, 3) initial_continuation = liveparam.getparam(self._video_id, 3)
self._listen(initial_continuation) self._listen(initial_continuation)
def _listen(self, continuation): def _listen(self, continuation):
@@ -152,13 +155,11 @@ class LiveChat:
with requests.Session() as session: with requests.Session() as session:
while(continuation and self._is_alive): while(continuation and self._is_alive):
continuation = self._check_pause(continuation) continuation = self._check_pause(continuation)
contents = self._get_contents( contents = self._get_contents(continuation, session, headers)
continuation, session, headers)
metadata, chatdata = self._parser.parse(contents) metadata, chatdata = self._parser.parse(contents)
timeout = metadata['timeoutMs'] / 1000 timeout = metadata['timeoutMs'] / 1000
chat_component = { chat_component = {
"video_id": self.video_id, "video_id": self._video_id,
"timeout": timeout, "timeout": timeout,
"chatdata": chatdata "chatdata": chatdata
} }
@@ -173,16 +174,17 @@ class LiveChat:
else: else:
self._buffer.put(chat_component) self._buffer.put(chat_component)
diff_time = timeout - (time.time() - time_mark) diff_time = timeout - (time.time() - time_mark)
time.sleep(diff_time if diff_time > 0 else 0) self._event.wait(diff_time if diff_time > 0 else 0)
continuation = metadata.get('continuation') continuation = metadata.get('continuation')
except ChatParseException as e: except exceptions.ChatParseException as e:
self._logger.debug(f"[{self.video_id}]{str(e)}") self._logger.debug(f"[{self._video_id}]{str(e)}")
return raise
except (TypeError, json.JSONDecodeError): except (TypeError, json.JSONDecodeError):
self._logger.error(f"{traceback.format_exc(limit=-1)}") self._logger.error(f"{traceback.format_exc(limit=-1)}")
return raise
self._logger.debug(f"[{self.video_id}]finished fetching chat.") self._logger.debug(f"[{self._video_id}]finished fetching chat.")
raise exceptions.ChatDataFinished
def _check_pause(self, continuation): def _check_pause(self, continuation):
if self._pauser.empty(): if self._pauser.empty():
@@ -193,7 +195,7 @@ class LiveChat:
''' '''
self._pauser.put_nowait(None) self._pauser.put_nowait(None)
if not self._is_replay: if not self._is_replay:
continuation = liveparam.getparam(self.video_id, 3) continuation = liveparam.getparam(self._video_id, 3)
return continuation return continuation
def _get_contents(self, continuation, session, headers): def _get_contents(self, continuation, session, headers):
@@ -215,9 +217,8 @@ class LiveChat:
self._parser.is_replay = True self._parser.is_replay = True
self._fetch_url = "live_chat_replay/get_live_chat_replay?continuation=" self._fetch_url = "live_chat_replay/get_live_chat_replay?continuation="
continuation = arcparam.getparam( continuation = arcparam.getparam(
self.video_id, self.seektime, self._topchat_only) self._video_id, self.seektime, self._topchat_only)
livechat_json = (self._get_livechat_json( livechat_json = (self._get_livechat_json(continuation, session, headers))
continuation, session, headers))
reload_continuation = self._parser.reload_continuation( reload_continuation = self._parser.reload_continuation(
self._parser.get_contents(livechat_json)) self._parser.get_contents(livechat_json))
if reload_continuation: if reload_continuation:
@@ -246,9 +247,9 @@ class LiveChat:
time.sleep(1) time.sleep(1)
continue continue
else: else:
self._logger.error(f"[{self.video_id}]" self._logger.error(f"[{self._video_id}]"
f"Exceeded retry count. status_code={status_code}") f"Exceeded retry count. status_code={status_code}")
return None raise exceptions.RetryExceedMaxCount()
return livechat_json return livechat_json
def _callback_loop(self, callback): def _callback_loop(self, callback):
@@ -276,9 +277,12 @@ class LiveChat:
: Processorによって加工されたチャットデータ : Processorによって加工されたチャットデータ
""" """
if self._callback is None: if self._callback is None:
items = self._buffer.get() if self.is_alive():
return self.processor.process(items) items = self._buffer.get()
raise IllegalFunctionCall( return self.processor.process(items)
else:
return []
raise exceptions.IllegalFunctionCall(
"既にcallbackを登録済みのため、get()は実行できません。") "既にcallbackを登録済みのため、get()は実行できません。")
def is_replay(self): def is_replay(self):
@@ -299,18 +303,34 @@ class LiveChat:
def is_alive(self): def is_alive(self):
return self._is_alive return self._is_alive
def finish(self, sender): def _finish(self, sender):
'''Listener終了時のコールバック''' '''Listener終了時のコールバック'''
try: try:
self.terminate() self._task_finished()
except CancelledError: except CancelledError:
self._logger.debug(f'[{self.video_id}]cancelled:{sender}') self._logger.debug(f'[{self._video_id}]cancelled:{sender}')
def terminate(self): def terminate(self):
if self._pauser.empty():
self._pauser.put_nowait(None)
self._is_alive = False
self._buffer.put({})
self._event.set()
def _task_finished(self):
''' '''
Listenerを終了する。 Listenerを終了する。
''' '''
if self.is_alive(): if self.is_alive():
self._is_alive = False self.terminate()
self._buffer.put({}) try:
self._logger.info(f'[{self.video_id}]終了しました') self.listen_task.result()
except Exception as e:
self.exception = e
if not isinstance(e, exceptions.ChatParseException):
self._logger.error(f'Internal exception - {type(e)}{str(e)}')
self._logger.info(f'[{self._video_id}]終了しました')
def raise_for_status(self):
if self.exception is not None:
raise self.exception

41
pytchat/exceptions.py
View File

@@ -5,13 +5,6 @@ class ChatParseException(Exception):
pass pass
class NoYtinitialdataException(ChatParseException):
'''
Thrown when the video is not found.
'''
pass
class ResponseContextError(ChatParseException): class ResponseContextError(ChatParseException):
''' '''
Thrown when chat data is invalid. Thrown when chat data is invalid.
@@ -19,21 +12,14 @@ class ResponseContextError(ChatParseException):
pass pass
class NoLivechatRendererException(ChatParseException): class NoContents(ChatParseException):
'''
Thrown when livechatRenderer is missing in JSON.
'''
pass
class NoContentsException(ChatParseException):
''' '''
Thrown when ContinuationContents is missing in JSON. Thrown when ContinuationContents is missing in JSON.
''' '''
pass pass
class NoContinuationsException(ChatParseException): class NoContinuation(ChatParseException):
''' '''
Thrown when continuation is missing in ContinuationContents. Thrown when continuation is missing in ContinuationContents.
''' '''
@@ -42,8 +28,8 @@ class NoContinuationsException(ChatParseException):
class IllegalFunctionCall(Exception): class IllegalFunctionCall(Exception):
''' '''
Thrown when get () is called even though Thrown when get() is called even though
set_callback () has been executed. set_callback() has been executed.
''' '''
pass pass
@@ -57,3 +43,22 @@ class InvalidVideoIdException(Exception):
class UnknownConnectionError(Exception): class UnknownConnectionError(Exception):
pass pass
class RetryExceedMaxCount(Exception):
'''
thrown when the number of retries exceeds the maximum value.
'''
pass
class ChatDataFinished(ChatParseException):
pass
class ReceivedUnknownContinuation(ChatParseException):
pass
class FailedExtractContinuation(ChatDataFinished):
pass

27
pytchat/parser/live.py
View File

@@ -4,11 +4,7 @@ pytchat.parser.live
Parser of live chat JSON. Parser of live chat JSON.
""" """
from .. exceptions import ( from .. import exceptions
ResponseContextError,
NoContentsException,
NoContinuationsException,
ChatParseException)
class Parser: class Parser:
@@ -20,9 +16,9 @@ class Parser:
def get_contents(self, jsn): def get_contents(self, jsn):
if jsn is None: if jsn is None:
raise ChatParseException('Called with none JSON object.') raise exceptions.IllegalFunctionCall('Called with none JSON object.')
if jsn['response']['responseContext'].get('errors'): if jsn['response']['responseContext'].get('errors'):
raise ResponseContextError( raise exceptions.ResponseContextError(
'The video_id would be wrong, or video is deleted or private.') 'The video_id would be wrong, or video is deleted or private.')
contents = jsn['response'].get('continuationContents') contents = jsn['response'].get('continuationContents')
return contents return contents
@@ -46,11 +42,11 @@ class Parser:
if contents is None: if contents is None:
'''Broadcasting end or cannot fetch chat stream''' '''Broadcasting end or cannot fetch chat stream'''
raise NoContentsException('Chat data stream is empty.') raise exceptions.NoContents('Chat data stream is empty.')
cont = contents['liveChatContinuation']['continuations'][0] cont = contents['liveChatContinuation']['continuations'][0]
if cont is None: if cont is None:
raise NoContinuationsException('No Continuation') raise exceptions.NoContinuation('No Continuation')
metadata = (cont.get('invalidationContinuationData') metadata = (cont.get('invalidationContinuationData')
or cont.get('timedContinuationData') or cont.get('timedContinuationData')
or cont.get('reloadContinuationData') or cont.get('reloadContinuationData')
@@ -58,22 +54,25 @@ class Parser:
) )
if metadata is None: if metadata is None:
if cont.get("playerSeekContinuationData"): if cont.get("playerSeekContinuationData"):
raise ChatParseException('Finished chat data') raise exceptions.ChatDataFinished('Finished chat data')
unknown = list(cont.keys())[0] unknown = list(cont.keys())[0]
if unknown: if unknown:
raise ChatParseException( raise exceptions.ReceivedUnknownContinuation(
f"Received unknown continuation type:{unknown}") f"Received unknown continuation type:{unknown}")
else: else:
raise ChatParseException('Cannot extract continuation data') raise exceptions.FailedExtractContinuation('Cannot extract continuation data')
return self._create_data(metadata, contents) return self._create_data(metadata, contents)
def reload_continuation(self, contents): def reload_continuation(self, contents):
""" """
When `seektime = 0` or seektime is abbreviated , When `seektime == 0` or seektime is abbreviated ,
check if fetched chat json has no chat data. check if fetched chat json has no chat data.
If so, try to fetch playerSeekContinuationData. If so, try to fetch playerSeekContinuationData.
This function must be run only first fetching. This function must be run only first fetching.
""" """
if contents is None:
'''Broadcasting end or cannot fetch chat stream'''
raise exceptions.NoContents('Chat data stream is empty.')
cont = contents['liveChatContinuation']['continuations'][0] cont = contents['liveChatContinuation']['continuations'][0]
if cont.get("liveChatReplayContinuationData"): if cont.get("liveChatReplayContinuationData"):
# chat data exist. # chat data exist.
@@ -82,7 +81,7 @@ class Parser:
init_cont = cont.get("playerSeekContinuationData") init_cont = cont.get("playerSeekContinuationData")
if init_cont: if init_cont:
return init_cont.get("continuation") return init_cont.get("continuation")
raise ChatParseException('Finished chat data') raise exceptions.ChatDataFinished('Finished chat data')
def _create_data(self, metadata, contents): def _create_data(self, metadata, contents):
actions = contents['liveChatContinuation'].get('actions') actions = contents['liveChatContinuation'].get('actions')

11
pytchat/tool/extract/parser.py
View File

@@ -1,8 +1,5 @@
from ... import config from ... import config
from ... exceptions import ( from ... import exceptions
ResponseContextError,
NoContentsException,
NoContinuationsException)
logger = config.logger(__name__) logger = config.logger(__name__)
@@ -23,15 +20,15 @@ def parse(jsn):
if jsn is None: if jsn is None:
raise ValueError("parameter JSON is None") raise ValueError("parameter JSON is None")
if jsn['response']['responseContext'].get('errors'): if jsn['response']['responseContext'].get('errors'):
raise ResponseContextError( raise exceptions.ResponseContextError(
'video_id is invalid or private/deleted.') 'video_id is invalid or private/deleted.')
contents = jsn['response'].get('continuationContents') contents = jsn['response'].get('continuationContents')
if contents is None: if contents is None:
raise NoContentsException('No chat data.') raise exceptions.NoContents('No chat data.')
cont = contents['liveChatContinuation']['continuations'][0] cont = contents['liveChatContinuation']['continuations'][0]
if cont is None: if cont is None:
raise NoContinuationsException('No Continuation') raise exceptions.NoContinuation('No Continuation')
metadata = cont.get('liveChatReplayContinuationData') metadata = cont.get('liveChatReplayContinuationData')
if metadata: if metadata:
continuation = metadata.get("continuation") continuation = metadata.get("continuation")

48
pytchat/tool/mining/parser.py
View File

@@ -1,12 +1,12 @@
import json import re
from ... import config from ... import config
from ... exceptions import ( from ... exceptions import (
ResponseContextError, ResponseContextError,
NoContentsException, NoContents, NoContinuation)
NoContinuationsException )
logger = config.logger(__name__) logger = config.logger(__name__)
def parse(jsn): def parse(jsn):
""" """
Parse replay chat data. Parse replay chat data.
@@ -20,45 +20,51 @@ def parse(jsn):
actions : list actions : list
""" """
if jsn is None: if jsn is None:
raise ValueError("parameter JSON is None") raise ValueError("parameter JSON is None")
if jsn['response']['responseContext'].get('errors'): if jsn['response']['responseContext'].get('errors'):
raise ResponseContextError( raise ResponseContextError(
'video_id is invalid or private/deleted.') 'video_id is invalid or private/deleted.')
contents=jsn["response"].get('continuationContents') contents = jsn["response"].get('continuationContents')
if contents is None: if contents is None:
raise NoContentsException('No chat data.') raise NoContents('No chat data.')
cont = contents['liveChatContinuation']['continuations'][0] cont = contents['liveChatContinuation']['continuations'][0]
if cont is None: if cont is None:
raise NoContinuationsException('No Continuation') raise NoContinuation('No Continuation')
metadata = cont.get('liveChatReplayContinuationData') metadata = cont.get('liveChatReplayContinuationData')
if metadata: if metadata:
continuation = metadata.get("continuation") continuation = metadata.get("continuation")
actions = contents['liveChatContinuation'].get('actions') actions = contents['liveChatContinuation'].get('actions')
if continuation: if continuation:
return continuation, [action["replayChatItemAction"]["actions"][0] return continuation, [action["replayChatItemAction"]["actions"][0]
for action in actions for action in actions
if list(action['replayChatItemAction']["actions"][0].values() if list(action['replayChatItemAction']["actions"][0].values()
)[0]['item'].get("liveChatPaidMessageRenderer") )[0]['item'].get("liveChatPaidMessageRenderer")
or list(action['replayChatItemAction']["actions"][0].values() or list(action['replayChatItemAction']["actions"][0].values()
)[0]['item'].get("liveChatPaidStickerRenderer") )[0]['item'].get("liveChatPaidStickerRenderer")
] ]
return None, [] return None, []
def get_offset(item): def get_offset(item):
return int(item['replayChatItemAction']["videoOffsetTimeMsec"]) return int(item['replayChatItemAction']["videoOffsetTimeMsec"])
def get_id(item): def get_id(item):
return list((list(item['replayChatItemAction']["actions"][0].values() return list((list(item['replayChatItemAction']["actions"][0].values()
)[0])['item'].values())[0].get('id') )[0])['item'].values())[0].get('id')
def get_type(item): def get_type(item):
return list((list(item['replayChatItemAction']["actions"][0].values() return list((list(item['replayChatItemAction']["actions"][0].values()
)[0])['item'].keys())[0] )[0])['item'].keys())[0]
import re
_REGEX_YTINIT = re.compile("window\\[\"ytInitialData\"\\]\\s*=\\s*({.+?});\\s+")
_REGEX_YTINIT = re.compile(
"window\\[\"ytInitialData\"\\]\\s*=\\s*({.+?});\\s+")
def extract(text): def extract(text):
match = re.findall(_REGEX_YTINIT, str(text)) match = re.findall(_REGEX_YTINIT, str(text))

25
tests/test_arcparam_mining.py
View File

@@ -1,40 +1,41 @@
import pytest
from pytchat.tool.mining import parser from pytchat.tool.mining import parser
import pytchat.config as config import pytchat.config as config
import requests, json import requests
import json
from pytchat.paramgen import arcparam_mining as arcparam from pytchat.paramgen import arcparam_mining as arcparam
def test_arcparam_e(mocker): def test_arcparam_e(mocker):
try: try:
arcparam.getparam("01234567890",-1) arcparam.getparam("01234567890", -1)
assert False assert False
except ValueError: except ValueError:
assert True assert True
def test_arcparam_0(mocker): def test_arcparam_0(mocker):
param = arcparam.getparam("01234567890",0) param = arcparam.getparam("01234567890", 0)
assert param =="op2w0wQsGiBDZzhhRFFvTE1ERXlNelExTmpjNE9UQWdBUSUzRCUzREABYARyAggBeAE%3D" assert param == "op2w0wQsGiBDZzhhRFFvTE1ERXlNelExTmpjNE9UQWdBUSUzRCUzREABYARyAggBeAE%3D"
def test_arcparam_1(mocker): def test_arcparam_1(mocker):
param = arcparam.getparam("01234567890", seektime = 100000) param = arcparam.getparam("01234567890", seektime=100000)
print(param) print(param)
assert param == "op2w0wQzGiBDZzhhRFFvTE1ERXlNelExTmpjNE9UQWdBUSUzRCUzREABWgUQgMLXL2AEcgIIAXgB" assert param == "op2w0wQzGiBDZzhhRFFvTE1ERXlNelExTmpjNE9UQWdBUSUzRCUzREABWgUQgMLXL2AEcgIIAXgB"
def test_arcparam_2(mocker): def test_arcparam_2(mocker):
param = arcparam.getparam("PZz9NB0-Z64",1) param = arcparam.getparam("PZz9NB0-Z64", 1)
url=f"https://www.youtube.com/live_chat_replay?continuation={param}&playerOffsetMs=1000&pbj=1" url = f"https://www.youtube.com/live_chat_replay?continuation={param}&playerOffsetMs=1000&pbj=1"
resp = requests.Session().get(url,headers = config.headers) resp = requests.Session().get(url, headers=config.headers)
jsn = json.loads(resp.text) jsn = json.loads(resp.text)
_ , chatdata = parser.parse(jsn[1]) _, chatdata = parser.parse(jsn[1])
test_id = chatdata[0]["addChatItemAction"]["item"]["liveChatPaidMessageRenderer"]["id"] test_id = chatdata[0]["addChatItemAction"]["item"]["liveChatPaidMessageRenderer"]["id"]
print(test_id) print(test_id)
assert test_id == "ChwKGkNKSGE0YnFJeWVBQ0ZWcUF3Z0VkdGIwRm9R" assert test_id == "ChwKGkNKSGE0YnFJeWVBQ0ZWcUF3Z0VkdGIwRm9R"
def test_arcparam_3(mocker): def test_arcparam_3(mocker):
param = arcparam.getparam("01234567890") param = arcparam.getparam("01234567890")
assert param == "op2w0wQsGiBDZzhhRFFvTE1ERXlNelExTmpjNE9UQWdBUSUzRCUzREABYARyAggBeAE%3D" assert param == "op2w0wQsGiBDZzhhRFFvTE1ERXlNelExTmpjNE9UQWdBUSUzRCUzREABYARyAggBeAE%3D"

61
tests/test_compatible_processor.py
View File

@@ -1,17 +1,6 @@
import json import json
import pytest
import asyncio
import aiohttp
from pytchat.parser.live import Parser from pytchat.parser.live import Parser
from pytchat.processors.compatible.processor import CompatibleProcessor from pytchat.processors.compatible.processor import CompatibleProcessor
from pytchat.exceptions import (
NoLivechatRendererException, NoYtinitialdataException,
ResponseContextError, NoContentsException)
from pytchat.processors.compatible.renderer.textmessage import LiveChatTextMessageRenderer
from pytchat.processors.compatible.renderer.paidmessage import LiveChatPaidMessageRenderer
from pytchat.processors.compatible.renderer.paidsticker import LiveChatPaidStickerRenderer
from pytchat.processors.compatible.renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
parser = Parser(is_replay=False) parser = Parser(is_replay=False)
@@ -31,21 +20,23 @@ def test_textmessage(mocker):
ret = processor.process([data]) ret = processor.process([data])
assert ret["kind"] == "youtube#liveChatMessageListResponse" assert ret["kind"] == "youtube#liveChatMessageListResponse"
assert ret["pollingIntervalMillis"] == data["timeout"]*1000 assert ret["pollingIntervalMillis"] == data["timeout"] * 1000
assert ret.keys() == { assert ret.keys() == {
"kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items" "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
} }
assert ret["pageInfo"].keys() == { assert ret["pageInfo"].keys() == {
"totalResults", "resultsPerPage" "totalResults", "resultsPerPage"
} }
assert ret["items"][0].keys() == { assert ret["items"][0].keys() == {
"kind", "etag", "id", "snippet", "authorDetails" "kind", "etag", "id", "snippet", "authorDetails"
} }
assert ret["items"][0]["snippet"].keys() == { assert ret["items"][0]["snippet"].keys() == {
'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage', 'textMessageDetails' 'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage',
'textMessageDetails'
} }
assert ret["items"][0]["authorDetails"].keys() == { assert ret["items"][0]["authorDetails"].keys() == {
'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator' 'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor',
'isChatModerator'
} }
assert ret["items"][0]["snippet"]["textMessageDetails"].keys() == { assert ret["items"][0]["snippet"]["textMessageDetails"].keys() == {
'messageText' 'messageText'
@@ -69,22 +60,23 @@ def test_newsponcer(mocker):
ret = processor.process([data]) ret = processor.process([data])
assert ret["kind"] == "youtube#liveChatMessageListResponse" assert ret["kind"] == "youtube#liveChatMessageListResponse"
assert ret["pollingIntervalMillis"] == data["timeout"]*1000 assert ret["pollingIntervalMillis"] == data["timeout"] * 1000
assert ret.keys() == { assert ret.keys() == {
"kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items" "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
} }
assert ret["pageInfo"].keys() == { assert ret["pageInfo"].keys() == {
"totalResults", "resultsPerPage" "totalResults", "resultsPerPage"
} }
assert ret["items"][0].keys() == { assert ret["items"][0].keys() == {
"kind", "etag", "id", "snippet", "authorDetails" "kind", "etag", "id", "snippet", "authorDetails"
} }
assert ret["items"][0]["snippet"].keys() == { assert ret["items"][0]["snippet"].keys() == {
'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage' 'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage'
} }
assert ret["items"][0]["authorDetails"].keys() == { assert ret["items"][0]["authorDetails"].keys() == {
'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator' 'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor',
'isChatModerator'
} }
assert "LCC." in ret["items"][0]["id"] assert "LCC." in ret["items"][0]["id"]
assert ret["items"][0]["snippet"]["type"] == "newSponsorEvent" assert ret["items"][0]["snippet"]["type"] == "newSponsorEvent"
@@ -105,22 +97,23 @@ def test_newsponcer_rev(mocker):
ret = processor.process([data]) ret = processor.process([data])
assert ret["kind"] == "youtube#liveChatMessageListResponse" assert ret["kind"] == "youtube#liveChatMessageListResponse"
assert ret["pollingIntervalMillis"] == data["timeout"]*1000 assert ret["pollingIntervalMillis"] == data["timeout"] * 1000
assert ret.keys() == { assert ret.keys() == {
"kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items" "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
} }
assert ret["pageInfo"].keys() == { assert ret["pageInfo"].keys() == {
"totalResults", "resultsPerPage" "totalResults", "resultsPerPage"
} }
assert ret["items"][0].keys() == { assert ret["items"][0].keys() == {
"kind", "etag", "id", "snippet", "authorDetails" "kind", "etag", "id", "snippet", "authorDetails"
} }
assert ret["items"][0]["snippet"].keys() == { assert ret["items"][0]["snippet"].keys() == {
'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage' 'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage'
} }
assert ret["items"][0]["authorDetails"].keys() == { assert ret["items"][0]["authorDetails"].keys() == {
'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator' 'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor',
'isChatModerator'
} }
assert "LCC." in ret["items"][0]["id"] assert "LCC." in ret["items"][0]["id"]
assert ret["items"][0]["snippet"]["type"] == "newSponsorEvent" assert ret["items"][0]["snippet"]["type"] == "newSponsorEvent"
@@ -141,21 +134,23 @@ def test_superchat(mocker):
ret = processor.process([data]) ret = processor.process([data])
assert ret["kind"] == "youtube#liveChatMessageListResponse" assert ret["kind"] == "youtube#liveChatMessageListResponse"
assert ret["pollingIntervalMillis"] == data["timeout"]*1000 assert ret["pollingIntervalMillis"] == data["timeout"] * 1000
assert ret.keys() == { assert ret.keys() == {
"kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items" "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
} }
assert ret["pageInfo"].keys() == { assert ret["pageInfo"].keys() == {
"totalResults", "resultsPerPage" "totalResults", "resultsPerPage"
} }
assert ret["items"][0].keys() == { assert ret["items"][0].keys() == {
"kind", "etag", "id", "snippet", "authorDetails" "kind", "etag", "id", "snippet", "authorDetails"
} }
assert ret["items"][0]["snippet"].keys() == { assert ret["items"][0]["snippet"].keys() == {
'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage', 'superChatDetails' 'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage',
'superChatDetails'
} }
assert ret["items"][0]["authorDetails"].keys() == { assert ret["items"][0]["authorDetails"].keys() == {
'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator' 'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor',
'isChatModerator'
} }
assert ret["items"][0]["snippet"]["superChatDetails"].keys() == { assert ret["items"][0]["snippet"]["superChatDetails"].keys() == {
'amountMicros', 'currency', 'amountDisplayString', 'tier', 'backgroundColor' 'amountMicros', 'currency', 'amountDisplayString', 'tier', 'backgroundColor'

25
tests/test_livechat.py
View File

@@ -1,30 +1,21 @@
import pytest
from pytchat.parser.live import Parser
import json import json
import asyncio,aiohttp
from aioresponses import aioresponses from aioresponses import aioresponses
from pytchat.core_async.livechat import LiveChatAsync from pytchat.core_async.livechat import LiveChatAsync
from pytchat.exceptions import ( from pytchat.exceptions import ResponseContextError
NoLivechatRendererException,NoYtinitialdataException,
ResponseContextError,NoContentsException)
from pytchat.core_multithread.livechat import LiveChat
import unittest
from unittest import TestCase
def _open_file(path): def _open_file(path):
with open(path,mode ='r',encoding = 'utf-8') as f: with open(path, mode='r', encoding='utf-8') as f:
return f.read() return f.read()
@aioresponses() @aioresponses()
def test_Async(*mock): def test_Async(*mock):
vid='' vid = ''
_text = _open_file('tests/testdata/paramgen_firstread.json') _text = _open_file('tests/testdata/paramgen_firstread.json')
_text = json.loads(_text) _text = json.loads(_text)
mock[0].get(f"https://www.youtube.com/live_chat?v={vid}&is_popout=1", status=200, body=_text) mock[0].get(
f"https://www.youtube.com/live_chat?v={vid}&is_popout=1", status=200, body=_text)
try: try:
chat = LiveChatAsync(video_id='') chat = LiveChatAsync(video_id='')
assert chat.is_alive() assert chat.is_alive()
@@ -33,6 +24,7 @@ def test_Async(*mock):
except ResponseContextError: except ResponseContextError:
assert not chat.is_alive() assert not chat.is_alive()
def test_MultiThread(mocker): def test_MultiThread(mocker):
_text = _open_file('tests/testdata/paramgen_firstread.json') _text = _open_file('tests/testdata/paramgen_firstread.json')
_text = json.loads(_text) _text = json.loads(_text)
@@ -48,6 +40,3 @@ def test_MultiThread(mocker):
except ResponseContextError: except ResponseContextError:
chat.terminate() chat.terminate()
assert not chat.is_alive() assert not chat.is_alive()

125
tests/test_livechat_2.py
View File

@@ -1,43 +1,43 @@
import asyncio, aiohttp import asyncio
import json
import pytest
import re import re
import requests
import sys
import time
from aioresponses import aioresponses from aioresponses import aioresponses
from concurrent.futures import CancelledError from concurrent.futures import CancelledError
from unittest import TestCase
from pytchat.core_multithread.livechat import LiveChat from pytchat.core_multithread.livechat import LiveChat
from pytchat.core_async.livechat import LiveChatAsync from pytchat.core_async.livechat import LiveChatAsync
from pytchat.exceptions import (
NoLivechatRendererException,NoYtinitialdataException,
ResponseContextError,NoContentsException)
from pytchat.parser.live import Parser
from pytchat.processors.dummy_processor import DummyProcessor from pytchat.processors.dummy_processor import DummyProcessor
def _open_file(path): def _open_file(path):
with open(path,mode ='r',encoding = 'utf-8') as f: with open(path, mode='r', encoding='utf-8') as f:
return f.read() return f.read()
@aioresponses() @aioresponses()
def test_async_live_stream(*mock): def test_async_live_stream(*mock):
async def test_loop(*mock): async def test_loop(*mock):
pattern = re.compile(r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$') pattern = re.compile(
r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$')
_text = _open_file('tests/testdata/test_stream.json') _text = _open_file('tests/testdata/test_stream.json')
mock[0].get(pattern, status=200, body=_text) mock[0].get(pattern, status=200, body=_text)
chat = LiveChatAsync(video_id='', processor = DummyProcessor()) chat = LiveChatAsync(video_id='', processor=DummyProcessor())
chats = await chat.get() chats = await chat.get()
rawdata = chats[0]["chatdata"] rawdata = chats[0]["chatdata"]
#assert fetching livachat data # assert fetching livachat data
assert list(rawdata[0]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer" assert list(rawdata[0]["addChatItemAction"]["item"].keys())[
assert list(rawdata[1]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer" 0] == "liveChatTextMessageRenderer"
assert list(rawdata[2]["addChatItemAction"]["item"].keys())[0] == "liveChatPlaceholderItemRenderer" assert list(rawdata[1]["addChatItemAction"]["item"].keys())[
assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerPaidMessageItemRenderer" 0] == "liveChatTextMessageRenderer"
assert list(rawdata[4]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer" assert list(rawdata[2]["addChatItemAction"]["item"].keys())[
assert list(rawdata[5]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidStickerRenderer" 0] == "liveChatPlaceholderItemRenderer"
assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerSponsorItemRenderer" assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[
0] == "liveChatTickerPaidMessageItemRenderer"
assert list(rawdata[4]["addChatItemAction"]["item"].keys())[
0] == "liveChatPaidMessageRenderer"
assert list(rawdata[5]["addChatItemAction"]["item"].keys())[
0] == "liveChatPaidStickerRenderer"
assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[
0] == "liveChatTickerSponsorItemRenderer"
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
try: try:
@@ -45,24 +45,29 @@ def test_async_live_stream(*mock):
except CancelledError: except CancelledError:
assert True assert True
@aioresponses()
@aioresponses()
def test_async_replay_stream(*mock): def test_async_replay_stream(*mock):
async def test_loop(*mock): async def test_loop(*mock):
pattern_live = re.compile(r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$') pattern_live = re.compile(
pattern_replay = re.compile(r'^https://www.youtube.com/live_chat_replay/get_live_chat_replay\?continuation=.*$') r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$')
#empty livechat -> switch to fetch replaychat pattern_replay = re.compile(
r'^https://www.youtube.com/live_chat_replay/get_live_chat_replay\?continuation=.*$')
# empty livechat -> switch to fetch replaychat
_text_live = _open_file('tests/testdata/finished_live.json') _text_live = _open_file('tests/testdata/finished_live.json')
_text_replay = _open_file('tests/testdata/chatreplay.json') _text_replay = _open_file('tests/testdata/chatreplay.json')
mock[0].get(pattern_live, status=200, body=_text_live) mock[0].get(pattern_live, status=200, body=_text_live)
mock[0].get(pattern_replay, status=200, body=_text_replay) mock[0].get(pattern_replay, status=200, body=_text_replay)
chat = LiveChatAsync(video_id='', processor = DummyProcessor()) chat = LiveChatAsync(video_id='', processor=DummyProcessor())
chats = await chat.get() chats = await chat.get()
rawdata = chats[0]["chatdata"] rawdata = chats[0]["chatdata"]
#assert fetching replaychat data # assert fetching replaychat data
assert list(rawdata[0]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer" assert list(rawdata[0]["addChatItemAction"]["item"].keys())[
assert list(rawdata[14]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer" 0] == "liveChatTextMessageRenderer"
assert list(rawdata[14]["addChatItemAction"]["item"].keys())[
0] == "liveChatPaidMessageRenderer"
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
try: try:
@@ -70,56 +75,66 @@ def test_async_replay_stream(*mock):
except CancelledError: except CancelledError:
assert True assert True
@aioresponses() @aioresponses()
def test_async_force_replay(*mock): def test_async_force_replay(*mock):
async def test_loop(*mock): async def test_loop(*mock):
pattern_live = re.compile(r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$') pattern_live = re.compile(
pattern_replay = re.compile(r'^https://www.youtube.com/live_chat_replay/get_live_chat_replay\?continuation=.*$') r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$')
#valid live data, but force_replay = True pattern_replay = re.compile(
r'^https://www.youtube.com/live_chat_replay/get_live_chat_replay\?continuation=.*$')
# valid live data, but force_replay = True
_text_live = _open_file('tests/testdata/test_stream.json') _text_live = _open_file('tests/testdata/test_stream.json')
#valid replay data # valid replay data
_text_replay = _open_file('tests/testdata/chatreplay.json') _text_replay = _open_file('tests/testdata/chatreplay.json')
mock[0].get(pattern_live, status=200, body=_text_live) mock[0].get(pattern_live, status=200, body=_text_live)
mock[0].get(pattern_replay, status=200, body=_text_replay) mock[0].get(pattern_replay, status=200, body=_text_replay)
#force replay # force replay
chat = LiveChatAsync(video_id='', processor = DummyProcessor(), force_replay = True) chat = LiveChatAsync(
video_id='', processor=DummyProcessor(), force_replay=True)
chats = await chat.get() chats = await chat.get()
rawdata = chats[0]["chatdata"] rawdata = chats[0]["chatdata"]
# assert fetching replaychat data # assert fetching replaychat data
assert list(rawdata[14]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer" assert list(rawdata[14]["addChatItemAction"]["item"].keys())[
0] == "liveChatPaidMessageRenderer"
# assert not mix livechat data # assert not mix livechat data
assert list(rawdata[2]["addChatItemAction"]["item"].keys())[0] != "liveChatPlaceholderItemRenderer" assert list(rawdata[2]["addChatItemAction"]["item"].keys())[
0] != "liveChatPlaceholderItemRenderer"
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
try: try:
loop.run_until_complete(test_loop(*mock)) loop.run_until_complete(test_loop(*mock))
except CancelledError: except CancelledError:
assert True assert True
def test_multithread_live_stream(mocker): def test_multithread_live_stream(mocker):
_text = _open_file('tests/testdata/test_stream.json') _text = _open_file('tests/testdata/test_stream.json')
responseMock = mocker.Mock() responseMock = mocker.Mock()
responseMock.status_code = 200 responseMock.status_code = 200
responseMock.text = _text responseMock.text = _text
mocker.patch('requests.Session.get').return_value.__enter__.return_value = responseMock mocker.patch(
'requests.Session.get').return_value.__enter__.return_value = responseMock
chat = LiveChat(video_id='test_id', processor = DummyProcessor()) chat = LiveChat(video_id='test_id', processor=DummyProcessor())
chats = chat.get() chats = chat.get()
rawdata = chats[0]["chatdata"] rawdata = chats[0]["chatdata"]
#assert fetching livachat data # assert fetching livachat data
assert list(rawdata[0]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer" assert list(rawdata[0]["addChatItemAction"]["item"].keys())[
assert list(rawdata[1]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer" 0] == "liveChatTextMessageRenderer"
assert list(rawdata[2]["addChatItemAction"]["item"].keys())[0] == "liveChatPlaceholderItemRenderer" assert list(rawdata[1]["addChatItemAction"]["item"].keys())[
assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerPaidMessageItemRenderer" 0] == "liveChatTextMessageRenderer"
assert list(rawdata[4]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer" assert list(rawdata[2]["addChatItemAction"]["item"].keys())[
assert list(rawdata[5]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidStickerRenderer" 0] == "liveChatPlaceholderItemRenderer"
assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerSponsorItemRenderer" assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[
0] == "liveChatTickerPaidMessageItemRenderer"
assert list(rawdata[4]["addChatItemAction"]["item"].keys())[
0] == "liveChatPaidMessageRenderer"
assert list(rawdata[5]["addChatItemAction"]["item"].keys())[
0] == "liveChatPaidStickerRenderer"
assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[
0] == "liveChatTickerSponsorItemRenderer"
chat.terminate() chat.terminate()

31
tests/test_parser.py
View File

@@ -1,17 +1,16 @@
import pytest
from pytchat.parser.live import Parser from pytchat.parser.live import Parser
import json import json
import asyncio,aiohttp
from aioresponses import aioresponses from aioresponses import aioresponses
from pytchat.exceptions import ( from pytchat.exceptions import NoContents
NoLivechatRendererException,NoYtinitialdataException,
ResponseContextError, NoContentsException)
def _open_file(path): def _open_file(path):
with open(path,mode ='r',encoding = 'utf-8') as f: with open(path, mode='r', encoding='utf-8') as f:
return f.read() return f.read()
parser = Parser(is_replay = False)
parser = Parser(is_replay=False)
@aioresponses() @aioresponses()
def test_finishedlive(*mock): def test_finishedlive(*mock):
@@ -20,12 +19,13 @@ def test_finishedlive(*mock):
_text = _open_file('tests/testdata/finished_live.json') _text = _open_file('tests/testdata/finished_live.json')
_text = json.loads(_text) _text = json.loads(_text)
try: try:
parser.parse(parser.get_contents(_text)) parser.parse(parser.get_contents(_text))
assert False assert False
except NoContentsException: except NoContents:
assert True assert True
@aioresponses() @aioresponses()
def test_parsejson(*mock): def test_parsejson(*mock):
'''jsonを正常にパースできるか''' '''jsonを正常にパースできるか'''
@@ -33,12 +33,13 @@ def test_parsejson(*mock):
_text = _open_file('tests/testdata/paramgen_firstread.json') _text = _open_file('tests/testdata/paramgen_firstread.json')
_text = json.loads(_text) _text = json.loads(_text)
try: try:
parser.parse(parser.get_contents(_text)) parser.parse(parser.get_contents(_text))
jsn = _text jsn = _text
timeout = jsn["response"]["continuationContents"]["liveChatContinuation"]["continuations"][0]["timedContinuationData"]["timeoutMs"] timeout = jsn["response"]["continuationContents"]["liveChatContinuation"]["continuations"][0]["timedContinuationData"]["timeoutMs"]
continuation = jsn["response"]["continuationContents"]["liveChatContinuation"]["continuations"][0]["timedContinuationData"]["continuation"] continuation = jsn["response"]["continuationContents"]["liveChatContinuation"][
assert 5035 == timeout "continuations"][0]["timedContinuationData"]["continuation"]
assert "0ofMyAPiARp8Q2c4S0RRb0xhelJMZDBsWFQwdERkalFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5YXpSTGQwbFhUMHREZGpRbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiPz5-Os-PkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgJqXjrPj5AJYA1CRwciOs-PkAli3pNq1k-PkAmgBggEECAEQAIgBAKABjbfnjrPj5AI%3D" == continuation assert timeout == 5035
except: assert continuation == "0ofMyAPiARp8Q2c4S0RRb0xhelJMZDBsWFQwdERkalFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5YXpSTGQwbFhUMHREZGpRbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiPz5-Os-PkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgJqXjrPj5AJYA1CRwciOs-PkAli3pNq1k-PkAmgBggEECAEQAIgBAKABjbfnjrPj5AI%3D"
assert False except Exception:
assert False

40
tests/test_speed_calculator.py
View File

@@ -1,15 +1,9 @@
import json import json
import pytest
import asyncio,aiohttp
from pytchat.parser.live import Parser from pytchat.parser.live import Parser
from pytchat.processors.compatible.processor import CompatibleProcessor
from pytchat.exceptions import (
NoLivechatRendererException,NoYtinitialdataException,
ResponseContextError, NoContentsException)
from pytchat.processors.speed.calculator import SpeedCalculator from pytchat.processors.speed.calculator import SpeedCalculator
parser = Parser(is_replay =False) parser = Parser(is_replay=False)
def test_speed_1(mocker): def test_speed_1(mocker):
'''test speed calculation with normal json. '''test speed calculation with normal json.
@@ -23,13 +17,14 @@ def test_speed_1(mocker):
_, chatdata = parser.parse(parser.get_contents(json.loads(_json))) _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
data = { data = {
"video_id" : "", "video_id": "",
"timeout" : 10, "timeout": 10,
"chatdata" : chatdata "chatdata": chatdata
} }
ret = processor.process([data]) ret = processor.process([data])
assert 30 == ret assert 30 == ret
def test_speed_2(mocker): def test_speed_2(mocker):
'''test speed calculation with no valid chat data. '''test speed calculation with no valid chat data.
''' '''
@@ -39,13 +34,14 @@ def test_speed_2(mocker):
_, chatdata = parser.parse(parser.get_contents(json.loads(_json))) _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
data = { data = {
"video_id" : "", "video_id": "",
"timeout" : 10, "timeout": 10,
"chatdata" : chatdata "chatdata": chatdata
} }
ret = processor.process([data]) ret = processor.process([data])
assert 0 == ret assert ret == 0
def test_speed_3(mocker): def test_speed_3(mocker):
'''test speed calculation with empty data. '''test speed calculation with empty data.
''' '''
@@ -55,14 +51,14 @@ def test_speed_3(mocker):
_, chatdata = parser.parse(parser.get_contents(json.loads(_json))) _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
data = { data = {
"video_id" : "", "video_id": "",
"timeout" : 10, "timeout": 10,
"chatdata" : chatdata "chatdata": chatdata
} }
ret = processor.process([data]) ret = processor.process([data])
assert 0 == ret assert ret == 0
def _open_file(path): def _open_file(path):
with open(path,mode ='r',encoding = 'utf-8') as f: with open(path, mode='r', encoding='utf-8') as f:
return f.read() return f.read()