ramforth/pytchat-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement raise_for_status()

Browse Source
This commit is contained in:
taizan-hokuto
2020-06-17 23:56:07 +09:00
parent 2474207691
commit 94d4eebd0f
13 changed files with 326 additions and 286 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pytchat/cli/__init__.py
View File

@@ -1,7 +1,7 @@
import argparse
from pathlib import Path
from .arguments import Arguments
from .. exceptions import InvalidVideoIdException, NoContentsException
from .. exceptions import InvalidVideoIdException, NoContents
from .. processors.html_archiver import HTMLArchiver
from .. tool.extract.extractor import Extractor
from .. tool.videoinfo import VideoInfo
@@ -50,7 +50,7 @@ def main():
callback=_disp_progress
).extract()
print("\nExtraction end.\n")
except (InvalidVideoIdException, NoContentsException) as e:
except (InvalidVideoIdException, NoContents) as e:
print(e)
return
parser.print_help()

90
pytchat/core_async/livechat.py
View File

@@ -11,7 +11,7 @@ from asyncio import Queue
from .buffer import Buffer
from ..parser.live import Parser
from .. import config
from ..exceptions import ChatParseException, IllegalFunctionCall
from .. import exceptions
from ..paramgen import liveparam, arcparam
from ..processors.default.processor import DefaultProcessor
from ..processors.combinator import Combinator
@@ -86,7 +86,7 @@ class LiveChatAsync:
topchat_only=False,
logger=config.logger(__name__),
):
self.video_id = video_id
self._video_id = video_id
self.seektime = seektime
if isinstance(processor, tuple):
self.processor = Combinator(processor)
@@ -102,28 +102,26 @@ class LiveChatAsync:
self._parser = Parser(is_replay=self._is_replay)
self._pauser = Queue()
self._pauser.put_nowait(None)
self._setup()
self._first_fetch = True
self._fetch_url = "live_chat/get_live_chat?continuation="
self._topchat_only = topchat_only
self._logger = logger
self.exception = None
LiveChatAsync._logger = logger
if not LiveChatAsync._setup_finished:
LiveChatAsync._setup_finished = True
if exception_handler:
self._set_exception_handler(exception_handler)
if interruptable:
signal.signal(signal.SIGINT,
(lambda a, b: asyncio.create_task(
LiveChatAsync.shutdown(None, signal.SIGINT, b))
))
if exception_handler:
self._set_exception_handler(exception_handler)
if interruptable:
signal.signal(signal.SIGINT,
(lambda a, b: asyncio.create_task(
LiveChatAsync.shutdown(None, signal.SIGINT, b))))
self._setup()
def _setup(self):
# direct modeがTrueでcallback未設定の場合例外発生。
if self._direct_mode:
if self._callback is None:
raise IllegalFunctionCall(
raise exceptions.IllegalFunctionCall(
"When direct_mode=True, callback parameter is required.")
else:
# direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成
@@ -138,18 +136,18 @@ class LiveChatAsync:
loop.create_task(self._callback_loop(self._callback))
# _listenループタスクの開始
loop = asyncio.get_event_loop()
listen_task = loop.create_task(self._startlisten())
self.listen_task = loop.create_task(self._startlisten())
# add_done_callbackの登録
if self._done_callback is None:
listen_task.add_done_callback(self.finish)
self.listen_task.add_done_callback(self._finish)
else:
listen_task.add_done_callback(self._done_callback)
self.listen_task.add_done_callback(self._done_callback)
async def _startlisten(self):
"""Fetch first continuation parameter,
create and start _listen loop.
"""
initial_continuation = liveparam.getparam(self.video_id, 3)
initial_continuation = liveparam.getparam(self._video_id, 3)
await self._listen(initial_continuation)
async def _listen(self, continuation):
@@ -171,7 +169,7 @@ class LiveChatAsync:
timeout = metadata['timeoutMs'] / 1000
chat_component = {
"video_id": self.video_id,
"video_id": self._video_id,
"timeout": timeout,
"chatdata": chatdata
}
@@ -188,14 +186,15 @@ class LiveChatAsync:
diff_time = timeout - (time.time() - time_mark)
await asyncio.sleep(diff_time)
continuation = metadata.get('continuation')
except ChatParseException as e:
self._logger.debug(f"[{self.video_id}]{str(e)}")
return
except exceptions.ChatParseException as e:
self._logger.debug(f"[{self._video_id}]{str(e)}")
raise
except (TypeError, json.JSONDecodeError):
self._logger.error(f"{traceback.format_exc(limit = -1)}")
return
raise
self._logger.debug(f"[{self.video_id}]finished fetching chat.")
self._logger.debug(f"[{self._video_id}]finished fetching chat.")
raise exceptions.ChatDataFinished
async def _check_pause(self, continuation):
if self._pauser.empty():
@@ -207,7 +206,7 @@ class LiveChatAsync:
self._pauser.put_nowait(None)
if not self._is_replay:
continuation = liveparam.getparam(
self.video_id, 3, self._topchat_only)
self._video_id, 3, self._topchat_only)
return continuation
async def _get_contents(self, continuation, session, headers):
@@ -227,7 +226,7 @@ class LiveChatAsync:
self._parser.is_replay = True
self._fetch_url = "live_chat_replay/get_live_chat_replay?continuation="
continuation = arcparam.getparam(
self.video_id, self.seektime, self._topchat_only)
self._video_id, self.seektime, self._topchat_only)
livechat_json = (await self._get_livechat_json(
continuation, session, headers))
reload_continuation = self._parser.reload_continuation(
@@ -258,7 +257,7 @@ class LiveChatAsync:
await asyncio.sleep(1)
continue
else:
self._logger.error(f"[{self.video_id}]"
self._logger.error(f"[{self._video_id}]"
f"Exceeded retry count. status_code={status_code}")
return None
return livechat_json
@@ -288,9 +287,12 @@ class LiveChatAsync:
: Processorによって加工されたチャットデータ
"""
if self._callback is None:
items = await self._buffer.get()
return self.processor.process(items)
raise IllegalFunctionCall(
if self.is_alive():
items = await self._buffer.get()
return self.processor.process(items)
else:
return []
raise exceptions.IllegalFunctionCall(
"既にcallbackを登録済みのため、get()は実行できません。")
def is_replay(self):
@@ -311,22 +313,36 @@ class LiveChatAsync:
def is_alive(self):
return self._is_alive
def finish(self, sender):
def _finish(self, sender):
'''Listener終了時のコールバック'''
try:
self.terminate()
self._task_finished()
except CancelledError:
self._logger.debug(f'[{self.video_id}]cancelled:{sender}')
self._logger.debug(f'[{self._video_id}]cancelled:{sender}')
def terminate(self):
if self._pauser.empty():
self._pauser.put_nowait(None)
self._is_alive = False
self._buffer.put_nowait({})
def _task_finished(self):
'''
Listenerを終了する。
'''
self._is_alive = False
if self._direct_mode is False:
# bufferにダミーオブジェクトを入れてis_alive()を判定させる
self._buffer.put_nowait({'chatdata': '', 'timeout': 0})
self._logger.info(f'[{self.video_id}]finished.')
if self.is_alive():
self.terminate()
try:
self.listen_task.result()
except Exception as e:
self.exception = e
if not isinstance(e, exceptions.ChatParseException):
self._logger.error(f'Internal exception - {type(e)}{str(e)}')
self._logger.info(f'[{self._video_id}]終了しました')
def raise_for_status(self):
if self.exception is not None:
raise self.exception
@classmethod
def _set_exception_handler(cls, handler):

84
pytchat/core_multithread/livechat.py
View File

@@ -6,10 +6,11 @@ import traceback
import urllib.parse
from concurrent.futures import CancelledError, ThreadPoolExecutor
from queue import Queue
from threading import Event
from .buffer import Buffer
from ..parser.live import Parser
from .. import config
from ..exceptions import ChatParseException, IllegalFunctionCall
from .. import exceptions
from ..paramgen import liveparam, arcparam
from ..processors.default.processor import DefaultProcessor
from ..processors.combinator import Combinator
@@ -83,7 +84,7 @@ class LiveChat:
topchat_only=False,
logger=config.logger(__name__)
):
self.video_id = video_id
self._video_id = video_id
self.seektime = seektime
if isinstance(processor, tuple):
self.processor = Combinator(processor)
@@ -102,7 +103,9 @@ class LiveChat:
self._first_fetch = True
self._fetch_url = "live_chat/get_live_chat?continuation="
self._topchat_only = topchat_only
self._event = Event()
self._logger = logger
self.exception = None
if interruptable:
signal.signal(signal.SIGINT, lambda a, b: self.terminate())
self._setup()
@@ -111,7 +114,7 @@ class LiveChat:
# direct modeがTrueでcallback未設定の場合例外発生。
if self._direct_mode:
if self._callback is None:
raise IllegalFunctionCall(
raise exceptions.IllegalFunctionCall(
"When direct_mode=True, callback parameter is required.")
else:
# direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成
@@ -124,19 +127,19 @@ class LiveChat:
# callbackを呼ぶループタスクの開始
self._executor.submit(self._callback_loop, self._callback)
# _listenループタスクの開始
listen_task = self._executor.submit(self._startlisten)
self.listen_task = self._executor.submit(self._startlisten)
# add_done_callbackの登録
if self._done_callback is None:
listen_task.add_done_callback(self.finish)
self.listen_task.add_done_callback(self._finish)
else:
listen_task.add_done_callback(self._done_callback)
self.listen_task.add_done_callback(self._done_callback)
def _startlisten(self):
time.sleep(0.1) # sleep shortly to prohibit skipping fetching data
"""Fetch first continuation parameter,
create and start _listen loop.
"""
initial_continuation = liveparam.getparam(self.video_id, 3)
initial_continuation = liveparam.getparam(self._video_id, 3)
self._listen(initial_continuation)
def _listen(self, continuation):
@@ -152,13 +155,11 @@ class LiveChat:
with requests.Session() as session:
while(continuation and self._is_alive):
continuation = self._check_pause(continuation)
contents = self._get_contents(
continuation, session, headers)
contents = self._get_contents(continuation, session, headers)
metadata, chatdata = self._parser.parse(contents)
timeout = metadata['timeoutMs'] / 1000
chat_component = {
"video_id": self.video_id,
"video_id": self._video_id,
"timeout": timeout,
"chatdata": chatdata
}
@@ -173,16 +174,17 @@ class LiveChat:
else:
self._buffer.put(chat_component)
diff_time = timeout - (time.time() - time_mark)
time.sleep(diff_time if diff_time > 0 else 0)
self._event.wait(diff_time if diff_time > 0 else 0)
continuation = metadata.get('continuation')
except ChatParseException as e:
self._logger.debug(f"[{self.video_id}]{str(e)}")
return
except exceptions.ChatParseException as e:
self._logger.debug(f"[{self._video_id}]{str(e)}")
raise
except (TypeError, json.JSONDecodeError):
self._logger.error(f"{traceback.format_exc(limit=-1)}")
return
raise
self._logger.debug(f"[{self.video_id}]finished fetching chat.")
self._logger.debug(f"[{self._video_id}]finished fetching chat.")
raise exceptions.ChatDataFinished
def _check_pause(self, continuation):
if self._pauser.empty():
@@ -193,7 +195,7 @@ class LiveChat:
'''
self._pauser.put_nowait(None)
if not self._is_replay:
continuation = liveparam.getparam(self.video_id, 3)
continuation = liveparam.getparam(self._video_id, 3)
return continuation
def _get_contents(self, continuation, session, headers):
@@ -215,9 +217,8 @@ class LiveChat:
self._parser.is_replay = True
self._fetch_url = "live_chat_replay/get_live_chat_replay?continuation="
continuation = arcparam.getparam(
self.video_id, self.seektime, self._topchat_only)
livechat_json = (self._get_livechat_json(
continuation, session, headers))
self._video_id, self.seektime, self._topchat_only)
livechat_json = (self._get_livechat_json(continuation, session, headers))
reload_continuation = self._parser.reload_continuation(
self._parser.get_contents(livechat_json))
if reload_continuation:
@@ -246,9 +247,9 @@ class LiveChat:
time.sleep(1)
continue
else:
self._logger.error(f"[{self.video_id}]"
self._logger.error(f"[{self._video_id}]"
f"Exceeded retry count. status_code={status_code}")
return None
raise exceptions.RetryExceedMaxCount()
return livechat_json
def _callback_loop(self, callback):
@@ -276,9 +277,12 @@ class LiveChat:
: Processorによって加工されたチャットデータ
"""
if self._callback is None:
items = self._buffer.get()
return self.processor.process(items)
raise IllegalFunctionCall(
if self.is_alive():
items = self._buffer.get()
return self.processor.process(items)
else:
return []
raise exceptions.IllegalFunctionCall(
"既にcallbackを登録済みのため、get()は実行できません。")
def is_replay(self):
@@ -299,18 +303,34 @@ class LiveChat:
def is_alive(self):
return self._is_alive
def finish(self, sender):
def _finish(self, sender):
'''Listener終了時のコールバック'''
try:
self.terminate()
self._task_finished()
except CancelledError:
self._logger.debug(f'[{self.video_id}]cancelled:{sender}')
self._logger.debug(f'[{self._video_id}]cancelled:{sender}')
def terminate(self):
if self._pauser.empty():
self._pauser.put_nowait(None)
self._is_alive = False
self._buffer.put({})
self._event.set()
def _task_finished(self):
'''
Listenerを終了する。
'''
if self.is_alive():
self._is_alive = False
self._buffer.put({})
self._logger.info(f'[{self.video_id}]終了しました')
self.terminate()
try:
self.listen_task.result()
except Exception as e:
self.exception = e
if not isinstance(e, exceptions.ChatParseException):
self._logger.error(f'Internal exception - {type(e)}{str(e)}')
self._logger.info(f'[{self._video_id}]終了しました')
def raise_for_status(self):
if self.exception is not None:
raise self.exception

41
pytchat/exceptions.py
View File

@@ -5,13 +5,6 @@ class ChatParseException(Exception):
pass
class NoYtinitialdataException(ChatParseException):
'''
Thrown when the video is not found.
'''
pass
class ResponseContextError(ChatParseException):
'''
Thrown when chat data is invalid.
@@ -19,21 +12,14 @@ class ResponseContextError(ChatParseException):
pass
class NoLivechatRendererException(ChatParseException):
'''
Thrown when livechatRenderer is missing in JSON.
'''
pass
class NoContentsException(ChatParseException):
class NoContents(ChatParseException):
'''
Thrown when ContinuationContents is missing in JSON.
'''
pass
class NoContinuationsException(ChatParseException):
class NoContinuation(ChatParseException):
'''
Thrown when continuation is missing in ContinuationContents.
'''
@@ -42,8 +28,8 @@ class NoContinuationsException(ChatParseException):
class IllegalFunctionCall(Exception):
'''
Thrown when get () is called even though
set_callback () has been executed.
Thrown when get() is called even though
set_callback() has been executed.
'''
pass
@@ -57,3 +43,22 @@ class InvalidVideoIdException(Exception):
class UnknownConnectionError(Exception):
pass
class RetryExceedMaxCount(Exception):
'''
thrown when the number of retries exceeds the maximum value.
'''
pass
class ChatDataFinished(ChatParseException):
pass
class ReceivedUnknownContinuation(ChatParseException):
pass
class FailedExtractContinuation(ChatDataFinished):
pass

27
pytchat/parser/live.py
View File

@@ -4,11 +4,7 @@ pytchat.parser.live
Parser of live chat JSON.
"""
from .. exceptions import (
ResponseContextError,
NoContentsException,
NoContinuationsException,
ChatParseException)
from .. import exceptions
class Parser:
@@ -20,9 +16,9 @@ class Parser:
def get_contents(self, jsn):
if jsn is None:
raise ChatParseException('Called with none JSON object.')
raise exceptions.IllegalFunctionCall('Called with none JSON object.')
if jsn['response']['responseContext'].get('errors'):
raise ResponseContextError(
raise exceptions.ResponseContextError(
'The video_id would be wrong, or video is deleted or private.')
contents = jsn['response'].get('continuationContents')
return contents
@@ -46,11 +42,11 @@ class Parser:
if contents is None:
'''Broadcasting end or cannot fetch chat stream'''
raise NoContentsException('Chat data stream is empty.')
raise exceptions.NoContents('Chat data stream is empty.')
cont = contents['liveChatContinuation']['continuations'][0]
if cont is None:
raise NoContinuationsException('No Continuation')
raise exceptions.NoContinuation('No Continuation')
metadata = (cont.get('invalidationContinuationData')
or cont.get('timedContinuationData')
or cont.get('reloadContinuationData')
@@ -58,22 +54,25 @@ class Parser:
)
if metadata is None:
if cont.get("playerSeekContinuationData"):
raise ChatParseException('Finished chat data')
raise exceptions.ChatDataFinished('Finished chat data')
unknown = list(cont.keys())[0]
if unknown:
raise ChatParseException(
raise exceptions.ReceivedUnknownContinuation(
f"Received unknown continuation type:{unknown}")
else:
raise ChatParseException('Cannot extract continuation data')
raise exceptions.FailedExtractContinuation('Cannot extract continuation data')
return self._create_data(metadata, contents)
def reload_continuation(self, contents):
"""
When `seektime = 0` or seektime is abbreviated ,
When `seektime == 0` or seektime is abbreviated ,
check if fetched chat json has no chat data.
If so, try to fetch playerSeekContinuationData.
This function must be run only first fetching.
"""
if contents is None:
'''Broadcasting end or cannot fetch chat stream'''
raise exceptions.NoContents('Chat data stream is empty.')
cont = contents['liveChatContinuation']['continuations'][0]
if cont.get("liveChatReplayContinuationData"):
# chat data exist.
@@ -82,7 +81,7 @@ class Parser:
init_cont = cont.get("playerSeekContinuationData")
if init_cont:
return init_cont.get("continuation")
raise ChatParseException('Finished chat data')
raise exceptions.ChatDataFinished('Finished chat data')
def _create_data(self, metadata, contents):
actions = contents['liveChatContinuation'].get('actions')

11
pytchat/tool/extract/parser.py
View File

@@ -1,8 +1,5 @@
from ... import config
from ... exceptions import (
ResponseContextError,
NoContentsException,
NoContinuationsException)
from ... import exceptions
logger = config.logger(__name__)
@@ -23,15 +20,15 @@ def parse(jsn):
if jsn is None:
raise ValueError("parameter JSON is None")
if jsn['response']['responseContext'].get('errors'):
raise ResponseContextError(
raise exceptions.ResponseContextError(
'video_id is invalid or private/deleted.')
contents = jsn['response'].get('continuationContents')
if contents is None:
raise NoContentsException('No chat data.')
raise exceptions.NoContents('No chat data.')
cont = contents['liveChatContinuation']['continuations'][0]
if cont is None:
raise NoContinuationsException('No Continuation')
raise exceptions.NoContinuation('No Continuation')
metadata = cont.get('liveChatReplayContinuationData')
if metadata:
continuation = metadata.get("continuation")

48
pytchat/tool/mining/parser.py
View File

@@ -1,12 +1,12 @@
import json
import re
from ... import config
from ... exceptions import (
ResponseContextError,
NoContentsException,
NoContinuationsException )
from ... exceptions import (
ResponseContextError,
NoContents, NoContinuation)
logger = config.logger(__name__)
def parse(jsn):
"""
Parse replay chat data.
@@ -20,45 +20,51 @@ def parse(jsn):
actions : list
"""
if jsn is None:
if jsn is None:
raise ValueError("parameter JSON is None")
if jsn['response']['responseContext'].get('errors'):
raise ResponseContextError(
'video_id is invalid or private/deleted.')
contents=jsn["response"].get('continuationContents')
'video_id is invalid or private/deleted.')
contents = jsn["response"].get('continuationContents')
if contents is None:
raise NoContentsException('No chat data.')
raise NoContents('No chat data.')
cont = contents['liveChatContinuation']['continuations'][0]
if cont is None:
raise NoContinuationsException('No Continuation')
raise NoContinuation('No Continuation')
metadata = cont.get('liveChatReplayContinuationData')
if metadata:
continuation = metadata.get("continuation")
actions = contents['liveChatContinuation'].get('actions')
if continuation:
return continuation, [action["replayChatItemAction"]["actions"][0]
for action in actions
if list(action['replayChatItemAction']["actions"][0].values()
)[0]['item'].get("liveChatPaidMessageRenderer")
or list(action['replayChatItemAction']["actions"][0].values()
)[0]['item'].get("liveChatPaidStickerRenderer")
]
return continuation, [action["replayChatItemAction"]["actions"][0]
for action in actions
if list(action['replayChatItemAction']["actions"][0].values()
)[0]['item'].get("liveChatPaidMessageRenderer")
or list(action['replayChatItemAction']["actions"][0].values()
)[0]['item'].get("liveChatPaidStickerRenderer")
]
return None, []
def get_offset(item):
return int(item['replayChatItemAction']["videoOffsetTimeMsec"])
def get_id(item):
return list((list(item['replayChatItemAction']["actions"][0].values()
)[0])['item'].values())[0].get('id')
)[0])['item'].values())[0].get('id')
def get_type(item):
return list((list(item['replayChatItemAction']["actions"][0].values()
)[0])['item'].keys())[0]
import re
_REGEX_YTINIT = re.compile("window\\[\"ytInitialData\"\\]\\s*=\\s*({.+?});\\s+")
)[0])['item'].keys())[0]
_REGEX_YTINIT = re.compile(
"window\\[\"ytInitialData\"\\]\\s*=\\s*({.+?});\\s+")
def extract(text):
match = re.findall(_REGEX_YTINIT, str(text))

25
tests/test_arcparam_mining.py
View File

@@ -1,40 +1,41 @@
import pytest
from pytchat.tool.mining import parser
import pytchat.config as config
import requests, json
import requests
import json
from pytchat.paramgen import arcparam_mining as arcparam
def test_arcparam_e(mocker):
try:
arcparam.getparam("01234567890",-1)
arcparam.getparam("01234567890", -1)
assert False
except ValueError:
assert True
def test_arcparam_0(mocker):
param = arcparam.getparam("01234567890",0)
param = arcparam.getparam("01234567890", 0)
assert param =="op2w0wQsGiBDZzhhRFFvTE1ERXlNelExTmpjNE9UQWdBUSUzRCUzREABYARyAggBeAE%3D"
assert param == "op2w0wQsGiBDZzhhRFFvTE1ERXlNelExTmpjNE9UQWdBUSUzRCUzREABYARyAggBeAE%3D"
def test_arcparam_1(mocker):
param = arcparam.getparam("01234567890", seektime = 100000)
param = arcparam.getparam("01234567890", seektime=100000)
print(param)
assert param == "op2w0wQzGiBDZzhhRFFvTE1ERXlNelExTmpjNE9UQWdBUSUzRCUzREABWgUQgMLXL2AEcgIIAXgB"
def test_arcparam_2(mocker):
param = arcparam.getparam("PZz9NB0-Z64",1)
url=f"https://www.youtube.com/live_chat_replay?continuation={param}&playerOffsetMs=1000&pbj=1"
resp = requests.Session().get(url,headers = config.headers)
param = arcparam.getparam("PZz9NB0-Z64", 1)
url = f"https://www.youtube.com/live_chat_replay?continuation={param}&playerOffsetMs=1000&pbj=1"
resp = requests.Session().get(url, headers=config.headers)
jsn = json.loads(resp.text)
_ , chatdata = parser.parse(jsn[1])
_, chatdata = parser.parse(jsn[1])
test_id = chatdata[0]["addChatItemAction"]["item"]["liveChatPaidMessageRenderer"]["id"]
print(test_id)
assert test_id == "ChwKGkNKSGE0YnFJeWVBQ0ZWcUF3Z0VkdGIwRm9R"
def test_arcparam_3(mocker):
param = arcparam.getparam("01234567890")
assert param == "op2w0wQsGiBDZzhhRFFvTE1ERXlNelExTmpjNE9UQWdBUSUzRCUzREABYARyAggBeAE%3D"

61
tests/test_compatible_processor.py
View File

@@ -1,17 +1,6 @@
import json
import pytest
import asyncio
import aiohttp
from pytchat.parser.live import Parser
from pytchat.processors.compatible.processor import CompatibleProcessor
from pytchat.exceptions import (
NoLivechatRendererException, NoYtinitialdataException,
ResponseContextError, NoContentsException)
from pytchat.processors.compatible.renderer.textmessage import LiveChatTextMessageRenderer
from pytchat.processors.compatible.renderer.paidmessage import LiveChatPaidMessageRenderer
from pytchat.processors.compatible.renderer.paidsticker import LiveChatPaidStickerRenderer
from pytchat.processors.compatible.renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
parser = Parser(is_replay=False)
@@ -31,21 +20,23 @@ def test_textmessage(mocker):
ret = processor.process([data])
assert ret["kind"] == "youtube#liveChatMessageListResponse"
assert ret["pollingIntervalMillis"] == data["timeout"]*1000
assert ret["pollingIntervalMillis"] == data["timeout"] * 1000
assert ret.keys() == {
"kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
"kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
}
assert ret["pageInfo"].keys() == {
"totalResults", "resultsPerPage"
"totalResults", "resultsPerPage"
}
assert ret["items"][0].keys() == {
"kind", "etag", "id", "snippet", "authorDetails"
"kind", "etag", "id", "snippet", "authorDetails"
}
assert ret["items"][0]["snippet"].keys() == {
'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage', 'textMessageDetails'
'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage',
'textMessageDetails'
}
assert ret["items"][0]["authorDetails"].keys() == {
'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator'
'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor',
'isChatModerator'
}
assert ret["items"][0]["snippet"]["textMessageDetails"].keys() == {
'messageText'
@@ -69,22 +60,23 @@ def test_newsponcer(mocker):
ret = processor.process([data])
assert ret["kind"] == "youtube#liveChatMessageListResponse"
assert ret["pollingIntervalMillis"] == data["timeout"]*1000
assert ret["pollingIntervalMillis"] == data["timeout"] * 1000
assert ret.keys() == {
"kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
"kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
}
assert ret["pageInfo"].keys() == {
"totalResults", "resultsPerPage"
"totalResults", "resultsPerPage"
}
assert ret["items"][0].keys() == {
"kind", "etag", "id", "snippet", "authorDetails"
"kind", "etag", "id", "snippet", "authorDetails"
}
assert ret["items"][0]["snippet"].keys() == {
'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage'
}
assert ret["items"][0]["authorDetails"].keys() == {
'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator'
'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor',
'isChatModerator'
}
assert "LCC." in ret["items"][0]["id"]
assert ret["items"][0]["snippet"]["type"] == "newSponsorEvent"
@@ -105,22 +97,23 @@ def test_newsponcer_rev(mocker):
ret = processor.process([data])
assert ret["kind"] == "youtube#liveChatMessageListResponse"
assert ret["pollingIntervalMillis"] == data["timeout"]*1000
assert ret["pollingIntervalMillis"] == data["timeout"] * 1000
assert ret.keys() == {
"kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
"kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
}
assert ret["pageInfo"].keys() == {
"totalResults", "resultsPerPage"
"totalResults", "resultsPerPage"
}
assert ret["items"][0].keys() == {
"kind", "etag", "id", "snippet", "authorDetails"
"kind", "etag", "id", "snippet", "authorDetails"
}
assert ret["items"][0]["snippet"].keys() == {
'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage'
}
assert ret["items"][0]["authorDetails"].keys() == {
'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator'
'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor',
'isChatModerator'
}
assert "LCC." in ret["items"][0]["id"]
assert ret["items"][0]["snippet"]["type"] == "newSponsorEvent"
@@ -141,21 +134,23 @@ def test_superchat(mocker):
ret = processor.process([data])
assert ret["kind"] == "youtube#liveChatMessageListResponse"
assert ret["pollingIntervalMillis"] == data["timeout"]*1000
assert ret["pollingIntervalMillis"] == data["timeout"] * 1000
assert ret.keys() == {
"kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
"kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
}
assert ret["pageInfo"].keys() == {
"totalResults", "resultsPerPage"
"totalResults", "resultsPerPage"
}
assert ret["items"][0].keys() == {
"kind", "etag", "id", "snippet", "authorDetails"
"kind", "etag", "id", "snippet", "authorDetails"
}
assert ret["items"][0]["snippet"].keys() == {
'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage', 'superChatDetails'
'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage',
'superChatDetails'
}
assert ret["items"][0]["authorDetails"].keys() == {
'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator'
'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor',
'isChatModerator'
}
assert ret["items"][0]["snippet"]["superChatDetails"].keys() == {
'amountMicros', 'currency', 'amountDisplayString', 'tier', 'backgroundColor'

25
tests/test_livechat.py
View File

@@ -1,30 +1,21 @@
import pytest
from pytchat.parser.live import Parser
import json
import asyncio,aiohttp
from aioresponses import aioresponses
from pytchat.core_async.livechat import LiveChatAsync
from pytchat.exceptions import (
NoLivechatRendererException,NoYtinitialdataException,
ResponseContextError,NoContentsException)
from pytchat.exceptions import ResponseContextError
from pytchat.core_multithread.livechat import LiveChat
import unittest
from unittest import TestCase
def _open_file(path):
with open(path,mode ='r',encoding = 'utf-8') as f:
with open(path, mode='r', encoding='utf-8') as f:
return f.read()
@aioresponses()
def test_Async(*mock):
vid=''
vid = ''
_text = _open_file('tests/testdata/paramgen_firstread.json')
_text = json.loads(_text)
mock[0].get(f"https://www.youtube.com/live_chat?v={vid}&is_popout=1", status=200, body=_text)
mock[0].get(
f"https://www.youtube.com/live_chat?v={vid}&is_popout=1", status=200, body=_text)
try:
chat = LiveChatAsync(video_id='')
assert chat.is_alive()
@@ -33,6 +24,7 @@ def test_Async(*mock):
except ResponseContextError:
assert not chat.is_alive()
def test_MultiThread(mocker):
_text = _open_file('tests/testdata/paramgen_firstread.json')
_text = json.loads(_text)
@@ -48,6 +40,3 @@ def test_MultiThread(mocker):
except ResponseContextError:
chat.terminate()
assert not chat.is_alive()

125
tests/test_livechat_2.py
View File

@@ -1,43 +1,43 @@
import asyncio, aiohttp
import json
import pytest
import asyncio
import re
import requests
import sys
import time
from aioresponses import aioresponses
from concurrent.futures import CancelledError
from unittest import TestCase
from pytchat.core_multithread.livechat import LiveChat
from pytchat.core_async.livechat import LiveChatAsync
from pytchat.exceptions import (
NoLivechatRendererException,NoYtinitialdataException,
ResponseContextError,NoContentsException)
from pytchat.parser.live import Parser
from pytchat.processors.dummy_processor import DummyProcessor
def _open_file(path):
with open(path,mode ='r',encoding = 'utf-8') as f:
with open(path, mode='r', encoding='utf-8') as f:
return f.read()
@aioresponses()
def test_async_live_stream(*mock):
async def test_loop(*mock):
pattern = re.compile(r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$')
pattern = re.compile(
r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$')
_text = _open_file('tests/testdata/test_stream.json')
mock[0].get(pattern, status=200, body=_text)
chat = LiveChatAsync(video_id='', processor = DummyProcessor())
chat = LiveChatAsync(video_id='', processor=DummyProcessor())
chats = await chat.get()
rawdata = chats[0]["chatdata"]
#assert fetching livachat data
assert list(rawdata[0]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer"
assert list(rawdata[1]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer"
assert list(rawdata[2]["addChatItemAction"]["item"].keys())[0] == "liveChatPlaceholderItemRenderer"
assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerPaidMessageItemRenderer"
assert list(rawdata[4]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer"
assert list(rawdata[5]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidStickerRenderer"
assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerSponsorItemRenderer"
# assert fetching livachat data
assert list(rawdata[0]["addChatItemAction"]["item"].keys())[
0] == "liveChatTextMessageRenderer"
assert list(rawdata[1]["addChatItemAction"]["item"].keys())[
0] == "liveChatTextMessageRenderer"
assert list(rawdata[2]["addChatItemAction"]["item"].keys())[
0] == "liveChatPlaceholderItemRenderer"
assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[
0] == "liveChatTickerPaidMessageItemRenderer"
assert list(rawdata[4]["addChatItemAction"]["item"].keys())[
0] == "liveChatPaidMessageRenderer"
assert list(rawdata[5]["addChatItemAction"]["item"].keys())[
0] == "liveChatPaidStickerRenderer"
assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[
0] == "liveChatTickerSponsorItemRenderer"
loop = asyncio.get_event_loop()
try:
@@ -45,24 +45,29 @@ def test_async_live_stream(*mock):
except CancelledError:
assert True
@aioresponses()
@aioresponses()
def test_async_replay_stream(*mock):
async def test_loop(*mock):
pattern_live = re.compile(r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$')
pattern_replay = re.compile(r'^https://www.youtube.com/live_chat_replay/get_live_chat_replay\?continuation=.*$')
#empty livechat -> switch to fetch replaychat
pattern_live = re.compile(
r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$')
pattern_replay = re.compile(
r'^https://www.youtube.com/live_chat_replay/get_live_chat_replay\?continuation=.*$')
# empty livechat -> switch to fetch replaychat
_text_live = _open_file('tests/testdata/finished_live.json')
_text_replay = _open_file('tests/testdata/chatreplay.json')
mock[0].get(pattern_live, status=200, body=_text_live)
mock[0].get(pattern_replay, status=200, body=_text_replay)
chat = LiveChatAsync(video_id='', processor = DummyProcessor())
chat = LiveChatAsync(video_id='', processor=DummyProcessor())
chats = await chat.get()
rawdata = chats[0]["chatdata"]
#assert fetching replaychat data
assert list(rawdata[0]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer"
assert list(rawdata[14]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer"
# assert fetching replaychat data
assert list(rawdata[0]["addChatItemAction"]["item"].keys())[
0] == "liveChatTextMessageRenderer"
assert list(rawdata[14]["addChatItemAction"]["item"].keys())[
0] == "liveChatPaidMessageRenderer"
loop = asyncio.get_event_loop()
try:
@@ -70,56 +75,66 @@ def test_async_replay_stream(*mock):
except CancelledError:
assert True
@aioresponses()
def test_async_force_replay(*mock):
async def test_loop(*mock):
pattern_live = re.compile(r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$')
pattern_replay = re.compile(r'^https://www.youtube.com/live_chat_replay/get_live_chat_replay\?continuation=.*$')
#valid live data, but force_replay = True
pattern_live = re.compile(
r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$')
pattern_replay = re.compile(
r'^https://www.youtube.com/live_chat_replay/get_live_chat_replay\?continuation=.*$')
# valid live data, but force_replay = True
_text_live = _open_file('tests/testdata/test_stream.json')
#valid replay data
# valid replay data
_text_replay = _open_file('tests/testdata/chatreplay.json')
mock[0].get(pattern_live, status=200, body=_text_live)
mock[0].get(pattern_replay, status=200, body=_text_replay)
#force replay
chat = LiveChatAsync(video_id='', processor = DummyProcessor(), force_replay = True)
# force replay
chat = LiveChatAsync(
video_id='', processor=DummyProcessor(), force_replay=True)
chats = await chat.get()
rawdata = chats[0]["chatdata"]
# assert fetching replaychat data
assert list(rawdata[14]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer"
assert list(rawdata[14]["addChatItemAction"]["item"].keys())[
0] == "liveChatPaidMessageRenderer"
# assert not mix livechat data
assert list(rawdata[2]["addChatItemAction"]["item"].keys())[0] != "liveChatPlaceholderItemRenderer"
assert list(rawdata[2]["addChatItemAction"]["item"].keys())[
0] != "liveChatPlaceholderItemRenderer"
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(test_loop(*mock))
except CancelledError:
assert True
def test_multithread_live_stream(mocker):
_text = _open_file('tests/testdata/test_stream.json')
responseMock = mocker.Mock()
responseMock.status_code = 200
responseMock.text = _text
mocker.patch('requests.Session.get').return_value.__enter__.return_value = responseMock
mocker.patch(
'requests.Session.get').return_value.__enter__.return_value = responseMock
chat = LiveChat(video_id='test_id', processor = DummyProcessor())
chat = LiveChat(video_id='test_id', processor=DummyProcessor())
chats = chat.get()
rawdata = chats[0]["chatdata"]
#assert fetching livachat data
assert list(rawdata[0]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer"
assert list(rawdata[1]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer"
assert list(rawdata[2]["addChatItemAction"]["item"].keys())[0] == "liveChatPlaceholderItemRenderer"
assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerPaidMessageItemRenderer"
assert list(rawdata[4]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer"
assert list(rawdata[5]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidStickerRenderer"
assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerSponsorItemRenderer"
# assert fetching livachat data
assert list(rawdata[0]["addChatItemAction"]["item"].keys())[
0] == "liveChatTextMessageRenderer"
assert list(rawdata[1]["addChatItemAction"]["item"].keys())[
0] == "liveChatTextMessageRenderer"
assert list(rawdata[2]["addChatItemAction"]["item"].keys())[
0] == "liveChatPlaceholderItemRenderer"
assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[
0] == "liveChatTickerPaidMessageItemRenderer"
assert list(rawdata[4]["addChatItemAction"]["item"].keys())[
0] == "liveChatPaidMessageRenderer"
assert list(rawdata[5]["addChatItemAction"]["item"].keys())[
0] == "liveChatPaidStickerRenderer"
assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[
0] == "liveChatTickerSponsorItemRenderer"
chat.terminate()

31
tests/test_parser.py
View File

@@ -1,17 +1,16 @@
import pytest
from pytchat.parser.live import Parser
import json
import asyncio,aiohttp
from aioresponses import aioresponses
from pytchat.exceptions import (
NoLivechatRendererException,NoYtinitialdataException,
ResponseContextError, NoContentsException)
from pytchat.exceptions import NoContents
def _open_file(path):
with open(path,mode ='r',encoding = 'utf-8') as f:
with open(path, mode='r', encoding='utf-8') as f:
return f.read()
parser = Parser(is_replay = False)
parser = Parser(is_replay=False)
@aioresponses()
def test_finishedlive(*mock):
@@ -20,12 +19,13 @@ def test_finishedlive(*mock):
_text = _open_file('tests/testdata/finished_live.json')
_text = json.loads(_text)
try:
try:
parser.parse(parser.get_contents(_text))
assert False
except NoContentsException:
except NoContents:
assert True
@aioresponses()
def test_parsejson(*mock):
'''jsonを正常にパースできるか'''
@@ -33,12 +33,13 @@ def test_parsejson(*mock):
_text = _open_file('tests/testdata/paramgen_firstread.json')
_text = json.loads(_text)
try:
try:
parser.parse(parser.get_contents(_text))
jsn = _text
timeout = jsn["response"]["continuationContents"]["liveChatContinuation"]["continuations"][0]["timedContinuationData"]["timeoutMs"]
continuation = jsn["response"]["continuationContents"]["liveChatContinuation"]["continuations"][0]["timedContinuationData"]["continuation"]
assert 5035 == timeout
assert "0ofMyAPiARp8Q2c4S0RRb0xhelJMZDBsWFQwdERkalFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5YXpSTGQwbFhUMHREZGpRbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiPz5-Os-PkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgJqXjrPj5AJYA1CRwciOs-PkAli3pNq1k-PkAmgBggEECAEQAIgBAKABjbfnjrPj5AI%3D" == continuation
except:
assert False
continuation = jsn["response"]["continuationContents"]["liveChatContinuation"][
"continuations"][0]["timedContinuationData"]["continuation"]
assert timeout == 5035
assert continuation == "0ofMyAPiARp8Q2c4S0RRb0xhelJMZDBsWFQwdERkalFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5YXpSTGQwbFhUMHREZGpRbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiPz5-Os-PkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgJqXjrPj5AJYA1CRwciOs-PkAli3pNq1k-PkAmgBggEECAEQAIgBAKABjbfnjrPj5AI%3D"
except Exception:
assert False

40
tests/test_speed_calculator.py
View File

@@ -1,15 +1,9 @@
import json
import pytest
import asyncio,aiohttp
from pytchat.parser.live import Parser
from pytchat.processors.compatible.processor import CompatibleProcessor
from pytchat.exceptions import (
NoLivechatRendererException,NoYtinitialdataException,
ResponseContextError, NoContentsException)
from pytchat.processors.speed.calculator import SpeedCalculator
parser = Parser(is_replay =False)
parser = Parser(is_replay=False)
def test_speed_1(mocker):
'''test speed calculation with normal json.
@@ -23,13 +17,14 @@ def test_speed_1(mocker):
_, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
data = {
"video_id" : "",
"timeout" : 10,
"chatdata" : chatdata
"video_id": "",
"timeout": 10,
"chatdata": chatdata
}
ret = processor.process([data])
assert 30 == ret
def test_speed_2(mocker):
'''test speed calculation with no valid chat data.
'''
@@ -39,13 +34,14 @@ def test_speed_2(mocker):
_, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
data = {
"video_id" : "",
"timeout" : 10,
"chatdata" : chatdata
"video_id": "",
"timeout": 10,
"chatdata": chatdata
}
ret = processor.process([data])
assert 0 == ret
assert ret == 0
def test_speed_3(mocker):
'''test speed calculation with empty data.
'''
@@ -55,14 +51,14 @@ def test_speed_3(mocker):
_, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
data = {
"video_id" : "",
"timeout" : 10,
"chatdata" : chatdata
"video_id": "",
"timeout": 10,
"chatdata": chatdata
}
ret = processor.process([data])
assert 0 == ret
assert ret == 0
def _open_file(path):
with open(path,mode ='r',encoding = 'utf-8') as f:
with open(path, mode='r', encoding='utf-8') as f:
return f.read()