From ce96d94e2363ba3446ffe55bc4099d526ab0c625 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Mon, 30 Dec 2019 17:10:34 +0900 Subject: [PATCH 1/3] Fix termination of fetching chat data --- pytchat/core_async/livechat.py | 10 ++++++---- pytchat/core_async/replaychat.py | 6 ++++-- pytchat/core_multithread/livechat.py | 23 +++++++++-------------- pytchat/core_multithread/replaychat.py | 19 +++++++++++-------- pytchat/parser/live.py | 2 +- pytchat/parser/replay.py | 8 ++++---- pytchat/util/__init__.py | 6 +++--- 7 files changed, 38 insertions(+), 36 deletions(-) diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index 8fbfbd4..639184f 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -18,9 +18,8 @@ from ..processors.default.processor import DefaultProcessor from ..processors.combinator import Combinator logger = config.logger(__name__) -MAX_RETRY = 10 headers = config.headers - +MAX_RETRY = 10 class LiveChatAsync: @@ -152,7 +151,7 @@ class LiveChatAsync: async def _listen(self, continuation): ''' continuationに紐付いたチャットデータを取得し - チャットデータを格納、 + Bufferにチャットデータを格納、 次のcontinuaitonを取得してループする。 Parameter @@ -184,9 +183,11 @@ class LiveChatAsync: await asyncio.sleep(diff_time) continuation = metadata.get('continuation') except ChatParseException as e: - logger.info(f"{str(e)}（video_id:\"{self.video_id}\"）") + self.terminate() + logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）") return except (TypeError , json.JSONDecodeError) : + self.terminate() logger.error(f"{traceback.format_exc(limit = -1)}") return @@ -215,6 +216,7 @@ class LiveChatAsync: else: logger.error(f"[{self.video_id}]" f"Exceeded retry count. status_code={status_code}") + self.terminate() return None return livechat_json diff --git a/pytchat/core_async/replaychat.py b/pytchat/core_async/replaychat.py index 9141e25..a36ae65 100644 --- a/pytchat/core_async/replaychat.py +++ b/pytchat/core_async/replaychat.py @@ -198,13 +198,15 @@ class ReplayChatAsync: await asyncio.sleep(diff_time) continuation = metadata.get('continuation') except ChatParseException as e: - logger.info(f"{str(e)}（video_id:\"{self.video_id}\"）") + logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）") return except (TypeError , json.JSONDecodeError) : logger.error(f"{traceback.format_exc(limit = -1)}") + self.terminate() return logger.debug(f"[{self.video_id}]チャット取得を終了しました。") + self.terminate() async def _get_livechat_json(self, continuation, session, headers): ''' @@ -286,7 +288,7 @@ class ReplayChatAsync: if self._direct_mode == False: #bufferにダミーオブジェクトを入れてis_alive()を判定させる self._buffer.put_nowait({'chatdata':'','timeout':1}) - logger.info(f'終了しました:[{self.video_id}]') + logger.info(f'[{self.video_id}]終了しました') @classmethod def _set_exception_handler(cls, handler): diff --git a/pytchat/core_multithread/livechat.py b/pytchat/core_multithread/livechat.py index 076a6f1..2ca3d5d 100644 --- a/pytchat/core_multithread/livechat.py +++ b/pytchat/core_multithread/livechat.py @@ -65,7 +65,7 @@ class LiveChat: _listeners= [] def __init__(self, video_id, processor = DefaultProcessor(), - buffer = Buffer(maxsize = 20), + buffer = None, interruptable = True, callback = None, done_callback = None, @@ -145,8 +145,8 @@ class LiveChat: def _listen(self, continuation): ''' continuationに紐付いたチャットデータを取得し - BUfferにチャットデータを格納、 - 次のcontinuaitonを取得してループする + Bufferにチャットデータを格納、 + 次のcontinuaitonを取得してループする。 Parameter --------- @@ -178,9 +178,11 @@ class LiveChat: time.sleep(diff_time) continuation = metadata.get('continuation') except ChatParseException as e: - logger.info(f"{str(e)}（video_id:\"{self.video_id}\"）") + self.terminate() + logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）") return except (TypeError , json.JSONDecodeError) : + self.terminate() logger.error(f"{traceback.format_exc(limit = -1)}") return @@ -209,6 +211,7 @@ class LiveChat: else: logger.error(f"[{self.video_id}]" f"Exceeded retry count. status_code={status_code}") + self.terminate() return None return livechat_json @@ -257,18 +260,10 @@ class LiveChat: if self._direct_mode == False: #bufferにダミーオブジェクトを入れてis_alive()を判定させる self._buffer.put({'chatdata':'','timeout':1}) - logger.info(f'終了しました:[{self.video_id}]') + logger.info(f'[{self.video_id}]終了しました') @classmethod def shutdown(cls, event, sig = None, handler=None): logger.debug("シャットダウンしています") for t in LiveChat._listeners: - t._is_alive = False - - - - - - - - + t._is_alive = False \ No newline at end of file diff --git a/pytchat/core_multithread/replaychat.py b/pytchat/core_multithread/replaychat.py index bc89aa1..448d6f6 100644 --- a/pytchat/core_multithread/replaychat.py +++ b/pytchat/core_multithread/replaychat.py @@ -18,9 +18,8 @@ from ..processors.default.processor import DefaultProcessor from ..processors.combinator import Combinator logger = config.logger(__name__) -MAX_RETRY = 10 headers = config.headers - +MAX_RETRY = 10 class ReplayChat: @@ -62,7 +61,7 @@ class ReplayChat: チャットデータ取得ループ（_listen）用のスレッド _is_alive : bool - チャット取得を終了したか + チャット取得を停止するためのフラグ ''' _setup_finished = False @@ -71,7 +70,7 @@ class ReplayChat: def __init__(self, video_id, seektime = 0, processor = DefaultProcessor(), - buffer = Buffer(maxsize = 20), + buffer = None, interruptable = True, callback = None, done_callback = None, @@ -94,6 +93,7 @@ class ReplayChat: self._pauser.put_nowait(None) self._setup() + if not ReplayChat._setup_finished: ReplayChat._setup_finished = True if interruptable: @@ -154,7 +154,7 @@ class ReplayChat: def _listen(self, continuation): ''' continuationに紐付いたチャットデータを取得し - にチャットデータを格納、 + BUfferにチャットデータを格納、 次のcontinuaitonを取得してループする Parameter @@ -193,9 +193,11 @@ class ReplayChat: time.sleep(diff_time) continuation = metadata.get('continuation') except ChatParseException as e: - logger.error(f"{str(e)}（動画ID:\"{self.video_id}\"）") + self.terminate() + logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）") return except (TypeError , json.JSONDecodeError) : + self.terminate() logger.error(f"{traceback.format_exc(limit = -1)}") return @@ -224,6 +226,7 @@ class ReplayChat: else: logger.error(f"[{self.video_id}]" f"Exceeded retry count. status_code={status_code}") + self.terminate() return None return livechat_json @@ -270,7 +273,7 @@ class ReplayChat: '''Listener終了時のコールバック''' try: self.terminate() - except CancelledError: + except RuntimeError: logger.debug(f'[{self.video_id}]cancelled:{sender}') def terminate(self): @@ -281,7 +284,7 @@ class ReplayChat: if self._direct_mode == False: #bufferにダミーオブジェクトを入れてis_alive()を判定させる self._buffer.put({'chatdata':'','timeout':1}) - logger.info(f'終了しました:[{self.video_id}]') + logger.info(f'[{self.video_id}]終了しました') @classmethod def shutdown(cls, event, sig = None, handler=None): diff --git a/pytchat/parser/live.py b/pytchat/parser/live.py index 4236c7f..908d1c6 100644 --- a/pytchat/parser/live.py +++ b/pytchat/parser/live.py @@ -59,7 +59,7 @@ class Parser: if metadata is None: unknown = list(cont.keys())[0] if unknown: - logger.error(f"Received unknown continuation type:{unknown}") + logger.debug(f"Received unknown continuation type:{unknown}") metadata = cont.get(unknown) metadata.setdefault('timeoutMs', 10000) chatdata = contents['liveChatContinuation'].get('actions') diff --git a/pytchat/parser/replay.py b/pytchat/parser/replay.py index 93a17b8..051e04e 100644 --- a/pytchat/parser/replay.py +++ b/pytchat/parser/replay.py @@ -9,7 +9,6 @@ from .. exceptions import ( logger = config.logger(__name__) - class Parser: def parse(self, jsn): """ @@ -53,12 +52,13 @@ class Parser: metadata = cont.get('liveChatReplayContinuationData') if metadata is None: unknown = list(cont.keys())[0] - if unknown: + if unknown != "playerSeekContinuationData": + logger.debug(f"Received unknown continuation type:{unknown}") metadata = cont.get(unknown) - actions = contents['liveChatContinuation'].get('actions') if actions is None: - raise NoContentsException('チャットデータを取得できませんでした。') + #後続のチャットデータなし + return {"continuation":None,"timeout":0,"chatdata":[]} interval = self.get_interval(actions) metadata.setdefault("timeoutMs",interval) """アーカイブ済みチャットはライブチャットと構造が異なっているため、以下の行により diff --git a/pytchat/util/__init__.py b/pytchat/util/__init__.py index 95ce56a..60be578 100644 --- a/pytchat/util/__init__.py +++ b/pytchat/util/__init__.py @@ -9,7 +9,7 @@ def download(url): json.dump(html.json(),f,ensure_ascii=False) -def save(data,filename): - with open(str(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S') - )+filename,mode ='w',encoding='utf-8') as f: +def save(data,filename,extention): + with open(filename+"_"+(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S') + )+extention,mode ='w',encoding='utf-8') as f: f.writelines(data) From b8bc00d880277cd22e0088118b516c332a5f86b0 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Mon, 30 Dec 2019 17:27:25 +0900 Subject: [PATCH 2/3] Fix README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e8bae28..218561d 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ from pytchat import ReplayChat def main(): chat = ReplayChat("ojes5ULOqhc", seektime = 60*30) - while True: + while chat.is_alive(): data = chat.get() for c in data.items: print(f"{c.elapsedTime} [{c.author.name}]-{c.message} {c.amountString}") From 582d0b749dcf702bdd31ed97000d852c281bf317 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Mon, 30 Dec 2019 17:32:23 +0900 Subject: [PATCH 3/3] Add comment --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 218561d..e8057cf 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ while chat.is_alive(): from pytchat import ReplayChat def main(): + #seektime (seconds): start position of chat. chat = ReplayChat("ojes5ULOqhc", seektime = 60*30) while chat.is_alive(): data = chat.get()