From 90b10a9f8fd60b13980624c6498bbf718b97303b Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Fri, 27 Dec 2019 02:19:08 +0900 Subject: [PATCH 1/8] Integrate rendering message and message_ex --- pytchat/processors/default/renderer/base.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/pytchat/processors/default/renderer/base.py b/pytchat/processors/default/renderer/base.py index e11e42f..4ea2a83 100644 --- a/pytchat/processors/default/renderer/base.py +++ b/pytchat/processors/default/renderer/base.py @@ -19,8 +19,7 @@ class BaseRenderer: else: self.elapsedTime = "" self.datetime = self.get_datetime(timestampUsec) - self.message = self.get_message(self.renderer) - self.messageEx = self.get_message_ex(self.renderer) + self.message ,self.messageEx = self.get_message(self.renderer) self.id = self.renderer.get('id') self.amountValue= 0.0 self.amountString = "" @@ -44,6 +43,7 @@ class BaseRenderer: def get_message(self,renderer): message = '' + message_ex = [] if renderer.get("message"): runs=renderer["message"].get("runs") if runs: @@ -51,22 +51,13 @@ class BaseRenderer: if r: if r.get('emoji'): message += r['emoji'].get('shortcuts',[''])[0] + message_ex.append(r['emoji']['image']['thumbnails'][1].get('url')) else: message += r.get('text','') - return message + message_ex.append(r.get('text','')) + return message, message_ex + - def get_message_ex(self,renderer): - message = [] - if renderer.get("message"): - runs=renderer["message"].get("runs") - if runs: - for r in runs: - if r: - if r.get('emoji'): - message.append(r['emoji']['image']['thumbnails'][1].get('url')) - else: - message.append(r.get('text','')) - return message def get_badges(self,renderer): isVerified = False From 4e956b8d84839d52f85ef29cfa3c1fae60c05f8b Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Mon, 30 Dec 2019 11:02:29 +0900 Subject: [PATCH 2/8] Implement Combinator, DummyProcessor --- pytchat/__init__.py | 5 ++-- pytchat/api.py | 1 + pytchat/core_async/livechat.py | 6 +++- pytchat/core_async/replaychat.py | 6 +++- pytchat/core_multithread/livechat.py | 6 +++- pytchat/core_multithread/replaychat.py | 6 +++- pytchat/processors/combinator.py | 39 ++++++++++++++++++++++++++ pytchat/processors/dummy_processor.py | 8 ++++++ 8 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 pytchat/processors/combinator.py create mode 100644 pytchat/processors/dummy_processor.py diff --git a/pytchat/__init__.py b/pytchat/__init__.py index 213ccdb..b87e086 100644 --- a/pytchat/__init__.py +++ b/pytchat/__init__.py @@ -19,5 +19,6 @@ from .api import ( CompatibleProcessor, SimpleDisplayProcessor, JsonfileArchiveProcessor, - SpeedCalculator - ) \ No newline at end of file + SpeedCalculator, + DummyProcessor +) \ No newline at end of file diff --git a/pytchat/api.py b/pytchat/api.py index 25c78e5..92164d6 100644 --- a/pytchat/api.py +++ b/pytchat/api.py @@ -8,3 +8,4 @@ from .processors.compatible.processor import CompatibleProcessor from .processors.simple_display_processor import SimpleDisplayProcessor from .processors.jsonfile_archive_processor import JsonfileArchiveProcessor from .processors.speed_calculator import SpeedCalculator +from .processors.dummy_processor import DummyProcessor diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index 77fbc3c..7d5e640 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -15,6 +15,7 @@ from .. import mylogger from ..exceptions import ChatParseException,IllegalFunctionCall from ..paramgen import liveparam from ..processors.default.processor import DefaultProcessor +from ..processors.combinator import Combinator logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) MAX_RETRY = 10 @@ -72,7 +73,10 @@ class LiveChatAsync: exception_handler = None, direct_mode = False): self.video_id = video_id - self.processor = processor + if isinstance(processor, tuple): + self.processor = Combinator(processor) + else: + self.processor = processor self._buffer = buffer self._callback = callback self._done_callback = done_callback diff --git a/pytchat/core_async/replaychat.py b/pytchat/core_async/replaychat.py index 80169d8..951c58e 100644 --- a/pytchat/core_async/replaychat.py +++ b/pytchat/core_async/replaychat.py @@ -16,6 +16,7 @@ from .. import mylogger from ..exceptions import ChatParseException,IllegalFunctionCall from ..paramgen import arcparam from ..processors.default.processor import DefaultProcessor +from ..processors.combinator import Combinator logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) MAX_RETRY = 10 @@ -78,7 +79,10 @@ class ReplayChatAsync: direct_mode = False): self.video_id = video_id self.seektime = seektime - self.processor = processor + if isinstance(processor, tuple): + self.processor = Combinator(processor) + else: + self.processor = processor self._buffer = buffer self._callback = callback self._done_callback = done_callback diff --git a/pytchat/core_multithread/livechat.py b/pytchat/core_multithread/livechat.py index 9d70b00..da9e52c 100644 --- a/pytchat/core_multithread/livechat.py +++ b/pytchat/core_multithread/livechat.py @@ -14,6 +14,7 @@ from .. import mylogger from ..exceptions import ChatParseException,IllegalFunctionCall from ..paramgen import liveparam from ..processors.default.processor import DefaultProcessor +from ..processors.combinator import Combinator logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) MAX_RETRY = 10 @@ -72,7 +73,10 @@ class LiveChat: direct_mode = False ): self.video_id = video_id - self.processor = processor + if isinstance(processor, tuple): + self.processor = Combinator(processor) + else: + self.processor = processor self._buffer = buffer self._callback = callback self._done_callback = done_callback diff --git a/pytchat/core_multithread/replaychat.py b/pytchat/core_multithread/replaychat.py index a72cf98..936f337 100644 --- a/pytchat/core_multithread/replaychat.py +++ b/pytchat/core_multithread/replaychat.py @@ -15,6 +15,7 @@ from .. import mylogger from ..exceptions import ChatParseException,IllegalFunctionCall from ..paramgen import arcparam from ..processors.default.processor import DefaultProcessor +from ..processors.combinator import Combinator logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) MAX_RETRY = 10 @@ -78,7 +79,10 @@ class ReplayChat: ): self.video_id = video_id self.seektime = seektime - self.processor = processor + if isinstance(processor, tuple): + self.processor = Combinator(processor) + else: + self.processor = processor self._buffer = buffer self._callback = callback self._done_callback = done_callback diff --git a/pytchat/processors/combinator.py b/pytchat/processors/combinator.py new file mode 100644 index 0000000..1b76df5 --- /dev/null +++ b/pytchat/processors/combinator.py @@ -0,0 +1,39 @@ +from .chat_processor import ChatProcessor + +class Combinator(ChatProcessor): + ''' + Combinator combines multiple chat processors. + Specify processors as tuple at `processor` params of LiveChat object. + + For example: + [constructor] + chat = LiveChat("video_id", processor = ( Processor1(), Processor2(), Processor3() ) + + [receive return values] + ret1, ret2, ret3 = chat.get() + + The return values are tuple of processed chat data, + the order of return depends on parameter order. + + Parameter + --------- + processors : Tuple[ChatProcessor] + multiple processors for processing chat data + ''' + + def __init__(self, processors: tuple): + self.processors = processors + + def process(self, chat_components: list): + ''' + Called from LiveChat.get() function by user, + or LiveChat._listen() automatically. + + Returns + ------- + Tuple of chat data processed by each chat processor. + ''' + return tuple([processor.process(chat_components) + for processor in self.processors]) + + diff --git a/pytchat/processors/dummy_processor.py b/pytchat/processors/dummy_processor.py new file mode 100644 index 0000000..e2e406d --- /dev/null +++ b/pytchat/processors/dummy_processor.py @@ -0,0 +1,8 @@ +from .chat_processor import ChatProcessor + +class DummyProcessor(ChatProcessor): + ''' + Dummy processor just returns received chat_components directly. + ''' + def process(self, chat_components: list): + return chat_components From 7305e4178b5842b1a2aade37bc81d4b6b8e427e4 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Mon, 30 Dec 2019 14:38:02 +0900 Subject: [PATCH 3/8] Change description of getting logger --- README.md | 69 ++++++++++------------ pytchat/__init__.py | 3 +- pytchat/api.py | 1 + pytchat/config/__init__.py | 11 +++- pytchat/config/mylogger.py | 32 ++++++++++ pytchat/core_async/livechat.py | 2 +- pytchat/core_async/replaychat.py | 2 +- pytchat/core_multithread/livechat.py | 5 +- pytchat/core_multithread/replaychat.py | 2 +- pytchat/parser/live.py | 2 +- pytchat/parser/replay.py | 2 +- pytchat/processors/compatible/processor.py | 2 +- pytchat/processors/default/processor.py | 2 +- 13 files changed, 85 insertions(+), 50 deletions(-) create mode 100644 pytchat/config/mylogger.py diff --git a/README.md b/README.md index ab904d4..e8bae28 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ pip install pytchat ```python from pytchat import LiveChat -chat = LiveChat("G1w62uEMZ74") +chat = LiveChat("rsHWP7IjMiw") while chat.is_alive(): data = chat.get() for c in data.items: @@ -40,17 +40,17 @@ while chat.is_alive(): from pytchat import LiveChat import time -def main() - chat = LiveChat("G1w62uEMZ74", callback = func) - while chat.is_alive(): - time.sleep(3) - #other background operation. - -#callback function is automatically called periodically. -def func(data): +#callback function is automatically called. +def display(data): for c in data.items: print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}") data.tick() + +#entry point +chat = LiveChat("rsHWP7IjMiw", callback = display) +while chat.is_alive(): + time.sleep(3) + #other background operation. ``` ### asyncio context: @@ -59,63 +59,56 @@ from pytchat import LiveChatAsync import asyncio async def main(): - chat = LiveChatAsync("G1w62uEMZ74", callback = func) + chat = LiveChatAsync("rsHWP7IjMiw", callback = func) while chat.is_alive(): await asyncio.sleep(3) #other background operation. -#callback function is automatically called periodically. +#callback function is automatically called. async def func(data): for c in data.items: print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}") await data.tick_async() -loop = asyncio.get_event_loop() -loop.run_until_complete(main()) +try: + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) +except CancelledError: + pass ``` ### youtube api compatible processor: ```python from pytchat import LiveChat, CompatibleProcessor +import time -chat = LiveChat("G1w62uEMZ74", +chat = LiveChat("rsHWP7IjMiw", processor = CompatibleProcessor() ) while chat.is_alive(): data = chat.get() - polling = data["pollingIntervalMillis"]/1000 - for c in data["items"]: - if c.get("snippet"): + polling = data['pollingIntervalMillis']/1000 + for c in data['items']: + if c.get('snippet'): print(f"[{c['authorDetails']['displayName']}]" f"-{c['snippet']['displayMessage']}") - time.sleep(polling/len(data["items"])) + time.sleep(polling/len(data['items'])) ``` ### replay: ```python -from pytchat import ReplayChatAsync -import asyncio +from pytchat import ReplayChat -async def main(): - chat = ReplayChatAsync("G1w62uEMZ74", seektime = 1000, callback = func) - while chat.is_alive(): - await asyncio.sleep(3) - #other background operation here. +def main(): + chat = ReplayChat("ojes5ULOqhc", seektime = 60*30) + while True: + data = chat.get() + for c in data.items: + print(f"{c.elapsedTime} [{c.author.name}]-{c.message} {c.amountString}") + data.tick() -#callback function is automatically called periodically. -async def func(data): - for count in range(0,len(data.items)): - c= data.items[count] - if count!=len(data.items): - tick=data.items[count+1].timestamp -data.items[count].timestamp - else: - tick=0 - print(f"<{c.elapsedTime}> [{c.author.name}]-{c.message} {c.amountString}") - await asyncio.sleep(tick/1000) - -loop = asyncio.get_event_loop() -loop.run_until_complete(main()) +main() ``` ## Structure of Default Processor diff --git a/pytchat/__init__.py b/pytchat/__init__.py index b87e086..65b61e4 100644 --- a/pytchat/__init__.py +++ b/pytchat/__init__.py @@ -2,7 +2,7 @@ pytchat is a python library for fetching youtube live chat without using yt api, Selenium, or BeautifulSoup. """ __copyright__ = 'Copyright (C) 2019 taizan-hokuto' -__version__ = '0.0.3.8' +__version__ = '0.0.3.9' __license__ = 'MIT' __author__ = 'taizan-hokuto' __author_email__ = '55448286+taizan-hokuto@users.noreply.github.com' @@ -11,6 +11,7 @@ __url__ = 'https://github.com/taizan-hokuto/pytchat' __all__ = ["core_async","core_multithread","processors"] from .api import ( + config, LiveChat, LiveChatAsync, ReplayChat, diff --git a/pytchat/api.py b/pytchat/api.py index 92164d6..10a7fba 100644 --- a/pytchat/api.py +++ b/pytchat/api.py @@ -9,3 +9,4 @@ from .processors.simple_display_processor import SimpleDisplayProcessor from .processors.jsonfile_archive_processor import JsonfileArchiveProcessor from .processors.speed_calculator import SpeedCalculator from .processors.dummy_processor import DummyProcessor +from . import config \ No newline at end of file diff --git a/pytchat/config/__init__.py b/pytchat/config/__init__.py index eb109d2..542551e 100644 --- a/pytchat/config/__init__.py +++ b/pytchat/config/__init__.py @@ -1,4 +1,13 @@ import logging -LOGGER_MODE = None +from . import mylogger + +LOGGER_MODE = logging.DEBUG + headers = { 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'} + +def logger(module_name: str): + module_logger = mylogger.get_logger(module_name, mode = LOGGER_MODE) + return module_logger + + diff --git a/pytchat/config/mylogger.py b/pytchat/config/mylogger.py new file mode 100644 index 0000000..83e325f --- /dev/null +++ b/pytchat/config/mylogger.py @@ -0,0 +1,32 @@ +from logging import NullHandler, getLogger, StreamHandler, FileHandler, Formatter +import logging +import datetime + + +def get_logger(modname,mode=logging.DEBUG): + logger = getLogger(modname) + if mode == None: + logger.addHandler(NullHandler()) + return logger + logger.setLevel(mode) + #create handler1 for showing info + handler1 = StreamHandler() + my_formatter = MyFormatter() + handler1.setFormatter(my_formatter) + + handler1.setLevel(mode) + logger.addHandler(handler1) + #create handler2 for recording log file + if mode <= logging.DEBUG: + handler2 = FileHandler(filename="log.txt", encoding='utf-8') + handler2.setLevel(logging.ERROR) + handler2.setFormatter(my_formatter) + + + logger.addHandler(handler2) + return logger + +class MyFormatter(logging.Formatter): + def format(self, record): + s =(datetime.datetime.fromtimestamp(record.created)).strftime("%m-%d %H:%M:%S")+'| '+ (record.module).ljust(15)+(' { '+record.funcName).ljust(20) +":"+str(record.lineno).rjust(4)+'} - '+record.getMessage() + return s diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index 7d5e640..8fbfbd4 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -17,7 +17,7 @@ from ..paramgen import liveparam from ..processors.default.processor import DefaultProcessor from ..processors.combinator import Combinator -logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) +logger = config.logger(__name__) MAX_RETRY = 10 headers = config.headers diff --git a/pytchat/core_async/replaychat.py b/pytchat/core_async/replaychat.py index 951c58e..9141e25 100644 --- a/pytchat/core_async/replaychat.py +++ b/pytchat/core_async/replaychat.py @@ -18,7 +18,7 @@ from ..paramgen import arcparam from ..processors.default.processor import DefaultProcessor from ..processors.combinator import Combinator -logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) +logger = config.logger(__name__) MAX_RETRY = 10 headers = config.headers diff --git a/pytchat/core_multithread/livechat.py b/pytchat/core_multithread/livechat.py index da9e52c..076a6f1 100644 --- a/pytchat/core_multithread/livechat.py +++ b/pytchat/core_multithread/livechat.py @@ -16,10 +16,9 @@ from ..paramgen import liveparam from ..processors.default.processor import DefaultProcessor from ..processors.combinator import Combinator -logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) -MAX_RETRY = 10 +logger = config.logger(__name__) headers = config.headers - +MAX_RETRY = 10 class LiveChat: diff --git a/pytchat/core_multithread/replaychat.py b/pytchat/core_multithread/replaychat.py index 936f337..bc89aa1 100644 --- a/pytchat/core_multithread/replaychat.py +++ b/pytchat/core_multithread/replaychat.py @@ -17,7 +17,7 @@ from ..paramgen import arcparam from ..processors.default.processor import DefaultProcessor from ..processors.combinator import Combinator -logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) +logger = config.logger(__name__) MAX_RETRY = 10 headers = config.headers diff --git a/pytchat/parser/live.py b/pytchat/parser/live.py index a8cf12c..4236c7f 100644 --- a/pytchat/parser/live.py +++ b/pytchat/parser/live.py @@ -13,7 +13,7 @@ from .. exceptions import ( NoContinuationsException ) -logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) +logger = config.logger(__name__) class Parser: diff --git a/pytchat/parser/replay.py b/pytchat/parser/replay.py index fc27a8d..93a17b8 100644 --- a/pytchat/parser/replay.py +++ b/pytchat/parser/replay.py @@ -7,7 +7,7 @@ from .. exceptions import ( NoContinuationsException ) -logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) +logger = config.logger(__name__) class Parser: diff --git a/pytchat/processors/compatible/processor.py b/pytchat/processors/compatible/processor.py index 31e1b15..32a3a64 100644 --- a/pytchat/processors/compatible/processor.py +++ b/pytchat/processors/compatible/processor.py @@ -7,7 +7,7 @@ from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer from .. chat_processor import ChatProcessor from ... import mylogger from ... import config -logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) +logger = config.logger(__name__) class CompatibleProcessor(ChatProcessor): diff --git a/pytchat/processors/default/processor.py b/pytchat/processors/default/processor.py index be3dfd6..4bff021 100644 --- a/pytchat/processors/default/processor.py +++ b/pytchat/processors/default/processor.py @@ -7,7 +7,7 @@ from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer from .. chat_processor import ChatProcessor from ... import config from ... import mylogger -logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) +logger = config.logger(__name__) class Chatdata: def __init__(self,chatlist:list, timeout:float): From ce96d94e2363ba3446ffe55bc4099d526ab0c625 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Mon, 30 Dec 2019 17:10:34 +0900 Subject: [PATCH 4/8] Fix termination of fetching chat data --- pytchat/core_async/livechat.py | 10 ++++++---- pytchat/core_async/replaychat.py | 6 ++++-- pytchat/core_multithread/livechat.py | 23 +++++++++-------------- pytchat/core_multithread/replaychat.py | 19 +++++++++++-------- pytchat/parser/live.py | 2 +- pytchat/parser/replay.py | 8 ++++---- pytchat/util/__init__.py | 6 +++--- 7 files changed, 38 insertions(+), 36 deletions(-) diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index 8fbfbd4..639184f 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -18,9 +18,8 @@ from ..processors.default.processor import DefaultProcessor from ..processors.combinator import Combinator logger = config.logger(__name__) -MAX_RETRY = 10 headers = config.headers - +MAX_RETRY = 10 class LiveChatAsync: @@ -152,7 +151,7 @@ class LiveChatAsync: async def _listen(self, continuation): ''' continuationに紐付いたチャットデータを取得し - チャットデータを格納、 + Bufferにチャットデータを格納、 次のcontinuaitonを取得してループする。 Parameter @@ -184,9 +183,11 @@ class LiveChatAsync: await asyncio.sleep(diff_time) continuation = metadata.get('continuation') except ChatParseException as e: - logger.info(f"{str(e)}（video_id:\"{self.video_id}\"）") + self.terminate() + logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）") return except (TypeError , json.JSONDecodeError) : + self.terminate() logger.error(f"{traceback.format_exc(limit = -1)}") return @@ -215,6 +216,7 @@ class LiveChatAsync: else: logger.error(f"[{self.video_id}]" f"Exceeded retry count. status_code={status_code}") + self.terminate() return None return livechat_json diff --git a/pytchat/core_async/replaychat.py b/pytchat/core_async/replaychat.py index 9141e25..a36ae65 100644 --- a/pytchat/core_async/replaychat.py +++ b/pytchat/core_async/replaychat.py @@ -198,13 +198,15 @@ class ReplayChatAsync: await asyncio.sleep(diff_time) continuation = metadata.get('continuation') except ChatParseException as e: - logger.info(f"{str(e)}（video_id:\"{self.video_id}\"）") + logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）") return except (TypeError , json.JSONDecodeError) : logger.error(f"{traceback.format_exc(limit = -1)}") + self.terminate() return logger.debug(f"[{self.video_id}]チャット取得を終了しました。") + self.terminate() async def _get_livechat_json(self, continuation, session, headers): ''' @@ -286,7 +288,7 @@ class ReplayChatAsync: if self._direct_mode == False: #bufferにダミーオブジェクトを入れてis_alive()を判定させる self._buffer.put_nowait({'chatdata':'','timeout':1}) - logger.info(f'終了しました:[{self.video_id}]') + logger.info(f'[{self.video_id}]終了しました') @classmethod def _set_exception_handler(cls, handler): diff --git a/pytchat/core_multithread/livechat.py b/pytchat/core_multithread/livechat.py index 076a6f1..2ca3d5d 100644 --- a/pytchat/core_multithread/livechat.py +++ b/pytchat/core_multithread/livechat.py @@ -65,7 +65,7 @@ class LiveChat: _listeners= [] def __init__(self, video_id, processor = DefaultProcessor(), - buffer = Buffer(maxsize = 20), + buffer = None, interruptable = True, callback = None, done_callback = None, @@ -145,8 +145,8 @@ class LiveChat: def _listen(self, continuation): ''' continuationに紐付いたチャットデータを取得し - BUfferにチャットデータを格納、 - 次のcontinuaitonを取得してループする + Bufferにチャットデータを格納、 + 次のcontinuaitonを取得してループする。 Parameter --------- @@ -178,9 +178,11 @@ class LiveChat: time.sleep(diff_time) continuation = metadata.get('continuation') except ChatParseException as e: - logger.info(f"{str(e)}（video_id:\"{self.video_id}\"）") + self.terminate() + logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）") return except (TypeError , json.JSONDecodeError) : + self.terminate() logger.error(f"{traceback.format_exc(limit = -1)}") return @@ -209,6 +211,7 @@ class LiveChat: else: logger.error(f"[{self.video_id}]" f"Exceeded retry count. status_code={status_code}") + self.terminate() return None return livechat_json @@ -257,18 +260,10 @@ class LiveChat: if self._direct_mode == False: #bufferにダミーオブジェクトを入れてis_alive()を判定させる self._buffer.put({'chatdata':'','timeout':1}) - logger.info(f'終了しました:[{self.video_id}]') + logger.info(f'[{self.video_id}]終了しました') @classmethod def shutdown(cls, event, sig = None, handler=None): logger.debug("シャットダウンしています") for t in LiveChat._listeners: - t._is_alive = False - - - - - - - - + t._is_alive = False \ No newline at end of file diff --git a/pytchat/core_multithread/replaychat.py b/pytchat/core_multithread/replaychat.py index bc89aa1..448d6f6 100644 --- a/pytchat/core_multithread/replaychat.py +++ b/pytchat/core_multithread/replaychat.py @@ -18,9 +18,8 @@ from ..processors.default.processor import DefaultProcessor from ..processors.combinator import Combinator logger = config.logger(__name__) -MAX_RETRY = 10 headers = config.headers - +MAX_RETRY = 10 class ReplayChat: @@ -62,7 +61,7 @@ class ReplayChat: チャットデータ取得ループ（_listen）用のスレッド _is_alive : bool - チャット取得を終了したか + チャット取得を停止するためのフラグ ''' _setup_finished = False @@ -71,7 +70,7 @@ class ReplayChat: def __init__(self, video_id, seektime = 0, processor = DefaultProcessor(), - buffer = Buffer(maxsize = 20), + buffer = None, interruptable = True, callback = None, done_callback = None, @@ -94,6 +93,7 @@ class ReplayChat: self._pauser.put_nowait(None) self._setup() + if not ReplayChat._setup_finished: ReplayChat._setup_finished = True if interruptable: @@ -154,7 +154,7 @@ class ReplayChat: def _listen(self, continuation): ''' continuationに紐付いたチャットデータを取得し - にチャットデータを格納、 + BUfferにチャットデータを格納、 次のcontinuaitonを取得してループする Parameter @@ -193,9 +193,11 @@ class ReplayChat: time.sleep(diff_time) continuation = metadata.get('continuation') except ChatParseException as e: - logger.error(f"{str(e)}（動画ID:\"{self.video_id}\"）") + self.terminate() + logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）") return except (TypeError , json.JSONDecodeError) : + self.terminate() logger.error(f"{traceback.format_exc(limit = -1)}") return @@ -224,6 +226,7 @@ class ReplayChat: else: logger.error(f"[{self.video_id}]" f"Exceeded retry count. status_code={status_code}") + self.terminate() return None return livechat_json @@ -270,7 +273,7 @@ class ReplayChat: '''Listener終了時のコールバック''' try: self.terminate() - except CancelledError: + except RuntimeError: logger.debug(f'[{self.video_id}]cancelled:{sender}') def terminate(self): @@ -281,7 +284,7 @@ class ReplayChat: if self._direct_mode == False: #bufferにダミーオブジェクトを入れてis_alive()を判定させる self._buffer.put({'chatdata':'','timeout':1}) - logger.info(f'終了しました:[{self.video_id}]') + logger.info(f'[{self.video_id}]終了しました') @classmethod def shutdown(cls, event, sig = None, handler=None): diff --git a/pytchat/parser/live.py b/pytchat/parser/live.py index 4236c7f..908d1c6 100644 --- a/pytchat/parser/live.py +++ b/pytchat/parser/live.py @@ -59,7 +59,7 @@ class Parser: if metadata is None: unknown = list(cont.keys())[0] if unknown: - logger.error(f"Received unknown continuation type:{unknown}") + logger.debug(f"Received unknown continuation type:{unknown}") metadata = cont.get(unknown) metadata.setdefault('timeoutMs', 10000) chatdata = contents['liveChatContinuation'].get('actions') diff --git a/pytchat/parser/replay.py b/pytchat/parser/replay.py index 93a17b8..051e04e 100644 --- a/pytchat/parser/replay.py +++ b/pytchat/parser/replay.py @@ -9,7 +9,6 @@ from .. exceptions import ( logger = config.logger(__name__) - class Parser: def parse(self, jsn): """ @@ -53,12 +52,13 @@ class Parser: metadata = cont.get('liveChatReplayContinuationData') if metadata is None: unknown = list(cont.keys())[0] - if unknown: + if unknown != "playerSeekContinuationData": + logger.debug(f"Received unknown continuation type:{unknown}") metadata = cont.get(unknown) - actions = contents['liveChatContinuation'].get('actions') if actions is None: - raise NoContentsException('チャットデータを取得できませんでした。') + #後続のチャットデータなし + return {"continuation":None,"timeout":0,"chatdata":[]} interval = self.get_interval(actions) metadata.setdefault("timeoutMs",interval) """アーカイブ済みチャットはライブチャットと構造が異なっているため、以下の行により diff --git a/pytchat/util/__init__.py b/pytchat/util/__init__.py index 95ce56a..60be578 100644 --- a/pytchat/util/__init__.py +++ b/pytchat/util/__init__.py @@ -9,7 +9,7 @@ def download(url): json.dump(html.json(),f,ensure_ascii=False) -def save(data,filename): - with open(str(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S') - )+filename,mode ='w',encoding='utf-8') as f: +def save(data,filename,extention): + with open(filename+"_"+(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S') + )+extention,mode ='w',encoding='utf-8') as f: f.writelines(data) From b8bc00d880277cd22e0088118b516c332a5f86b0 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Mon, 30 Dec 2019 17:27:25 +0900 Subject: [PATCH 5/8] Fix README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e8bae28..218561d 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ from pytchat import ReplayChat def main(): chat = ReplayChat("ojes5ULOqhc", seektime = 60*30) - while True: + while chat.is_alive(): data = chat.get() for c in data.items: print(f"{c.elapsedTime} [{c.author.name}]-{c.message} {c.amountString}") From 582d0b749dcf702bdd31ed97000d852c281bf317 Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Mon, 30 Dec 2019 17:32:23 +0900 Subject: [PATCH 6/8] Add comment --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 218561d..e8057cf 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ while chat.is_alive(): from pytchat import ReplayChat def main(): + #seektime (seconds): start position of chat. chat = ReplayChat("ojes5ULOqhc", seektime = 60*30) while chat.is_alive(): data = chat.get() From 52689618541c3e1fb131d8ff8b7970c01c599caa Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Mon, 30 Dec 2019 17:54:53 +0900 Subject: [PATCH 7/8] Delete unnecessary lines of old logger --- pytchat/core_async/livechat.py | 1 - pytchat/core_async/replaychat.py | 1 - pytchat/core_multithread/livechat.py | 1 - pytchat/core_multithread/replaychat.py | 1 - pytchat/parser/live.py | 1 - pytchat/parser/replay.py | 1 - pytchat/processors/compatible/processor.py | 1 - pytchat/processors/default/processor.py | 1 - 8 files changed, 8 deletions(-) diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index 639184f..c671d1a 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -11,7 +11,6 @@ from concurrent.futures import CancelledError from .buffer import Buffer from ..parser.live import Parser from .. import config -from .. import mylogger from ..exceptions import ChatParseException,IllegalFunctionCall from ..paramgen import liveparam from ..processors.default.processor import DefaultProcessor diff --git a/pytchat/core_async/replaychat.py b/pytchat/core_async/replaychat.py index a36ae65..95499fe 100644 --- a/pytchat/core_async/replaychat.py +++ b/pytchat/core_async/replaychat.py @@ -12,7 +12,6 @@ from queue import Queue from .buffer import Buffer from ..parser.replay import Parser from .. import config -from .. import mylogger from ..exceptions import ChatParseException,IllegalFunctionCall from ..paramgen import arcparam from ..processors.default.processor import DefaultProcessor diff --git a/pytchat/core_multithread/livechat.py b/pytchat/core_multithread/livechat.py index 2ca3d5d..30b9249 100644 --- a/pytchat/core_multithread/livechat.py +++ b/pytchat/core_multithread/livechat.py @@ -10,7 +10,6 @@ from concurrent.futures import CancelledError, ThreadPoolExecutor from .buffer import Buffer from ..parser.live import Parser from .. import config -from .. import mylogger from ..exceptions import ChatParseException,IllegalFunctionCall from ..paramgen import liveparam from ..processors.default.processor import DefaultProcessor diff --git a/pytchat/core_multithread/replaychat.py b/pytchat/core_multithread/replaychat.py index 448d6f6..43b5e2e 100644 --- a/pytchat/core_multithread/replaychat.py +++ b/pytchat/core_multithread/replaychat.py @@ -11,7 +11,6 @@ from queue import Queue from .buffer import Buffer from ..parser.replay import Parser from .. import config -from .. import mylogger from ..exceptions import ChatParseException,IllegalFunctionCall from ..paramgen import arcparam from ..processors.default.processor import DefaultProcessor diff --git a/pytchat/parser/live.py b/pytchat/parser/live.py index 908d1c6..e32177f 100644 --- a/pytchat/parser/live.py +++ b/pytchat/parser/live.py @@ -6,7 +6,6 @@ This module is parser of live chat JSON. import json from .. import config -from .. import mylogger from .. exceptions import ( ResponseContextError, NoContentsException, diff --git a/pytchat/parser/replay.py b/pytchat/parser/replay.py index 051e04e..7399238 100644 --- a/pytchat/parser/replay.py +++ b/pytchat/parser/replay.py @@ -1,6 +1,5 @@ import json from .. import config -from .. import mylogger from .. exceptions import ( ResponseContextError, NoContentsException, diff --git a/pytchat/processors/compatible/processor.py b/pytchat/processors/compatible/processor.py index 32a3a64..9b799d3 100644 --- a/pytchat/processors/compatible/processor.py +++ b/pytchat/processors/compatible/processor.py @@ -5,7 +5,6 @@ from .renderer.paidmessage import LiveChatPaidMessageRenderer from .renderer.paidsticker import LiveChatPaidStickerRenderer from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer from .. chat_processor import ChatProcessor -from ... import mylogger from ... import config logger = config.logger(__name__) diff --git a/pytchat/processors/default/processor.py b/pytchat/processors/default/processor.py index 4bff021..8c33c8e 100644 --- a/pytchat/processors/default/processor.py +++ b/pytchat/processors/default/processor.py @@ -6,7 +6,6 @@ from .renderer.paidsticker import LiveChatPaidStickerRenderer from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer from .. chat_processor import ChatProcessor from ... import config -from ... import mylogger logger = config.logger(__name__) class Chatdata: From f24c5f9e301bc794add8e7d679a48f4bbfb3c39a Mon Sep 17 00:00:00 2001 From: taizan-hokuto <55448286+taizan-hokuto@users.noreply.github.com> Date: Mon, 30 Dec 2019 18:32:17 +0900 Subject: [PATCH 8/8] Increment version --- pytchat/__init__.py | 2 +- pytchat/config/__init__.py | 2 +- setup.py | 14 ++++++++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pytchat/__init__.py b/pytchat/__init__.py index 65b61e4..5287d42 100644 --- a/pytchat/__init__.py +++ b/pytchat/__init__.py @@ -2,7 +2,7 @@ pytchat is a python library for fetching youtube live chat without using yt api, Selenium, or BeautifulSoup. """ __copyright__ = 'Copyright (C) 2019 taizan-hokuto' -__version__ = '0.0.3.9' +__version__ = '0.0.4.0' __license__ = 'MIT' __author__ = 'taizan-hokuto' __author_email__ = '55448286+taizan-hokuto@users.noreply.github.com' diff --git a/pytchat/config/__init__.py b/pytchat/config/__init__.py index 542551e..1a84b59 100644 --- a/pytchat/config/__init__.py +++ b/pytchat/config/__init__.py @@ -1,7 +1,7 @@ import logging from . import mylogger -LOGGER_MODE = logging.DEBUG +LOGGER_MODE = None headers = { 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'} diff --git a/setup.py b/setup.py index 858fd1a..9728c33 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages, Command -from codecs import open -from os import path, system +#from codecs import open as open_c +from os import path, system, remove, rename import re package_name = "pytchat" @@ -28,6 +28,16 @@ assert author assert author_email assert url + + +with open('README.MD', 'r', encoding='utf-8') as f: + txt = f.read() + +with open('README1.MD', 'w', encoding='utf-8', newline='\n') as f: + f.write(txt) + +remove("README.MD") +rename("README1.MD","README.MD") with open('README.md', encoding='utf-8') as f: long_description = f.read()