diff --git a/README.md b/README.md index ab904d4..e8057cf 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ pip install pytchat ```python from pytchat import LiveChat -chat = LiveChat("G1w62uEMZ74") +chat = LiveChat("rsHWP7IjMiw") while chat.is_alive(): data = chat.get() for c in data.items: @@ -40,17 +40,17 @@ while chat.is_alive(): from pytchat import LiveChat import time -def main() - chat = LiveChat("G1w62uEMZ74", callback = func) - while chat.is_alive(): - time.sleep(3) - #other background operation. - -#callback function is automatically called periodically. -def func(data): +#callback function is automatically called. +def display(data): for c in data.items: print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}") data.tick() + +#entry point +chat = LiveChat("rsHWP7IjMiw", callback = display) +while chat.is_alive(): + time.sleep(3) + #other background operation. ``` ### asyncio context: @@ -59,63 +59,57 @@ from pytchat import LiveChatAsync import asyncio async def main(): - chat = LiveChatAsync("G1w62uEMZ74", callback = func) + chat = LiveChatAsync("rsHWP7IjMiw", callback = func) while chat.is_alive(): await asyncio.sleep(3) #other background operation. -#callback function is automatically called periodically. +#callback function is automatically called. async def func(data): for c in data.items: print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}") await data.tick_async() -loop = asyncio.get_event_loop() -loop.run_until_complete(main()) +try: + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) +except CancelledError: + pass ``` ### youtube api compatible processor: ```python from pytchat import LiveChat, CompatibleProcessor +import time -chat = LiveChat("G1w62uEMZ74", +chat = LiveChat("rsHWP7IjMiw", processor = CompatibleProcessor() ) while chat.is_alive(): data = chat.get() - polling = data["pollingIntervalMillis"]/1000 - for c in data["items"]: - if c.get("snippet"): + polling = data['pollingIntervalMillis']/1000 + for c in data['items']: + if c.get('snippet'): print(f"[{c['authorDetails']['displayName']}]" f"-{c['snippet']['displayMessage']}") - time.sleep(polling/len(data["items"])) + time.sleep(polling/len(data['items'])) ``` ### replay: ```python -from pytchat import ReplayChatAsync -import asyncio +from pytchat import ReplayChat -async def main(): - chat = ReplayChatAsync("G1w62uEMZ74", seektime = 1000, callback = func) +def main(): + #seektime (seconds): start position of chat. + chat = ReplayChat("ojes5ULOqhc", seektime = 60*30) while chat.is_alive(): - await asyncio.sleep(3) - #other background operation here. + data = chat.get() + for c in data.items: + print(f"{c.elapsedTime} [{c.author.name}]-{c.message} {c.amountString}") + data.tick() -#callback function is automatically called periodically. -async def func(data): - for count in range(0,len(data.items)): - c= data.items[count] - if count!=len(data.items): - tick=data.items[count+1].timestamp -data.items[count].timestamp - else: - tick=0 - print(f"<{c.elapsedTime}> [{c.author.name}]-{c.message} {c.amountString}") - await asyncio.sleep(tick/1000) - -loop = asyncio.get_event_loop() -loop.run_until_complete(main()) +main() ``` ## Structure of Default Processor diff --git a/pytchat/__init__.py b/pytchat/__init__.py index 213ccdb..5287d42 100644 --- a/pytchat/__init__.py +++ b/pytchat/__init__.py @@ -2,7 +2,7 @@ pytchat is a python library for fetching youtube live chat without using yt api, Selenium, or BeautifulSoup. """ __copyright__ = 'Copyright (C) 2019 taizan-hokuto' -__version__ = '0.0.3.8' +__version__ = '0.0.4.0' __license__ = 'MIT' __author__ = 'taizan-hokuto' __author_email__ = '55448286+taizan-hokuto@users.noreply.github.com' @@ -11,6 +11,7 @@ __url__ = 'https://github.com/taizan-hokuto/pytchat' __all__ = ["core_async","core_multithread","processors"] from .api import ( + config, LiveChat, LiveChatAsync, ReplayChat, @@ -19,5 +20,6 @@ from .api import ( CompatibleProcessor, SimpleDisplayProcessor, JsonfileArchiveProcessor, - SpeedCalculator - ) \ No newline at end of file + SpeedCalculator, + DummyProcessor +) \ No newline at end of file diff --git a/pytchat/api.py b/pytchat/api.py index 25c78e5..10a7fba 100644 --- a/pytchat/api.py +++ b/pytchat/api.py @@ -8,3 +8,5 @@ from .processors.compatible.processor import CompatibleProcessor from .processors.simple_display_processor import SimpleDisplayProcessor from .processors.jsonfile_archive_processor import JsonfileArchiveProcessor from .processors.speed_calculator import SpeedCalculator +from .processors.dummy_processor import DummyProcessor +from . import config \ No newline at end of file diff --git a/pytchat/config/__init__.py b/pytchat/config/__init__.py index eb109d2..1a84b59 100644 --- a/pytchat/config/__init__.py +++ b/pytchat/config/__init__.py @@ -1,4 +1,13 @@ import logging +from . import mylogger + LOGGER_MODE = None + headers = { 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'} + +def logger(module_name: str): + module_logger = mylogger.get_logger(module_name, mode = LOGGER_MODE) + return module_logger + + diff --git a/pytchat/config/mylogger.py b/pytchat/config/mylogger.py new file mode 100644 index 0000000..83e325f --- /dev/null +++ b/pytchat/config/mylogger.py @@ -0,0 +1,32 @@ +from logging import NullHandler, getLogger, StreamHandler, FileHandler, Formatter +import logging +import datetime + + +def get_logger(modname,mode=logging.DEBUG): + logger = getLogger(modname) + if mode == None: + logger.addHandler(NullHandler()) + return logger + logger.setLevel(mode) + #create handler1 for showing info + handler1 = StreamHandler() + my_formatter = MyFormatter() + handler1.setFormatter(my_formatter) + + handler1.setLevel(mode) + logger.addHandler(handler1) + #create handler2 for recording log file + if mode <= logging.DEBUG: + handler2 = FileHandler(filename="log.txt", encoding='utf-8') + handler2.setLevel(logging.ERROR) + handler2.setFormatter(my_formatter) + + + logger.addHandler(handler2) + return logger + +class MyFormatter(logging.Formatter): + def format(self, record): + s =(datetime.datetime.fromtimestamp(record.created)).strftime("%m-%d %H:%M:%S")+'| '+ (record.module).ljust(15)+(' { '+record.funcName).ljust(20) +":"+str(record.lineno).rjust(4)+'} - '+record.getMessage() + return s diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index 77fbc3c..c671d1a 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -11,15 +11,14 @@ from concurrent.futures import CancelledError from .buffer import Buffer from ..parser.live import Parser from .. import config -from .. import mylogger from ..exceptions import ChatParseException,IllegalFunctionCall from ..paramgen import liveparam from ..processors.default.processor import DefaultProcessor +from ..processors.combinator import Combinator -logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) -MAX_RETRY = 10 +logger = config.logger(__name__) headers = config.headers - +MAX_RETRY = 10 class LiveChatAsync: @@ -72,7 +71,10 @@ class LiveChatAsync: exception_handler = None, direct_mode = False): self.video_id = video_id - self.processor = processor + if isinstance(processor, tuple): + self.processor = Combinator(processor) + else: + self.processor = processor self._buffer = buffer self._callback = callback self._done_callback = done_callback @@ -148,7 +150,7 @@ class LiveChatAsync: async def _listen(self, continuation): ''' continuationに紐付いたチャットデータを取得し - チャットデータを格納、 + Bufferにチャットデータを格納、 次のcontinuaitonを取得してループする。 Parameter @@ -180,9 +182,11 @@ class LiveChatAsync: await asyncio.sleep(diff_time) continuation = metadata.get('continuation') except ChatParseException as e: - logger.info(f"{str(e)}（video_id:\"{self.video_id}\"）") + self.terminate() + logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）") return except (TypeError , json.JSONDecodeError) : + self.terminate() logger.error(f"{traceback.format_exc(limit = -1)}") return @@ -211,6 +215,7 @@ class LiveChatAsync: else: logger.error(f"[{self.video_id}]" f"Exceeded retry count. status_code={status_code}") + self.terminate() return None return livechat_json diff --git a/pytchat/core_async/replaychat.py b/pytchat/core_async/replaychat.py index 80169d8..95499fe 100644 --- a/pytchat/core_async/replaychat.py +++ b/pytchat/core_async/replaychat.py @@ -12,12 +12,12 @@ from queue import Queue from .buffer import Buffer from ..parser.replay import Parser from .. import config -from .. import mylogger from ..exceptions import ChatParseException,IllegalFunctionCall from ..paramgen import arcparam from ..processors.default.processor import DefaultProcessor +from ..processors.combinator import Combinator -logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) +logger = config.logger(__name__) MAX_RETRY = 10 headers = config.headers @@ -78,7 +78,10 @@ class ReplayChatAsync: direct_mode = False): self.video_id = video_id self.seektime = seektime - self.processor = processor + if isinstance(processor, tuple): + self.processor = Combinator(processor) + else: + self.processor = processor self._buffer = buffer self._callback = callback self._done_callback = done_callback @@ -194,13 +197,15 @@ class ReplayChatAsync: await asyncio.sleep(diff_time) continuation = metadata.get('continuation') except ChatParseException as e: - logger.info(f"{str(e)}（video_id:\"{self.video_id}\"）") + logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）") return except (TypeError , json.JSONDecodeError) : logger.error(f"{traceback.format_exc(limit = -1)}") + self.terminate() return logger.debug(f"[{self.video_id}]チャット取得を終了しました。") + self.terminate() async def _get_livechat_json(self, continuation, session, headers): ''' @@ -282,7 +287,7 @@ class ReplayChatAsync: if self._direct_mode == False: #bufferにダミーオブジェクトを入れてis_alive()を判定させる self._buffer.put_nowait({'chatdata':'','timeout':1}) - logger.info(f'終了しました:[{self.video_id}]') + logger.info(f'[{self.video_id}]終了しました') @classmethod def _set_exception_handler(cls, handler): diff --git a/pytchat/core_multithread/livechat.py b/pytchat/core_multithread/livechat.py index 9d70b00..30b9249 100644 --- a/pytchat/core_multithread/livechat.py +++ b/pytchat/core_multithread/livechat.py @@ -10,15 +10,14 @@ from concurrent.futures import CancelledError, ThreadPoolExecutor from .buffer import Buffer from ..parser.live import Parser from .. import config -from .. import mylogger from ..exceptions import ChatParseException,IllegalFunctionCall from ..paramgen import liveparam from ..processors.default.processor import DefaultProcessor +from ..processors.combinator import Combinator -logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) -MAX_RETRY = 10 +logger = config.logger(__name__) headers = config.headers - +MAX_RETRY = 10 class LiveChat: @@ -65,14 +64,17 @@ class LiveChat: _listeners= [] def __init__(self, video_id, processor = DefaultProcessor(), - buffer = Buffer(maxsize = 20), + buffer = None, interruptable = True, callback = None, done_callback = None, direct_mode = False ): self.video_id = video_id - self.processor = processor + if isinstance(processor, tuple): + self.processor = Combinator(processor) + else: + self.processor = processor self._buffer = buffer self._callback = callback self._done_callback = done_callback @@ -142,8 +144,8 @@ class LiveChat: def _listen(self, continuation): ''' continuationに紐付いたチャットデータを取得し - BUfferにチャットデータを格納、 - 次のcontinuaitonを取得してループする + Bufferにチャットデータを格納、 + 次のcontinuaitonを取得してループする。 Parameter --------- @@ -175,9 +177,11 @@ class LiveChat: time.sleep(diff_time) continuation = metadata.get('continuation') except ChatParseException as e: - logger.info(f"{str(e)}（video_id:\"{self.video_id}\"）") + self.terminate() + logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）") return except (TypeError , json.JSONDecodeError) : + self.terminate() logger.error(f"{traceback.format_exc(limit = -1)}") return @@ -206,6 +210,7 @@ class LiveChat: else: logger.error(f"[{self.video_id}]" f"Exceeded retry count. status_code={status_code}") + self.terminate() return None return livechat_json @@ -254,18 +259,10 @@ class LiveChat: if self._direct_mode == False: #bufferにダミーオブジェクトを入れてis_alive()を判定させる self._buffer.put({'chatdata':'','timeout':1}) - logger.info(f'終了しました:[{self.video_id}]') + logger.info(f'[{self.video_id}]終了しました') @classmethod def shutdown(cls, event, sig = None, handler=None): logger.debug("シャットダウンしています") for t in LiveChat._listeners: - t._is_alive = False - - - - - - - - + t._is_alive = False \ No newline at end of file diff --git a/pytchat/core_multithread/replaychat.py b/pytchat/core_multithread/replaychat.py index a72cf98..43b5e2e 100644 --- a/pytchat/core_multithread/replaychat.py +++ b/pytchat/core_multithread/replaychat.py @@ -11,15 +11,14 @@ from queue import Queue from .buffer import Buffer from ..parser.replay import Parser from .. import config -from .. import mylogger from ..exceptions import ChatParseException,IllegalFunctionCall from ..paramgen import arcparam from ..processors.default.processor import DefaultProcessor +from ..processors.combinator import Combinator -logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) -MAX_RETRY = 10 +logger = config.logger(__name__) headers = config.headers - +MAX_RETRY = 10 class ReplayChat: @@ -61,7 +60,7 @@ class ReplayChat: チャットデータ取得ループ（_listen）用のスレッド _is_alive : bool - チャット取得を終了したか + チャット取得を停止するためのフラグ ''' _setup_finished = False @@ -70,7 +69,7 @@ class ReplayChat: def __init__(self, video_id, seektime = 0, processor = DefaultProcessor(), - buffer = Buffer(maxsize = 20), + buffer = None, interruptable = True, callback = None, done_callback = None, @@ -78,7 +77,10 @@ class ReplayChat: ): self.video_id = video_id self.seektime = seektime - self.processor = processor + if isinstance(processor, tuple): + self.processor = Combinator(processor) + else: + self.processor = processor self._buffer = buffer self._callback = callback self._done_callback = done_callback @@ -90,6 +92,7 @@ class ReplayChat: self._pauser.put_nowait(None) self._setup() + if not ReplayChat._setup_finished: ReplayChat._setup_finished = True if interruptable: @@ -150,7 +153,7 @@ class ReplayChat: def _listen(self, continuation): ''' continuationに紐付いたチャットデータを取得し - にチャットデータを格納、 + BUfferにチャットデータを格納、 次のcontinuaitonを取得してループする Parameter @@ -189,9 +192,11 @@ class ReplayChat: time.sleep(diff_time) continuation = metadata.get('continuation') except ChatParseException as e: - logger.error(f"{str(e)}（動画ID:\"{self.video_id}\"）") + self.terminate() + logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）") return except (TypeError , json.JSONDecodeError) : + self.terminate() logger.error(f"{traceback.format_exc(limit = -1)}") return @@ -220,6 +225,7 @@ class ReplayChat: else: logger.error(f"[{self.video_id}]" f"Exceeded retry count. status_code={status_code}") + self.terminate() return None return livechat_json @@ -266,7 +272,7 @@ class ReplayChat: '''Listener終了時のコールバック''' try: self.terminate() - except CancelledError: + except RuntimeError: logger.debug(f'[{self.video_id}]cancelled:{sender}') def terminate(self): @@ -277,7 +283,7 @@ class ReplayChat: if self._direct_mode == False: #bufferにダミーオブジェクトを入れてis_alive()を判定させる self._buffer.put({'chatdata':'','timeout':1}) - logger.info(f'終了しました:[{self.video_id}]') + logger.info(f'[{self.video_id}]終了しました') @classmethod def shutdown(cls, event, sig = None, handler=None): diff --git a/pytchat/parser/live.py b/pytchat/parser/live.py index a8cf12c..e32177f 100644 --- a/pytchat/parser/live.py +++ b/pytchat/parser/live.py @@ -6,14 +6,13 @@ This module is parser of live chat JSON. import json from .. import config -from .. import mylogger from .. exceptions import ( ResponseContextError, NoContentsException, NoContinuationsException ) -logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) +logger = config.logger(__name__) class Parser: @@ -59,7 +58,7 @@ class Parser: if metadata is None: unknown = list(cont.keys())[0] if unknown: - logger.error(f"Received unknown continuation type:{unknown}") + logger.debug(f"Received unknown continuation type:{unknown}") metadata = cont.get(unknown) metadata.setdefault('timeoutMs', 10000) chatdata = contents['liveChatContinuation'].get('actions') diff --git a/pytchat/parser/replay.py b/pytchat/parser/replay.py index fc27a8d..7399238 100644 --- a/pytchat/parser/replay.py +++ b/pytchat/parser/replay.py @@ -1,14 +1,12 @@ import json from .. import config -from .. import mylogger from .. exceptions import ( ResponseContextError, NoContentsException, NoContinuationsException ) -logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) - +logger = config.logger(__name__) class Parser: def parse(self, jsn): @@ -53,12 +51,13 @@ class Parser: metadata = cont.get('liveChatReplayContinuationData') if metadata is None: unknown = list(cont.keys())[0] - if unknown: + if unknown != "playerSeekContinuationData": + logger.debug(f"Received unknown continuation type:{unknown}") metadata = cont.get(unknown) - actions = contents['liveChatContinuation'].get('actions') if actions is None: - raise NoContentsException('チャットデータを取得できませんでした。') + #後続のチャットデータなし + return {"continuation":None,"timeout":0,"chatdata":[]} interval = self.get_interval(actions) metadata.setdefault("timeoutMs",interval) """アーカイブ済みチャットはライブチャットと構造が異なっているため、以下の行により diff --git a/pytchat/processors/combinator.py b/pytchat/processors/combinator.py new file mode 100644 index 0000000..1b76df5 --- /dev/null +++ b/pytchat/processors/combinator.py @@ -0,0 +1,39 @@ +from .chat_processor import ChatProcessor + +class Combinator(ChatProcessor): + ''' + Combinator combines multiple chat processors. + Specify processors as tuple at `processor` params of LiveChat object. + + For example: + [constructor] + chat = LiveChat("video_id", processor = ( Processor1(), Processor2(), Processor3() ) + + [receive return values] + ret1, ret2, ret3 = chat.get() + + The return values are tuple of processed chat data, + the order of return depends on parameter order. + + Parameter + --------- + processors : Tuple[ChatProcessor] + multiple processors for processing chat data + ''' + + def __init__(self, processors: tuple): + self.processors = processors + + def process(self, chat_components: list): + ''' + Called from LiveChat.get() function by user, + or LiveChat._listen() automatically. + + Returns + ------- + Tuple of chat data processed by each chat processor. + ''' + return tuple([processor.process(chat_components) + for processor in self.processors]) + + diff --git a/pytchat/processors/compatible/processor.py b/pytchat/processors/compatible/processor.py index 31e1b15..9b799d3 100644 --- a/pytchat/processors/compatible/processor.py +++ b/pytchat/processors/compatible/processor.py @@ -5,9 +5,8 @@ from .renderer.paidmessage import LiveChatPaidMessageRenderer from .renderer.paidsticker import LiveChatPaidStickerRenderer from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer from .. chat_processor import ChatProcessor -from ... import mylogger from ... import config -logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) +logger = config.logger(__name__) class CompatibleProcessor(ChatProcessor): diff --git a/pytchat/processors/default/processor.py b/pytchat/processors/default/processor.py index be3dfd6..8c33c8e 100644 --- a/pytchat/processors/default/processor.py +++ b/pytchat/processors/default/processor.py @@ -6,8 +6,7 @@ from .renderer.paidsticker import LiveChatPaidStickerRenderer from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer from .. chat_processor import ChatProcessor from ... import config -from ... import mylogger -logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) +logger = config.logger(__name__) class Chatdata: def __init__(self,chatlist:list, timeout:float): diff --git a/pytchat/processors/default/renderer/base.py b/pytchat/processors/default/renderer/base.py index e11e42f..4ea2a83 100644 --- a/pytchat/processors/default/renderer/base.py +++ b/pytchat/processors/default/renderer/base.py @@ -19,8 +19,7 @@ class BaseRenderer: else: self.elapsedTime = "" self.datetime = self.get_datetime(timestampUsec) - self.message = self.get_message(self.renderer) - self.messageEx = self.get_message_ex(self.renderer) + self.message ,self.messageEx = self.get_message(self.renderer) self.id = self.renderer.get('id') self.amountValue= 0.0 self.amountString = "" @@ -44,6 +43,7 @@ class BaseRenderer: def get_message(self,renderer): message = '' + message_ex = [] if renderer.get("message"): runs=renderer["message"].get("runs") if runs: @@ -51,22 +51,13 @@ class BaseRenderer: if r: if r.get('emoji'): message += r['emoji'].get('shortcuts',[''])[0] + message_ex.append(r['emoji']['image']['thumbnails'][1].get('url')) else: message += r.get('text','') - return message + message_ex.append(r.get('text','')) + return message, message_ex + - def get_message_ex(self,renderer): - message = [] - if renderer.get("message"): - runs=renderer["message"].get("runs") - if runs: - for r in runs: - if r: - if r.get('emoji'): - message.append(r['emoji']['image']['thumbnails'][1].get('url')) - else: - message.append(r.get('text','')) - return message def get_badges(self,renderer): isVerified = False diff --git a/pytchat/processors/dummy_processor.py b/pytchat/processors/dummy_processor.py new file mode 100644 index 0000000..e2e406d --- /dev/null +++ b/pytchat/processors/dummy_processor.py @@ -0,0 +1,8 @@ +from .chat_processor import ChatProcessor + +class DummyProcessor(ChatProcessor): + ''' + Dummy processor just returns received chat_components directly. + ''' + def process(self, chat_components: list): + return chat_components diff --git a/pytchat/util/__init__.py b/pytchat/util/__init__.py index 95ce56a..60be578 100644 --- a/pytchat/util/__init__.py +++ b/pytchat/util/__init__.py @@ -9,7 +9,7 @@ def download(url): json.dump(html.json(),f,ensure_ascii=False) -def save(data,filename): - with open(str(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S') - )+filename,mode ='w',encoding='utf-8') as f: +def save(data,filename,extention): + with open(filename+"_"+(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S') + )+extention,mode ='w',encoding='utf-8') as f: f.writelines(data) diff --git a/setup.py b/setup.py index 858fd1a..9728c33 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages, Command -from codecs import open -from os import path, system +#from codecs import open as open_c +from os import path, system, remove, rename import re package_name = "pytchat" @@ -28,6 +28,16 @@ assert author assert author_email assert url + + +with open('README.MD', 'r', encoding='utf-8') as f: + txt = f.read() + +with open('README1.MD', 'w', encoding='utf-8', newline='\n') as f: + f.write(txt) + +remove("README.MD") +rename("README1.MD","README.MD") with open('README.md', encoding='utf-8') as f: long_description = f.read()