From 3b580690c7c83ef47b573ddc3fea93d24859eb29 Mon Sep 17 00:00:00 2001 From: "55448286+taizan-hokuto@users.noreply.github.com" Date: Sun, 3 Nov 2019 21:57:30 +0900 Subject: [PATCH 1/7] Delete listen_manager --- pytchat/core_async/listen_manager.py | 184 --------------------------- 1 file changed, 184 deletions(-) delete mode 100644 pytchat/core_async/listen_manager.py diff --git a/pytchat/core_async/listen_manager.py b/pytchat/core_async/listen_manager.py deleted file mode 100644 index c35e6bd..0000000 --- a/pytchat/core_async/listen_manager.py +++ /dev/null @@ -1,184 +0,0 @@ -import asyncio -from .listener import AsyncListener -from .. import config -from .. import mylogger -import datetime -import os -import aiohttp -import signal -import threading -from .buffer import Buffer -from concurrent.futures import CancelledError -logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) - -class ListenManager: - - ''' - 動画IDまたは動画IDのリストを受け取り、 - 動画IDに対応したListenerを生成・保持する。 - - #Attributes - ---------- - _listeners: dict - ListenManegerがつかんでいるListener達のリスト. - key:動画ID value:動画IDに対応するListener - _queue: Queue - 動画IDを外部から受け渡しするためのキュー - _queueが空である間は、ノンブロッキングで他のタスクを実行 - _queueに動画IDが投入されると、_dequeueメソッドで - 直ちにListenerを生成し返す。 - _event: threading.Event - キーボードのCTRL+Cを検知するためのEventオブジェクト - ''' - def __init__(self,interruptable = True): - #チャット監視中の動画リスト - self._listeners={} - self._tasks = [] - #外部からvideoを受け取るためのキュー - self._queue = asyncio.Queue() - self._event = threading.Event() - self._ready_queue() - self._is_alive = True - #キーボードのCtrl+cを押したとき、_hundler関数を呼び出すように設定 - signal.signal(signal.SIGINT, (lambda a, b: self._handler(self._event, a, b))) - - def is_alive(self)->bool: - ''' - ListenManagerが稼働中であるか。 - True->稼働中 - False->Ctrl+Cが押されて終了 - ''' - logger.debug(f'check is_alive() :{self._is_alive}') - return self._is_alive - - def _handler(self, event, sig, handler): - ''' - Ctrl+Cが押下されたとき、終了フラグをセットする。 - ''' - logger.debug('Ctrl+c pushed') - self._is_alive = False - logger.debug('terminating listeners.') - for listener in self._listeners.values(): - listener.terminate() - logger.debug('end.') - - - def _ready_queue(self): - #loop = asyncio.get_event_loop() - self._tasks.append( - asyncio.create_task(self._dequeue()) - ) - - - async def set_video_ids(self,video_ids:list): - for video_id in video_ids: - if video_id: - await self._queue.put(video_id) - - - async def get_listener(self,video_id) -> AsyncListener: - return await self._create_listener(video_id) - - # async def getlivechat(self,video_id): - # ''' - # 指定された動画IDのチャットデータを返す - - # Parameter - # ---------- - # video_id: str - # 動画ID - - # Return - # ---------- - # 引数で受け取った動画IDに対応する - # Listenerオブジェクトへの参照 - - # ''' - # logger.debug('manager get/create listener') - # listener = await self._create_listener(video_id) - # ''' - # 上が完了しないうちに、下が呼び出される - # ''' - # if not listener._initialized: - # await asyncio.sleep(2) - # return [] - # if listener: - # #listener._isfirstrun=False - # return await listener.getlivechat() - - - - async def _dequeue(self): - ''' - キューに入った動画IDを - Listener登録に回す。 - - ''' - while True: - video_id = await self._queue.get() - #listenerを登録、タスクとして実行する - logger.debug(f'deque got [{video_id}]') - await self._create_listener(video_id) - - async def _create_listener(self, video_id) -> AsyncListener: - ''' - Listenerを作成しチャット取得中リストに加え、 - Listenerを返す - ''' - if video_id is None or not isinstance(video_id, str): - raise TypeError('video_idは文字列でなければなりません') - if video_id in self._listeners: - return self._listeners[video_id] - else: - #listenerを登録する - listener = AsyncListener(video_id,interruptable = False,buffer = Buffer()) - self._listeners.setdefault(video_id,listener) - #task = asyncio.ensure_future(listener.initialize()) - #await asyncio.gather(listener.initialize()) - #task.add_done_callback(self.finish) - #await listener.initialize() - #self._tasks.append(task) - - return listener - - - def finish(self,sender): - try: - if sender.result(): - video_id = sender.result()[0] - message = sender.result()[1] - - #listener終了時のコールバック - #sender.result()[]でデータを取得できる - logger.info(f'終了しました VIDEO_ID:[{video_id}] message:{message}') - #logger.info(f'終了しました') - if video_id in self._listeners: - self._listeners.pop(video_id) - except CancelledError: - logger.debug('cancelled.') - - def get_listeners(self): - return self._listeners - - def shutdown(self): - ''' - ListenManegerを終了する - ''' - logger.debug("start shutdown") - self._is_alive =False - try: - #Listenerを停止する。 - for listener in self._listeners.values(): - listener.terminate() - #taskをキャンセルする。 - for task in self._tasks: - if not task.done(): - #print(task) - task.cancel() - except Exception as er: - logger.info(str(er),type(er)) - - logger.debug("finished.") - - def get_tasks(self): - return self._tasks \ No newline at end of file From 8bc209fde8b64959793c87bb876cd821ddac5a24 Mon Sep 17 00:00:00 2001 From: "55448286+taizan-hokuto@users.noreply.github.com" Date: Sun, 3 Nov 2019 22:26:47 +0900 Subject: [PATCH 2/7] Fix renderer type check --- pytchat/processors/simple_display_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytchat/processors/simple_display_processor.py b/pytchat/processors/simple_display_processor.py index 281b3df..1ca01ce 100644 --- a/pytchat/processors/simple_display_processor.py +++ b/pytchat/processors/simple_display_processor.py @@ -31,7 +31,7 @@ class SimpleDisplayProcessor(ChatProcessor): purchase_amount_text = '' else: root = ( action['addChatItemAction']['item'].get('liveChatPaidMessageRenderer') or - action['addChatItemAction']['item'].get('liveChatPaidMessageRenderer') ) + action['addChatItemAction']['item'].get('liveChatPaidStickerRenderer') ) if root: author_name = root['authorName']['simpleText'] message = self._parse_message(root.get('message')) From 3912758a5242701ef818d7a9a2d8e27c2efaeb3e Mon Sep 17 00:00:00 2001 From: "55448286+taizan-hokuto@users.noreply.github.com" Date: Sun, 3 Nov 2019 22:43:41 +0900 Subject: [PATCH 3/7] Update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0d31592..9ecfde9 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Other features: + Quick fetching of initial chat data by generating continuation params instead of web scraping. -For more detailed information, see [wiki](https://github.com/taizan-hokuto/pytchat/wiki) +より詳細な説明は [wiki](https://github.com/taizan-hokuto/pytchat/wiki) をご参照ください。 ## Install ```python From 7c6e12cbe5d7e999b149f7572d67560db2d888dd Mon Sep 17 00:00:00 2001 From: "55448286+taizan-hokuto@users.noreply.github.com" Date: Mon, 4 Nov 2019 23:29:00 +0900 Subject: [PATCH 4/7] Change format of multithread parser to class --- pytchat/processors/compatible/parser.py | 67 +++++++++++----------- pytchat/processors/compatible/processor.py | 11 ++-- 2 files changed, 41 insertions(+), 37 deletions(-) diff --git a/pytchat/processors/compatible/parser.py b/pytchat/processors/compatible/parser.py index 08cd7a1..b08d975 100644 --- a/pytchat/processors/compatible/parser.py +++ b/pytchat/processors/compatible/parser.py @@ -4,40 +4,41 @@ from .renderer.paidmessage import LiveChatPaidMessageRenderer from .renderer.paidsticker import LiveChatPaidStickerRenderer from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer -def parse(sitem): +class Parser: + def parse(self, sitem): - action = sitem.get("addChatItemAction") - if action: - item = action.get("item") - if item is None: return None - rd={} - try: - renderer = get_renderer(item) - if renderer == None: + action = sitem.get("addChatItemAction") + if action: + item = action.get("item") + if item is None: return None + rd={} + try: + renderer = self.get_renderer(item) + if renderer == None: + return None + + rd["kind"] = "youtube#liveChatMessage" + rd["etag"] = "" + rd["id"] = 'LCC.' + renderer.get_id() + rd["snippet"] = renderer.get_snippet() + rd["authorDetails"] = renderer.get_authordetails() + except (KeyError,TypeError,AttributeError) as e: + print(f"------{str(type(e))}-{str(e)}----------") + print(sitem) return None + + return rd - rd["kind"] = "youtube#liveChatMessage" - rd["etag"] = "" - rd["id"] = 'LCC.' + renderer.get_id() - rd["snippet"] = renderer.get_snippet() - rd["authorDetails"] = renderer.get_authordetails() - except (KeyError,TypeError,AttributeError) as e: - print(f"------{str(type(e))}-{str(e)}----------") - print(sitem) - return None - - return rd - -def get_renderer(item): - if item.get("liveChatTextMessageRenderer"): - renderer = LiveChatTextMessageRenderer(item) - elif item.get("liveChatPaidMessageRenderer"): - renderer = LiveChatPaidMessageRenderer(item) - elif item.get( "liveChatPaidStickerRenderer"): - renderer = LiveChatPaidStickerRenderer(item) - elif item.get("liveChatLegacyPaidMessageRenderer"): - renderer = LiveChatLegacyPaidMessageRenderer(item) - else: - renderer = None - return renderer + def get_renderer(self, item): + if item.get("liveChatTextMessageRenderer"): + renderer = LiveChatTextMessageRenderer(item) + elif item.get("liveChatPaidMessageRenderer"): + renderer = LiveChatPaidMessageRenderer(item) + elif item.get( "liveChatPaidStickerRenderer"): + renderer = LiveChatPaidStickerRenderer(item) + elif item.get("liveChatLegacyPaidMessageRenderer"): + renderer = LiveChatLegacyPaidMessageRenderer(item) + else: + renderer = None + return renderer diff --git a/pytchat/processors/compatible/processor.py b/pytchat/processors/compatible/processor.py index 277cbcf..1cb3dc7 100644 --- a/pytchat/processors/compatible/processor.py +++ b/pytchat/processors/compatible/processor.py @@ -1,11 +1,14 @@ -from . import parser +from . parser import Parser import json import os import traceback import datetime import time -class CompatibleProcessor(): - + +class CompatibleProcessor: + def __init__(self): + self.parser = Parser() + def process(self, chat_components: list): chatlist = [] @@ -26,7 +29,7 @@ class CompatibleProcessor(): if action.get('addChatItemAction') is None: continue if action['addChatItemAction'].get('item') is None: continue - chat = parser.parse(action) + chat = self.parser.parse(action) if chat: chatlist.append(chat) ret["pollingIntervalMillis"] = int(timeout*1000) From 64ec413bcada6ee6f8de2d923acb0d84f884c294 Mon Sep 17 00:00:00 2001 From: "55448286+taizan-hokuto@users.noreply.github.com" Date: Mon, 4 Nov 2019 23:43:00 +0900 Subject: [PATCH 5/7] Move parser functions to processor --- pytchat/processors/compatible/parser.py | 44 ----------------- pytchat/processors/compatible/processor.py | 55 ++++++++++++++++++---- pytchat/processors/default/parser.py | 39 --------------- pytchat/processors/default/processor.py | 39 ++++++++++++++- 4 files changed, 84 insertions(+), 93 deletions(-) delete mode 100644 pytchat/processors/compatible/parser.py delete mode 100644 pytchat/processors/default/parser.py diff --git a/pytchat/processors/compatible/parser.py b/pytchat/processors/compatible/parser.py deleted file mode 100644 index b08d975..0000000 --- a/pytchat/processors/compatible/parser.py +++ /dev/null @@ -1,44 +0,0 @@ - -from .renderer.textmessage import LiveChatTextMessageRenderer -from .renderer.paidmessage import LiveChatPaidMessageRenderer -from .renderer.paidsticker import LiveChatPaidStickerRenderer -from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer - -class Parser: - def parse(self, sitem): - - action = sitem.get("addChatItemAction") - if action: - item = action.get("item") - if item is None: return None - rd={} - try: - renderer = self.get_renderer(item) - if renderer == None: - return None - - rd["kind"] = "youtube#liveChatMessage" - rd["etag"] = "" - rd["id"] = 'LCC.' + renderer.get_id() - rd["snippet"] = renderer.get_snippet() - rd["authorDetails"] = renderer.get_authordetails() - except (KeyError,TypeError,AttributeError) as e: - print(f"------{str(type(e))}-{str(e)}----------") - print(sitem) - return None - - return rd - - def get_renderer(self, item): - if item.get("liveChatTextMessageRenderer"): - renderer = LiveChatTextMessageRenderer(item) - elif item.get("liveChatPaidMessageRenderer"): - renderer = LiveChatPaidMessageRenderer(item) - elif item.get( "liveChatPaidStickerRenderer"): - renderer = LiveChatPaidStickerRenderer(item) - elif item.get("liveChatLegacyPaidMessageRenderer"): - renderer = LiveChatLegacyPaidMessageRenderer(item) - else: - renderer = None - return renderer - diff --git a/pytchat/processors/compatible/processor.py b/pytchat/processors/compatible/processor.py index 1cb3dc7..35be738 100644 --- a/pytchat/processors/compatible/processor.py +++ b/pytchat/processors/compatible/processor.py @@ -1,13 +1,14 @@ -from . parser import Parser -import json -import os -import traceback import datetime import time +from .renderer.textmessage import LiveChatTextMessageRenderer +from .renderer.paidmessage import LiveChatPaidMessageRenderer +from .renderer.paidsticker import LiveChatPaidStickerRenderer +from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer +from ... import mylogger +from ... import config +logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE) class CompatibleProcessor: - def __init__(self): - self.parser = Parser() def process(self, chat_components: list): @@ -29,7 +30,7 @@ class CompatibleProcessor: if action.get('addChatItemAction') is None: continue if action['addChatItemAction'].get('item') is None: continue - chat = self.parser.parse(action) + chat = self.parse(action) if chat: chatlist.append(chat) ret["pollingIntervalMillis"] = int(timeout*1000) @@ -39,4 +40,42 @@ class CompatibleProcessor: } ret["items"] = chatlist - return ret \ No newline at end of file + return ret + + def parse(self, sitem): + + action = sitem.get("addChatItemAction") + if action: + item = action.get("item") + if item is None: return None + rd={} + try: + renderer = self.get_renderer(item) + if renderer == None: + return None + + rd["kind"] = "youtube#liveChatMessage" + rd["etag"] = "" + rd["id"] = 'LCC.' + renderer.get_id() + rd["snippet"] = renderer.get_snippet() + rd["authorDetails"] = renderer.get_authordetails() + except (KeyError,TypeError,AttributeError) as e: + print(f"------{str(type(e))}-{str(e)}----------") + print(sitem) + return None + + return rd + + def get_renderer(self, item): + if item.get("liveChatTextMessageRenderer"): + renderer = LiveChatTextMessageRenderer(item) + elif item.get("liveChatPaidMessageRenderer"): + renderer = LiveChatPaidMessageRenderer(item) + elif item.get( "liveChatPaidStickerRenderer"): + renderer = LiveChatPaidStickerRenderer(item) + elif item.get("liveChatLegacyPaidMessageRenderer"): + renderer = LiveChatLegacyPaidMessageRenderer(item) + else: + renderer = None + return renderer + diff --git a/pytchat/processors/default/parser.py b/pytchat/processors/default/parser.py deleted file mode 100644 index eec4274..0000000 --- a/pytchat/processors/default/parser.py +++ /dev/null @@ -1,39 +0,0 @@ - -from .renderer.textmessage import LiveChatTextMessageRenderer -from .renderer.paidmessage import LiveChatPaidMessageRenderer -from .renderer.paidsticker import LiveChatPaidStickerRenderer -from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer - -def parse(sitem): - - action = sitem.get("addChatItemAction") - if action: - item = action.get("item") - if item is None: return None - try: - renderer = get_renderer(item) - if renderer == None: - return None - - renderer.get_snippet() - renderer.get_authordetails() - except (KeyError,TypeError,AttributeError) as e: - print(f"------{str(type(e))}-{str(e)}----------") - print(sitem) - return None - - return renderer - -def get_renderer(item): - if item.get("liveChatTextMessageRenderer"): - renderer = LiveChatTextMessageRenderer(item) - elif item.get("liveChatPaidMessageRenderer"): - renderer = LiveChatPaidMessageRenderer(item) - elif item.get( "liveChatPaidStickerRenderer"): - renderer = LiveChatPaidStickerRenderer(item) - elif item.get("liveChatLegacyPaidMessageRenderer"): - renderer = LiveChatLegacyPaidMessageRenderer(item) - else: - renderer = None - return renderer - diff --git a/pytchat/processors/default/processor.py b/pytchat/processors/default/processor.py index 2c1834c..7059901 100644 --- a/pytchat/processors/default/processor.py +++ b/pytchat/processors/default/processor.py @@ -1,6 +1,9 @@ -from . import parser import asyncio import time +from .renderer.textmessage import LiveChatTextMessageRenderer +from .renderer.paidmessage import LiveChatPaidMessageRenderer +from .renderer.paidsticker import LiveChatPaidStickerRenderer +from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer class Chatdata: @@ -37,8 +40,40 @@ class DefaultProcessor: if action.get('addChatItemAction') is None: continue if action['addChatItemAction'].get('item') is None: continue - chat = parser.parse(action) + chat = self.parse(action) if chat: chatlist.append(chat) return Chatdata(chatlist, float(timeout)) + + def parse(self, sitem): + + action = sitem.get("addChatItemAction") + if action: + item = action.get("item") + if item is None: return None + try: + renderer = self.get_renderer(item) + if renderer == None: + return None + + renderer.get_snippet() + renderer.get_authordetails() + except (KeyError,TypeError,AttributeError) as e: + print(f"------{str(type(e))}-{str(e)}----------") + print(sitem) + return None + return renderer + + def get_renderer(self, item): + if item.get("liveChatTextMessageRenderer"): + renderer = LiveChatTextMessageRenderer(item) + elif item.get("liveChatPaidMessageRenderer"): + renderer = LiveChatPaidMessageRenderer(item) + elif item.get( "liveChatPaidStickerRenderer"): + renderer = LiveChatPaidStickerRenderer(item) + elif item.get("liveChatLegacyPaidMessageRenderer"): + renderer = LiveChatLegacyPaidMessageRenderer(item) + else: + renderer = None + return renderer \ No newline at end of file From 432825b5edaa981da6081ad1b893a6a5a00390c5 Mon Sep 17 00:00:00 2001 From: "55448286+taizan-hokuto@users.noreply.github.com" Date: Mon, 4 Nov 2019 23:45:10 +0900 Subject: [PATCH 6/7] Use logger when errors occur --- pytchat/processors/compatible/processor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytchat/processors/compatible/processor.py b/pytchat/processors/compatible/processor.py index 35be738..124af35 100644 --- a/pytchat/processors/compatible/processor.py +++ b/pytchat/processors/compatible/processor.py @@ -60,8 +60,8 @@ class CompatibleProcessor: rd["snippet"] = renderer.get_snippet() rd["authorDetails"] = renderer.get_authordetails() except (KeyError,TypeError,AttributeError) as e: - print(f"------{str(type(e))}-{str(e)}----------") - print(sitem) + logger.error(f"Error: {str(type(e))}-{str(e)}") + logger.error(f"item: {sitem}") return None return rd From 517f41f5fe42163b1578bb880f475684931685e3 Mon Sep 17 00:00:00 2001 From: "55448286+taizan-hokuto@users.noreply.github.com" Date: Mon, 4 Nov 2019 23:50:41 +0900 Subject: [PATCH 7/7] Increment version --- pytchat/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytchat/__init__.py b/pytchat/__init__.py index b16080c..60da97f 100644 --- a/pytchat/__init__.py +++ b/pytchat/__init__.py @@ -2,7 +2,7 @@ pytchat is a python library for fetching youtube live chat. """ __copyright__ = 'Copyright (C) 2019 taizan-hokuto' -__version__ = '0.0.2.2' +__version__ = '0.0.2.3' __license__ = 'MIT' __author__ = 'taizan-hokuto' __author_email__ = '55448286+taizan-hokuto@users.noreply.github.com'