From 328889689f305ddac05134c466e241543583a7d1 Mon Sep 17 00:00:00 2001 From: taizan-hokouto <55448286+taizan-hokuto@users.noreply.github.com> Date: Sat, 24 Jul 2021 21:48:13 +0900 Subject: [PATCH] Make it possible to use customized client --- pytchat/core/pytchat.py | 62 ++++++++++--------- pytchat/core_async/livechat.py | 20 +++--- pytchat/core_multithread/livechat.py | 15 +++-- pytchat/processors/default/processor.py | 4 +- .../processors/default/renderer/donation.py | 6 ++ pytchat/processors/speed/calculator.py | 43 ++++--------- 6 files changed, 79 insertions(+), 71 deletions(-) create mode 100644 pytchat/processors/default/renderer/donation.py diff --git a/pytchat/core/pytchat.py b/pytchat/core/pytchat.py index 93605e9..3168f55 100644 --- a/pytchat/core/pytchat.py +++ b/pytchat/core/pytchat.py @@ -29,9 +29,13 @@ class PytchatCore: processor : ChatProcessor + client : httpx.Client + The client for connecting youtube. + You can specify any customized httpx client (e.g. coolies, user agent). + interruptable : bool Allows keyboard interrupts. - Set this parameter to False if your own threading program causes + Set this parameter to False if your own multi-threading program causes the problem. force_replay : bool @@ -57,6 +61,7 @@ class PytchatCore: def __init__(self, video_id, seektime=-1, processor=DefaultProcessor(), + client = httpx.Client(http2=True), interruptable=True, force_replay=False, topchat_only=False, @@ -64,6 +69,7 @@ class PytchatCore: logger=config.logger(__name__), replay_continuation=None ): + self._client = client self._video_id = util.extract_video_id(video_id) self.seektime = seektime if isinstance(processor, tuple): @@ -97,7 +103,7 @@ class PytchatCore: """ self.continuation = liveparam.getparam( self._video_id, - channel_id=util.get_channelid(httpx.Client(http2=True), self._video_id), + channel_id=util.get_channelid(self._client, self._video_id), past_sec=3) def _get_chat_component(self): @@ -110,19 +116,18 @@ class PytchatCore: parameter for next chat data ''' try: - with httpx.Client(http2=True) as client: - if self.continuation and self._is_alive: - contents = self._get_contents(self.continuation, client, headers) - metadata, chatdata = self._parser.parse(contents) - timeout = metadata['timeoutMs'] / 1000 - chat_component = { - "video_id": self._video_id, - "timeout": timeout, - "chatdata": chatdata - } - self.continuation = metadata.get('continuation') - self._last_offset_ms = metadata.get('last_offset_ms', 0) - return chat_component + if self.continuation and self._is_alive: + contents = self._get_contents(self.continuation, self._client, headers) + metadata, chatdata = self._parser.parse(contents) + timeout = metadata['timeoutMs'] / 1000 + chat_component = { + "video_id": self._video_id, + "timeout": timeout, + "chatdata": chatdata + } + self.continuation = metadata.get('continuation') + self._last_offset_ms = metadata.get('last_offset_ms', 0) + return chat_component except exceptions.ChatParseException as e: self._logger.debug(f"[{self._video_id}]{str(e)}") self._raise_exception(e) @@ -139,9 +144,8 @@ class PytchatCore: ------- 'continuationContents' which includes metadata & chat data. ''' - livechat_json = ( - self._get_livechat_json(continuation, client, replay=self._is_replay, offset_ms=self._last_offset_ms) - ) + livechat_json = self._get_livechat_json( + continuation, client, replay=self._is_replay, offset_ms=self._last_offset_ms) contents, dat = self._parser.get_contents(livechat_json) if self._dat == '' and dat: self._dat = dat @@ -152,7 +156,8 @@ class PytchatCore: self._fetch_url = config._smr continuation = arcparam.getparam( self._video_id, self.seektime, self._topchat_only, util.get_channelid(client, self._video_id)) - livechat_json = self._get_livechat_json(continuation, client, replay=True, offset_ms=self.seektime * 1000) + livechat_json = self._get_livechat_json( + continuation, client, replay=True, offset_ms=self.seektime * 1000) reload_continuation = self._parser.reload_continuation( self._parser.get_contents(livechat_json)[0]) if reload_continuation: @@ -173,15 +178,14 @@ class PytchatCore: offset_ms = 0 param = util.get_param(continuation, dat=self._dat, replay=replay, offsetms=offset_ms) for _ in range(MAX_RETRY + 1): - with httpx.Client(http2=True) as client: - try: - response = client.post(self._fetch_url, json=param) - livechat_json = json.loads(response.text) - break - except (json.JSONDecodeError, httpx.ConnectTimeout, httpx.ReadTimeout, httpx.ConnectError) as e: - err = e - time.sleep(2) - continue + try: + response = client.post(self._fetch_url, json=param) + livechat_json = response.json() + break + except (json.JSONDecodeError, httpx.ConnectTimeout, httpx.ReadTimeout, httpx.ConnectError) as e: + err = e + time.sleep(2) + continue else: self._logger.error(f"[{self._video_id}]" f"Exceeded retry count. Last error: {str(err)}") @@ -202,6 +206,8 @@ class PytchatCore: return self._is_alive def terminate(self): + if not self.is_alive(): + return self._is_alive = False self.processor.finalize() diff --git a/pytchat/core_async/livechat.py b/pytchat/core_async/livechat.py index 655f43c..7351470 100644 --- a/pytchat/core_async/livechat.py +++ b/pytchat/core_async/livechat.py @@ -78,6 +78,7 @@ class LiveChatAsync: seektime=-1, processor=DefaultProcessor(), buffer=None, + client = httpx.AsyncClient(http2=True), interruptable=True, callback=None, done_callback=None, @@ -88,6 +89,7 @@ class LiveChatAsync: logger=config.logger(__name__), replay_continuation=None ): + self._client:httpx.AsyncClient = client self._video_id = util.extract_video_id(video_id) self.seektime = seektime if isinstance(processor, tuple): @@ -152,9 +154,10 @@ class LiveChatAsync: create and start _listen loop. """ if not self.continuation: + channel_id = await util.get_channelid_async(self._client, self._video_id) self.continuation = liveparam.getparam( self._video_id, - channel_id=util.get_channelid(httpx.Client(http2=True), self._video_id), + channel_id, past_sec=3) await self._listen(self.continuation) @@ -169,10 +172,10 @@ class LiveChatAsync: parameter for next chat data ''' try: - async with httpx.AsyncClient(http2=True) as client: + async with self._client as client: while(continuation and self._is_alive): continuation = await self._check_pause(continuation) - contents = await self._get_contents(continuation, client, headers) + contents = await self._get_contents(continuation, client, headers) #Q# metadata, chatdata = self._parser.parse(contents) continuation = metadata.get('continuation') if continuation: @@ -214,9 +217,10 @@ class LiveChatAsync: ''' self._pauser.put_nowait(None) if not self._is_replay: - async with httpx.AsyncClient(http2=True) as client: - continuation = await liveparam.getparam(self._video_id, - channel_id=util.get_channelid_async(client, self.video_id), + async with self._client as client: + channel_id = await util.get_channelid_async(client, self.video_id) + continuation = liveparam.getparam(self._video_id, + channel_id, past_sec=3) return continuation @@ -338,12 +342,14 @@ class LiveChatAsync: self._logger.debug(f'[{self._video_id}] cancelled:{sender}') def terminate(self): + if not self.is_alive(): + return if self._pauser.empty(): self._pauser.put_nowait(None) self._is_alive = False self._buffer.put_nowait({}) self.processor.finalize() - + def _keyboard_interrupt(self): self.exception = exceptions.ChatDataFinished() self.terminate() diff --git a/pytchat/core_multithread/livechat.py b/pytchat/core_multithread/livechat.py index 153dce1..534d0e3 100644 --- a/pytchat/core_multithread/livechat.py +++ b/pytchat/core_multithread/livechat.py @@ -78,6 +78,7 @@ class LiveChat: def __init__(self, video_id, seektime=-1, processor=DefaultProcessor(), + client = httpx.Client(http2=True), buffer=None, interruptable=True, callback=None, @@ -88,6 +89,7 @@ class LiveChat: logger=config.logger(__name__), replay_continuation=None ): + self._client = client self._video_id = util.extract_video_id(video_id) self.seektime = seektime if isinstance(processor, tuple): @@ -150,7 +152,7 @@ class LiveChat: if not self.continuation: self.continuation = liveparam.getparam( self._video_id, - channel_id=util.get_channelid(httpx.Client(http2=True), self._video_id), + channel_id=util.get_channelid(self._client, self._video_id), past_sec=3) self._listen(self.continuation) @@ -164,7 +166,7 @@ class LiveChat: parameter for next chat data ''' try: - with httpx.Client(http2=True) as client: + with self._client as client: while(continuation and self._is_alive): continuation = self._check_pause(continuation) contents = self._get_contents(continuation, client, headers) @@ -224,7 +226,8 @@ class LiveChat: ------- 'continuationContents' which includes metadata & chat data. ''' - livechat_json = self._get_livechat_json(continuation, client, replay=self._is_replay, offset_ms=self._last_offset_ms) + livechat_json = self._get_livechat_json( + continuation, client, replay=self._is_replay, offset_ms=self._last_offset_ms) contents, dat = self._parser.get_contents(livechat_json) if self._dat == '' and dat: self._dat = dat @@ -235,8 +238,8 @@ class LiveChat: self._fetch_url = config._smr continuation = arcparam.getparam( self._video_id, self.seektime, self._topchat_only, util.get_channelid(client, self._video_id)) - livechat_json = (self._get_livechat_json( - continuation, client, replay=True, offset_ms=self.seektime * 1000)) + livechat_json = self._get_livechat_json( + continuation, client, replay=True, offset_ms=self.seektime * 1000) reload_continuation = self._parser.reload_continuation( self._parser.get_contents(livechat_json)[0]) if reload_continuation: @@ -331,6 +334,8 @@ class LiveChat: self._logger.debug(f'[{self._video_id}] cancelled:{sender}') def terminate(self): + if not self.is_alive(): + return if self._pauser.empty(): self._pauser.put_nowait(None) self._is_alive = False diff --git a/pytchat/processors/default/processor.py b/pytchat/processors/default/processor.py index e28da0e..2121a35 100644 --- a/pytchat/processors/default/processor.py +++ b/pytchat/processors/default/processor.py @@ -7,6 +7,7 @@ from .renderer.paidmessage import LiveChatPaidMessageRenderer from .renderer.paidsticker import LiveChatPaidStickerRenderer from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer from .renderer.membership import LiveChatMembershipItemRenderer +from .renderer.donation import LiveChatDonationAnnouncementRenderer from .. chat_processor import ChatProcessor from ... import config @@ -124,7 +125,8 @@ class DefaultProcessor(ChatProcessor): "liveChatPaidMessageRenderer": LiveChatPaidMessageRenderer(), "liveChatPaidStickerRenderer": LiveChatPaidStickerRenderer(), "liveChatLegacyPaidMessageRenderer": LiveChatLegacyPaidMessageRenderer(), - "liveChatMembershipItemRenderer": LiveChatMembershipItemRenderer() + "liveChatMembershipItemRenderer": LiveChatMembershipItemRenderer(), + "liveChatDonationAnnouncementRenderer": LiveChatDonationAnnouncementRenderer(), } def process(self, chat_components: list): diff --git a/pytchat/processors/default/renderer/donation.py b/pytchat/processors/default/renderer/donation.py new file mode 100644 index 0000000..7f654e5 --- /dev/null +++ b/pytchat/processors/default/renderer/donation.py @@ -0,0 +1,6 @@ +from .base import BaseRenderer + + +class LiveChatDonationAnnouncementRenderer(BaseRenderer): + def settype(self): + self.chat.type = "donation" \ No newline at end of file diff --git a/pytchat/processors/speed/calculator.py b/pytchat/processors/speed/calculator.py index 52d57df..3bef2ba 100644 --- a/pytchat/processors/speed/calculator.py +++ b/pytchat/processors/speed/calculator.py @@ -82,16 +82,17 @@ class RingQueue: class SpeedCalculator(ChatProcessor, RingQueue): """ - チャットの勢いを計算する。 + Calculate the momentum of the chat. - 一定期間のチャットデータのうち、最初のチャットの投稿時刻と - 最後のチャットの投稿時刻の差を、チャット数で割り返し - 1分あたりの速度に換算する。 + Divide the difference between the time of the first chat and + the time of the last chat in the chat data over a period of + time by the number of chats and convert it to speed per minute. Parameter ---------- capacity : int - RingQueueに格納するチャット勢い算出用データの最大数 + Maximum number of data for calculating chat momentum + to be stored in RingQueue. """ def __init__(self, capacity=10): @@ -111,17 +112,17 @@ class SpeedCalculator(ChatProcessor, RingQueue): def _calc_speed(self): """ - RingQueue内のチャット勢い算出用データリストを元に、 - チャット速度を計算して返す + Calculates the chat speed based on the data list for calculating + the chat momentum in RingQueue. Return --------------------------- - チャット速度（１分間で換算したチャット数） + Chat speed (number of chats converted per minute) """ try: - # キュー内の総チャット数 + # Total number of chats in the queue total = sum(item['chat_count'] for item in self.items) - # キュー内の最初と最後のチャットの時間差 + # Interval between the first and last chats in the queue duration = (self.items[self.last_pos]['endtime'] - self.items[self.first_pos]['starttime']) if duration != 0: return int(total * 60 / duration) @@ -131,19 +132,12 @@ class SpeedCalculator(ChatProcessor, RingQueue): def _put_chatdata(self, actions): """ - チャットデータからタイムスタンプを読み取り、勢い測定用のデータを組み立て、 - RingQueueに投入する。 - 200円以上のスパチャはtickerとmessageの2つのデータが生成されるが、 - tickerの方は時刻データの場所が異なることを利用し、勢いの集計から除外している。 Parameter --------- actions : List[dict] - チャットデータ(addChatItemAction) のリスト + List of addChatItemActions """ def _put_emptydata(): - ''' - チャットデータがない場合に空のデータをキューに投入する。 - ''' timestamp_now = int(time.time()) self.put({ 'chat_count': 0, @@ -152,9 +146,6 @@ class SpeedCalculator(ChatProcessor, RingQueue): }) def _get_timestamp(action: dict): - """ - チャットデータから時刻データを取り出す。 - """ try: item = action['addChatItemAction']['item'] timestamp = int(item[list(item.keys())[0]]['timestampUsec']) @@ -166,32 +157,24 @@ class SpeedCalculator(ChatProcessor, RingQueue): _put_emptydata() return - # actions内の時刻データを持つチャットデータの数 counter = 0 - # actions内の最初のチャットデータの時刻 starttime = None - # actions内の最後のチャットデータの時刻 endtime = None for action in actions: - # チャットデータからtimestampUsecを読み取る + # Get timestampUsec from chatdata gettime = _get_timestamp(action) - # 時刻のないデータだった場合は次の行のデータで読み取り試行 if gettime is None: continue - # 最初に有効な時刻を持つデータのtimestampをstarttimeに設定 if starttime is None: starttime = gettime - # 最後のtimestampを設定(途中で時刻のないデータの場合もあるので上書きしていく) endtime = gettime - # チャットの数をインクリメント counter += 1 - # チャット速度用のデータをRingQueueに送る if starttime is None or endtime is None: _put_emptydata() return