ramforth/pytchat-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make it possible to use customized client

Browse Source
This commit is contained in:
taizan-hokouto
2021-07-24 21:48:13 +09:00
parent 6d581c22f9
commit 328889689f
6 changed files with 79 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
pytchat/core/pytchat.py
View File

@@ -29,9 +29,13 @@ class PytchatCore:
processor : ChatProcessor processor : ChatProcessor
client : httpx.Client
The client for connecting youtube.
You can specify any customized httpx client (e.g. coolies, user agent).
interruptable : bool interruptable : bool
Allows keyboard interrupts. Allows keyboard interrupts.
Set this parameter to False if your own threading program causes Set this parameter to False if your own multi-threading program causes
the problem. the problem.
force_replay : bool force_replay : bool
@@ -57,6 +61,7 @@ class PytchatCore:
def __init__(self, video_id, def __init__(self, video_id,
seektime=-1, seektime=-1,
processor=DefaultProcessor(), processor=DefaultProcessor(),
client = httpx.Client(http2=True),
interruptable=True, interruptable=True,
force_replay=False, force_replay=False,
topchat_only=False, topchat_only=False,
@@ -64,6 +69,7 @@ class PytchatCore:
logger=config.logger(__name__), logger=config.logger(__name__),
replay_continuation=None replay_continuation=None
): ):
self._client = client
self._video_id = util.extract_video_id(video_id) self._video_id = util.extract_video_id(video_id)
self.seektime = seektime self.seektime = seektime
if isinstance(processor, tuple): if isinstance(processor, tuple):
@@ -97,7 +103,7 @@ class PytchatCore:
""" """
self.continuation = liveparam.getparam( self.continuation = liveparam.getparam(
self._video_id, self._video_id,
channel_id=util.get_channelid(httpx.Client(http2=True), self._video_id), channel_id=util.get_channelid(self._client, self._video_id),
past_sec=3) past_sec=3)
def _get_chat_component(self): def _get_chat_component(self):
@@ -110,19 +116,18 @@ class PytchatCore:
parameter for next chat data parameter for next chat data
''' '''
try: try:
with httpx.Client(http2=True) as client: if self.continuation and self._is_alive:
if self.continuation and self._is_alive: contents = self._get_contents(self.continuation, self._client, headers)
contents = self._get_contents(self.continuation, client, headers) metadata, chatdata = self._parser.parse(contents)
metadata, chatdata = self._parser.parse(contents) timeout = metadata['timeoutMs'] / 1000
timeout = metadata['timeoutMs'] / 1000 chat_component = {
chat_component = { "video_id": self._video_id,
"video_id": self._video_id, "timeout": timeout,
"timeout": timeout, "chatdata": chatdata
"chatdata": chatdata }
} self.continuation = metadata.get('continuation')
self.continuation = metadata.get('continuation') self._last_offset_ms = metadata.get('last_offset_ms', 0)
self._last_offset_ms = metadata.get('last_offset_ms', 0) return chat_component
return chat_component
except exceptions.ChatParseException as e: except exceptions.ChatParseException as e:
self._logger.debug(f"[{self._video_id}]{str(e)}") self._logger.debug(f"[{self._video_id}]{str(e)}")
self._raise_exception(e) self._raise_exception(e)
@@ -139,9 +144,8 @@ class PytchatCore:
------- -------
'continuationContents' which includes metadata & chat data. 'continuationContents' which includes metadata & chat data.
''' '''
livechat_json = ( livechat_json = self._get_livechat_json(
self._get_livechat_json(continuation, client, replay=self._is_replay, offset_ms=self._last_offset_ms) continuation, client, replay=self._is_replay, offset_ms=self._last_offset_ms)
)
contents, dat = self._parser.get_contents(livechat_json) contents, dat = self._parser.get_contents(livechat_json)
if self._dat == '' and dat: if self._dat == '' and dat:
self._dat = dat self._dat = dat
@@ -152,7 +156,8 @@ class PytchatCore:
self._fetch_url = config._smr self._fetch_url = config._smr
continuation = arcparam.getparam( continuation = arcparam.getparam(
self._video_id, self.seektime, self._topchat_only, util.get_channelid(client, self._video_id)) self._video_id, self.seektime, self._topchat_only, util.get_channelid(client, self._video_id))
livechat_json = self._get_livechat_json(continuation, client, replay=True, offset_ms=self.seektime * 1000) livechat_json = self._get_livechat_json(
continuation, client, replay=True, offset_ms=self.seektime * 1000)
reload_continuation = self._parser.reload_continuation( reload_continuation = self._parser.reload_continuation(
self._parser.get_contents(livechat_json)[0]) self._parser.get_contents(livechat_json)[0])
if reload_continuation: if reload_continuation:
@@ -173,15 +178,14 @@ class PytchatCore:
offset_ms = 0 offset_ms = 0
param = util.get_param(continuation, dat=self._dat, replay=replay, offsetms=offset_ms) param = util.get_param(continuation, dat=self._dat, replay=replay, offsetms=offset_ms)
for _ in range(MAX_RETRY + 1): for _ in range(MAX_RETRY + 1):
with httpx.Client(http2=True) as client: try:
try: response = client.post(self._fetch_url, json=param)
response = client.post(self._fetch_url, json=param) livechat_json = response.json()
livechat_json = json.loads(response.text) break
break except (json.JSONDecodeError, httpx.ConnectTimeout, httpx.ReadTimeout, httpx.ConnectError) as e:
except (json.JSONDecodeError, httpx.ConnectTimeout, httpx.ReadTimeout, httpx.ConnectError) as e: err = e
err = e time.sleep(2)
time.sleep(2) continue
continue
else: else:
self._logger.error(f"[{self._video_id}]" self._logger.error(f"[{self._video_id}]"
f"Exceeded retry count. Last error: {str(err)}") f"Exceeded retry count. Last error: {str(err)}")
@@ -202,6 +206,8 @@ class PytchatCore:
return self._is_alive return self._is_alive
def terminate(self): def terminate(self):
if not self.is_alive():
return
self._is_alive = False self._is_alive = False
self.processor.finalize() self.processor.finalize()

18
pytchat/core_async/livechat.py
View File

@@ -78,6 +78,7 @@ class LiveChatAsync:
seektime=-1, seektime=-1,
processor=DefaultProcessor(), processor=DefaultProcessor(),
buffer=None, buffer=None,
client = httpx.AsyncClient(http2=True),
interruptable=True, interruptable=True,
callback=None, callback=None,
done_callback=None, done_callback=None,
@@ -88,6 +89,7 @@ class LiveChatAsync:
logger=config.logger(__name__), logger=config.logger(__name__),
replay_continuation=None replay_continuation=None
): ):
self._client:httpx.AsyncClient = client
self._video_id = util.extract_video_id(video_id) self._video_id = util.extract_video_id(video_id)
self.seektime = seektime self.seektime = seektime
if isinstance(processor, tuple): if isinstance(processor, tuple):
@@ -152,9 +154,10 @@ class LiveChatAsync:
create and start _listen loop. create and start _listen loop.
""" """
if not self.continuation: if not self.continuation:
channel_id = await util.get_channelid_async(self._client, self._video_id)
self.continuation = liveparam.getparam( self.continuation = liveparam.getparam(
self._video_id, self._video_id,
channel_id=util.get_channelid(httpx.Client(http2=True), self._video_id), channel_id,
past_sec=3) past_sec=3)
await self._listen(self.continuation) await self._listen(self.continuation)
@@ -169,10 +172,10 @@ class LiveChatAsync:
parameter for next chat data parameter for next chat data
''' '''
try: try:
async with httpx.AsyncClient(http2=True) as client: async with self._client as client:
while(continuation and self._is_alive): while(continuation and self._is_alive):
continuation = await self._check_pause(continuation) continuation = await self._check_pause(continuation)
contents = await self._get_contents(continuation, client, headers) contents = await self._get_contents(continuation, client, headers) #Q#
metadata, chatdata = self._parser.parse(contents) metadata, chatdata = self._parser.parse(contents)
continuation = metadata.get('continuation') continuation = metadata.get('continuation')
if continuation: if continuation:
@@ -214,9 +217,10 @@ class LiveChatAsync:
''' '''
self._pauser.put_nowait(None) self._pauser.put_nowait(None)
if not self._is_replay: if not self._is_replay:
async with httpx.AsyncClient(http2=True) as client: async with self._client as client:
continuation = await liveparam.getparam(self._video_id, channel_id = await util.get_channelid_async(client, self.video_id)
channel_id=util.get_channelid_async(client, self.video_id), continuation = liveparam.getparam(self._video_id,
channel_id,
past_sec=3) past_sec=3)
return continuation return continuation
@@ -338,6 +342,8 @@ class LiveChatAsync:
self._logger.debug(f'[{self._video_id}] cancelled:{sender}') self._logger.debug(f'[{self._video_id}] cancelled:{sender}')
def terminate(self): def terminate(self):
if not self.is_alive():
return
if self._pauser.empty(): if self._pauser.empty():
self._pauser.put_nowait(None) self._pauser.put_nowait(None)
self._is_alive = False self._is_alive = False

15
pytchat/core_multithread/livechat.py
View File

@@ -78,6 +78,7 @@ class LiveChat:
def __init__(self, video_id, def __init__(self, video_id,
seektime=-1, seektime=-1,
processor=DefaultProcessor(), processor=DefaultProcessor(),
client = httpx.Client(http2=True),
buffer=None, buffer=None,
interruptable=True, interruptable=True,
callback=None, callback=None,
@@ -88,6 +89,7 @@ class LiveChat:
logger=config.logger(__name__), logger=config.logger(__name__),
replay_continuation=None replay_continuation=None
): ):
self._client = client
self._video_id = util.extract_video_id(video_id) self._video_id = util.extract_video_id(video_id)
self.seektime = seektime self.seektime = seektime
if isinstance(processor, tuple): if isinstance(processor, tuple):
@@ -150,7 +152,7 @@ class LiveChat:
if not self.continuation: if not self.continuation:
self.continuation = liveparam.getparam( self.continuation = liveparam.getparam(
self._video_id, self._video_id,
channel_id=util.get_channelid(httpx.Client(http2=True), self._video_id), channel_id=util.get_channelid(self._client, self._video_id),
past_sec=3) past_sec=3)
self._listen(self.continuation) self._listen(self.continuation)
@@ -164,7 +166,7 @@ class LiveChat:
parameter for next chat data parameter for next chat data
''' '''
try: try:
with httpx.Client(http2=True) as client: with self._client as client:
while(continuation and self._is_alive): while(continuation and self._is_alive):
continuation = self._check_pause(continuation) continuation = self._check_pause(continuation)
contents = self._get_contents(continuation, client, headers) contents = self._get_contents(continuation, client, headers)
@@ -224,7 +226,8 @@ class LiveChat:
------- -------
'continuationContents' which includes metadata & chat data. 'continuationContents' which includes metadata & chat data.
''' '''
livechat_json = self._get_livechat_json(continuation, client, replay=self._is_replay, offset_ms=self._last_offset_ms) livechat_json = self._get_livechat_json(
continuation, client, replay=self._is_replay, offset_ms=self._last_offset_ms)
contents, dat = self._parser.get_contents(livechat_json) contents, dat = self._parser.get_contents(livechat_json)
if self._dat == '' and dat: if self._dat == '' and dat:
self._dat = dat self._dat = dat
@@ -235,8 +238,8 @@ class LiveChat:
self._fetch_url = config._smr self._fetch_url = config._smr
continuation = arcparam.getparam( continuation = arcparam.getparam(
self._video_id, self.seektime, self._topchat_only, util.get_channelid(client, self._video_id)) self._video_id, self.seektime, self._topchat_only, util.get_channelid(client, self._video_id))
livechat_json = (self._get_livechat_json( livechat_json = self._get_livechat_json(
continuation, client, replay=True, offset_ms=self.seektime * 1000)) continuation, client, replay=True, offset_ms=self.seektime * 1000)
reload_continuation = self._parser.reload_continuation( reload_continuation = self._parser.reload_continuation(
self._parser.get_contents(livechat_json)[0]) self._parser.get_contents(livechat_json)[0])
if reload_continuation: if reload_continuation:
@@ -331,6 +334,8 @@ class LiveChat:
self._logger.debug(f'[{self._video_id}] cancelled:{sender}') self._logger.debug(f'[{self._video_id}] cancelled:{sender}')
def terminate(self): def terminate(self):
if not self.is_alive():
return
if self._pauser.empty(): if self._pauser.empty():
self._pauser.put_nowait(None) self._pauser.put_nowait(None)
self._is_alive = False self._is_alive = False

4
pytchat/processors/default/processor.py
View File

@@ -7,6 +7,7 @@ from .renderer.paidmessage import LiveChatPaidMessageRenderer
from .renderer.paidsticker import LiveChatPaidStickerRenderer from .renderer.paidsticker import LiveChatPaidStickerRenderer
from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
from .renderer.membership import LiveChatMembershipItemRenderer from .renderer.membership import LiveChatMembershipItemRenderer
from .renderer.donation import LiveChatDonationAnnouncementRenderer
from .. chat_processor import ChatProcessor from .. chat_processor import ChatProcessor
from ... import config from ... import config
@@ -124,7 +125,8 @@ class DefaultProcessor(ChatProcessor):
"liveChatPaidMessageRenderer": LiveChatPaidMessageRenderer(), "liveChatPaidMessageRenderer": LiveChatPaidMessageRenderer(),
"liveChatPaidStickerRenderer": LiveChatPaidStickerRenderer(), "liveChatPaidStickerRenderer": LiveChatPaidStickerRenderer(),
"liveChatLegacyPaidMessageRenderer": LiveChatLegacyPaidMessageRenderer(), "liveChatLegacyPaidMessageRenderer": LiveChatLegacyPaidMessageRenderer(),
"liveChatMembershipItemRenderer": LiveChatMembershipItemRenderer() "liveChatMembershipItemRenderer": LiveChatMembershipItemRenderer(),
"liveChatDonationAnnouncementRenderer": LiveChatDonationAnnouncementRenderer(),
} }
def process(self, chat_components: list): def process(self, chat_components: list):

6
pytchat/processors/default/renderer/donation.py Normal file
View File

@@ -0,0 +1,6 @@
from .base import BaseRenderer
class LiveChatDonationAnnouncementRenderer(BaseRenderer):
def settype(self):
self.chat.type = "donation"

43
pytchat/processors/speed/calculator.py
View File

@@ -82,16 +82,17 @@ class RingQueue:
class SpeedCalculator(ChatProcessor, RingQueue): class SpeedCalculator(ChatProcessor, RingQueue):
""" """
チャットの勢いを計算する。 Calculate the momentum of the chat.
一定期間のチャットデータのうち、最初のチャットの投稿時刻と Divide the difference between the time of the first chat and
最後のチャットの投稿時刻の差を、チャット数で割り返し the time of the last chat in the chat data over a period of
1分あたりの速度に換算する。 time by the number of chats and convert it to speed per minute.
Parameter Parameter
---------- ----------
capacity : int capacity : int
RingQueueに格納するチャット勢い算出用データの最大数 Maximum number of data for calculating chat momentum
to be stored in RingQueue.
""" """
def __init__(self, capacity=10): def __init__(self, capacity=10):
@@ -111,17 +112,17 @@ class SpeedCalculator(ChatProcessor, RingQueue):
def _calc_speed(self): def _calc_speed(self):
""" """
RingQueue内のチャット勢い算出用データリストを元に、 Calculates the chat speed based on the data list for calculating
チャット速度を計算して返す the chat momentum in RingQueue.
Return Return
--------------------------- ---------------------------
チャット速度（１分間で換算したチャット数） Chat speed (number of chats converted per minute)
""" """
try: try:
# キュー内の総チャット数 # Total number of chats in the queue
total = sum(item['chat_count'] for item in self.items) total = sum(item['chat_count'] for item in self.items)
# キュー内の最初と最後のチャットの時間差 # Interval between the first and last chats in the queue
duration = (self.items[self.last_pos]['endtime'] - self.items[self.first_pos]['starttime']) duration = (self.items[self.last_pos]['endtime'] - self.items[self.first_pos]['starttime'])
if duration != 0: if duration != 0:
return int(total * 60 / duration) return int(total * 60 / duration)
@@ -131,19 +132,12 @@ class SpeedCalculator(ChatProcessor, RingQueue):
def _put_chatdata(self, actions): def _put_chatdata(self, actions):
""" """
チャットデータからタイムスタンプを読み取り、勢い測定用のデータを組み立て、
RingQueueに投入する。
200円以上のスパチャはtickerとmessageの2つのデータが生成されるが、
tickerの方は時刻データの場所が異なることを利用し、勢いの集計から除外している。
Parameter Parameter
--------- ---------
actions : List[dict] actions : List[dict]
チャットデータ(addChatItemAction) のリスト List of addChatItemActions
""" """
def _put_emptydata(): def _put_emptydata():
'''
チャットデータがない場合に空のデータをキューに投入する。
'''
timestamp_now = int(time.time()) timestamp_now = int(time.time())
self.put({ self.put({
'chat_count': 0, 'chat_count': 0,
@@ -152,9 +146,6 @@ class SpeedCalculator(ChatProcessor, RingQueue):
}) })
def _get_timestamp(action: dict): def _get_timestamp(action: dict):
"""
チャットデータから時刻データを取り出す。
"""
try: try:
item = action['addChatItemAction']['item'] item = action['addChatItemAction']['item']
timestamp = int(item[list(item.keys())[0]]['timestampUsec']) timestamp = int(item[list(item.keys())[0]]['timestampUsec'])
@@ -166,32 +157,24 @@ class SpeedCalculator(ChatProcessor, RingQueue):
_put_emptydata() _put_emptydata()
return return
# actions内の時刻データを持つチャットデータの数
counter = 0 counter = 0
# actions内の最初のチャットデータの時刻
starttime = None starttime = None
# actions内の最後のチャットデータの時刻
endtime = None endtime = None
for action in actions: for action in actions:
# チャットデータからtimestampUsecを読み取る # Get timestampUsec from chatdata
gettime = _get_timestamp(action) gettime = _get_timestamp(action)
# 時刻のないデータだった場合は次の行のデータで読み取り試行
if gettime is None: if gettime is None:
continue continue
# 最初に有効な時刻を持つデータのtimestampをstarttimeに設定
if starttime is None: if starttime is None:
starttime = gettime starttime = gettime
# 最後のtimestampを設定(途中で時刻のないデータの場合もあるので上書きしていく)
endtime = gettime endtime = gettime
# チャットの数をインクリメント
counter += 1 counter += 1
# チャット速度用のデータをRingQueueに送る
if starttime is None or endtime is None: if starttime is None or endtime is None:
_put_emptydata() _put_emptydata()
return return