Compare commits
21 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f0a1a509a0 | ||
|
|
5ebca605ac | ||
|
|
a46c82d3c0 | ||
|
|
3826b32ab9 | ||
|
|
206d052907 | ||
|
|
04457eaa5c | ||
|
|
141d7a9299 | ||
|
|
bd32c75833 | ||
|
|
84bae4ad2a | ||
|
|
3243d69d7a | ||
|
|
d72608bf0a | ||
|
|
6e1b735ebc | ||
|
|
c54481dad5 | ||
|
|
78604c84d4 | ||
|
|
21d93613a2 | ||
|
|
56bf721330 | ||
|
|
5f50598f79 | ||
|
|
5e8c438c6b | ||
|
|
23e47f6fb0 | ||
|
|
74dfe0a612 | ||
|
|
725af25d81 |
@@ -7,10 +7,10 @@ pytchat is a python library for fetching youtube live chat.
|
|||||||
pytchat is a python library for fetching youtube live chat
|
pytchat is a python library for fetching youtube live chat
|
||||||
without using youtube api, Selenium or BeautifulSoup.
|
without using youtube api, Selenium or BeautifulSoup.
|
||||||
|
|
||||||
pytchatはAPIを使わずにYouTubeチャットを取得するための軽量pythonライブラリです。
|
pytchatはAPIを使わずにYouTubeチャットを取得するためのpythonライブラリです。
|
||||||
|
|
||||||
Other features:
|
Other features:
|
||||||
+ Customizable chat data processors including youtube api compatible one.
|
+ Customizable [chat data processors](https://github.com/taizan-hokuto/pytchat/wiki/ChatProcessor) including youtube api compatible one.
|
||||||
+ Available on asyncio context.
|
+ Available on asyncio context.
|
||||||
+ Quick fetching of initial chat data by generating continuation params
|
+ Quick fetching of initial chat data by generating continuation params
|
||||||
instead of web scraping.
|
instead of web scraping.
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
pytchat is a python library for fetching youtube live chat without using yt api, Selenium, or BeautifulSoup.
|
pytchat is a python library for fetching youtube live chat without using yt api, Selenium, or BeautifulSoup.
|
||||||
"""
|
"""
|
||||||
__copyright__ = 'Copyright (C) 2019 taizan-hokuto'
|
__copyright__ = 'Copyright (C) 2019 taizan-hokuto'
|
||||||
__version__ = '0.0.6.4'
|
__version__ = '0.0.7'
|
||||||
__license__ = 'MIT'
|
__license__ = 'MIT'
|
||||||
__author__ = 'taizan-hokuto'
|
__author__ = 'taizan-hokuto'
|
||||||
__author_email__ = '55448286+taizan-hokuto@users.noreply.github.com'
|
__author_email__ = '55448286+taizan-hokuto@users.noreply.github.com'
|
||||||
|
|||||||
@@ -22,9 +22,10 @@ def main():
|
|||||||
# Arguments
|
# Arguments
|
||||||
parser = argparse.ArgumentParser(description=f'pytchat v{__version__}')
|
parser = argparse.ArgumentParser(description=f'pytchat v{__version__}')
|
||||||
parser.add_argument('-v', f'--{Arguments.Name.VIDEO}', type=str,
|
parser.add_argument('-v', f'--{Arguments.Name.VIDEO}', type=str,
|
||||||
help='Video IDs separated by commas without space')
|
help='Video IDs separated by commas without space.\n'
|
||||||
|
'If ID starts with a hyphen (-), enclose the ID in square brackets.')
|
||||||
parser.add_argument('-o', f'--{Arguments.Name.OUTPUT}', type=str,
|
parser.add_argument('-o', f'--{Arguments.Name.OUTPUT}', type=str,
|
||||||
help='Output directory (end with "/")', default='./')
|
help='Output directory (end with "/"). default="./"', default='./')
|
||||||
parser.add_argument(f'--{Arguments.Name.VERSION}', action='store_true',
|
parser.add_argument(f'--{Arguments.Name.VERSION}', action='store_true',
|
||||||
help='Settings version')
|
help='Settings version')
|
||||||
Arguments(parser.parse_args().__dict__)
|
Arguments(parser.parse_args().__dict__)
|
||||||
@@ -35,17 +36,25 @@ def main():
|
|||||||
# Extractor
|
# Extractor
|
||||||
if Arguments().video_ids:
|
if Arguments().video_ids:
|
||||||
for video_id in Arguments().video_ids:
|
for video_id in Arguments().video_ids:
|
||||||
|
if '[' in video_id:
|
||||||
|
video_id = video_id.replace('[','').replace(']','')
|
||||||
try:
|
try:
|
||||||
info = VideoInfo(video_id)
|
info = VideoInfo(video_id)
|
||||||
print(f"Extracting...\n"
|
print(f"Extracting...\n"
|
||||||
f" video_id: {video_id}\n"
|
f" video_id: {video_id}\n"
|
||||||
f" channel: {info.get_channel_name()}\n"
|
f" channel: {info.get_channel_name()}\n"
|
||||||
f" title: {info.get_title()}")
|
f" title: {info.get_title()}")
|
||||||
|
path = Path(Arguments().output+video_id+'.html')
|
||||||
|
print(f"output path: {path.resolve()}")
|
||||||
Extractor(video_id,
|
Extractor(video_id,
|
||||||
processor = HTMLArchiver(Arguments().output+video_id+'.html')
|
processor = HTMLArchiver(Arguments().output+video_id+'.html'),
|
||||||
|
callback = _disp_progress
|
||||||
).extract()
|
).extract()
|
||||||
print("Extraction end.\n")
|
print("\nExtraction end.\n")
|
||||||
except (InvalidVideoIdException, NoContentsException) as e:
|
except (InvalidVideoIdException, NoContentsException) as e:
|
||||||
print(e)
|
print(e)
|
||||||
return
|
return
|
||||||
parser.print_help()
|
parser.print_help()
|
||||||
|
|
||||||
|
def _disp_progress(a,b):
|
||||||
|
print('.',end="",flush=True)
|
||||||
|
|||||||
@@ -72,8 +72,6 @@ class LiveChat:
|
|||||||
'''
|
'''
|
||||||
|
|
||||||
_setup_finished = False
|
_setup_finished = False
|
||||||
#チャット監視中のListenerのリスト
|
|
||||||
_listeners = []
|
|
||||||
|
|
||||||
def __init__(self, video_id,
|
def __init__(self, video_id,
|
||||||
seektime=0,
|
seektime=0,
|
||||||
@@ -103,19 +101,13 @@ class LiveChat:
|
|||||||
self._parser = Parser(is_replay=self._is_replay)
|
self._parser = Parser(is_replay=self._is_replay)
|
||||||
self._pauser = Queue()
|
self._pauser = Queue()
|
||||||
self._pauser.put_nowait(None)
|
self._pauser.put_nowait(None)
|
||||||
self._setup()
|
|
||||||
self._first_fetch = True
|
self._first_fetch = True
|
||||||
self._fetch_url = "live_chat/get_live_chat?continuation="
|
self._fetch_url = "live_chat/get_live_chat?continuation="
|
||||||
self._topchat_only = topchat_only
|
self._topchat_only = topchat_only
|
||||||
self._logger = logger
|
self._logger = logger
|
||||||
LiveChat._logger = logger
|
|
||||||
if not LiveChat._setup_finished:
|
|
||||||
LiveChat._setup_finished = True
|
|
||||||
if interruptable:
|
if interruptable:
|
||||||
signal.signal(signal.SIGINT, (lambda a, b:
|
signal.signal(signal.SIGINT, lambda a, b: self.terminate())
|
||||||
(LiveChat.shutdown(None,signal.SIGINT,b))
|
self._setup()
|
||||||
))
|
|
||||||
LiveChat._listeners.append(self)
|
|
||||||
|
|
||||||
def _setup(self):
|
def _setup(self):
|
||||||
# direct modeがTrueでcallback未設定の場合例外発生。
|
# direct modeがTrueでcallback未設定の場合例外発生。
|
||||||
@@ -174,7 +166,8 @@ class LiveChat:
|
|||||||
}
|
}
|
||||||
time_mark = time.time()
|
time_mark = time.time()
|
||||||
if self._direct_mode:
|
if self._direct_mode:
|
||||||
processed_chat = self.processor.process([chat_component])
|
processed_chat = self.processor.process(
|
||||||
|
[chat_component])
|
||||||
if isinstance(processed_chat, tuple):
|
if isinstance(processed_chat, tuple):
|
||||||
self._callback(*processed_chat)
|
self._callback(*processed_chat)
|
||||||
else:
|
else:
|
||||||
@@ -319,14 +312,7 @@ class LiveChat:
|
|||||||
'''
|
'''
|
||||||
Listenerを終了する。
|
Listenerを終了する。
|
||||||
'''
|
'''
|
||||||
|
if self.is_alive():
|
||||||
self._is_alive = False
|
self._is_alive = False
|
||||||
if self._direct_mode == False:
|
self._buffer.put({})
|
||||||
#bufferにダミーオブジェクトを入れてis_alive()を判定させる
|
self._logger.info(f'[{self.video_id}]終了しました')
|
||||||
self._buffer.put({'chatdata':'','timeout':0})
|
|
||||||
self._logger.info(f'[{self.video_id}]finished.')
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def shutdown(cls, event, sig = None, handler=None):
|
|
||||||
cls._logger.debug("shutdown...")
|
|
||||||
for t in LiveChat._listeners:
|
|
||||||
t._is_alive = False
|
|
||||||
|
|||||||
@@ -1,46 +1,52 @@
|
|||||||
class ChatParseException(Exception):
|
class ChatParseException(Exception):
|
||||||
'''
|
'''
|
||||||
チャットデータをパースするライブラリが投げる例外の基底クラス
|
Base exception thrown by the parser
|
||||||
'''
|
'''
|
||||||
pass
|
pass
|
||||||
|
|
||||||
class NoYtinitialdataException(ChatParseException):
|
class NoYtinitialdataException(ChatParseException):
|
||||||
'''
|
'''
|
||||||
配信ページ内にチャットデータurlが見つからないときに投げる例外
|
Thrown when the video is not found.
|
||||||
'''
|
'''
|
||||||
pass
|
pass
|
||||||
|
|
||||||
class ResponseContextError(ChatParseException):
|
class ResponseContextError(ChatParseException):
|
||||||
'''
|
'''
|
||||||
配信ページでチャットデータ無効の時に投げる例外
|
Thrown when chat data is invalid.
|
||||||
'''
|
'''
|
||||||
pass
|
pass
|
||||||
|
|
||||||
class NoLivechatRendererException(ChatParseException):
|
class NoLivechatRendererException(ChatParseException):
|
||||||
'''
|
'''
|
||||||
チャットデータのJSON中にlivechatRendererがない時に投げる例外
|
Thrown when livechatRenderer is missing in JSON.
|
||||||
'''
|
'''
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
class NoContentsException(ChatParseException):
|
class NoContentsException(ChatParseException):
|
||||||
'''
|
'''
|
||||||
チャットデータのJSON中にContinuationContentsがない時に投げる例外
|
Thrown when ContinuationContents is missing in JSON.
|
||||||
'''
|
'''
|
||||||
pass
|
pass
|
||||||
|
|
||||||
class NoContinuationsException(ChatParseException):
|
class NoContinuationsException(ChatParseException):
|
||||||
'''
|
'''
|
||||||
チャットデータのContinuationContents中にcontinuationがない時に投げる例外
|
Thrown when continuation is missing in ContinuationContents.
|
||||||
'''
|
'''
|
||||||
pass
|
pass
|
||||||
|
|
||||||
class IllegalFunctionCall(Exception):
|
class IllegalFunctionCall(Exception):
|
||||||
'''
|
'''
|
||||||
set_callback()を実行済みにもかかわらず
|
Thrown when get () is called even though
|
||||||
get()を呼び出した場合の例外
|
set_callback () has been executed.
|
||||||
'''
|
'''
|
||||||
pass
|
pass
|
||||||
|
|
||||||
class InvalidVideoIdException(Exception):
|
class InvalidVideoIdException(Exception):
|
||||||
|
'''
|
||||||
|
Thrown when the video_id is not exist (VideoInfo).
|
||||||
|
'''
|
||||||
|
pass
|
||||||
|
|
||||||
|
class UnknownConnectionError(Exception):
|
||||||
pass
|
pass
|
||||||
@@ -12,6 +12,7 @@ Author: taizan-hokuto (2019) @taizan205
|
|||||||
ver 0.0.1 2019.10.05
|
ver 0.0.1 2019.10.05
|
||||||
'''
|
'''
|
||||||
|
|
||||||
|
|
||||||
def _gen_vid(video_id):
|
def _gen_vid(video_id):
|
||||||
"""generate video_id parameter.
|
"""generate video_id parameter.
|
||||||
Parameter
|
Parameter
|
||||||
@@ -40,9 +41,11 @@ def _gen_vid(video_id):
|
|||||||
b64enc(reduce(lambda x, y: x+y, item)).decode()
|
b64enc(reduce(lambda x, y: x+y, item)).decode()
|
||||||
).encode()
|
).encode()
|
||||||
|
|
||||||
|
|
||||||
def _nval(val):
|
def _nval(val):
|
||||||
"""convert value to byte array"""
|
"""convert value to byte array"""
|
||||||
if val<0: raise ValueError
|
if val < 0:
|
||||||
|
raise ValueError
|
||||||
buf = b''
|
buf = b''
|
||||||
while val >> 7:
|
while val >> 7:
|
||||||
m = val & 0xFF | 0x80
|
m = val & 0xFF | 0x80
|
||||||
@@ -51,6 +54,7 @@ def _nval(val):
|
|||||||
buf += val.to_bytes(1, 'big')
|
buf += val.to_bytes(1, 'big')
|
||||||
return buf
|
return buf
|
||||||
|
|
||||||
|
|
||||||
def _build(video_id, seektime, topchat_only):
|
def _build(video_id, seektime, topchat_only):
|
||||||
switch_01 = b'\x04' if topchat_only else b'\x01'
|
switch_01 = b'\x04' if topchat_only else b'\x01'
|
||||||
if seektime < 0:
|
if seektime < 0:
|
||||||
@@ -75,7 +79,8 @@ def _build(video_id, seektime, topchat_only):
|
|||||||
sep_3 = b'\x00\x58\x03\x60'
|
sep_3 = b'\x00\x58\x03\x60'
|
||||||
sep_4 = b'\x68' + parity + b'\x72\x04\x08'
|
sep_4 = b'\x68' + parity + b'\x72\x04\x08'
|
||||||
sep_5 = b'\x10' + parity + b'\x78\x00'
|
sep_5 = b'\x10' + parity + b'\x78\x00'
|
||||||
body = [
|
|
||||||
|
body = b''.join([
|
||||||
sep_0,
|
sep_0,
|
||||||
_nval(len(vid)),
|
_nval(len(vid)),
|
||||||
vid,
|
vid,
|
||||||
@@ -90,9 +95,7 @@ def _build(video_id, seektime, topchat_only):
|
|||||||
sep_4,
|
sep_4,
|
||||||
switch_01,
|
switch_01,
|
||||||
sep_5
|
sep_5
|
||||||
]
|
])
|
||||||
|
|
||||||
body = reduce(lambda x, y: x+y, body)
|
|
||||||
|
|
||||||
return urllib.parse.quote(
|
return urllib.parse.quote(
|
||||||
b64enc(header_magic +
|
b64enc(header_magic +
|
||||||
@@ -101,6 +104,7 @@ def _build(video_id, seektime, topchat_only):
|
|||||||
).decode()
|
).decode()
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def getparam(video_id, seektime=0, topchat_only=False):
|
def getparam(video_id, seektime=0, topchat_only=False):
|
||||||
'''
|
'''
|
||||||
Parameter
|
Parameter
|
||||||
|
|||||||
@@ -11,6 +11,8 @@ Author: taizan-hokuto (2019) @taizan205
|
|||||||
|
|
||||||
ver 0.0.1 2019.10.05
|
ver 0.0.1 2019.10.05
|
||||||
'''
|
'''
|
||||||
|
|
||||||
|
|
||||||
def _gen_vid(video_id):
|
def _gen_vid(video_id):
|
||||||
"""generate video_id parameter.
|
"""generate video_id parameter.
|
||||||
Parameter
|
Parameter
|
||||||
@@ -44,6 +46,7 @@ def _gen_vid(video_id):
|
|||||||
b64enc(reduce(lambda x, y: x+y, item)).decode()
|
b64enc(reduce(lambda x, y: x+y, item)).decode()
|
||||||
).encode()
|
).encode()
|
||||||
|
|
||||||
|
|
||||||
def _tzparity(video_id, times):
|
def _tzparity(video_id, times):
|
||||||
t = 0
|
t = 0
|
||||||
for i, s in enumerate(video_id):
|
for i, s in enumerate(video_id):
|
||||||
@@ -55,9 +58,11 @@ def _tzparity(video_id,times):
|
|||||||
|
|
||||||
return ((times ^ t) % 2).to_bytes(1, 'big')
|
return ((times ^ t) % 2).to_bytes(1, 'big')
|
||||||
|
|
||||||
|
|
||||||
def _nval(val):
|
def _nval(val):
|
||||||
"""convert value to byte array"""
|
"""convert value to byte array"""
|
||||||
if val<0: raise ValueError
|
if val < 0:
|
||||||
|
raise ValueError
|
||||||
buf = b''
|
buf = b''
|
||||||
while val >> 7:
|
while val >> 7:
|
||||||
m = val & 0xFF | 0x80
|
m = val & 0xFF | 0x80
|
||||||
@@ -66,6 +71,7 @@ def _nval(val):
|
|||||||
buf += val.to_bytes(1, 'big')
|
buf += val.to_bytes(1, 'big')
|
||||||
return buf
|
return buf
|
||||||
|
|
||||||
|
|
||||||
def _build(video_id, _ts1, _ts2, _ts3, _ts4, _ts5, topchat_only):
|
def _build(video_id, _ts1, _ts2, _ts3, _ts4, _ts5, topchat_only):
|
||||||
# _short_type2
|
# _short_type2
|
||||||
switch_01 = b'\x04' if topchat_only else b'\x01'
|
switch_01 = b'\x04' if topchat_only else b'\x01'
|
||||||
@@ -99,7 +105,7 @@ def _build(video_id, _ts1, _ts2, _ts3, _ts4, _ts5, topchat_only):
|
|||||||
sep_9 = b'\x88\x01\x00\xA0\x01'
|
sep_9 = b'\x88\x01\x00\xA0\x01'
|
||||||
timestamp5 = _nval(_ts5)
|
timestamp5 = _nval(_ts5)
|
||||||
|
|
||||||
body = [
|
body = b''.join([
|
||||||
sep_0,
|
sep_0,
|
||||||
_nval(len(vid)),
|
_nval(len(vid)),
|
||||||
vid,
|
vid,
|
||||||
@@ -121,15 +127,13 @@ def _build(video_id, _ts1, _ts2, _ts3, _ts4, _ts5, topchat_only):
|
|||||||
ts_3_end,
|
ts_3_end,
|
||||||
timestamp4,
|
timestamp4,
|
||||||
sep_6,
|
sep_6,
|
||||||
switch_01,#
|
switch_01,
|
||||||
sep_7,
|
sep_7,
|
||||||
switch_01,#
|
switch_01,
|
||||||
sep_8,
|
sep_8,
|
||||||
sep_9,
|
sep_9,
|
||||||
timestamp5
|
timestamp5
|
||||||
]
|
])
|
||||||
|
|
||||||
body = reduce(lambda x, y: x+y, body)
|
|
||||||
|
|
||||||
return urllib.parse.quote(
|
return urllib.parse.quote(
|
||||||
b64enc(header_magic +
|
b64enc(header_magic +
|
||||||
@@ -161,4 +165,3 @@ def getparam(video_id, past_sec = 0, topchat_only = False):
|
|||||||
if True, fetch only 'top chat'
|
if True, fetch only 'top chat'
|
||||||
'''
|
'''
|
||||||
return _build(video_id, *_times(past_sec), topchat_only)
|
return _build(video_id, *_times(past_sec), topchat_only)
|
||||||
|
|
||||||
|
|||||||
@@ -4,10 +4,12 @@ from .renderer.textmessage import LiveChatTextMessageRenderer
|
|||||||
from .renderer.paidmessage import LiveChatPaidMessageRenderer
|
from .renderer.paidmessage import LiveChatPaidMessageRenderer
|
||||||
from .renderer.paidsticker import LiveChatPaidStickerRenderer
|
from .renderer.paidsticker import LiveChatPaidStickerRenderer
|
||||||
from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
|
from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
|
||||||
|
from .renderer.membership import LiveChatMembershipItemRenderer
|
||||||
from .. chat_processor import ChatProcessor
|
from .. chat_processor import ChatProcessor
|
||||||
from ... import config
|
from ... import config
|
||||||
logger = config.logger(__name__)
|
logger = config.logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class CompatibleProcessor(ChatProcessor):
|
class CompatibleProcessor(ChatProcessor):
|
||||||
|
|
||||||
def process(self, chat_components: list):
|
def process(self, chat_components: list):
|
||||||
@@ -24,11 +26,15 @@ class CompatibleProcessor(ChatProcessor):
|
|||||||
timeout += chat_component.get('timeout', 0)
|
timeout += chat_component.get('timeout', 0)
|
||||||
chatdata = chat_component.get('chatdata')
|
chatdata = chat_component.get('chatdata')
|
||||||
|
|
||||||
if chatdata is None: break
|
if chatdata is None:
|
||||||
|
break
|
||||||
for action in chatdata:
|
for action in chatdata:
|
||||||
if action is None: continue
|
if action is None:
|
||||||
if action.get('addChatItemAction') is None: continue
|
continue
|
||||||
if action['addChatItemAction'].get('item') is None: continue
|
if action.get('addChatItemAction') is None:
|
||||||
|
continue
|
||||||
|
if action['addChatItemAction'].get('item') is None:
|
||||||
|
continue
|
||||||
|
|
||||||
chat = self.parse(action)
|
chat = self.parse(action)
|
||||||
if chat:
|
if chat:
|
||||||
@@ -47,7 +53,8 @@ class CompatibleProcessor(ChatProcessor):
|
|||||||
action = sitem.get("addChatItemAction")
|
action = sitem.get("addChatItemAction")
|
||||||
if action:
|
if action:
|
||||||
item = action.get("item")
|
item = action.get("item")
|
||||||
if item is None: return None
|
if item is None:
|
||||||
|
return None
|
||||||
rd = {}
|
rd = {}
|
||||||
try:
|
try:
|
||||||
renderer = self.get_renderer(item)
|
renderer = self.get_renderer(item)
|
||||||
@@ -75,7 +82,8 @@ class CompatibleProcessor(ChatProcessor):
|
|||||||
renderer = LiveChatPaidStickerRenderer(item)
|
renderer = LiveChatPaidStickerRenderer(item)
|
||||||
elif item.get("liveChatLegacyPaidMessageRenderer"):
|
elif item.get("liveChatLegacyPaidMessageRenderer"):
|
||||||
renderer = LiveChatLegacyPaidMessageRenderer(item)
|
renderer = LiveChatLegacyPaidMessageRenderer(item)
|
||||||
|
elif item.get("liveChatMembershipItemRenderer"):
|
||||||
|
renderer = LiveChatMembershipItemRenderer(item)
|
||||||
else:
|
else:
|
||||||
renderer = None
|
renderer = None
|
||||||
return renderer
|
return renderer
|
||||||
|
|
||||||
|
|||||||
40
pytchat/processors/compatible/renderer/membership.py
Normal file
40
pytchat/processors/compatible/renderer/membership.py
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
from .base import BaseRenderer
|
||||||
|
|
||||||
|
|
||||||
|
class LiveChatMembershipItemRenderer(BaseRenderer):
|
||||||
|
def __init__(self, item):
|
||||||
|
super().__init__(item, "newSponsorEvent")
|
||||||
|
|
||||||
|
def get_snippet(self):
|
||||||
|
message = self.get_message(self.renderer)
|
||||||
|
return {
|
||||||
|
"type": self.chattype,
|
||||||
|
"liveChatId": "",
|
||||||
|
"authorChannelId": self.renderer.get("authorExternalChannelId"),
|
||||||
|
"publishedAt": self.get_publishedat(self.renderer.get("timestampUsec", 0)),
|
||||||
|
"hasDisplayContent": True,
|
||||||
|
"displayMessage": message,
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
def get_authordetails(self):
|
||||||
|
authorExternalChannelId = self.renderer.get("authorExternalChannelId")
|
||||||
|
# parse subscriber type
|
||||||
|
isVerified, isChatOwner, _, isChatModerator = (
|
||||||
|
self.get_badges(self.renderer)
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"channelId": authorExternalChannelId,
|
||||||
|
"channelUrl": "http://www.youtube.com/channel/"+authorExternalChannelId,
|
||||||
|
"displayName": self.renderer["authorName"]["simpleText"],
|
||||||
|
"profileImageUrl": self.renderer["authorPhoto"]["thumbnails"][1]["url"],
|
||||||
|
"isVerified": isVerified,
|
||||||
|
"isChatOwner": isChatOwner,
|
||||||
|
"isChatSponsor": True,
|
||||||
|
"isChatModerator": isChatModerator
|
||||||
|
}
|
||||||
|
|
||||||
|
def get_message(self, renderer):
|
||||||
|
message = (renderer["headerSubtext"]["runs"][0]["text"]
|
||||||
|
)+' / '+(renderer["authorName"]["simpleText"])
|
||||||
|
return message
|
||||||
@@ -4,10 +4,13 @@ from .renderer.textmessage import LiveChatTextMessageRenderer
|
|||||||
from .renderer.paidmessage import LiveChatPaidMessageRenderer
|
from .renderer.paidmessage import LiveChatPaidMessageRenderer
|
||||||
from .renderer.paidsticker import LiveChatPaidStickerRenderer
|
from .renderer.paidsticker import LiveChatPaidStickerRenderer
|
||||||
from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
|
from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
|
||||||
|
from .renderer.membership import LiveChatMembershipItemRenderer
|
||||||
from .. chat_processor import ChatProcessor
|
from .. chat_processor import ChatProcessor
|
||||||
from ... import config
|
from ... import config
|
||||||
|
|
||||||
logger = config.logger(__name__)
|
logger = config.logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class Chatdata:
|
class Chatdata:
|
||||||
def __init__(self, chatlist: list, timeout: float):
|
def __init__(self, chatlist: list, timeout: float):
|
||||||
self.items = chatlist
|
self.items = chatlist
|
||||||
@@ -25,6 +28,7 @@ class Chatdata:
|
|||||||
return
|
return
|
||||||
await asyncio.sleep(self.interval/len(self.items))
|
await asyncio.sleep(self.interval/len(self.items))
|
||||||
|
|
||||||
|
|
||||||
class DefaultProcessor(ChatProcessor):
|
class DefaultProcessor(ChatProcessor):
|
||||||
def process(self, chat_components: list):
|
def process(self, chat_components: list):
|
||||||
|
|
||||||
@@ -35,24 +39,27 @@ class DefaultProcessor(ChatProcessor):
|
|||||||
for component in chat_components:
|
for component in chat_components:
|
||||||
timeout += component.get('timeout', 0)
|
timeout += component.get('timeout', 0)
|
||||||
chatdata = component.get('chatdata')
|
chatdata = component.get('chatdata')
|
||||||
if chatdata is None: continue
|
if chatdata is None:
|
||||||
|
continue
|
||||||
for action in chatdata:
|
for action in chatdata:
|
||||||
if action is None: continue
|
if action is None:
|
||||||
if action.get('addChatItemAction') is None: continue
|
continue
|
||||||
if action['addChatItemAction'].get('item') is None: continue
|
if action.get('addChatItemAction') is None:
|
||||||
|
continue
|
||||||
|
if action['addChatItemAction'].get('item') is None:
|
||||||
|
continue
|
||||||
|
|
||||||
chat = self._parse(action)
|
chat = self._parse(action)
|
||||||
if chat:
|
if chat:
|
||||||
chatlist.append(chat)
|
chatlist.append(chat)
|
||||||
return Chatdata(chatlist, float(timeout))
|
return Chatdata(chatlist, float(timeout))
|
||||||
|
|
||||||
|
|
||||||
def _parse(self, sitem):
|
def _parse(self, sitem):
|
||||||
|
|
||||||
action = sitem.get("addChatItemAction")
|
action = sitem.get("addChatItemAction")
|
||||||
if action:
|
if action:
|
||||||
item = action.get("item")
|
item = action.get("item")
|
||||||
if item is None: return None
|
if item is None:
|
||||||
|
return None
|
||||||
try:
|
try:
|
||||||
renderer = self._get_renderer(item)
|
renderer = self._get_renderer(item)
|
||||||
if renderer == None:
|
if renderer == None:
|
||||||
@@ -74,6 +81,8 @@ class DefaultProcessor(ChatProcessor):
|
|||||||
renderer = LiveChatPaidStickerRenderer(item)
|
renderer = LiveChatPaidStickerRenderer(item)
|
||||||
elif item.get("liveChatLegacyPaidMessageRenderer"):
|
elif item.get("liveChatLegacyPaidMessageRenderer"):
|
||||||
renderer = LiveChatLegacyPaidMessageRenderer(item)
|
renderer = LiveChatLegacyPaidMessageRenderer(item)
|
||||||
|
elif item.get("liveChatMembershipItemRenderer"):
|
||||||
|
renderer = LiveChatMembershipItemRenderer(item)
|
||||||
else:
|
else:
|
||||||
renderer = None
|
renderer = None
|
||||||
return renderer
|
return renderer
|
||||||
15
pytchat/processors/default/renderer/membership.py
Normal file
15
pytchat/processors/default/renderer/membership.py
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
from .base import BaseRenderer
|
||||||
|
|
||||||
|
|
||||||
|
class LiveChatMembershipItemRenderer(BaseRenderer):
|
||||||
|
def __init__(self, item):
|
||||||
|
super().__init__(item, "newSponsor")
|
||||||
|
|
||||||
|
def get_authordetails(self):
|
||||||
|
super().get_authordetails()
|
||||||
|
self.author.isChatSponsor = True
|
||||||
|
|
||||||
|
def get_message(self, renderer):
|
||||||
|
message = (renderer["headerSubtext"]["runs"][0]["text"]
|
||||||
|
)+' / '+(renderer["authorName"]["simpleText"])
|
||||||
|
return message
|
||||||
@@ -8,6 +8,11 @@ PATTERN = re.compile(r"(.*)\(([0-9]+)\)$")
|
|||||||
fmt_headers = ['datetime','elapsed','authorName','message','superchat'
|
fmt_headers = ['datetime','elapsed','authorName','message','superchat'
|
||||||
,'type','authorChannel']
|
,'type','authorChannel']
|
||||||
|
|
||||||
|
HEADER_HTML = '''
|
||||||
|
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
|
||||||
|
<meta http-equiv="Content-Type" content="text/html;charset=UTF-8">
|
||||||
|
'''
|
||||||
|
|
||||||
class HTMLArchiver(ChatProcessor):
|
class HTMLArchiver(ChatProcessor):
|
||||||
'''
|
'''
|
||||||
HtmlArchiver saves chat data as HTML table format.
|
HtmlArchiver saves chat data as HTML table format.
|
||||||
@@ -17,6 +22,7 @@ class HTMLArchiver(ChatProcessor):
|
|||||||
super().__init__()
|
super().__init__()
|
||||||
self.save_path = self._checkpath(save_path)
|
self.save_path = self._checkpath(save_path)
|
||||||
with open(self.save_path, mode='a', encoding = 'utf-8') as f:
|
with open(self.save_path, mode='a', encoding = 'utf-8') as f:
|
||||||
|
f.write(HEADER_HTML)
|
||||||
f.write('<table border="1" style="border-collapse: collapse">')
|
f.write('<table border="1" style="border-collapse: collapse">')
|
||||||
f.writelines(self._parse_html_header(fmt_headers))
|
f.writelines(self._parse_html_header(fmt_headers))
|
||||||
self.processor = DefaultProcessor()
|
self.processor = DefaultProcessor()
|
||||||
|
|||||||
@@ -7,12 +7,15 @@ from . worker import ExtractWorker
|
|||||||
from . patch import Patch
|
from . patch import Patch
|
||||||
from ... import config
|
from ... import config
|
||||||
from ... paramgen import arcparam
|
from ... paramgen import arcparam
|
||||||
|
from ... exceptions import UnknownConnectionError
|
||||||
from concurrent.futures import CancelledError
|
from concurrent.futures import CancelledError
|
||||||
|
from json import JSONDecodeError
|
||||||
from urllib.parse import quote
|
from urllib.parse import quote
|
||||||
|
|
||||||
headers = config.headers
|
headers = config.headers
|
||||||
REPLAY_URL = "https://www.youtube.com/live_chat_replay/" \
|
REPLAY_URL = "https://www.youtube.com/live_chat_replay/" \
|
||||||
"get_live_chat_replay?continuation="
|
"get_live_chat_replay?continuation="
|
||||||
|
MAX_RETRY_COUNT = 3
|
||||||
|
|
||||||
def _split(start, end, count, min_interval_sec = 120):
|
def _split(start, end, count, min_interval_sec = 120):
|
||||||
"""
|
"""
|
||||||
@@ -57,9 +60,18 @@ def ready_blocks(video_id, duration, div, callback):
|
|||||||
async def _create_block(session, video_id, seektime, callback):
|
async def _create_block(session, video_id, seektime, callback):
|
||||||
continuation = arcparam.getparam(video_id, seektime = seektime)
|
continuation = arcparam.getparam(video_id, seektime = seektime)
|
||||||
url = f"{REPLAY_URL}{quote(continuation)}&pbj=1"
|
url = f"{REPLAY_URL}{quote(continuation)}&pbj=1"
|
||||||
|
for _ in range(MAX_RETRY_COUNT):
|
||||||
|
try :
|
||||||
async with session.get(url, headers = headers) as resp:
|
async with session.get(url, headers = headers) as resp:
|
||||||
text = await resp.text()
|
text = await resp.text()
|
||||||
next_continuation, actions = parser.parse(json.loads(text))
|
next_continuation, actions = parser.parse(json.loads(text))
|
||||||
|
break
|
||||||
|
except JSONDecodeError:
|
||||||
|
await asyncio.sleep(3)
|
||||||
|
else:
|
||||||
|
cancel()
|
||||||
|
raise UnknownConnectionError("Abort: Unknown connection error.")
|
||||||
|
|
||||||
if actions:
|
if actions:
|
||||||
first = parser.get_offset(actions[0])
|
first = parser.get_offset(actions[0])
|
||||||
last = parser.get_offset(actions[-1])
|
last = parser.get_offset(actions[-1])
|
||||||
@@ -71,6 +83,7 @@ def ready_blocks(video_id, duration, div, callback):
|
|||||||
first = first,
|
first = first,
|
||||||
last = last
|
last = last
|
||||||
)
|
)
|
||||||
|
|
||||||
"""
|
"""
|
||||||
fetch initial blocks.
|
fetch initial blocks.
|
||||||
"""
|
"""
|
||||||
@@ -95,9 +108,18 @@ def fetch_patch(callback, blocks, video_id):
|
|||||||
|
|
||||||
async def _fetch(continuation,session) -> Patch:
|
async def _fetch(continuation,session) -> Patch:
|
||||||
url = f"{REPLAY_URL}{quote(continuation)}&pbj=1"
|
url = f"{REPLAY_URL}{quote(continuation)}&pbj=1"
|
||||||
|
for _ in range(MAX_RETRY_COUNT):
|
||||||
|
try:
|
||||||
async with session.get(url,headers = config.headers) as resp:
|
async with session.get(url,headers = config.headers) as resp:
|
||||||
chat_json = await resp.text()
|
chat_json = await resp.text()
|
||||||
continuation, actions = parser.parse(json.loads(chat_json))
|
continuation, actions = parser.parse(json.loads(chat_json))
|
||||||
|
break
|
||||||
|
except JSONDecodeError:
|
||||||
|
await asyncio.sleep(3)
|
||||||
|
else:
|
||||||
|
cancel()
|
||||||
|
raise UnknownConnectionError("Abort: Unknown connection error.")
|
||||||
|
|
||||||
if actions:
|
if actions:
|
||||||
last = parser.get_offset(actions[-1])
|
last = parser.get_offset(actions[-1])
|
||||||
first = parser.get_offset(actions[0])
|
first = parser.get_offset(actions[0])
|
||||||
@@ -105,6 +127,7 @@ def fetch_patch(callback, blocks, video_id):
|
|||||||
callback(actions, last - first)
|
callback(actions, last - first)
|
||||||
return Patch(actions, continuation, first, last)
|
return Patch(actions, continuation, first, last)
|
||||||
return Patch(continuation = continuation)
|
return Patch(continuation = continuation)
|
||||||
|
|
||||||
"""
|
"""
|
||||||
allocate workers and assign blocks.
|
allocate workers and assign blocks.
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
aiohttp
|
aiohttp
|
||||||
argumentparser
|
|
||||||
pytz
|
pytz
|
||||||
requests
|
requests
|
||||||
urllib3
|
urllib3
|
||||||
@@ -36,7 +36,7 @@ def test_process_0():
|
|||||||
chat_component = {
|
chat_component = {
|
||||||
'video_id':'',
|
'video_id':'',
|
||||||
'timeout':10,
|
'timeout':10,
|
||||||
'chatdata':load_chatdata(r"tests\testdata\calculator\superchat_0.json")
|
'chatdata':load_chatdata(r"tests/testdata/calculator/superchat_0.json")
|
||||||
}
|
}
|
||||||
assert SuperchatCalculator().process([chat_component])=={'¥': 6800.0, '€': 2.0}
|
assert SuperchatCalculator().process([chat_component])=={'¥': 6800.0, '€': 2.0}
|
||||||
|
|
||||||
@@ -47,7 +47,7 @@ def test_process_1():
|
|||||||
chat_component = {
|
chat_component = {
|
||||||
'video_id':'',
|
'video_id':'',
|
||||||
'timeout':10,
|
'timeout':10,
|
||||||
'chatdata':load_chatdata(r"tests\testdata\calculator\text_only.json")
|
'chatdata':load_chatdata(r"tests/testdata/calculator/text_only.json")
|
||||||
}
|
}
|
||||||
assert SuperchatCalculator().process([chat_component])=={}
|
assert SuperchatCalculator().process([chat_component])=={}
|
||||||
|
|
||||||
@@ -59,7 +59,7 @@ def test_process_2():
|
|||||||
chat_component = {
|
chat_component = {
|
||||||
'video_id':'',
|
'video_id':'',
|
||||||
'timeout':10,
|
'timeout':10,
|
||||||
'chatdata':load_chatdata(r"tests\testdata\calculator\replay_end.json")
|
'chatdata':load_chatdata(r"tests/testdata/calculator/replay_end.json")
|
||||||
}
|
}
|
||||||
assert False
|
assert False
|
||||||
SuperchatCalculator().process([chat_component])
|
SuperchatCalculator().process([chat_component])
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import json
|
import json
|
||||||
import pytest
|
import pytest
|
||||||
import asyncio,aiohttp
|
import asyncio
|
||||||
|
import aiohttp
|
||||||
from pytchat.parser.live import Parser
|
from pytchat.parser.live import Parser
|
||||||
from pytchat.processors.compatible.processor import CompatibleProcessor
|
from pytchat.processors.compatible.processor import CompatibleProcessor
|
||||||
from pytchat.exceptions import (
|
from pytchat.exceptions import (
|
||||||
@@ -14,6 +15,7 @@ from pytchat.processors.compatible.renderer.legacypaid import LiveChatLegacyPaid
|
|||||||
|
|
||||||
parser = Parser(is_replay=False)
|
parser = Parser(is_replay=False)
|
||||||
|
|
||||||
|
|
||||||
def test_textmessage(mocker):
|
def test_textmessage(mocker):
|
||||||
'''api互換processorのテスト:通常テキストメッセージ'''
|
'''api互換processorのテスト:通常テキストメッセージ'''
|
||||||
processor = CompatibleProcessor()
|
processor = CompatibleProcessor()
|
||||||
@@ -51,6 +53,7 @@ def test_textmessage(mocker):
|
|||||||
assert "LCC." in ret["items"][0]["id"]
|
assert "LCC." in ret["items"][0]["id"]
|
||||||
assert ret["items"][0]["snippet"]["type"] == "textMessageEvent"
|
assert ret["items"][0]["snippet"]["type"] == "textMessageEvent"
|
||||||
|
|
||||||
|
|
||||||
def test_newsponcer(mocker):
|
def test_newsponcer(mocker):
|
||||||
'''api互換processorのテスト:メンバ新規登録'''
|
'''api互換processorのテスト:メンバ新規登録'''
|
||||||
processor = CompatibleProcessor()
|
processor = CompatibleProcessor()
|
||||||
@@ -87,6 +90,42 @@ def test_newsponcer(mocker):
|
|||||||
assert ret["items"][0]["snippet"]["type"] == "newSponsorEvent"
|
assert ret["items"][0]["snippet"]["type"] == "newSponsorEvent"
|
||||||
|
|
||||||
|
|
||||||
|
def test_newsponcer_rev(mocker):
|
||||||
|
'''api互換processorのテスト:メンバ新規登録'''
|
||||||
|
processor = CompatibleProcessor()
|
||||||
|
|
||||||
|
_json = _open_file("tests/testdata/compatible/newSponsor_rev.json")
|
||||||
|
|
||||||
|
_, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
|
||||||
|
data = {
|
||||||
|
"video_id": "",
|
||||||
|
"timeout": 7,
|
||||||
|
"chatdata": chatdata
|
||||||
|
}
|
||||||
|
ret = processor.process([data])
|
||||||
|
|
||||||
|
assert ret["kind"] == "youtube#liveChatMessageListResponse"
|
||||||
|
assert ret["pollingIntervalMillis"] == data["timeout"]*1000
|
||||||
|
assert ret.keys() == {
|
||||||
|
"kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
|
||||||
|
}
|
||||||
|
assert ret["pageInfo"].keys() == {
|
||||||
|
"totalResults", "resultsPerPage"
|
||||||
|
}
|
||||||
|
assert ret["items"][0].keys() == {
|
||||||
|
"kind", "etag", "id", "snippet", "authorDetails"
|
||||||
|
}
|
||||||
|
assert ret["items"][0]["snippet"].keys() == {
|
||||||
|
'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage'
|
||||||
|
|
||||||
|
}
|
||||||
|
assert ret["items"][0]["authorDetails"].keys() == {
|
||||||
|
'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator'
|
||||||
|
}
|
||||||
|
assert "LCC." in ret["items"][0]["id"]
|
||||||
|
assert ret["items"][0]["snippet"]["type"] == "newSponsorEvent"
|
||||||
|
|
||||||
|
|
||||||
def test_superchat(mocker):
|
def test_superchat(mocker):
|
||||||
'''api互換processorのテスト:スパチャ'''
|
'''api互換processorのテスト:スパチャ'''
|
||||||
processor = CompatibleProcessor()
|
processor = CompatibleProcessor()
|
||||||
@@ -124,6 +163,7 @@ def test_superchat(mocker):
|
|||||||
assert "LCC." in ret["items"][0]["id"]
|
assert "LCC." in ret["items"][0]["id"]
|
||||||
assert ret["items"][0]["snippet"]["type"] == "superChatEvent"
|
assert ret["items"][0]["snippet"]["type"] == "superChatEvent"
|
||||||
|
|
||||||
|
|
||||||
def test_unregistered_currency(mocker):
|
def test_unregistered_currency(mocker):
|
||||||
processor = CompatibleProcessor()
|
processor = CompatibleProcessor()
|
||||||
|
|
||||||
|
|||||||
1823
tests/testdata/compatible/newSponsor_rev.json
vendored
Normal file
1823
tests/testdata/compatible/newSponsor_rev.json
vendored
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user