ramforth/pytchat-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'hotfix/termination'

Browse Source
This commit is contained in:
taizan-hokuto
2020-05-05 21:18:46 +09:00
parent 3243d69d7a bd32c75833
commit 04457eaa5c
3 changed files with 166 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
pytchat/core_multithread/livechat.py
View File

@@ -11,7 +11,7 @@ from queue import Queue
from .buffer import Buffer from .buffer import Buffer
from ..parser.live import Parser from ..parser.live import Parser
from .. import config from .. import config
from ..exceptions import ChatParseException,IllegalFunctionCall from ..exceptions import ChatParseException, IllegalFunctionCall
from ..paramgen import liveparam, arcparam from ..paramgen import liveparam, arcparam
from ..processors.default.processor import DefaultProcessor from ..processors.default.processor import DefaultProcessor
from ..processors.combinator import Combinator from ..processors.combinator import Combinator
@@ -72,20 +72,18 @@ class LiveChat:
''' '''
_setup_finished = False _setup_finished = False
#チャット監視中のListenerのリスト
_listeners = []
def __init__(self, video_id, def __init__(self, video_id,
seektime = 0, seektime=0,
processor = DefaultProcessor(), processor=DefaultProcessor(),
buffer = None, buffer=None,
interruptable = True, interruptable=True,
callback = None, callback=None,
done_callback = None, done_callback=None,
direct_mode = False, direct_mode=False,
force_replay = False, force_replay=False,
topchat_only = False, topchat_only=False,
logger = config.logger(__name__) logger=config.logger(__name__)
): ):
self.video_id = video_id self.video_id = video_id
self.seektime = seektime self.seektime = seektime
@@ -100,53 +98,47 @@ class LiveChat:
self._direct_mode = direct_mode self._direct_mode = direct_mode
self._is_alive = True self._is_alive = True
self._is_replay = force_replay self._is_replay = force_replay
self._parser = Parser(is_replay = self._is_replay) self._parser = Parser(is_replay=self._is_replay)
self._pauser = Queue() self._pauser = Queue()
self._pauser.put_nowait(None) self._pauser.put_nowait(None)
self._setup()
self._first_fetch = True self._first_fetch = True
self._fetch_url = "live_chat/get_live_chat?continuation=" self._fetch_url = "live_chat/get_live_chat?continuation="
self._topchat_only = topchat_only self._topchat_only = topchat_only
self._logger = logger self._logger = logger
LiveChat._logger = logger
if not LiveChat._setup_finished:
LiveChat._setup_finished = True
if interruptable: if interruptable:
signal.signal(signal.SIGINT, (lambda a, b: signal.signal(signal.SIGINT, lambda a, b: self.terminate())
(LiveChat.shutdown(None,signal.SIGINT,b)) self._setup()
))
LiveChat._listeners.append(self)
def _setup(self): def _setup(self):
#direct modeがTrueでcallback未設定の場合例外発生。 # direct modeがTrueでcallback未設定の場合例外発生。
if self._direct_mode: if self._direct_mode:
if self._callback is None: if self._callback is None:
raise IllegalFunctionCall( raise IllegalFunctionCall(
"When direct_mode=True, callback parameter is required.") "When direct_mode=True, callback parameter is required.")
else: else:
#direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成 # direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成
if self._buffer is None: if self._buffer is None:
self._buffer = Buffer(maxsize = 20) self._buffer = Buffer(maxsize=20)
#callbackが指定されている場合はcallbackを呼ぶループタスクを作成 # callbackが指定されている場合はcallbackを呼ぶループタスクを作成
if self._callback is None: if self._callback is None:
pass pass
else: else:
#callbackを呼ぶループタスクの開始 # callbackを呼ぶループタスクの開始
self._executor.submit(self._callback_loop,self._callback) self._executor.submit(self._callback_loop, self._callback)
#_listenループタスクの開始 # _listenループタスクの開始
listen_task = self._executor.submit(self._startlisten) listen_task = self._executor.submit(self._startlisten)
#add_done_callbackの登録 # add_done_callbackの登録
if self._done_callback is None: if self._done_callback is None:
listen_task.add_done_callback(self.finish) listen_task.add_done_callback(self.finish)
else: else:
listen_task.add_done_callback(self._done_callback) listen_task.add_done_callback(self._done_callback)
def _startlisten(self): def _startlisten(self):
time.sleep(0.1) #sleep shortly to prohibit skipping fetching data time.sleep(0.1) # sleep shortly to prohibit skipping fetching data
"""Fetch first continuation parameter, """Fetch first continuation parameter,
create and start _listen loop. create and start _listen loop.
""" """
initial_continuation = liveparam.getparam(self.video_id,3) initial_continuation = liveparam.getparam(self.video_id, 3)
self._listen(initial_continuation) self._listen(initial_continuation)
def _listen(self, continuation): def _listen(self, continuation):
@@ -168,14 +160,15 @@ class LiveChat:
timeout = metadata['timeoutMs']/1000 timeout = metadata['timeoutMs']/1000
chat_component = { chat_component = {
"video_id" : self.video_id, "video_id": self.video_id,
"timeout" : timeout, "timeout": timeout,
"chatdata" : chatdata "chatdata": chatdata
} }
time_mark =time.time() time_mark = time.time()
if self._direct_mode: if self._direct_mode:
processed_chat = self.processor.process([chat_component]) processed_chat = self.processor.process(
if isinstance(processed_chat,tuple): [chat_component])
if isinstance(processed_chat, tuple):
self._callback(*processed_chat) self._callback(*processed_chat)
else: else:
self._callback(processed_chat) self._callback(processed_chat)
@@ -187,7 +180,7 @@ class LiveChat:
except ChatParseException as e: except ChatParseException as e:
self._logger.debug(f"[{self.video_id}]{str(e)}") self._logger.debug(f"[{self.video_id}]{str(e)}")
return return
except (TypeError , json.JSONDecodeError) : except (TypeError, json.JSONDecodeError):
self._logger.error(f"{traceback.format_exc(limit = -1)}") self._logger.error(f"{traceback.format_exc(limit = -1)}")
return return
@@ -202,7 +195,7 @@ class LiveChat:
''' '''
self._pauser.put_nowait(None) self._pauser.put_nowait(None)
if not self._is_replay: if not self._is_replay:
continuation = liveparam.getparam(self.video_id,3) continuation = liveparam.getparam(self.video_id, 3)
return continuation return continuation
def _get_contents(self, continuation, session, headers): def _get_contents(self, continuation, session, headers):
@@ -225,7 +218,7 @@ class LiveChat:
self._fetch_url = "live_chat_replay/get_live_chat_replay?continuation=" self._fetch_url = "live_chat_replay/get_live_chat_replay?continuation="
continuation = arcparam.getparam( continuation = arcparam.getparam(
self.video_id, self.seektime, self._topchat_only) self.video_id, self.seektime, self._topchat_only)
livechat_json = ( self._get_livechat_json( livechat_json = (self._get_livechat_json(
continuation, session, headers)) continuation, session, headers))
reload_continuation = self._parser.reload_continuation( reload_continuation = self._parser.reload_continuation(
self._parser.get_contents(livechat_json)) self._parser.get_contents(livechat_json))
@@ -244,14 +237,14 @@ class LiveChat:
continuation = urllib.parse.quote(continuation) continuation = urllib.parse.quote(continuation)
livechat_json = None livechat_json = None
status_code = 0 status_code = 0
url =f"https://www.youtube.com/{self._fetch_url}{continuation}&pbj=1" url = f"https://www.youtube.com/{self._fetch_url}{continuation}&pbj=1"
for _ in range(MAX_RETRY + 1): for _ in range(MAX_RETRY + 1):
with session.get(url ,headers = headers) as resp: with session.get(url, headers=headers) as resp:
try: try:
text = resp.text text = resp.text
livechat_json = json.loads(text) livechat_json = json.loads(text)
break break
except json.JSONDecodeError : except json.JSONDecodeError:
time.sleep(1) time.sleep(1)
continue continue
else: else:
@@ -260,7 +253,7 @@ class LiveChat:
return None return None
return livechat_json return livechat_json
def _callback_loop(self,callback): def _callback_loop(self, callback):
""" コンストラクタでcallbackを指定している場合、バックグラウンドで """ コンストラクタでcallbackを指定している場合、バックグラウンドで
callbackに指定された関数に一定間隔でチャットデータを投げる。 callbackに指定された関数に一定間隔でチャットデータを投げる。
@@ -308,7 +301,7 @@ class LiveChat:
def is_alive(self): def is_alive(self):
return self._is_alive return self._is_alive
def finish(self,sender): def finish(self, sender):
'''Listener終了時のコールバック''' '''Listener終了時のコールバック'''
try: try:
self.terminate() self.terminate()
@@ -319,14 +312,7 @@ class LiveChat:
''' '''
Listenerを終了する。 Listenerを終了する。
''' '''
if self.is_alive():
self._is_alive = False self._is_alive = False
if self._direct_mode == False: self._buffer.put({})
#bufferにダミーオブジェクトを入れてis_alive()を判定させる self._logger.info(f'[{self.video_id}]終了しました')
self._buffer.put({'chatdata':'','timeout':0})
self._logger.info(f'[{self.video_id}]finished.')
@classmethod
def shutdown(cls, event, sig = None, handler=None):
cls._logger.debug("shutdown...")
for t in LiveChat._listeners:
t._is_alive = False

30
pytchat/paramgen/arcparam.py
View File

@@ -12,6 +12,7 @@ Author: taizan-hokuto (2019) @taizan205
ver 0.0.1 2019.10.05 ver 0.0.1 2019.10.05
''' '''
def _gen_vid(video_id): def _gen_vid(video_id):
"""generate video_id parameter. """generate video_id parameter.
Parameter Parameter
@@ -40,31 +41,34 @@ def _gen_vid(video_id):
b64enc(reduce(lambda x, y: x+y, item)).decode() b64enc(reduce(lambda x, y: x+y, item)).decode()
).encode() ).encode()
def _nval(val): def _nval(val):
"""convert value to byte array""" """convert value to byte array"""
if val<0: raise ValueError if val < 0:
raise ValueError
buf = b'' buf = b''
while val >> 7: while val >> 7:
m = val & 0xFF | 0x80 m = val & 0xFF | 0x80
buf += m.to_bytes(1,'big') buf += m.to_bytes(1, 'big')
val >>= 7 val >>= 7
buf += val.to_bytes(1,'big') buf += val.to_bytes(1, 'big')
return buf return buf
def _build(video_id, seektime, topchat_only): def _build(video_id, seektime, topchat_only):
switch_01 = b'\x04' if topchat_only else b'\x01' switch_01 = b'\x04' if topchat_only else b'\x01'
if seektime < 0: if seektime < 0:
times =_nval(0) times = _nval(0)
switch = b'\x04' switch = b'\x04'
elif seektime == 0: elif seektime == 0:
times =_nval(1) times = _nval(1)
switch = b'\x03' switch = b'\x03'
else: else:
times =_nval(int(seektime*1000000)) times = _nval(int(seektime*1000000))
switch = b'\x03' switch = b'\x03'
parity = b'\x00' parity = b'\x00'
header_magic= b'\xA2\x9D\xB0\xD3\x04' header_magic = b'\xA2\x9D\xB0\xD3\x04'
sep_0 = b'\x1A' sep_0 = b'\x1A'
vid = _gen_vid(video_id) vid = _gen_vid(video_id)
time_tag = b'\x28' time_tag = b'\x28'
@@ -75,7 +79,8 @@ def _build(video_id, seektime, topchat_only):
sep_3 = b'\x00\x58\x03\x60' sep_3 = b'\x00\x58\x03\x60'
sep_4 = b'\x68' + parity + b'\x72\x04\x08' sep_4 = b'\x68' + parity + b'\x72\x04\x08'
sep_5 = b'\x10' + parity + b'\x78\x00' sep_5 = b'\x10' + parity + b'\x78\x00'
body = [
body = b''.join([
sep_0, sep_0,
_nval(len(vid)), _nval(len(vid)),
vid, vid,
@@ -90,18 +95,17 @@ def _build(video_id, seektime, topchat_only):
sep_4, sep_4,
switch_01, switch_01,
sep_5 sep_5
] ])
body = reduce(lambda x, y: x+y, body)
return urllib.parse.quote( return urllib.parse.quote(
b64enc( header_magic + b64enc(header_magic +
_nval(len(body)) + _nval(len(body)) +
body body
).decode() ).decode()
) )
def getparam(video_id, seektime = 0, topchat_only = False):
def getparam(video_id, seektime=0, topchat_only=False):
''' '''
Parameter Parameter
--------- ---------

63
pytchat/paramgen/liveparam.py
View File

@@ -11,6 +11,8 @@ Author: taizan-hokuto (2019) @taizan205
ver 0.0.1 2019.10.05 ver 0.0.1 2019.10.05
''' '''
def _gen_vid(video_id): def _gen_vid(video_id):
"""generate video_id parameter. """generate video_id parameter.
Parameter Parameter
@@ -44,34 +46,38 @@ def _gen_vid(video_id):
b64enc(reduce(lambda x, y: x+y, item)).decode() b64enc(reduce(lambda x, y: x+y, item)).decode()
).encode() ).encode()
def _tzparity(video_id,times):
t=0 def _tzparity(video_id, times):
for i,s in enumerate(video_id): t = 0
for i, s in enumerate(video_id):
ss = ord(s) ss = ord(s)
if(ss % 2 == 0): if(ss % 2 == 0):
t += ss*(12-i) t += ss*(12-i)
else: else:
t ^= ss*i t ^= ss*i
return ((times^t) % 2).to_bytes(1,'big') return ((times ^ t) % 2).to_bytes(1, 'big')
def _nval(val): def _nval(val):
"""convert value to byte array""" """convert value to byte array"""
if val<0: raise ValueError if val < 0:
raise ValueError
buf = b'' buf = b''
while val >> 7: while val >> 7:
m = val & 0xFF | 0x80 m = val & 0xFF | 0x80
buf += m.to_bytes(1,'big') buf += m.to_bytes(1, 'big')
val >>= 7 val >>= 7
buf += val.to_bytes(1,'big') buf += val.to_bytes(1, 'big')
return buf return buf
def _build(video_id, _ts1, _ts2, _ts3, _ts4, _ts5, topchat_only):
#_short_type2
switch_01 = b'\x04' if topchat_only else b'\x01'
parity = _tzparity(video_id, _ts1^_ts2^_ts3^_ts4^_ts5)
header_magic= b'\xD2\x87\xCC\xC8\x03' def _build(video_id, _ts1, _ts2, _ts3, _ts4, _ts5, topchat_only):
# _short_type2
switch_01 = b'\x04' if topchat_only else b'\x01'
parity = _tzparity(video_id, _ts1 ^ _ts2 ^ _ts3 ^ _ts4 ^ _ts5)
header_magic = b'\xD2\x87\xCC\xC8\x03'
sep_0 = b'\x1A' sep_0 = b'\x1A'
vid = _gen_vid(video_id) vid = _gen_vid(video_id)
time_tag = b'\x28' time_tag = b'\x28'
@@ -92,14 +98,14 @@ def _build(video_id, _ts1, _ts2, _ts3, _ts4, _ts5, topchat_only):
ts_3_end = b'\x58' ts_3_end = b'\x58'
timestamp4 = _nval(_ts4) timestamp4 = _nval(_ts4)
sep_6 = b'\x68' sep_6 = b'\x68'
#switch # switch
sep_7 = b'\x82\x01\x04\x08' sep_7 = b'\x82\x01\x04\x08'
#switch # switch
sep_8 = b'\x10\x00' sep_8 = b'\x10\x00'
sep_9 = b'\x88\x01\x00\xA0\x01' sep_9 = b'\x88\x01\x00\xA0\x01'
timestamp5 = _nval(_ts5) timestamp5 = _nval(_ts5)
body = [ body = b''.join([
sep_0, sep_0,
_nval(len(vid)), _nval(len(vid)),
vid, vid,
@@ -121,18 +127,16 @@ def _build(video_id, _ts1, _ts2, _ts3, _ts4, _ts5, topchat_only):
ts_3_end, ts_3_end,
timestamp4, timestamp4,
sep_6, sep_6,
switch_01,# switch_01,
sep_7, sep_7,
switch_01,# switch_01,
sep_8, sep_8,
sep_9, sep_9,
timestamp5 timestamp5
] ])
body = reduce(lambda x, y: x+y, body)
return urllib.parse.quote( return urllib.parse.quote(
b64enc( header_magic + b64enc(header_magic +
_nval(len(body)) + _nval(len(body)) +
body body
).decode() ).decode()
@@ -143,15 +147,15 @@ def _times(past_sec):
n = int(time.time()) n = int(time.time())
_ts1= n - random.uniform(0,1*3) _ts1 = n - random.uniform(0, 1*3)
_ts2= n - random.uniform(0.01,0.99) _ts2 = n - random.uniform(0.01, 0.99)
_ts3= n - past_sec + random.uniform(0,1) _ts3 = n - past_sec + random.uniform(0, 1)
_ts4= n - random.uniform(10*60,60*60) _ts4 = n - random.uniform(10*60, 60*60)
_ts5= n - random.uniform(0.01,0.99) _ts5 = n - random.uniform(0.01, 0.99)
return list(map(lambda x:int(x*1000000),[_ts1,_ts2,_ts3,_ts4,_ts5])) return list(map(lambda x: int(x*1000000), [_ts1, _ts2, _ts3, _ts4, _ts5]))
def getparam(video_id, past_sec = 0, topchat_only = False): def getparam(video_id, past_sec=0, topchat_only=False):
''' '''
Parameter Parameter
--------- ---------
@@ -160,5 +164,4 @@ def getparam(video_id, past_sec = 0, topchat_only = False):
topchat_only : bool topchat_only : bool
if True, fetch only 'top chat' if True, fetch only 'top chat'
''' '''
return _build(video_id,*_times(past_sec),topchat_only) return _build(video_id, *_times(past_sec), topchat_only)