ramforth/pytchat-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lint

Browse Source
This commit is contained in:
taizan-hokuto
2020-05-31 19:45:01 +09:00
parent 6eb848f1c9
commit 141dbcd2da
4 changed files with 120 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
pytchat/core_async/livechat.py
View File

@@ -1,7 +1,6 @@
import aiohttp, asyncio import aiohttp
import datetime import asyncio
import json import json
import random
import signal import signal
import time import time
import traceback import traceback
@@ -12,7 +11,7 @@ from asyncio import Queue
from .buffer import Buffer from .buffer import Buffer
from ..parser.live import Parser from ..parser.live import Parser
from .. import config from .. import config
from ..exceptions import ChatParseException,IllegalFunctionCall from ..exceptions import ChatParseException, IllegalFunctionCall
from ..paramgen import liveparam, arcparam from ..paramgen import liveparam, arcparam
from ..processors.default.processor import DefaultProcessor from ..processors.default.processor import DefaultProcessor
from ..processors.combinator import Combinator from ..processors.combinator import Combinator
@@ -75,17 +74,17 @@ class LiveChatAsync:
_setup_finished = False _setup_finished = False
def __init__(self, video_id, def __init__(self, video_id,
seektime = 0, seektime=-1,
processor = DefaultProcessor(), processor=DefaultProcessor(),
buffer = None, buffer=None,
interruptable = True, interruptable=True,
callback = None, callback=None,
done_callback = None, done_callback=None,
exception_handler = None, exception_handler=None,
direct_mode = False, direct_mode=False,
force_replay = False, force_replay=False,
topchat_only = False, topchat_only=False,
logger = config.logger(__name__), logger=config.logger(__name__),
): ):
self.video_id = video_id self.video_id = video_id
self.seektime = seektime self.seektime = seektime
@@ -100,7 +99,7 @@ class LiveChatAsync:
self._direct_mode = direct_mode self._direct_mode = direct_mode
self._is_alive = True self._is_alive = True
self._is_replay = force_replay self._is_replay = force_replay
self._parser = Parser(is_replay = self._is_replay) self._parser = Parser(is_replay=self._is_replay)
self._pauser = Queue() self._pauser = Queue()
self._pauser.put_nowait(None) self._pauser.put_nowait(None)
self._setup() self._setup()
@@ -116,31 +115,31 @@ class LiveChatAsync:
self._set_exception_handler(exception_handler) self._set_exception_handler(exception_handler)
if interruptable: if interruptable:
signal.signal(signal.SIGINT, signal.signal(signal.SIGINT,
(lambda a, b:asyncio.create_task( (lambda a, b: asyncio.create_task(
LiveChatAsync.shutdown(None,signal.SIGINT,b)) LiveChatAsync.shutdown(None, signal.SIGINT, b))
)) ))
def _setup(self): def _setup(self):
#direct modeがTrueでcallback未設定の場合例外発生。 # direct modeがTrueでcallback未設定の場合例外発生。
if self._direct_mode: if self._direct_mode:
if self._callback is None: if self._callback is None:
raise IllegalFunctionCall( raise IllegalFunctionCall(
"When direct_mode=True, callback parameter is required.") "When direct_mode=True, callback parameter is required.")
else: else:
#direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成 # direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成
if self._buffer is None: if self._buffer is None:
self._buffer = Buffer(maxsize = 20) self._buffer = Buffer(maxsize=20)
#callbackが指定されている場合はcallbackを呼ぶループタスクを作成 # callbackが指定されている場合はcallbackを呼ぶループタスクを作成
if self._callback is None: if self._callback is None:
pass pass
else: else:
#callbackを呼ぶループタスクの開始 # callbackを呼ぶループタスクの開始
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
loop.create_task(self._callback_loop(self._callback)) loop.create_task(self._callback_loop(self._callback))
#_listenループタスクの開始 # _listenループタスクの開始
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
listen_task = loop.create_task(self._startlisten()) listen_task = loop.create_task(self._startlisten())
#add_done_callbackの登録 # add_done_callbackの登録
if self._done_callback is None: if self._done_callback is None:
listen_task.add_done_callback(self.finish) listen_task.add_done_callback(self.finish)
else: else:
@@ -150,7 +149,7 @@ class LiveChatAsync:
"""Fetch first continuation parameter, """Fetch first continuation parameter,
create and start _listen loop. create and start _listen loop.
""" """
initial_continuation = liveparam.getparam(self.video_id,3) initial_continuation = liveparam.getparam(self.video_id, 3)
await self._listen(initial_continuation) await self._listen(initial_continuation)
async def _listen(self, continuation): async def _listen(self, continuation):
@@ -172,14 +171,14 @@ class LiveChatAsync:
timeout = metadata['timeoutMs']/1000 timeout = metadata['timeoutMs']/1000
chat_component = { chat_component = {
"video_id" : self.video_id, "video_id": self.video_id,
"timeout" : timeout, "timeout": timeout,
"chatdata" : chatdata "chatdata": chatdata
} }
time_mark =time.time() time_mark = time.time()
if self._direct_mode: if self._direct_mode:
processed_chat = self.processor.process([chat_component]) processed_chat = self.processor.process([chat_component])
if isinstance(processed_chat,tuple): if isinstance(processed_chat, tuple):
await self._callback(*processed_chat) await self._callback(*processed_chat)
else: else:
await self._callback(processed_chat) await self._callback(processed_chat)
@@ -191,7 +190,7 @@ class LiveChatAsync:
except ChatParseException as e: except ChatParseException as e:
self._logger.debug(f"[{self.video_id}]{str(e)}") self._logger.debug(f"[{self.video_id}]{str(e)}")
return return
except (TypeError , json.JSONDecodeError) : except (TypeError, json.JSONDecodeError):
self._logger.error(f"{traceback.format_exc(limit = -1)}") self._logger.error(f"{traceback.format_exc(limit = -1)}")
return return
@@ -219,9 +218,7 @@ class LiveChatAsync:
------- -------
'continuationContents' which includes metadata & chatdata. 'continuationContents' which includes metadata & chatdata.
''' '''
livechat_json = (await livechat_json = await self._get_livechat_json(continuation, session, headers)
self._get_livechat_json(continuation, session, headers)
)
contents = self._parser.get_contents(livechat_json) contents = self._parser.get_contents(livechat_json)
if self._first_fetch: if self._first_fetch:
if contents is None or self._is_replay: if contents is None or self._is_replay:
@@ -249,14 +246,14 @@ class LiveChatAsync:
continuation = urllib.parse.quote(continuation) continuation = urllib.parse.quote(continuation)
livechat_json = None livechat_json = None
status_code = 0 status_code = 0
url =f"https://www.youtube.com/{self._fetch_url}{continuation}&pbj=1" url = f"https://www.youtube.com/{self._fetch_url}{continuation}&pbj=1"
for _ in range(MAX_RETRY + 1): for _ in range(MAX_RETRY + 1):
async with session.get(url ,headers = headers) as resp: async with session.get(url, headers=headers) as resp:
try: try:
text = await resp.text() text = await resp.text()
livechat_json = json.loads(text) livechat_json = json.loads(text)
break break
except (ClientConnectorError,json.JSONDecodeError) : except (ClientConnectorError, json.JSONDecodeError):
await asyncio.sleep(1) await asyncio.sleep(1)
continue continue
else: else:
@@ -265,7 +262,7 @@ class LiveChatAsync:
return None return None
return livechat_json return livechat_json
async def _callback_loop(self,callback): async def _callback_loop(self, callback):
""" コンストラクタでcallbackを指定している場合、バックグラウンドで """ コンストラクタでcallbackを指定している場合、バックグラウンドで
callbackに指定された関数に一定間隔でチャットデータを投げる。 callbackに指定された関数に一定間隔でチャットデータを投げる。
@@ -313,7 +310,7 @@ class LiveChatAsync:
def is_alive(self): def is_alive(self):
return self._is_alive return self._is_alive
def finish(self,sender): def finish(self, sender):
'''Listener終了時のコールバック''' '''Listener終了時のコールバック'''
try: try:
self.terminate() self.terminate()
@@ -325,9 +322,9 @@ class LiveChatAsync:
Listenerを終了する。 Listenerを終了する。
''' '''
self._is_alive = False self._is_alive = False
if self._direct_mode == False: if self._direct_mode is False:
#bufferにダミーオブジェクトを入れてis_alive()を判定させる # bufferにダミーオブジェクトを入れてis_alive()を判定させる
self._buffer.put_nowait({'chatdata':'','timeout':0}) self._buffer.put_nowait({'chatdata': '', 'timeout': 0})
self._logger.info(f'[{self.video_id}]finished.') self._logger.info(f'[{self.video_id}]finished.')
@classmethod @classmethod
@@ -336,13 +333,13 @@ class LiveChatAsync:
loop.set_exception_handler(handler) loop.set_exception_handler(handler)
@classmethod @classmethod
async def shutdown(cls, event, sig = None, handler=None): async def shutdown(cls, event, sig=None, handler=None):
cls._logger.debug("shutdown...") cls._logger.debug("shutdown...")
tasks = [t for t in asyncio.all_tasks() if t is not tasks = [t for t in asyncio.all_tasks() if t is not
asyncio.current_task()] asyncio.current_task()]
[task.cancel() for task in tasks] [task.cancel() for task in tasks]
cls._logger.debug(f"complete remaining tasks...") cls._logger.debug("complete remaining tasks...")
await asyncio.gather(*tasks,return_exceptions=True) await asyncio.gather(*tasks, return_exceptions=True)
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
loop.stop() loop.stop()

6
pytchat/core_multithread/livechat.py
View File

@@ -1,7 +1,5 @@
import requests import requests
import datetime
import json import json
import random
import signal import signal
import time import time
import traceback import traceback
@@ -74,7 +72,7 @@ class LiveChat:
_setup_finished = False _setup_finished = False
def __init__(self, video_id, def __init__(self, video_id,
seektime=0, seektime=-1,
processor=DefaultProcessor(), processor=DefaultProcessor(),
buffer=None, buffer=None,
interruptable=True, interruptable=True,
@@ -181,7 +179,7 @@ class LiveChat:
self._logger.debug(f"[{self.video_id}]{str(e)}") self._logger.debug(f"[{self.video_id}]{str(e)}")
return return
except (TypeError, json.JSONDecodeError): except (TypeError, json.JSONDecodeError):
self._logger.error(f"{traceback.format_exc(limit = -1)}") self._logger.error(f"{traceback.format_exc(limit=-1)}")
return return
self._logger.debug(f"[{self.video_id}]finished fetching chat.") self._logger.debug(f"[{self.video_id}]finished fetching chat.")

22
pytchat/paramgen/arcparam_mining.py
View File

@@ -1,7 +1,5 @@
from base64 import urlsafe_b64encode as b64enc from base64 import urlsafe_b64encode as b64enc
from functools import reduce from functools import reduce
import math
import random
import urllib.parse import urllib.parse
''' '''
@@ -12,6 +10,7 @@ Author: taizan-hokuto (2019) @taizan205
ver 0.0.1 2019.10.05 ver 0.0.1 2019.10.05
''' '''
def _gen_vid_long(video_id): def _gen_vid_long(video_id):
"""generate video_id parameter. """generate video_id parameter.
Parameter Parameter
@@ -67,15 +66,17 @@ def _gen_vid(video_id):
def _nval(val): def _nval(val):
"""convert value to byte array""" """convert value to byte array"""
if val<0: raise ValueError if val < 0:
raise ValueError
buf = b'' buf = b''
while val >> 7: while val >> 7:
m = val & 0xFF | 0x80 m = val & 0xFF | 0x80
buf += m.to_bytes(1,'big') buf += m.to_bytes(1, 'big')
val >>= 7 val >>= 7
buf += val.to_bytes(1,'big') buf += val.to_bytes(1, 'big')
return buf return buf
def _build(video_id, seektime, topchat_only): def _build(video_id, seektime, topchat_only):
switch_01 = b'\x04' if topchat_only else b'\x01' switch_01 = b'\x04' if topchat_only else b'\x01'
if seektime < 0: if seektime < 0:
@@ -83,11 +84,9 @@ def _build(video_id, seektime, topchat_only):
if seektime == 0: if seektime == 0:
times = b'' times = b''
else: else:
times =_nval(int(seektime*1000)) times = _nval(int(seektime*1000))
if seektime > 0: if seektime > 0:
_len_time = ( b'\x5A' _len_time = b'\x5A' + (len(times)+1).to_bytes(1, 'big') + b'\x10'
+ (len(times)+1).to_bytes(1,'big')
+ b'\x10')
else: else:
_len_time = b'' _len_time = b''
@@ -114,13 +113,14 @@ def _build(video_id, seektime, topchat_only):
body = reduce(lambda x, y: x+y, body) body = reduce(lambda x, y: x+y, body)
return urllib.parse.quote( return urllib.parse.quote(
b64enc( header_magic + b64enc(header_magic +
_nval(len(body)) + _nval(len(body)) +
body body
).decode() ).decode()
) )
def getparam(video_id, seektime = 0.0, topchat_only = False):
def getparam(video_id, seektime=0.0, topchat_only=False):
''' '''
Parameter Parameter
--------- ---------

15
pytchat/parser/live.py
View File

@@ -4,12 +4,12 @@ pytchat.parser.live
Parser of live chat JSON. Parser of live chat JSON.
""" """
import json
from .. exceptions import ( from .. exceptions import (
ResponseContextError, ResponseContextError,
NoContentsException, NoContentsException,
NoContinuationsException, NoContinuationsException,
ChatParseException ) ChatParseException)
class Parser: class Parser:
@@ -22,9 +22,8 @@ class Parser:
if jsn is None: if jsn is None:
raise ChatParseException('Called with none JSON object.') raise ChatParseException('Called with none JSON object.')
if jsn['response']['responseContext'].get('errors'): if jsn['response']['responseContext'].get('errors'):
raise ResponseContextError('The video_id would be wrong,' raise ResponseContextError('The video_id would be wrong, or video is deleted or private.')
'or video is deleted or private.') contents = jsn['response'].get('continuationContents')
contents=jsn['response'].get('continuationContents')
return contents return contents
def parse(self, contents): def parse(self, contents):
@@ -75,9 +74,9 @@ class Parser:
""" """
cont = contents['liveChatContinuation']['continuations'][0] cont = contents['liveChatContinuation']['continuations'][0]
if cont.get("liveChatReplayContinuationData"): if cont.get("liveChatReplayContinuationData"):
#chat data exist. # chat data exist.
return None return None
#chat data do not exist, get playerSeekContinuationData. # chat data do not exist, get playerSeekContinuationData.
init_cont = cont.get("playerSeekContinuationData") init_cont = cont.get("playerSeekContinuationData")
if init_cont: if init_cont:
return init_cont.get("continuation") return init_cont.get("continuation")
@@ -87,7 +86,7 @@ class Parser:
actions = contents['liveChatContinuation'].get('actions') actions = contents['liveChatContinuation'].get('actions')
if self.is_replay: if self.is_replay:
interval = self._get_interval(actions) interval = self._get_interval(actions)
metadata.setdefault("timeoutMs",interval) metadata.setdefault("timeoutMs", interval)
"""Archived chat has different structures than live chat, """Archived chat has different structures than live chat,
so make it the same format.""" so make it the same format."""
chatdata = [action["replayChatItemAction"]["actions"][0] chatdata = [action["replayChatItemAction"]["actions"][0]