ramforth/pytchat-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lint

Browse Source
This commit is contained in:
taizan-hokuto
2020-05-31 19:45:01 +09:00
parent 6eb848f1c9
commit 141dbcd2da
4 changed files with 120 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
pytchat/core_async/livechat.py
View File

@@ -1,7 +1,6 @@
import aiohttp, asyncio
import datetime
import aiohttp
import asyncio
import json
import random
import signal
import time
import traceback
@@ -12,7 +11,7 @@ from asyncio import Queue
from .buffer import Buffer
from ..parser.live import Parser
from .. import config
from ..exceptions import ChatParseException,IllegalFunctionCall
from ..exceptions import ChatParseException, IllegalFunctionCall
from ..paramgen import liveparam, arcparam
from ..processors.default.processor import DefaultProcessor
from ..processors.combinator import Combinator
@@ -75,17 +74,17 @@ class LiveChatAsync:
_setup_finished = False
def __init__(self, video_id,
seektime = 0,
processor = DefaultProcessor(),
buffer = None,
interruptable = True,
callback = None,
done_callback = None,
exception_handler = None,
direct_mode = False,
force_replay = False,
topchat_only = False,
logger = config.logger(__name__),
seektime=-1,
processor=DefaultProcessor(),
buffer=None,
interruptable=True,
callback=None,
done_callback=None,
exception_handler=None,
direct_mode=False,
force_replay=False,
topchat_only=False,
logger=config.logger(__name__),
):
self.video_id = video_id
self.seektime = seektime
@@ -100,7 +99,7 @@ class LiveChatAsync:
self._direct_mode = direct_mode
self._is_alive = True
self._is_replay = force_replay
self._parser = Parser(is_replay = self._is_replay)
self._parser = Parser(is_replay=self._is_replay)
self._pauser = Queue()
self._pauser.put_nowait(None)
self._setup()
@@ -116,31 +115,31 @@ class LiveChatAsync:
self._set_exception_handler(exception_handler)
if interruptable:
signal.signal(signal.SIGINT,
(lambda a, b:asyncio.create_task(
LiveChatAsync.shutdown(None,signal.SIGINT,b))
(lambda a, b: asyncio.create_task(
LiveChatAsync.shutdown(None, signal.SIGINT, b))
))
def _setup(self):
#direct modeがTrueでcallback未設定の場合例外発生。
# direct modeがTrueでcallback未設定の場合例外発生。
if self._direct_mode:
if self._callback is None:
raise IllegalFunctionCall(
"When direct_mode=True, callback parameter is required.")
else:
#direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成
# direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成
if self._buffer is None:
self._buffer = Buffer(maxsize = 20)
#callbackが指定されている場合はcallbackを呼ぶループタスクを作成
self._buffer = Buffer(maxsize=20)
# callbackが指定されている場合はcallbackを呼ぶループタスクを作成
if self._callback is None:
pass
else:
#callbackを呼ぶループタスクの開始
# callbackを呼ぶループタスクの開始
loop = asyncio.get_event_loop()
loop.create_task(self._callback_loop(self._callback))
#_listenループタスクの開始
# _listenループタスクの開始
loop = asyncio.get_event_loop()
listen_task = loop.create_task(self._startlisten())
#add_done_callbackの登録
# add_done_callbackの登録
if self._done_callback is None:
listen_task.add_done_callback(self.finish)
else:
@@ -150,7 +149,7 @@ class LiveChatAsync:
"""Fetch first continuation parameter,
create and start _listen loop.
"""
initial_continuation = liveparam.getparam(self.video_id,3)
initial_continuation = liveparam.getparam(self.video_id, 3)
await self._listen(initial_continuation)
async def _listen(self, continuation):
@@ -172,14 +171,14 @@ class LiveChatAsync:
timeout = metadata['timeoutMs']/1000
chat_component = {
"video_id" : self.video_id,
"timeout" : timeout,
"chatdata" : chatdata
"video_id": self.video_id,
"timeout": timeout,
"chatdata": chatdata
}
time_mark =time.time()
time_mark = time.time()
if self._direct_mode:
processed_chat = self.processor.process([chat_component])
if isinstance(processed_chat,tuple):
if isinstance(processed_chat, tuple):
await self._callback(*processed_chat)
else:
await self._callback(processed_chat)
@@ -191,7 +190,7 @@ class LiveChatAsync:
except ChatParseException as e:
self._logger.debug(f"[{self.video_id}]{str(e)}")
return
except (TypeError , json.JSONDecodeError) :
except (TypeError, json.JSONDecodeError):
self._logger.error(f"{traceback.format_exc(limit = -1)}")
return
@@ -219,9 +218,7 @@ class LiveChatAsync:
-------
'continuationContents' which includes metadata & chatdata.
'''
livechat_json = (await
self._get_livechat_json(continuation, session, headers)
)
livechat_json = await self._get_livechat_json(continuation, session, headers)
contents = self._parser.get_contents(livechat_json)
if self._first_fetch:
if contents is None or self._is_replay:
@@ -249,14 +246,14 @@ class LiveChatAsync:
continuation = urllib.parse.quote(continuation)
livechat_json = None
status_code = 0
url =f"https://www.youtube.com/{self._fetch_url}{continuation}&pbj=1"
url = f"https://www.youtube.com/{self._fetch_url}{continuation}&pbj=1"
for _ in range(MAX_RETRY + 1):
async with session.get(url ,headers = headers) as resp:
async with session.get(url, headers=headers) as resp:
try:
text = await resp.text()
livechat_json = json.loads(text)
break
except (ClientConnectorError,json.JSONDecodeError) :
except (ClientConnectorError, json.JSONDecodeError):
await asyncio.sleep(1)
continue
else:
@@ -265,7 +262,7 @@ class LiveChatAsync:
return None
return livechat_json
async def _callback_loop(self,callback):
async def _callback_loop(self, callback):
""" コンストラクタでcallbackを指定している場合、バックグラウンドで
callbackに指定された関数に一定間隔でチャットデータを投げる。
@@ -313,7 +310,7 @@ class LiveChatAsync:
def is_alive(self):
return self._is_alive
def finish(self,sender):
def finish(self, sender):
'''Listener終了時のコールバック'''
try:
self.terminate()
@@ -325,9 +322,9 @@ class LiveChatAsync:
Listenerを終了する。
'''
self._is_alive = False
if self._direct_mode == False:
#bufferにダミーオブジェクトを入れてis_alive()を判定させる
self._buffer.put_nowait({'chatdata':'','timeout':0})
if self._direct_mode is False:
# bufferにダミーオブジェクトを入れてis_alive()を判定させる
self._buffer.put_nowait({'chatdata': '', 'timeout': 0})
self._logger.info(f'[{self.video_id}]finished.')
@classmethod
@@ -336,13 +333,13 @@ class LiveChatAsync:
loop.set_exception_handler(handler)
@classmethod
async def shutdown(cls, event, sig = None, handler=None):
async def shutdown(cls, event, sig=None, handler=None):
cls._logger.debug("shutdown...")
tasks = [t for t in asyncio.all_tasks() if t is not
asyncio.current_task()]
[task.cancel() for task in tasks]
cls._logger.debug(f"complete remaining tasks...")
await asyncio.gather(*tasks,return_exceptions=True)
cls._logger.debug("complete remaining tasks...")
await asyncio.gather(*tasks, return_exceptions=True)
loop = asyncio.get_event_loop()
loop.stop()

6
pytchat/core_multithread/livechat.py
View File

@@ -1,7 +1,5 @@
import requests
import datetime
import json
import random
import signal
import time
import traceback
@@ -74,7 +72,7 @@ class LiveChat:
_setup_finished = False
def __init__(self, video_id,
seektime=0,
seektime=-1,
processor=DefaultProcessor(),
buffer=None,
interruptable=True,
@@ -181,7 +179,7 @@ class LiveChat:
self._logger.debug(f"[{self.video_id}]{str(e)}")
return
except (TypeError, json.JSONDecodeError):
self._logger.error(f"{traceback.format_exc(limit = -1)}")
self._logger.error(f"{traceback.format_exc(limit=-1)}")
return
self._logger.debug(f"[{self.video_id}]finished fetching chat.")

22
pytchat/paramgen/arcparam_mining.py
View File

@@ -1,7 +1,5 @@
from base64 import urlsafe_b64encode as b64enc
from functools import reduce
import math
import random
import urllib.parse
'''
@@ -12,6 +10,7 @@ Author: taizan-hokuto (2019) @taizan205
ver 0.0.1 2019.10.05
'''
def _gen_vid_long(video_id):
"""generate video_id parameter.
Parameter
@@ -67,15 +66,17 @@ def _gen_vid(video_id):
def _nval(val):
"""convert value to byte array"""
if val<0: raise ValueError
if val < 0:
raise ValueError
buf = b''
while val >> 7:
m = val & 0xFF | 0x80
buf += m.to_bytes(1,'big')
buf += m.to_bytes(1, 'big')
val >>= 7
buf += val.to_bytes(1,'big')
buf += val.to_bytes(1, 'big')
return buf
def _build(video_id, seektime, topchat_only):
switch_01 = b'\x04' if topchat_only else b'\x01'
if seektime < 0:
@@ -83,11 +84,9 @@ def _build(video_id, seektime, topchat_only):
if seektime == 0:
times = b''
else:
times =_nval(int(seektime*1000))
times = _nval(int(seektime*1000))
if seektime > 0:
_len_time = ( b'\x5A'
+ (len(times)+1).to_bytes(1,'big')
+ b'\x10')
_len_time = b'\x5A' + (len(times)+1).to_bytes(1, 'big') + b'\x10'
else:
_len_time = b''
@@ -114,13 +113,14 @@ def _build(video_id, seektime, topchat_only):
body = reduce(lambda x, y: x+y, body)
return urllib.parse.quote(
b64enc( header_magic +
b64enc(header_magic +
_nval(len(body)) +
body
).decode()
)
def getparam(video_id, seektime = 0.0, topchat_only = False):
def getparam(video_id, seektime=0.0, topchat_only=False):
'''
Parameter
---------

15
pytchat/parser/live.py
View File

@@ -4,12 +4,12 @@ pytchat.parser.live
Parser of live chat JSON.
"""
import json
from .. exceptions import (
ResponseContextError,
NoContentsException,
NoContinuationsException,
ChatParseException )
ChatParseException)
class Parser:
@@ -22,9 +22,8 @@ class Parser:
if jsn is None:
raise ChatParseException('Called with none JSON object.')
if jsn['response']['responseContext'].get('errors'):
raise ResponseContextError('The video_id would be wrong,'
'or video is deleted or private.')
contents=jsn['response'].get('continuationContents')
raise ResponseContextError('The video_id would be wrong, or video is deleted or private.')
contents = jsn['response'].get('continuationContents')
return contents
def parse(self, contents):
@@ -75,9 +74,9 @@ class Parser:
"""
cont = contents['liveChatContinuation']['continuations'][0]
if cont.get("liveChatReplayContinuationData"):
#chat data exist.
# chat data exist.
return None
#chat data do not exist, get playerSeekContinuationData.
# chat data do not exist, get playerSeekContinuationData.
init_cont = cont.get("playerSeekContinuationData")
if init_cont:
return init_cont.get("continuation")
@@ -87,7 +86,7 @@ class Parser:
actions = contents['liveChatContinuation'].get('actions')
if self.is_replay:
interval = self._get_interval(actions)
metadata.setdefault("timeoutMs",interval)
metadata.setdefault("timeoutMs", interval)
"""Archived chat has different structures than live chat,
so make it the same format."""
chatdata = [action["replayChatItemAction"]["actions"][0]