ramforth/pytchat-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'release/v0.5.2'

Browse Source
This commit is contained in:
taizan-hokouto
2021-01-17 22:41:20 +09:00
parent 808e599be6 da79895e55
commit b3ebe3879d
4 changed files with 54 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pytchat/__init__.py
View File

@@ -2,7 +2,7 @@
pytchat is a lightweight python library to browse youtube livechat without Selenium or BeautifulSoup. pytchat is a lightweight python library to browse youtube livechat without Selenium or BeautifulSoup.
""" """
__copyright__ = 'Copyright (C) 2019, 2020 taizan-hokuto' __copyright__ = 'Copyright (C) 2019, 2020 taizan-hokuto'
__version__ = '0.5.1' __version__ = '0.5.2'
__license__ = 'MIT' __license__ = 'MIT'
__author__ = 'taizan-hokuto' __author__ = 'taizan-hokuto'
__author_email__ = '55448286+taizan-hokuto@users.noreply.github.com' __author_email__ = '55448286+taizan-hokuto@users.noreply.github.com'

28
pytchat/core/pytchat.py
View File

@@ -44,6 +44,10 @@ class PytchatCore:
If True, when exceptions occur, the exception is held internally, If True, when exceptions occur, the exception is held internally,
and can be raised by raise_for_status(). and can be raised by raise_for_status().
replay_continuation : str
If this parameter is not None, the processor will attempt to get chat data from continuation.
This parameter is only allowed in archived mode.
Attributes Attributes
--------- ---------
_is_alive : bool _is_alive : bool
@@ -58,6 +62,7 @@ class PytchatCore:
topchat_only=False, topchat_only=False,
hold_exception=True, hold_exception=True,
logger=config.logger(__name__), logger=config.logger(__name__),
replay_continuation=None
): ):
self._video_id = util.extract_video_id(video_id) self._video_id = util.extract_video_id(video_id)
self.seektime = seektime self.seektime = seektime
@@ -66,32 +71,33 @@ class PytchatCore:
else: else:
self.processor = processor self.processor = processor
self._is_alive = True self._is_alive = True
self._is_replay = force_replay self._is_replay = force_replay or (replay_continuation is not None)
self._hold_exception = hold_exception self._hold_exception = hold_exception
self._exception_holder = None self._exception_holder = None
self._parser = Parser( self._parser = Parser(
is_replay=self._is_replay, is_replay=self._is_replay,
exception_holder=self._exception_holder exception_holder=self._exception_holder
) )
self._first_fetch = True self._first_fetch = replay_continuation is None
self._fetch_url = config._sml self._fetch_url = config._sml if replay_continuation is None else config._smr
self._topchat_only = topchat_only self._topchat_only = topchat_only
self._dat = '' self._dat = ''
self._last_offset_ms = 0 self._last_offset_ms = 0
self._logger = logger self._logger = logger
self.continuation = replay_continuation
if interruptable: if interruptable:
signal.signal(signal.SIGINT, lambda a, b: self.terminate()) signal.signal(signal.SIGINT, lambda a, b: self.terminate())
self._setup() self._setup()
def _setup(self): def _setup(self):
time.sleep(0.1) # sleep shortly to prohibit skipping fetching data if not self.continuation:
"""Fetch first continuation parameter, time.sleep(0.1) # sleep shortly to prohibit skipping fetching data
create and start _listen loop. """Fetch first continuation parameter,
""" create and start _listen loop.
self.continuation = liveparam.getparam(self._video_id, past_sec=3) """
self.continuation = liveparam.getparam(self._video_id, past_sec=3)
def _get_chat_component(self): def _get_chat_component(self):
''' Fetch chat data and store them into buffer, ''' Fetch chat data and store them into buffer,
get next continuaiton parameter and loop. get next continuaiton parameter and loop.
@@ -178,7 +184,7 @@ class PytchatCore:
f"Exceeded retry count. Last error: {str(err)}") f"Exceeded retry count. Last error: {str(err)}")
self._raise_exception(exceptions.RetryExceedMaxCount()) self._raise_exception(exceptions.RetryExceedMaxCount())
return livechat_json return livechat_json
def get(self): def get(self):
if self.is_alive(): if self.is_alive():
chat_component = self._get_chat_component() chat_component = self._get_chat_component()

24
pytchat/core_async/livechat.py
View File

@@ -62,6 +62,10 @@ class LiveChatAsync:
topchat_only : bool topchat_only : bool
If True, get only top chat. If True, get only top chat.
replay_continuation : str
If this parameter is not None, the processor will attempt to get chat data from continuation.
This parameter is only allowed in archived mode.
Attributes Attributes
--------- ---------
_is_alive : bool _is_alive : bool
@@ -81,7 +85,8 @@ class LiveChatAsync:
direct_mode=False, direct_mode=False,
force_replay=False, force_replay=False,
topchat_only=False, topchat_only=False,
logger=config.logger(__name__) logger=config.logger(__name__),
replay_continuation=None
): ):
self._video_id = util.extract_video_id(video_id) self._video_id = util.extract_video_id(video_id)
self.seektime = seektime self.seektime = seektime
@@ -95,17 +100,18 @@ class LiveChatAsync:
self._exception_handler = exception_handler self._exception_handler = exception_handler
self._direct_mode = direct_mode self._direct_mode = direct_mode
self._is_alive = True self._is_alive = True
self._is_replay = force_replay self._is_replay = force_replay or (replay_continuation is not None)
self._parser = Parser(is_replay=self._is_replay) self._parser = Parser(is_replay=self._is_replay)
self._pauser = Queue() self._pauser = Queue()
self._pauser.put_nowait(None) self._pauser.put_nowait(None)
self._first_fetch = True self._first_fetch = replay_continuation is None
self._fetch_url = config._sml self._fetch_url = config._sml if replay_continuation is None else config._smr
self._topchat_only = topchat_only self._topchat_only = topchat_only
self._dat = '' self._dat = ''
self._last_offset_ms = 0 self._last_offset_ms = 0
self._logger = logger self._logger = logger
self.exception = None self.exception = None
self.continuation = replay_continuation
LiveChatAsync._logger = logger LiveChatAsync._logger = logger
if exception_handler: if exception_handler:
@@ -145,8 +151,9 @@ class LiveChatAsync:
"""Fetch first continuation parameter, """Fetch first continuation parameter,
create and start _listen loop. create and start _listen loop.
""" """
initial_continuation = liveparam.getparam(self._video_id, 3) if not self.continuation:
await self._listen(initial_continuation) self.continuation = liveparam.getparam(self._video_id, 3)
await self._listen(self.continuation)
async def _listen(self, continuation): async def _listen(self, continuation):
''' Fetch chat data and store them into buffer, ''' Fetch chat data and store them into buffer,
@@ -163,6 +170,9 @@ class LiveChatAsync:
continuation = await self._check_pause(continuation) continuation = await self._check_pause(continuation)
contents = await self._get_contents(continuation, client, headers) contents = await self._get_contents(continuation, client, headers)
metadata, chatdata = self._parser.parse(contents) metadata, chatdata = self._parser.parse(contents)
continuation = metadata.get('continuation')
if continuation:
self.continuation = continuation
timeout = metadata['timeoutMs'] / 1000 timeout = metadata['timeoutMs'] / 1000
chat_component = { chat_component = {
"video_id": self._video_id, "video_id": self._video_id,
@@ -181,7 +191,6 @@ class LiveChatAsync:
await self._buffer.put(chat_component) await self._buffer.put(chat_component)
diff_time = timeout - (time.time() - time_mark) diff_time = timeout - (time.time() - time_mark)
await asyncio.sleep(diff_time) await asyncio.sleep(diff_time)
continuation = metadata.get('continuation')
self._last_offset_ms = metadata.get('last_offset_ms', 0) self._last_offset_ms = metadata.get('last_offset_ms', 0)
except exceptions.ChatParseException as e: except exceptions.ChatParseException as e:
self._logger.debug(f"[{self._video_id}]{str(e)}") self._logger.debug(f"[{self._video_id}]{str(e)}")
@@ -242,7 +251,6 @@ class LiveChatAsync:
''' '''
Get json which includes chat data. Get json which includes chat data.
''' '''
# continuation = urllib.parse.quote(continuation)
livechat_json = None livechat_json = None
if offset_ms < 0: if offset_ms < 0:
offset_ms = 0 offset_ms = 0

30
pytchat/core_multithread/livechat.py
View File

@@ -60,6 +60,10 @@ class LiveChat:
topchat_only : bool topchat_only : bool
If True, get only top chat. If True, get only top chat.
replay_continuation : str
If this parameter is not None, the processor will attempt to get chat data from continuation.
This parameter is only allowed in archived mode.
Attributes Attributes
--------- ---------
_executor : ThreadPoolExecutor _executor : ThreadPoolExecutor
@@ -81,7 +85,8 @@ class LiveChat:
direct_mode=False, direct_mode=False,
force_replay=False, force_replay=False,
topchat_only=False, topchat_only=False,
logger=config.logger(__name__) logger=config.logger(__name__),
replay_continuation=None
): ):
self._video_id = util.extract_video_id(video_id) self._video_id = util.extract_video_id(video_id)
self.seektime = seektime self.seektime = seektime
@@ -95,17 +100,19 @@ class LiveChat:
self._executor = ThreadPoolExecutor(max_workers=2) self._executor = ThreadPoolExecutor(max_workers=2)
self._direct_mode = direct_mode self._direct_mode = direct_mode
self._is_alive = True self._is_alive = True
self._is_replay = force_replay self._is_replay = force_replay or (replay_continuation is not None)
self._parser = Parser(is_replay=self._is_replay) self._parser = Parser(is_replay=self._is_replay)
self._pauser = Queue() self._pauser = Queue()
self._pauser.put_nowait(None) self._pauser.put_nowait(None)
self._first_fetch = True self._first_fetch = replay_continuation is None
self._fetch_url = config._sml self._fetch_url = config._sml if replay_continuation is None else config._smr
self._topchat_only = topchat_only self._topchat_only = topchat_only
self._dat = '' self._dat = ''
self._last_offset_ms = 0 self._last_offset_ms = 0
self._event = Event()
self._logger = logger self._logger = logger
self._event = Event()
self.continuation = replay_continuation
self.exception = None self.exception = None
if interruptable: if interruptable:
signal.signal(signal.SIGINT, lambda a, b: self.terminate()) signal.signal(signal.SIGINT, lambda a, b: self.terminate())
@@ -140,8 +147,9 @@ class LiveChat:
"""Fetch first continuation parameter, """Fetch first continuation parameter,
create and start _listen loop. create and start _listen loop.
""" """
initial_continuation = liveparam.getparam(self._video_id, 3) if not self.continuation:
self._listen(initial_continuation) self.continuation = liveparam.getparam(self._video_id, 3)
self._listen(self.continuation)
def _listen(self, continuation): def _listen(self, continuation):
''' Fetch chat data and store them into buffer, ''' Fetch chat data and store them into buffer,
@@ -158,6 +166,9 @@ class LiveChat:
continuation = self._check_pause(continuation) continuation = self._check_pause(continuation)
contents = self._get_contents(continuation, client, headers) contents = self._get_contents(continuation, client, headers)
metadata, chatdata = self._parser.parse(contents) metadata, chatdata = self._parser.parse(contents)
continuation = metadata.get('continuation')
if continuation:
self.continuation = continuation
timeout = metadata['timeoutMs'] / 1000 timeout = metadata['timeoutMs'] / 1000
chat_component = { chat_component = {
"video_id": self._video_id, "video_id": self._video_id,
@@ -176,7 +187,6 @@ class LiveChat:
self._buffer.put(chat_component) self._buffer.put(chat_component)
diff_time = timeout - (time.time() - time_mark) diff_time = timeout - (time.time() - time_mark)
self._event.wait(diff_time if diff_time > 0 else 0) self._event.wait(diff_time if diff_time > 0 else 0)
continuation = metadata.get('continuation')
self._last_offset_ms = metadata.get('last_offset_ms', 0) self._last_offset_ms = metadata.get('last_offset_ms', 0)
except exceptions.ChatParseException as e: except exceptions.ChatParseException as e:
self._logger.debug(f"[{self._video_id}]{str(e)}") self._logger.debug(f"[{self._video_id}]{str(e)}")
@@ -196,7 +206,8 @@ class LiveChat:
''' '''
self._pauser.put_nowait(None) self._pauser.put_nowait(None)
if not self._is_replay: if not self._is_replay:
continuation = liveparam.getparam(self._video_id, 3) continuation = liveparam.getparam(
self._video_id, 3, self._topchat_only)
return continuation return continuation
def _get_contents(self, continuation, client, headers): def _get_contents(self, continuation, client, headers):
@@ -235,7 +246,6 @@ class LiveChat:
''' '''
Get json which includes chat data. Get json which includes chat data.
''' '''
# continuation = urllib.parse.quote(continuation)
livechat_json = None livechat_json = None
if offset_ms < 0: if offset_ms < 0:
offset_ms = 0 offset_ms = 0