ramforth/pytchat-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'release/v0.5.2'

Browse Source
This commit is contained in:
taizan-hokouto
2021-01-17 22:41:20 +09:00
parent 808e599be6 da79895e55
commit b3ebe3879d
4 changed files with 54 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pytchat/__init__.py
View File

@@ -2,7 +2,7 @@
pytchat is a lightweight python library to browse youtube livechat without Selenium or BeautifulSoup. pytchat is a lightweight python library to browse youtube livechat without Selenium or BeautifulSoup.
""" """
__copyright__ = 'Copyright (C) 2019, 2020 taizan-hokuto' __copyright__ = 'Copyright (C) 2019, 2020 taizan-hokuto'
__version__ = '0.5.1' __version__ = '0.5.2'
__license__ = 'MIT' __license__ = 'MIT'
__author__ = 'taizan-hokuto' __author__ = 'taizan-hokuto'
__author_email__ = '55448286+taizan-hokuto@users.noreply.github.com' __author_email__ = '55448286+taizan-hokuto@users.noreply.github.com'

24
pytchat/core/pytchat.py
View File

@@ -44,6 +44,10 @@ class PytchatCore:
If True, when exceptions occur, the exception is held internally, If True, when exceptions occur, the exception is held internally,
and can be raised by raise_for_status(). and can be raised by raise_for_status().
replay_continuation : str
If this parameter is not None, the processor will attempt to get chat data from continuation.
This parameter is only allowed in archived mode.
Attributes Attributes
--------- ---------
_is_alive : bool _is_alive : bool
@@ -58,6 +62,7 @@ class PytchatCore:
topchat_only=False, topchat_only=False,
hold_exception=True, hold_exception=True,
logger=config.logger(__name__), logger=config.logger(__name__),
replay_continuation=None
): ):
self._video_id = util.extract_video_id(video_id) self._video_id = util.extract_video_id(video_id)
self.seektime = seektime self.seektime = seektime
@@ -66,32 +71,33 @@ class PytchatCore:
else: else:
self.processor = processor self.processor = processor
self._is_alive = True self._is_alive = True
self._is_replay = force_replay self._is_replay = force_replay or (replay_continuation is not None)
self._hold_exception = hold_exception self._hold_exception = hold_exception
self._exception_holder = None self._exception_holder = None
self._parser = Parser( self._parser = Parser(
is_replay=self._is_replay, is_replay=self._is_replay,
exception_holder=self._exception_holder exception_holder=self._exception_holder
) )
self._first_fetch = True self._first_fetch = replay_continuation is None
self._fetch_url = config._sml self._fetch_url = config._sml if replay_continuation is None else config._smr
self._topchat_only = topchat_only self._topchat_only = topchat_only
self._dat = '' self._dat = ''
self._last_offset_ms = 0 self._last_offset_ms = 0
self._logger = logger self._logger = logger
self.continuation = replay_continuation
if interruptable: if interruptable:
signal.signal(signal.SIGINT, lambda a, b: self.terminate()) signal.signal(signal.SIGINT, lambda a, b: self.terminate())
self._setup() self._setup()
def _setup(self): def _setup(self):
time.sleep(0.1) # sleep shortly to prohibit skipping fetching data if not self.continuation:
"""Fetch first continuation parameter, time.sleep(0.1) # sleep shortly to prohibit skipping fetching data
create and start _listen loop. """Fetch first continuation parameter,
""" create and start _listen loop.
self.continuation = liveparam.getparam(self._video_id, past_sec=3) """
self.continuation = liveparam.getparam(self._video_id, past_sec=3)
def _get_chat_component(self): def _get_chat_component(self):
''' Fetch chat data and store them into buffer, ''' Fetch chat data and store them into buffer,
get next continuaiton parameter and loop. get next continuaiton parameter and loop.

24
pytchat/core_async/livechat.py
View File

@@ -62,6 +62,10 @@ class LiveChatAsync:
topchat_only : bool topchat_only : bool
If True, get only top chat. If True, get only top chat.
replay_continuation : str
If this parameter is not None, the processor will attempt to get chat data from continuation.
This parameter is only allowed in archived mode.
Attributes Attributes
--------- ---------
_is_alive : bool _is_alive : bool
@@ -81,7 +85,8 @@ class LiveChatAsync:
direct_mode=False, direct_mode=False,
force_replay=False, force_replay=False,
topchat_only=False, topchat_only=False,
logger=config.logger(__name__) logger=config.logger(__name__),
replay_continuation=None
): ):
self._video_id = util.extract_video_id(video_id) self._video_id = util.extract_video_id(video_id)
self.seektime = seektime self.seektime = seektime
@@ -95,17 +100,18 @@ class LiveChatAsync:
self._exception_handler = exception_handler self._exception_handler = exception_handler
self._direct_mode = direct_mode self._direct_mode = direct_mode
self._is_alive = True self._is_alive = True
self._is_replay = force_replay self._is_replay = force_replay or (replay_continuation is not None)
self._parser = Parser(is_replay=self._is_replay) self._parser = Parser(is_replay=self._is_replay)
self._pauser = Queue() self._pauser = Queue()
self._pauser.put_nowait(None) self._pauser.put_nowait(None)
self._first_fetch = True self._first_fetch = replay_continuation is None
self._fetch_url = config._sml self._fetch_url = config._sml if replay_continuation is None else config._smr
self._topchat_only = topchat_only self._topchat_only = topchat_only
self._dat = '' self._dat = ''
self._last_offset_ms = 0 self._last_offset_ms = 0
self._logger = logger self._logger = logger
self.exception = None self.exception = None
self.continuation = replay_continuation
LiveChatAsync._logger = logger LiveChatAsync._logger = logger
if exception_handler: if exception_handler:
@@ -145,8 +151,9 @@ class LiveChatAsync:
"""Fetch first continuation parameter, """Fetch first continuation parameter,
create and start _listen loop. create and start _listen loop.
""" """
initial_continuation = liveparam.getparam(self._video_id, 3) if not self.continuation:
await self._listen(initial_continuation) self.continuation = liveparam.getparam(self._video_id, 3)
await self._listen(self.continuation)
async def _listen(self, continuation): async def _listen(self, continuation):
''' Fetch chat data and store them into buffer, ''' Fetch chat data and store them into buffer,
@@ -163,6 +170,9 @@ class LiveChatAsync:
continuation = await self._check_pause(continuation) continuation = await self._check_pause(continuation)
contents = await self._get_contents(continuation, client, headers) contents = await self._get_contents(continuation, client, headers)
metadata, chatdata = self._parser.parse(contents) metadata, chatdata = self._parser.parse(contents)
continuation = metadata.get('continuation')
if continuation:
self.continuation = continuation
timeout = metadata['timeoutMs'] / 1000 timeout = metadata['timeoutMs'] / 1000
chat_component = { chat_component = {
"video_id": self._video_id, "video_id": self._video_id,
@@ -181,7 +191,6 @@ class LiveChatAsync:
await self._buffer.put(chat_component) await self._buffer.put(chat_component)
diff_time = timeout - (time.time() - time_mark) diff_time = timeout - (time.time() - time_mark)
await asyncio.sleep(diff_time) await asyncio.sleep(diff_time)
continuation = metadata.get('continuation')
self._last_offset_ms = metadata.get('last_offset_ms', 0) self._last_offset_ms = metadata.get('last_offset_ms', 0)
except exceptions.ChatParseException as e: except exceptions.ChatParseException as e:
self._logger.debug(f"[{self._video_id}]{str(e)}") self._logger.debug(f"[{self._video_id}]{str(e)}")
@@ -242,7 +251,6 @@ class LiveChatAsync:
''' '''
Get json which includes chat data. Get json which includes chat data.
''' '''
# continuation = urllib.parse.quote(continuation)
livechat_json = None livechat_json = None
if offset_ms < 0: if offset_ms < 0:
offset_ms = 0 offset_ms = 0

30
pytchat/core_multithread/livechat.py
View File

@@ -60,6 +60,10 @@ class LiveChat:
topchat_only : bool topchat_only : bool
If True, get only top chat. If True, get only top chat.
replay_continuation : str
If this parameter is not None, the processor will attempt to get chat data from continuation.
This parameter is only allowed in archived mode.
Attributes Attributes
--------- ---------
_executor : ThreadPoolExecutor _executor : ThreadPoolExecutor
@@ -81,7 +85,8 @@ class LiveChat:
direct_mode=False, direct_mode=False,
force_replay=False, force_replay=False,
topchat_only=False, topchat_only=False,
logger=config.logger(__name__) logger=config.logger(__name__),
replay_continuation=None
): ):
self._video_id = util.extract_video_id(video_id) self._video_id = util.extract_video_id(video_id)
self.seektime = seektime self.seektime = seektime
@@ -95,17 +100,19 @@ class LiveChat:
self._executor = ThreadPoolExecutor(max_workers=2) self._executor = ThreadPoolExecutor(max_workers=2)
self._direct_mode = direct_mode self._direct_mode = direct_mode
self._is_alive = True self._is_alive = True
self._is_replay = force_replay self._is_replay = force_replay or (replay_continuation is not None)
self._parser = Parser(is_replay=self._is_replay) self._parser = Parser(is_replay=self._is_replay)
self._pauser = Queue() self._pauser = Queue()
self._pauser.put_nowait(None) self._pauser.put_nowait(None)
self._first_fetch = True self._first_fetch = replay_continuation is None
self._fetch_url = config._sml self._fetch_url = config._sml if replay_continuation is None else config._smr
self._topchat_only = topchat_only self._topchat_only = topchat_only
self._dat = '' self._dat = ''
self._last_offset_ms = 0 self._last_offset_ms = 0
self._event = Event()
self._logger = logger self._logger = logger
self._event = Event()
self.continuation = replay_continuation
self.exception = None self.exception = None
if interruptable: if interruptable:
signal.signal(signal.SIGINT, lambda a, b: self.terminate()) signal.signal(signal.SIGINT, lambda a, b: self.terminate())
@@ -140,8 +147,9 @@ class LiveChat:
"""Fetch first continuation parameter, """Fetch first continuation parameter,
create and start _listen loop. create and start _listen loop.
""" """
initial_continuation = liveparam.getparam(self._video_id, 3) if not self.continuation:
self._listen(initial_continuation) self.continuation = liveparam.getparam(self._video_id, 3)
self._listen(self.continuation)
def _listen(self, continuation): def _listen(self, continuation):
''' Fetch chat data and store them into buffer, ''' Fetch chat data and store them into buffer,
@@ -158,6 +166,9 @@ class LiveChat:
continuation = self._check_pause(continuation) continuation = self._check_pause(continuation)
contents = self._get_contents(continuation, client, headers) contents = self._get_contents(continuation, client, headers)
metadata, chatdata = self._parser.parse(contents) metadata, chatdata = self._parser.parse(contents)
continuation = metadata.get('continuation')
if continuation:
self.continuation = continuation
timeout = metadata['timeoutMs'] / 1000 timeout = metadata['timeoutMs'] / 1000
chat_component = { chat_component = {
"video_id": self._video_id, "video_id": self._video_id,
@@ -176,7 +187,6 @@ class LiveChat:
self._buffer.put(chat_component) self._buffer.put(chat_component)
diff_time = timeout - (time.time() - time_mark) diff_time = timeout - (time.time() - time_mark)
self._event.wait(diff_time if diff_time > 0 else 0) self._event.wait(diff_time if diff_time > 0 else 0)
continuation = metadata.get('continuation')
self._last_offset_ms = metadata.get('last_offset_ms', 0) self._last_offset_ms = metadata.get('last_offset_ms', 0)
except exceptions.ChatParseException as e: except exceptions.ChatParseException as e:
self._logger.debug(f"[{self._video_id}]{str(e)}") self._logger.debug(f"[{self._video_id}]{str(e)}")
@@ -196,7 +206,8 @@ class LiveChat:
''' '''
self._pauser.put_nowait(None) self._pauser.put_nowait(None)
if not self._is_replay: if not self._is_replay:
continuation = liveparam.getparam(self._video_id, 3) continuation = liveparam.getparam(
self._video_id, 3, self._topchat_only)
return continuation return continuation
def _get_contents(self, continuation, client, headers): def _get_contents(self, continuation, client, headers):
@@ -235,7 +246,6 @@ class LiveChat:
''' '''
Get json which includes chat data. Get json which includes chat data.
''' '''
# continuation = urllib.parse.quote(continuation)
livechat_json = None livechat_json = None
if offset_ms < 0: if offset_ms < 0:
offset_ms = 0 offset_ms = 0