ramforth/pytchat-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'release/v0.0.5.1.3'

Browse Source
This commit is contained in:
taizan-hokuto
2020-01-18 14:45:33 +09:00
parent fb0edef136 76b126faf2
commit f7d1830226
10 changed files with 100 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
README.md
View File

@@ -7,13 +7,16 @@ pytchat is a python library for fetching youtube live chat.
pytchat is a python library for fetching youtube live chat pytchat is a python library for fetching youtube live chat
without using youtube api, Selenium or BeautifulSoup. without using youtube api, Selenium or BeautifulSoup.
pytchatはAPIを使わずにYouTubeチャットを取得するための軽量pythonライブラリです。
Other features: Other features:
+ Customizable chat data processors including youtube api compatible one. + Customizable chat data processors including youtube api compatible one.
+ Available on asyncio context. + Available on asyncio context.
+ Quick fetching of initial chat data by generating continuation params + Quick fetching of initial chat data by generating continuation params
instead of web scraping. instead of web scraping.
For more detailed information, see [wiki](https://github.com/taizan-hokuto/pytchat/wiki). For more detailed information, see [wiki](https://github.com/taizan-hokuto/pytchat/wiki). <br>
より詳細な解説は[wiki](https://github.com/taizan-hokuto/pytchat/wiki/Home-:)を参照してください。
## Install ## Install
```python ```python
@@ -26,13 +29,17 @@ pip install pytchat
### on-demand mode ### on-demand mode
```python ```python
from pytchat import LiveChat from pytchat import LiveChat
livechat = LiveChat(video_id = "Zvp1pJpie4I")
chat = LiveChat("DSGyEsJ17cI") while livechat.is_alive():
while chat.is_alive(): try:
data = chat.get() chatdata = livechat.get()
for c in data.items: for c in chatdata.items:
print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}") print(f"{c.datetime} [{c.author.name}]- {c.message}")
data.tick() chatdata.tick()
except KeyboardInterrupt:
livechat.terminate()
break
``` ```
### callback mode ### callback mode
@@ -40,17 +47,21 @@ while chat.is_alive():
from pytchat import LiveChat from pytchat import LiveChat
import time import time
#callback function is automatically called. def main():
def display(data): livechat = LiveChat(video_id = "Zvp1pJpie4I", callback = disp)
for c in data.items: while livechat.is_alive():
print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}") #other background operation.
data.tick() time.sleep(1)
livechat.terminate()
#callback function (automatically called)
def disp(chatdata):
for c in chatdata.items:
print(f"{c.datetime} [{c.author.name}]- {c.message}")
chatdata.tick()
if __name__ == '__main__': if __name__ == '__main__':
chat = LiveChat("DSGyEsJ17cI", callback = display) main()
while chat.is_alive():
#other background operation.
time.sleep(3)
``` ```
@@ -61,16 +72,16 @@ from concurrent.futures import CancelledError
import asyncio import asyncio
async def main(): async def main():
chat = LiveChatAsync("DSGyEsJ17cI", callback = func) livechat = LiveChatAsync("Zvp1pJpie4I", callback = func)
while chat.is_alive(): while livechat.is_alive():
#other background operation. #other background operation.
await asyncio.sleep(3) await asyncio.sleep(3)
#callback function is automatically called. #callback function is automatically called.
async def func(data): async def func(chatdata):
for c in data.items: for c in chatdata.items:
print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}") print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}")
await data.tick_async() await chatdata.tick_async()
if __name__ == '__main__': if __name__ == '__main__':
try: try:
@@ -86,18 +97,20 @@ if __name__ == '__main__':
from pytchat import LiveChat, CompatibleProcessor from pytchat import LiveChat, CompatibleProcessor
import time import time
chat = LiveChat("DSGyEsJ17cI", chat = LiveChat("Zvp1pJpie4I",
processor = CompatibleProcessor() ) processor = CompatibleProcessor() )
while chat.is_alive(): while chat.is_alive():
data = chat.get() try:
polling = data['pollingIntervalMillis']/1000 data = chat.get()
for c in data['items']: polling = data['pollingIntervalMillis']/1000
if c.get('snippet'): for c in data['items']:
print(f"[{c['authorDetails']['displayName']}]" if c.get('snippet'):
f"-{c['snippet']['displayMessage']}") print(f"[{c['authorDetails']['displayName']}]"
time.sleep(polling/len(data['items'])) f"-{c['snippet']['displayMessage']}")
time.sleep(polling/len(data['items']))
except KeyboardInterrupt:
chat.terminate()
``` ```
### replay: ### replay:
If specified video is not live, If specified video is not live,
@@ -110,18 +123,21 @@ def main():
#seektime (seconds): start position of chat. #seektime (seconds): start position of chat.
chat = LiveChat("ojes5ULOqhc", seektime = 60*30) chat = LiveChat("ojes5ULOqhc", seektime = 60*30)
print('Replay from 30:00') print('Replay from 30:00')
while chat.is_alive(): try:
data = chat.get() while chat.is_alive():
for c in data.items: data = chat.get()
print(f"{c.elapsedTime} [{c.author.name}]-{c.message} {c.amountString}") for c in data.items:
data.tick() print(f"{c.elapsedTime} [{c.author.name}]-{c.message} {c.amountString}")
data.tick()
except KeyboardInterrupt:
chat.terminate()
if __name__ == '__main__': if __name__ == '__main__':
main() main()
``` ```
## Structure of Default Processor ## Structure of Default Processor
Each item can be got with items() function. Each item can be got with `items` function.
<table> <table>
<tr> <tr>
<th>name</th> <th>name</th>

2
pytchat/__init__.py
View File

@@ -2,7 +2,7 @@
pytchat is a python library for fetching youtube live chat without using yt api, Selenium, or BeautifulSoup. pytchat is a python library for fetching youtube live chat without using yt api, Selenium, or BeautifulSoup.
""" """
__copyright__ = 'Copyright (C) 2019 taizan-hokuto' __copyright__ = 'Copyright (C) 2019 taizan-hokuto'
__version__ = '0.0.5.0' __version__ = '0.0.5.1.3'
__license__ = 'MIT' __license__ = 'MIT'
__author__ = 'taizan-hokuto' __author__ = 'taizan-hokuto'
__author_email__ = '55448286+taizan-hokuto@users.noreply.github.com' __author_email__ = '55448286+taizan-hokuto@users.noreply.github.com'

6
pytchat/config/__init__.py
View File

@@ -1,13 +1,11 @@
import logging import logging
from . import mylogger from . import mylogger
LOGGER_MODE = None
headers = { headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'} 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'}
def logger(module_name: str): def logger(module_name: str, loglevel = None):
module_logger = mylogger.get_logger(module_name, mode = LOGGER_MODE) module_logger = mylogger.get_logger(module_name, loglevel = loglevel)
return module_logger return module_logger

10
pytchat/config/mylogger.py
View File

@@ -3,21 +3,21 @@ import logging
import datetime import datetime
def get_logger(modname,mode=logging.DEBUG): def get_logger(modname,loglevel=logging.DEBUG):
logger = getLogger(modname) logger = getLogger(modname)
if mode == None: if loglevel == None:
logger.addHandler(NullHandler()) logger.addHandler(NullHandler())
return logger return logger
logger.setLevel(mode) logger.setLevel(loglevel)
#create handler1 for showing info #create handler1 for showing info
handler1 = StreamHandler() handler1 = StreamHandler()
my_formatter = MyFormatter() my_formatter = MyFormatter()
handler1.setFormatter(my_formatter) handler1.setFormatter(my_formatter)
handler1.setLevel(mode) handler1.setLevel(loglevel)
logger.addHandler(handler1) logger.addHandler(handler1)
#create handler2 for recording log file #create handler2 for recording log file
if mode <= logging.DEBUG: if loglevel <= logging.DEBUG:
handler2 = FileHandler(filename="log.txt", encoding='utf-8') handler2 = FileHandler(filename="log.txt", encoding='utf-8')
handler2.setLevel(logging.ERROR) handler2.setLevel(logging.ERROR)
handler2.setFormatter(my_formatter) handler2.setFormatter(my_formatter)

35
pytchat/core_async/livechat.py
View File

@@ -17,7 +17,6 @@ from ..paramgen import liveparam, arcparam
from ..processors.default.processor import DefaultProcessor from ..processors.default.processor import DefaultProcessor
from ..processors.combinator import Combinator from ..processors.combinator import Combinator
logger = config.logger(__name__)
headers = config.headers headers = config.headers
MAX_RETRY = 10 MAX_RETRY = 10
@@ -74,6 +73,7 @@ class LiveChatAsync:
''' '''
_setup_finished = False _setup_finished = False
_logger = config.logger(__name__)
def __init__(self, video_id, def __init__(self, video_id,
seektime = 0, seektime = 0,
@@ -85,7 +85,8 @@ class LiveChatAsync:
exception_handler = None, exception_handler = None,
direct_mode = False, direct_mode = False,
force_replay = False, force_replay = False,
topchat_only = False topchat_only = False,
logger = config.logger(__name__),
): ):
self.video_id = video_id self.video_id = video_id
self.seektime = seektime self.seektime = seektime
@@ -107,11 +108,12 @@ class LiveChatAsync:
self._first_fetch = True self._first_fetch = True
self._fetch_url = "live_chat/get_live_chat?continuation=" self._fetch_url = "live_chat/get_live_chat?continuation="
self._topchat_only = topchat_only self._topchat_only = topchat_only
self._logger = logger
LiveChatAsync._logger = logger
if not LiveChatAsync._setup_finished: if not LiveChatAsync._setup_finished:
LiveChatAsync._setup_finished = True LiveChatAsync._setup_finished = True
if exception_handler == None: if exception_handler:
self._set_exception_handler(self._handle_exception)
else:
self._set_exception_handler(exception_handler) self._set_exception_handler(exception_handler)
if interruptable: if interruptable:
signal.signal(signal.SIGINT, signal.signal(signal.SIGINT,
@@ -187,14 +189,14 @@ class LiveChatAsync:
continuation = metadata.get('continuation') continuation = metadata.get('continuation')
except ChatParseException as e: except ChatParseException as e:
#self.terminate() #self.terminate()
logger.debug(f"[{self.video_id}]{str(e)}") self._logger.debug(f"[{self.video_id}]{str(e)}")
return return
except (TypeError , json.JSONDecodeError) : except (TypeError , json.JSONDecodeError) :
#self.terminate() #self.terminate()
logger.error(f"{traceback.format_exc(limit = -1)}") self._logger.error(f"{traceback.format_exc(limit = -1)}")
return return
logger.debug(f"[{self.video_id}]finished fetching chat.") self._logger.debug(f"[{self.video_id}]finished fetching chat.")
async def _check_pause(self, continuation): async def _check_pause(self, continuation):
if self._pauser.empty(): if self._pauser.empty():
@@ -254,7 +256,7 @@ class LiveChatAsync:
await asyncio.sleep(1) await asyncio.sleep(1)
continue continue
else: else:
logger.error(f"[{self.video_id}]" self._logger.error(f"[{self.video_id}]"
f"Exceeded retry count. status_code={status_code}") f"Exceeded retry count. status_code={status_code}")
return None return None
return livechat_json return livechat_json
@@ -309,7 +311,7 @@ class LiveChatAsync:
try: try:
self.terminate() self.terminate()
except CancelledError: except CancelledError:
logger.debug(f'[{self.video_id}]cancelled:{sender}') self._logger.debug(f'[{self.video_id}]cancelled:{sender}')
def terminate(self): def terminate(self):
''' '''
@@ -319,28 +321,21 @@ class LiveChatAsync:
if self._direct_mode == False: if self._direct_mode == False:
#bufferにダミーオブジェクトを入れてis_alive()を判定させる #bufferにダミーオブジェクトを入れてis_alive()を判定させる
self._buffer.put_nowait({'chatdata':'','timeout':0}) self._buffer.put_nowait({'chatdata':'','timeout':0})
logger.info(f'[{self.video_id}]finished.') self._logger.info(f'[{self.video_id}]finished.')
@classmethod @classmethod
def _set_exception_handler(cls, handler): def _set_exception_handler(cls, handler):
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
loop.set_exception_handler(handler) loop.set_exception_handler(handler)
@classmethod
def _handle_exception(cls, loop, context):
if not isinstance(context["exception"],CancelledError):
logger.error(f"Caught exception: {context}")
loop= asyncio.get_event_loop()
loop.create_task(cls.shutdown(None,None,None))
@classmethod @classmethod
async def shutdown(cls, event, sig = None, handler=None): async def shutdown(cls, event, sig = None, handler=None):
logger.debug("shutdown...") cls._logger.debug("shutdown...")
tasks = [t for t in asyncio.all_tasks() if t is not tasks = [t for t in asyncio.all_tasks() if t is not
asyncio.current_task()] asyncio.current_task()]
[task.cancel() for task in tasks] [task.cancel() for task in tasks]
logger.debug(f"complete remaining tasks...") cls._logger.debug(f"complete remaining tasks...")
await asyncio.gather(*tasks,return_exceptions=True) await asyncio.gather(*tasks,return_exceptions=True)
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
loop.stop() loop.stop()

25
pytchat/core_multithread/livechat.py
View File

@@ -16,7 +16,6 @@ from ..paramgen import liveparam, arcparam
from ..processors.default.processor import DefaultProcessor from ..processors.default.processor import DefaultProcessor
from ..processors.combinator import Combinator from ..processors.combinator import Combinator
logger = config.logger(__name__)
headers = config.headers headers = config.headers
MAX_RETRY = 10 MAX_RETRY = 10
@@ -74,7 +73,9 @@ class LiveChat:
_setup_finished = False _setup_finished = False
#チャット監視中のListenerのリスト #チャット監視中のListenerのリスト
_listeners= [] _listeners = []
_logger = config.logger(__name__)
def __init__(self, video_id, def __init__(self, video_id,
seektime = 0, seektime = 0,
processor = DefaultProcessor(), processor = DefaultProcessor(),
@@ -84,7 +85,8 @@ class LiveChat:
done_callback = None, done_callback = None,
direct_mode = False, direct_mode = False,
force_replay = False, force_replay = False,
topchat_only = False topchat_only = False,
logger = config.logger(__name__)
): ):
self.video_id = video_id self.video_id = video_id
self.seektime = seektime self.seektime = seektime
@@ -106,6 +108,8 @@ class LiveChat:
self._first_fetch = True self._first_fetch = True
self._fetch_url = "live_chat/get_live_chat?continuation=" self._fetch_url = "live_chat/get_live_chat?continuation="
self._topchat_only = topchat_only self._topchat_only = topchat_only
self._logger = logger
LiveChat._logger = logger
if not LiveChat._setup_finished: if not LiveChat._setup_finished:
LiveChat._setup_finished = True LiveChat._setup_finished = True
if interruptable: if interruptable:
@@ -115,7 +119,6 @@ class LiveChat:
LiveChat._listeners.append(self) LiveChat._listeners.append(self)
def _setup(self): def _setup(self):
#logger.debug("setup")
#direct modeがTrueでcallback未設定の場合例外発生。 #direct modeがTrueでcallback未設定の場合例外発生。
if self._direct_mode: if self._direct_mode:
if self._callback is None: if self._callback is None:
@@ -181,13 +184,13 @@ class LiveChat:
time.sleep(diff_time if diff_time > 0 else 0) time.sleep(diff_time if diff_time > 0 else 0)
continuation = metadata.get('continuation') continuation = metadata.get('continuation')
except ChatParseException as e: except ChatParseException as e:
logger.debug(f"[{self.video_id}]{str(e)}") self._logger.debug(f"[{self.video_id}]{str(e)}")
return return
except (TypeError , json.JSONDecodeError) : except (TypeError , json.JSONDecodeError) :
logger.error(f"{traceback.format_exc(limit = -1)}") self._logger.error(f"{traceback.format_exc(limit = -1)}")
return return
logger.debug(f"[{self.video_id}]finished fetching chat.") self._logger.debug(f"[{self.video_id}]finished fetching chat.")
def _check_pause(self, continuation): def _check_pause(self, continuation):
if self._pauser.empty(): if self._pauser.empty():
@@ -244,7 +247,7 @@ class LiveChat:
time.sleep(1) time.sleep(1)
continue continue
else: else:
logger.error(f"[{self.video_id}]" self._logger.error(f"[{self.video_id}]"
f"Exceeded retry count. status_code={status_code}") f"Exceeded retry count. status_code={status_code}")
return None return None
return livechat_json return livechat_json
@@ -299,7 +302,7 @@ class LiveChat:
try: try:
self.terminate() self.terminate()
except CancelledError: except CancelledError:
logger.debug(f'[{self.video_id}]cancelled:{sender}') self._logger.debug(f'[{self.video_id}]cancelled:{sender}')
def terminate(self): def terminate(self):
''' '''
@@ -309,10 +312,10 @@ class LiveChat:
if self._direct_mode == False: if self._direct_mode == False:
#bufferにダミーオブジェクトを入れてis_alive()を判定させる #bufferにダミーオブジェクトを入れてis_alive()を判定させる
self._buffer.put({'chatdata':'','timeout':0}) self._buffer.put({'chatdata':'','timeout':0})
logger.info(f'[{self.video_id}]finished.') self._logger.info(f'[{self.video_id}]finished.')
@classmethod @classmethod
def shutdown(cls, event, sig = None, handler=None): def shutdown(cls, event, sig = None, handler=None):
logger.debug("shutdown...") cls._logger.debug("shutdown...")
for t in LiveChat._listeners: for t in LiveChat._listeners:
t._is_alive = False t._is_alive = False

7
pytchat/parser/live.py
View File

@@ -5,16 +5,12 @@ Parser of live chat JSON.
""" """
import json import json
from .. import config
from .. exceptions import ( from .. exceptions import (
ResponseContextError, ResponseContextError,
NoContentsException, NoContentsException,
NoContinuationsException, NoContinuationsException,
ChatParseException ) ChatParseException )
logger = config.logger(__name__)
class Parser: class Parser:
__slots__ = ['is_replay'] __slots__ = ['is_replay']
@@ -65,8 +61,7 @@ class Parser:
raise ChatParseException('Finished chat data') raise ChatParseException('Finished chat data')
unknown = list(cont.keys())[0] unknown = list(cont.keys())[0]
if unknown: if unknown:
logger.debug(f"Received unknown continuation type:{unknown}") raise ChatParseException(f"Received unknown continuation type:{unknown}")
metadata = cont.get(unknown)
else: else:
raise ChatParseException('Cannot extract continuation data') raise ChatParseException('Cannot extract continuation data')
return self._create_data(metadata, contents) return self._create_data(metadata, contents)

2
pytchat/processors/default/processor.py
View File

@@ -61,7 +61,7 @@ class DefaultProcessor(ChatProcessor):
renderer.get_snippet() renderer.get_snippet()
renderer.get_authordetails() renderer.get_authordetails()
except (KeyError,TypeError,AttributeError) as e: except (KeyError,TypeError) as e:
logger.error(f"{str(type(e))}-{str(e)} sitem:{str(sitem)}") logger.error(f"{str(type(e))}-{str(e)} sitem:{str(sitem)}")
return None return None
return renderer return renderer

18
pytchat/processors/default/renderer/base.py
View File

@@ -1,5 +1,4 @@
from datetime import datetime from datetime import datetime
class Author: class Author:
pass pass
class BaseRenderer: class BaseRenderer:
@@ -67,16 +66,17 @@ class BaseRenderer:
badges=renderer.get("authorBadges") badges=renderer.get("authorBadges")
if badges: if badges:
for badge in badges: for badge in badges:
author_type = badge["liveChatAuthorBadgeRenderer"]["accessibility"]["accessibilityData"]["label"] if badge["liveChatAuthorBadgeRenderer"].get("icon"):
if author_type == '確認済み': author_type = badge["liveChatAuthorBadgeRenderer"]["icon"]["iconType"]
isVerified = True if author_type == 'VERIFIED':
if author_type == '所有者': isVerified = True
isChatOwner = True if author_type == 'OWNER':
if 'メンバー' in author_type: isChatOwner = True
if author_type == 'MODERATOR':
isChatModerator = True
if badge["liveChatAuthorBadgeRenderer"].get("customThumbnail"):
isChatSponsor = True isChatSponsor = True
self.get_badgeurl(badge) self.get_badgeurl(badge)
if author_type == 'モデレーター':
isChatModerator = True
return isVerified, isChatOwner, isChatSponsor, isChatModerator return isVerified, isChatOwner, isChatSponsor, isChatModerator

13
pytchat/processors/json_display_processor.py
View File

@@ -1,13 +0,0 @@
import json
from .chat_processor import ChatProcessor
class JsonDisplayProcessor(ChatProcessor):
def process(self,chat_components: list):
if chat_components:
for component in chat_components:
chatdata = component.get('chatdata')
if chatdata:
for chat in chatdata:
print(json.dumps(chat,ensure_ascii=False)[:200])