ramforth/pytchat-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'release/v0.0.5.1.3'

Browse Source
This commit is contained in:
taizan-hokuto
2020-01-18 14:45:33 +09:00
parent fb0edef136 76b126faf2
commit f7d1830226
10 changed files with 100 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
README.md
View File

@@ -7,13 +7,16 @@ pytchat is a python library for fetching youtube live chat.
pytchat is a python library for fetching youtube live chat
without using youtube api, Selenium or BeautifulSoup.
pytchatはAPIを使わずにYouTubeチャットを取得するための軽量pythonライブラリです。
Other features:
+ Customizable chat data processors including youtube api compatible one.
+ Available on asyncio context.
+ Quick fetching of initial chat data by generating continuation params
instead of web scraping.
For more detailed information, see [wiki](https://github.com/taizan-hokuto/pytchat/wiki).
For more detailed information, see [wiki](https://github.com/taizan-hokuto/pytchat/wiki). <br>
より詳細な解説は[wiki](https://github.com/taizan-hokuto/pytchat/wiki/Home-:)を参照してください。
## Install
```python
@@ -26,13 +29,17 @@ pip install pytchat
### on-demand mode
```python
from pytchat import LiveChat
livechat = LiveChat(video_id = "Zvp1pJpie4I")
chat = LiveChat("DSGyEsJ17cI")
while chat.is_alive():
data = chat.get()
for c in data.items:
print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}")
data.tick()
while livechat.is_alive():
try:
chatdata = livechat.get()
for c in chatdata.items:
print(f"{c.datetime} [{c.author.name}]- {c.message}")
chatdata.tick()
except KeyboardInterrupt:
livechat.terminate()
break
```
### callback mode
@@ -40,17 +47,21 @@ while chat.is_alive():
from pytchat import LiveChat
import time
#callback function is automatically called.
def display(data):
for c in data.items:
print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}")
data.tick()
def main():
livechat = LiveChat(video_id = "Zvp1pJpie4I", callback = disp)
while livechat.is_alive():
#other background operation.
time.sleep(1)
livechat.terminate()
#callback function (automatically called)
def disp(chatdata):
for c in chatdata.items:
print(f"{c.datetime} [{c.author.name}]- {c.message}")
chatdata.tick()
if __name__ == '__main__':
chat = LiveChat("DSGyEsJ17cI", callback = display)
while chat.is_alive():
#other background operation.
time.sleep(3)
main()
```
@@ -61,16 +72,16 @@ from concurrent.futures import CancelledError
import asyncio
async def main():
chat = LiveChatAsync("DSGyEsJ17cI", callback = func)
while chat.is_alive():
livechat = LiveChatAsync("Zvp1pJpie4I", callback = func)
while livechat.is_alive():
#other background operation.
await asyncio.sleep(3)
#callback function is automatically called.
async def func(data):
for c in data.items:
async def func(chatdata):
for c in chatdata.items:
print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}")
await data.tick_async()
await chatdata.tick_async()
if __name__ == '__main__':
try:
@@ -86,10 +97,11 @@ if __name__ == '__main__':
from pytchat import LiveChat, CompatibleProcessor
import time
chat = LiveChat("DSGyEsJ17cI",
chat = LiveChat("Zvp1pJpie4I",
processor = CompatibleProcessor() )
while chat.is_alive():
try:
data = chat.get()
polling = data['pollingIntervalMillis']/1000
for c in data['items']:
@@ -97,7 +109,8 @@ while chat.is_alive():
print(f"[{c['authorDetails']['displayName']}]"
f"-{c['snippet']['displayMessage']}")
time.sleep(polling/len(data['items']))
except KeyboardInterrupt:
chat.terminate()
```
### replay:
If specified video is not live,
@@ -110,18 +123,21 @@ def main():
#seektime (seconds): start position of chat.
chat = LiveChat("ojes5ULOqhc", seektime = 60*30)
print('Replay from 30:00')
try:
while chat.is_alive():
data = chat.get()
for c in data.items:
print(f"{c.elapsedTime} [{c.author.name}]-{c.message} {c.amountString}")
data.tick()
except KeyboardInterrupt:
chat.terminate()
if __name__ == '__main__':
main()
```
## Structure of Default Processor
Each item can be got with items() function.
Each item can be got with `items` function.
<table>
<tr>
<th>name</th>

2
pytchat/__init__.py
View File

@@ -2,7 +2,7 @@
pytchat is a python library for fetching youtube live chat without using yt api, Selenium, or BeautifulSoup.
"""
__copyright__ = 'Copyright (C) 2019 taizan-hokuto'
__version__ = '0.0.5.0'
__version__ = '0.0.5.1.3'
__license__ = 'MIT'
__author__ = 'taizan-hokuto'
__author_email__ = '55448286+taizan-hokuto@users.noreply.github.com'

6
pytchat/config/__init__.py
View File

@@ -1,13 +1,11 @@
import logging
from . import mylogger
LOGGER_MODE = None
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'}
def logger(module_name: str):
module_logger = mylogger.get_logger(module_name, mode = LOGGER_MODE)
def logger(module_name: str, loglevel = None):
module_logger = mylogger.get_logger(module_name, loglevel = loglevel)
return module_logger

10
pytchat/config/mylogger.py
View File

@@ -3,21 +3,21 @@ import logging
import datetime
def get_logger(modname,mode=logging.DEBUG):
def get_logger(modname,loglevel=logging.DEBUG):
logger = getLogger(modname)
if mode == None:
if loglevel == None:
logger.addHandler(NullHandler())
return logger
logger.setLevel(mode)
logger.setLevel(loglevel)
#create handler1 for showing info
handler1 = StreamHandler()
my_formatter = MyFormatter()
handler1.setFormatter(my_formatter)
handler1.setLevel(mode)
handler1.setLevel(loglevel)
logger.addHandler(handler1)
#create handler2 for recording log file
if mode <= logging.DEBUG:
if loglevel <= logging.DEBUG:
handler2 = FileHandler(filename="log.txt", encoding='utf-8')
handler2.setLevel(logging.ERROR)
handler2.setFormatter(my_formatter)

35
pytchat/core_async/livechat.py
View File

@@ -17,7 +17,6 @@ from ..paramgen import liveparam, arcparam
from ..processors.default.processor import DefaultProcessor
from ..processors.combinator import Combinator
logger = config.logger(__name__)
headers = config.headers
MAX_RETRY = 10
@@ -74,6 +73,7 @@ class LiveChatAsync:
'''
_setup_finished = False
_logger = config.logger(__name__)
def __init__(self, video_id,
seektime = 0,
@@ -85,7 +85,8 @@ class LiveChatAsync:
exception_handler = None,
direct_mode = False,
force_replay = False,
topchat_only = False
topchat_only = False,
logger = config.logger(__name__),
):
self.video_id = video_id
self.seektime = seektime
@@ -107,11 +108,12 @@ class LiveChatAsync:
self._first_fetch = True
self._fetch_url = "live_chat/get_live_chat?continuation="
self._topchat_only = topchat_only
self._logger = logger
LiveChatAsync._logger = logger
if not LiveChatAsync._setup_finished:
LiveChatAsync._setup_finished = True
if exception_handler == None:
self._set_exception_handler(self._handle_exception)
else:
if exception_handler:
self._set_exception_handler(exception_handler)
if interruptable:
signal.signal(signal.SIGINT,
@@ -187,14 +189,14 @@ class LiveChatAsync:
continuation = metadata.get('continuation')
except ChatParseException as e:
#self.terminate()
logger.debug(f"[{self.video_id}]{str(e)}")
self._logger.debug(f"[{self.video_id}]{str(e)}")
return
except (TypeError , json.JSONDecodeError) :
#self.terminate()
logger.error(f"{traceback.format_exc(limit = -1)}")
self._logger.error(f"{traceback.format_exc(limit = -1)}")
return
logger.debug(f"[{self.video_id}]finished fetching chat.")
self._logger.debug(f"[{self.video_id}]finished fetching chat.")
async def _check_pause(self, continuation):
if self._pauser.empty():
@@ -254,7 +256,7 @@ class LiveChatAsync:
await asyncio.sleep(1)
continue
else:
logger.error(f"[{self.video_id}]"
self._logger.error(f"[{self.video_id}]"
f"Exceeded retry count. status_code={status_code}")
return None
return livechat_json
@@ -309,7 +311,7 @@ class LiveChatAsync:
try:
self.terminate()
except CancelledError:
logger.debug(f'[{self.video_id}]cancelled:{sender}')
self._logger.debug(f'[{self.video_id}]cancelled:{sender}')
def terminate(self):
'''
@@ -319,28 +321,21 @@ class LiveChatAsync:
if self._direct_mode == False:
#bufferにダミーオブジェクトを入れてis_alive()を判定させる
self._buffer.put_nowait({'chatdata':'','timeout':0})
logger.info(f'[{self.video_id}]finished.')
self._logger.info(f'[{self.video_id}]finished.')
@classmethod
def _set_exception_handler(cls, handler):
loop = asyncio.get_event_loop()
loop.set_exception_handler(handler)
@classmethod
def _handle_exception(cls, loop, context):
if not isinstance(context["exception"],CancelledError):
logger.error(f"Caught exception: {context}")
loop= asyncio.get_event_loop()
loop.create_task(cls.shutdown(None,None,None))
@classmethod
async def shutdown(cls, event, sig = None, handler=None):
logger.debug("shutdown...")
cls._logger.debug("shutdown...")
tasks = [t for t in asyncio.all_tasks() if t is not
asyncio.current_task()]
[task.cancel() for task in tasks]
logger.debug(f"complete remaining tasks...")
cls._logger.debug(f"complete remaining tasks...")
await asyncio.gather(*tasks,return_exceptions=True)
loop = asyncio.get_event_loop()
loop.stop()

25
pytchat/core_multithread/livechat.py
View File

@@ -16,7 +16,6 @@ from ..paramgen import liveparam, arcparam
from ..processors.default.processor import DefaultProcessor
from ..processors.combinator import Combinator
logger = config.logger(__name__)
headers = config.headers
MAX_RETRY = 10
@@ -74,7 +73,9 @@ class LiveChat:
_setup_finished = False
#チャット監視中のListenerのリスト
_listeners= []
_listeners = []
_logger = config.logger(__name__)
def __init__(self, video_id,
seektime = 0,
processor = DefaultProcessor(),
@@ -84,7 +85,8 @@ class LiveChat:
done_callback = None,
direct_mode = False,
force_replay = False,
topchat_only = False
topchat_only = False,
logger = config.logger(__name__)
):
self.video_id = video_id
self.seektime = seektime
@@ -106,6 +108,8 @@ class LiveChat:
self._first_fetch = True
self._fetch_url = "live_chat/get_live_chat?continuation="
self._topchat_only = topchat_only
self._logger = logger
LiveChat._logger = logger
if not LiveChat._setup_finished:
LiveChat._setup_finished = True
if interruptable:
@@ -115,7 +119,6 @@ class LiveChat:
LiveChat._listeners.append(self)
def _setup(self):
#logger.debug("setup")
#direct modeがTrueでcallback未設定の場合例外発生。
if self._direct_mode:
if self._callback is None:
@@ -181,13 +184,13 @@ class LiveChat:
time.sleep(diff_time if diff_time > 0 else 0)
continuation = metadata.get('continuation')
except ChatParseException as e:
logger.debug(f"[{self.video_id}]{str(e)}")
self._logger.debug(f"[{self.video_id}]{str(e)}")
return
except (TypeError , json.JSONDecodeError) :
logger.error(f"{traceback.format_exc(limit = -1)}")
self._logger.error(f"{traceback.format_exc(limit = -1)}")
return
logger.debug(f"[{self.video_id}]finished fetching chat.")
self._logger.debug(f"[{self.video_id}]finished fetching chat.")
def _check_pause(self, continuation):
if self._pauser.empty():
@@ -244,7 +247,7 @@ class LiveChat:
time.sleep(1)
continue
else:
logger.error(f"[{self.video_id}]"
self._logger.error(f"[{self.video_id}]"
f"Exceeded retry count. status_code={status_code}")
return None
return livechat_json
@@ -299,7 +302,7 @@ class LiveChat:
try:
self.terminate()
except CancelledError:
logger.debug(f'[{self.video_id}]cancelled:{sender}')
self._logger.debug(f'[{self.video_id}]cancelled:{sender}')
def terminate(self):
'''
@@ -309,10 +312,10 @@ class LiveChat:
if self._direct_mode == False:
#bufferにダミーオブジェクトを入れてis_alive()を判定させる
self._buffer.put({'chatdata':'','timeout':0})
logger.info(f'[{self.video_id}]finished.')
self._logger.info(f'[{self.video_id}]finished.')
@classmethod
def shutdown(cls, event, sig = None, handler=None):
logger.debug("shutdown...")
cls._logger.debug("shutdown...")
for t in LiveChat._listeners:
t._is_alive = False

7
pytchat/parser/live.py
View File

@@ -5,16 +5,12 @@ Parser of live chat JSON.
"""
import json
from .. import config
from .. exceptions import (
ResponseContextError,
NoContentsException,
NoContinuationsException,
ChatParseException )
logger = config.logger(__name__)
class Parser:
__slots__ = ['is_replay']
@@ -65,8 +61,7 @@ class Parser:
raise ChatParseException('Finished chat data')
unknown = list(cont.keys())[0]
if unknown:
logger.debug(f"Received unknown continuation type:{unknown}")
metadata = cont.get(unknown)
raise ChatParseException(f"Received unknown continuation type:{unknown}")
else:
raise ChatParseException('Cannot extract continuation data')
return self._create_data(metadata, contents)

2
pytchat/processors/default/processor.py
View File

@@ -61,7 +61,7 @@ class DefaultProcessor(ChatProcessor):
renderer.get_snippet()
renderer.get_authordetails()
except (KeyError,TypeError,AttributeError) as e:
except (KeyError,TypeError) as e:
logger.error(f"{str(type(e))}-{str(e)} sitem:{str(sitem)}")
return None
return renderer

14
pytchat/processors/default/renderer/base.py
View File

@@ -1,5 +1,4 @@
from datetime import datetime
class Author:
pass
class BaseRenderer:
@@ -67,16 +66,17 @@ class BaseRenderer:
badges=renderer.get("authorBadges")
if badges:
for badge in badges:
author_type = badge["liveChatAuthorBadgeRenderer"]["accessibility"]["accessibilityData"]["label"]
if author_type == '確認済み':
if badge["liveChatAuthorBadgeRenderer"].get("icon"):
author_type = badge["liveChatAuthorBadgeRenderer"]["icon"]["iconType"]
if author_type == 'VERIFIED':
isVerified = True
if author_type == '所有者':
if author_type == 'OWNER':
isChatOwner = True
if 'メンバー' in author_type:
if author_type == 'MODERATOR':
isChatModerator = True
if badge["liveChatAuthorBadgeRenderer"].get("customThumbnail"):
isChatSponsor = True
self.get_badgeurl(badge)
if author_type == 'モデレーター':
isChatModerator = True
return isVerified, isChatOwner, isChatSponsor, isChatModerator

13
pytchat/processors/json_display_processor.py
View File

@@ -1,13 +0,0 @@
import json
from .chat_processor import ChatProcessor
class JsonDisplayProcessor(ChatProcessor):
def process(self,chat_components: list):
if chat_components:
for component in chat_components:
chatdata = component.get('chatdata')
if chatdata:
for chat in chatdata:
print(json.dumps(chat,ensure_ascii=False)[:200])