ramforth/pytchat-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop'

Browse Source
This commit is contained in:
taizan-hokuto
2019-12-22 02:56:54 +09:00
parent ee0ff7fe74 91aa944df5
commit dfada86caf
13 changed files with 591 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -8,7 +8,7 @@ pytchat is a python library for fetching youtube live chat
without using youtube api, Selenium or BeautifulSoup.
Other features:
+ Customizable chat data processors including yt api compatible one.
+ Customizable chat data processors including youtube api compatible one.
+ Available on asyncio context.
+ Quick fetching of initial chat data by generating continuation params
instead of web scraping.

5
pytchat/__init__.py
View File

@@ -2,7 +2,7 @@
pytchat is a python library for fetching youtube live chat without using yt api, Selenium, or BeautifulSoup.
"""
__copyright__ = 'Copyright (C) 2019 taizan-hokuto'
__version__ = '0.0.3.6'
__version__ = '0.0.3.7'
__license__ = 'MIT'
__author__ = 'taizan-hokuto'
__author_email__ = '55448286+taizan-hokuto@users.noreply.github.com'
@@ -18,5 +18,6 @@ from .api import (
ChatProcessor,
CompatibleProcessor,
SimpleDisplayProcessor,
JsonfileArchiveProcessor
JsonfileArchiveProcessor,
SpeedCalculator
)

2
pytchat/api.py
View File

@@ -7,4 +7,4 @@ from .processors.default.processor import DefaultProcessor
from .processors.compatible.processor import CompatibleProcessor
from .processors.simple_display_processor import SimpleDisplayProcessor
from .processors.jsonfile_archive_processor import JsonfileArchiveProcessor
from .processors.speed_calculator import SpeedCalculator

2
pytchat/config/__init__.py
View File

@@ -1,4 +1,4 @@
import logging
LOGGER_MODE = logging.ERROR
LOGGER_MODE = None
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'}

6
pytchat/processors/chat_processor.py
View File

@@ -10,14 +10,14 @@ class ChatProcessor:
Parameter
----------
chat_components: [LIST:component]
chat_components: List[component]
component : dict {
"video_id" : str
動画ID
"timeout" : int
次のチャットの再読み込みまでの時間(秒）
"chatdata" : list<object>
チャットデータ（actions）のリスト
"chatdata" : List[dict]
チャットデータのリスト
}
'''
pass

3
pytchat/processors/compatible/processor.py
View File

@@ -4,11 +4,12 @@ from .renderer.textmessage import LiveChatTextMessageRenderer
from .renderer.paidmessage import LiveChatPaidMessageRenderer
from .renderer.paidsticker import LiveChatPaidStickerRenderer
from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
from .. chat_processor import ChatProcessor
from ... import mylogger
from ... import config
logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
class CompatibleProcessor:
class CompatibleProcessor(ChatProcessor):
def process(self, chat_components: list):

3
pytchat/processors/default/processor.py
View File

@@ -4,6 +4,7 @@ from .renderer.textmessage import LiveChatTextMessageRenderer
from .renderer.paidmessage import LiveChatPaidMessageRenderer
from .renderer.paidsticker import LiveChatPaidStickerRenderer
from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
from .. chat_processor import ChatProcessor
from ... import config
from ... import mylogger
logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
@@ -25,7 +26,7 @@ class Chatdata:
return
await asyncio.sleep(self.interval/len(self.items))
class DefaultProcessor:
class DefaultProcessor(ChatProcessor):
def process(self, chat_components: list):
chatlist = []

86
pytchat/processors/speed_calculator.py
View File

@@ -4,7 +4,7 @@ speedmeter.py
Calculate speed of chat.
"""
import calendar, datetime, pytz
from .chat_processor import ChatProcessor
class RingQueue:
"""
リング型キュー
@@ -21,7 +21,7 @@ class RingQueue:
キュー内に余裕があるか。キュー内のアイテム個数が、キューの最大個数未満であればTrue。
"""
def __init__(self, capacity = 10):
def __init__(self, capacity):
"""
コンストラクタ
@@ -77,34 +77,39 @@ class RingQueue:
def item_count(self):
return len(self.items)
class SpeedCalculator(RingQueue):
class SpeedCalculator(ChatProcessor, RingQueue):
"""
チャットの勢いを計算するクラス
チャットの勢いを計算する。
一定期間のチャットデータのうち、最初のチャットの投稿時刻と
最後のチャットの投稿時刻の差を、チャット数で割り返し
1分あたりの速度に換算する。
Parameter
----------
格納するチャットブロックの数
capacity : int
RingQueueに格納するチャット勢い算出用データの最大数
"""
def __init__(self, capacity, video_id):
def __init__(self, capacity = 10):
super().__init__(capacity)
self.video_id=video_id
self.speed = 0
def process(self, chat_components: list):
chatdata = []
if chat_components:
for component in chat_components:
if component.get("chatdata"):
chatdata.extend(component.get("chatdata"))
chatdata = component.get('chatdata')
if chatdata is None:
return self.speed
self.speed = self.calc(chatdata)
self._put_chatdata(chatdata)
self.speed = self._calc_speed()
return self.speed
def _value(self):
def _calc_speed(self):
"""
ActionsQueue内のチャットデータリストから、
RingQueue内のチャット勢い算出用データリストを元に、
チャット速度を計算して返す
Return
@@ -112,7 +117,7 @@ class SpeedCalculator(RingQueue):
チャット速度（１分間で換算したチャット数）
"""
try:
#キュー内のactionsの総チャット数
#キュー内の総チャット数
total = sum(item['chat_count'] for item in self.items)
#キュー内の最初と最後のチャットの時間差
duration = (self.items[self.last_pos]['endtime']
@@ -123,24 +128,20 @@ class SpeedCalculator(RingQueue):
except IndexError:
return 0
def _get_timestamp(self, action :dict):
def _put_chatdata(self, actions):
"""
チャットデータのtimestampUsecを読み取る
liveChatTickerSponsorItemRenderer等のtickerデータは時刻格納位置が
異なるため、時刻データなしとして扱う
チャットデータからタイムスタンプを読み取り、勢い測定用のデータを組み立て、
RingQueueに投入する。
200円以上のスパチャはtickerとmessageの2つのデータが生成されるが、
tickerの方は時刻データの場所が異なることを利用し、勢いの集計から除外している。
Parameter
---------
actions : List[dict]
チャットデータ(addChatItemAction) のリスト
"""
try:
item = action['addChatItemAction']['item']
timestamp = int(item[list(item.keys())[0]]['timestampUsec'])
except (KeyError,TypeError):
return None
return timestamp
def calc(self,actions):
def empty_data():
def _put_emptydata():
'''
データがない場合にゼロのデータをリングキューに入れる
チャットデータがない場合に空のデータをキューに投入する。
'''
timestamp_now = calendar.timegm(datetime.datetime.
now(pytz.utc).utctimetuple())
@@ -149,12 +150,23 @@ class SpeedCalculator(RingQueue):
'starttime':int(timestamp_now),
'endtime':int(timestamp_now)
})
return self._value()
def _get_timestamp(action :dict):
"""
チャットデータから時刻データを取り出す。
"""
try:
item = action['addChatItemAction']['item']
timestamp = int(item[list(item.keys())[0]]['timestampUsec'])
except (KeyError,TypeError):
return None
return timestamp
if actions is None or len(actions)==0:
return empty_data
_put_emptydata()
return
#actions内の時刻データを持つチャットデータの数（tickerは除く）
#actions内の時刻データを持つチャットデータの数
counter=0
#actions内の最初のチャットデータの時刻
starttime= None
@@ -163,7 +175,7 @@ class SpeedCalculator(RingQueue):
for action in actions:
#チャットデータからtimestampUsecを読み取る
gettime = self._get_timestamp(action)
gettime = _get_timestamp(action)
#時刻のないデータだった場合は次の行のデータで読み取り試行
if gettime is None:
@@ -179,9 +191,10 @@ class SpeedCalculator(RingQueue):
#チャットの数をインクリメント
counter += 1
#チャット速度用のデータをリングキューに送る
#チャット速度用のデータをRingQueueに送る
if starttime is None or endtime is None:
return empty_data
_put_emptydata()
return
self.put({
'chat_count':counter,
@@ -189,4 +202,3 @@ class SpeedCalculator(RingQueue):
'endtime':int(endtime/1000000)
})
return self._value()

68
tests/test_speed_calculator.py Normal file
View File

@@ -0,0 +1,68 @@
import json
import pytest
import asyncio,aiohttp
from pytchat.parser.live import Parser
from pytchat.processors.compatible.processor import CompatibleProcessor
from pytchat.exceptions import (
NoLivechatRendererException,NoYtinitialdataException,
ResponseContextError, NoContentsException)
from pytchat.processors.speed_calculator import SpeedCalculator
parser = Parser()
def test_speed_1(mocker):
'''test speed calculation with normal json.
test json has 15 chatdata, duration is 30 seconds,
so the speed of chatdata is 30 chats/minute.
'''
processor = SpeedCalculator(capacity=30)
_json = _open_file("tests/testdata/speed/speedtest_normal.json")
_, chatdata = parser.parse(json.loads(_json))
data = {
"video_id" : "",
"timeout" : 10,
"chatdata" : chatdata
}
ret = processor.process([data])
assert 30 == ret
def test_speed_2(mocker):
'''test speed calculation with no valid chat data.
'''
processor = SpeedCalculator(capacity=30)
_json = _open_file("tests/testdata/speed/speedtest_undefined.json")
_, chatdata = parser.parse(json.loads(_json))
data = {
"video_id" : "",
"timeout" : 10,
"chatdata" : chatdata
}
ret = processor.process([data])
assert 0 == ret
def test_speed_3(mocker):
'''test speed calculation with empty data.
'''
processor = SpeedCalculator(capacity=30)
_json = _open_file("tests/testdata/speed/speedtest_empty.json")
_, chatdata = parser.parse(json.loads(_json))
data = {
"video_id" : "",
"timeout" : 10,
"chatdata" : chatdata
}
ret = processor.process([data])
assert 0 == ret
def _open_file(path):
with open(path,mode ='r',encoding = 'utf-8') as f:
return f.read()

164
tests/testdata/chat.json vendored Normal file
View File

@@ -0,0 +1,164 @@
{
"timing": {
"info": {
"st": 164
}
},
"csn": "",
"response": {
"responseContext": {
"serviceTrackingParams": [{
"service": "CSI",
"params": [{
"key": "GetLiveChat_rid",
"value": ""
}, {
"key": "c",
"value": "WEB"
}, {
"key": "cver",
"value": "2.20191219.03.01"
}, {
"key": "yt_li",
"value": "0"
}]
}, {
"service": "GFEEDBACK",
"params": [{
"key": "e",
"value": ""
}, {
"key": "logged_in",
"value": "0"
}]
}, {
"service": "GUIDED_HELP",
"params": [{
"key": "logged_in",
"value": "0"
}]
}, {
"service": "ECATCHER",
"params": [{
"key": "client.name",
"value": "WEB"
}, {
"key": "client.version",
"value": "2.2"
}, {
"key": "innertube.build.changelist",
"value": "228"
}, {
"key": "innertube.build.experiments.source_version",
"value": "2858"
}, {
"key": "innertube.build.label",
"value": "youtube.ytfe.innertube_"
}, {
"key": "innertube.build.timestamp",
"value": "154"
}, {
"key": "innertube.build.variants.checksum",
"value": "e"
}, {
"key": "innertube.run.job",
"value": "ytfe-innertube-replica-only.ytfe"
}]
}],
"webResponseContextExtensionData": {
"ytConfigData": {
"csn": "ADw",
"visitorData": "%3D%3D"
}
}
},
"continuationContents": {
"liveChatContinuation": {
"continuations": [{
"timedContinuationData": {
"timeoutMs": 10000,
"continuation": "continuation"
}
}],
"actions": [{
"addChatItemAction": {
"item": {
"liveChatTextMessageRenderer": {
"message": {
"runs": [{
"text": "message"
}]
},
"authorName": {
"simpleText": "authorName"
},
"authorPhoto": {
"thumbnails": [{
"url": "https://yt3.ggpht.com/photo.jpg",
"width": 32,
"height": 32
}, {
"url": "https://yt3.ggpht.com/photo.jpg",
"width": 64,
"height": 64
}]
},
"contextMenuEndpoint": {
"commandMetadata": {
"webCommandMetadata": {
"ignoreNavigation": true
}
},
"liveChatItemContextMenuEndpoint": {
"params": "params"
}
},
"id": "id",
"timestampUsec": "1576851922945411",
"authorBadges": [{
"liveChatAuthorBadgeRenderer": {
"customThumbnail": {
"thumbnails": [{
"url": "https://yt3.ggpht.com/photo.jpg"
}, {
"url": "https://yt3.ggpht.com/photo.jpg"
}]
},
"tooltip": "メンバー（6 か月）",
"accessibility": {
"accessibilityData": {
"label": "メンバー（6 か月）"
}
}
}
}],
"authorExternalChannelId": "UC",
"contextMenuAccessibility": {
"accessibilityData": {
"label": "コメントの操作"
}
}
}
},
"clientId": "00000000000000000000"
}
}
]}
},
"xsrf_token": "xsrf_token",
"url": "/live_chat/get_live_chat?continuation=0",
"endpoint": {
"commandMetadata": {
"webCommandMetadata": {
"url": "/live_chat/get_live_chat?continuation=0",
"rootVe": 0
}
},
"urlEndpoint": {
"url": "/live_chat/get_live_chat?continuation=0"
}
}
}
}

24
tests/testdata/speed/speedtest_empty.json vendored Normal file
View File

@@ -0,0 +1,24 @@
{
"timing": {
"info": {
"st": 164
}
},
"csn": "",
"response": {
"responseContext": {
},
"continuationContents": {
"liveChatContinuation": {
"continuations": [{
"timedContinuationData": {
"timeoutMs": 10000,
"continuation": "continuation"
}
}]
}
}
}
}

188
tests/testdata/speed/speedtest_normal.json vendored Normal file
View File

@@ -0,0 +1,188 @@
{
"timing": {
"info": {
"st": 164
}
},
"csn": "",
"response": {
"responseContext": {
},
"continuationContents": {
"liveChatContinuation": {
"continuations": [{
"timedContinuationData": {
"timeoutMs": 10000,
"continuation": "continuation"
}
}],
"actions": [{
"addChatItemAction": {
"item": {
"liveChatTextMessageRenderer": {
"timestampUsec": "1500000000000000"
}
}
}
},
{
"addChatItemAction": {
"item": {
"liveChatTextMessageRenderer": {
"timestampUsec": "1500000000000000"
}
}
}
},
{
"addChatItemAction": {
"item": {
"liveChatTextMessageRenderer": {
"timestampUsec": "1500000000000000"
}
}
}
},
{
"addChatItemAction": {
"item": {
"liveChatTextMessageRenderer": {
"timestampUsec": "1500000000000000"
}
}
}
},
{
"addChatItemAction": {
"item": {
"liveChatTextMessageRenderer": {
"timestampUsec": "1500000000000000"
}
}
}
},
{
"addChatItemAction": {
"item": {
"liveChatTextMessageRenderer": {
"timestampUsec": "1500000000000000"
}
}
}
},
{
"addChatItemAction": {
"item": {
"liveChatTextMessageRenderer": {
"timestampUsec": "1500000000000000"
}
}
}
},
{
"addChatItemAction": {
"item": {
"liveChatTextMessageRenderer": {
"timestampUsec": "1500000000000000"
}
}
}
},
{
"addChatItemAction": {
"item": {
"liveChatTextMessageRenderer": {
"timestampUsec": "1500000000000000"
}
}
}
},
{
"addChatItemAction": {
"item": {
"liveChatTextMessageRenderer": {
"timestampUsec": "1500000000000000"
}
}
}
},
{
"addChatItemAction": {
"item": {
"liveChatTextMessageRenderer": {
"timestampUsec": "1500000000000000"
}
}
}
},
{
"addChatItemAction": {
"item": {
"liveChatTextMessageRenderer": {
"timestampUsec": "1500000000000000"
}
}
}
},
{
"addChatItemAction": {
"item": {
"liveChatTextMessageRenderer": {
"timestampUsec": "1500000000000000"
}
}
}
},
{
"addChatItemAction": {
"item": {
"liveChatTextMessageRenderer": {
"timestampUsec": "1500000000000000"
}
}
}
},
{
"addChatItemAction": {
"item": {
"liveChatTextMessageRenderer": {
"timestampUsec": "1500000030000000"
}
}
}
}
]}
}
}
}

42
tests/testdata/speed/speedtest_undefined.json vendored Normal file
View File

@@ -0,0 +1,42 @@
{
"timing": {
"info": {
"st": 164
}
},
"csn": "",
"response": {
"responseContext": {
},
"continuationContents": {
"liveChatContinuation": {
"continuations": [{
"timedContinuationData": {
"timeoutMs": 10000,
"continuation": "continuation"
}
}],
"actions": [{
"addChatItemAction": {
"liveChatPlaceholderItemRenderer": {
"id": "",
"timestampUsec": "1500000000000000"
}
}
},
{
"addChatItemAction": {
"item": {
"liveChatPlaceholderItemRenderer": {
"id": "",
"timestampUsec": "1500000030000000"
}
}
}
}
]}
}
}
}