ramforth/pytchat-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement Superchat Calculator

Browse Source
This commit is contained in:
taizan-hokuto
2020-02-24 13:56:58 +09:00
parent af4afb4636
commit 3c95242ddf
14 changed files with 3744 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
pytchat/core_async/livechat.py
View File

@@ -178,20 +178,20 @@ class LiveChatAsync:
}
time_mark =time.time()
if self._direct_mode:
await self._callback(
self.processor.process([chat_component])
)
processed_chat = self.processor.process([chat_component])
if isinstance(processed_chat,tuple):
await self._callback(*processed_chat)
else:
await self._callback(processed_chat)
else:
await self._buffer.put(chat_component)
diff_time = timeout - (time.time()-time_mark)
await asyncio.sleep(diff_time)
continuation = metadata.get('continuation')
except ChatParseException as e:
#self.terminate()
self._logger.debug(f"[{self.video_id}]{str(e)}")
return
except (TypeError , json.JSONDecodeError) :
#self.terminate()
self._logger.error(f"{traceback.format_exc(limit = -1)}")
return
@@ -211,13 +211,13 @@ class LiveChatAsync:
return continuation
async def _get_contents(self, continuation, session, headers):
'''Get 'contents' dict from livechat json.
'''Get 'continuationContents' from livechat json.
If contents is None at first fetching,
try to fetch archive chat data.
Return:
-------
'contents' dict which includes metadata & chatdata.
'continuationContents' which includes metadata & chatdata.
'''
livechat_json = (await
self._get_livechat_json(continuation, session, headers)
@@ -275,8 +275,11 @@ class LiveChatAsync:
"""
while self.is_alive():
items = await self._buffer.get()
data = self.processor.process(items)
await callback(data)
processed_chat = self.processor.process(items)
if isinstance(processed_chat, tuple):
await self._callback(*processed_chat)
else:
await self._callback(processed_chat)
async def get(self):
""" bufferからデータを取り出し、processorに投げ、

19
pytchat/core_multithread/livechat.py
View File

@@ -174,9 +174,11 @@ class LiveChat:
}
time_mark =time.time()
if self._direct_mode:
self._callback(
self.processor.process([chat_component])
)
processed_chat = self.processor.process([chat_component])
if isinstance(processed_chat,tuple):
self._callback(*processed_chat)
else:
self._callback(processed_chat)
else:
self._buffer.put(chat_component)
diff_time = timeout - (time.time()-time_mark)
@@ -204,13 +206,13 @@ class LiveChat:
return continuation
def _get_contents(self, continuation, session, headers):
'''Get 'contents' dict from livechat json.
'''Get 'continuationContents' from livechat json.
If contents is None at first fetching,
try to fetch archive chat data.
Return:
-------
'contents' dict which includes metadata & chatdata.
'continuationContents' which includes metadata & chat data.
'''
livechat_json = (
self._get_livechat_json(continuation, session, headers)
@@ -268,8 +270,11 @@ class LiveChat:
"""
while self.is_alive():
items = self._buffer.get()
data = self.processor.process(items)
callback(data)
processed_chat = self.processor.process(items)
if isinstance(processed_chat, tuple):
self._callback(*processed_chat)
else:
self._callback(processed_chat)
def get(self):
""" bufferからデータを取り出し、processorに投げ、

13
pytchat/processors/chat_processor.py
View File

@@ -1,23 +1,22 @@
class ChatProcessor:
'''
Listenerからチャットデータ（actions）を受け取り
チャットデータを加工するクラスの抽象クラス
Abstract class that processes chat data.
Receive chat data (actions) from Listener.
'''
def process(self, chat_components: list):
'''
チャットデータの加工を表すインターフェース。
LiveChatオブジェクトから呼び出される。
Interface that represents processing of chat data.
Called from LiveChat object.
Parameter
----------
chat_components: List[component]
component : dict {
"video_id" : str
動画ID
"timeout" : int
次のチャットの再読み込みまでの時間(秒）
Time to fetch next chat (seconds)
"chatdata" : List[dict]
チャットデータのリスト
List of chat data.
}
'''
pass

1
pytchat/processors/default/processor.py
View File

@@ -35,7 +35,6 @@ class DefaultProcessor(ChatProcessor):
for component in chat_components:
timeout += component.get('timeout', 0)
chatdata = component.get('chatdata')
if chatdata is None: continue
for action in chatdata:
if action is None: continue

73
pytchat/processors/superchat/calculator.py Normal file
View File

@@ -0,0 +1,73 @@
import re
from pytchat.processors.chat_processor import ChatProcessor
superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
items_paid = [
'addChatItemAction',
'item',
'liveChatPaidMessageRenderer'
]
items_sticker = [
'addChatItemAction',
'item',
'liveChatPaidStickerRenderer'
]
class Calculator(ChatProcessor):
"""
Calculate the amount of SuperChat by currency.
"""
def __init__(self):
self.results = {}
def process(self, chat_components: list):
"""
Return
------------
results : dict :
List of amount by currency.
key: currency symbol, value: total amount.
"""
if chat_components is None:
return self.results
for component in chat_components:
chatdata = component.get('chatdata')
if chatdata is None: continue
for action in chatdata:
renderer = self._get_item(action, items_paid) or \
self._get_item(action, items_sticker)
if renderer is None: continue
symbol, amount = self._parse(renderer)
self.results.setdefault(symbol,0)
self.results[symbol]+=amount
return self.results
def _parse(self, renderer):
purchase_amount_text = renderer["purchaseAmountText"]["simpleText"]
m = superchat_regex.search(purchase_amount_text)
if m:
symbol = m.group(1)
amount = float(m.group(2).replace(',',''))
else:
symbol = ""
amount = 0.0
return symbol, amount
def _get_item(self, dict_body, items: list):
for item in items:
if dict_body is None:
break
if isinstance(dict_body, dict):
dict_body = dict_body.get(item)
continue
if isinstance(item, int) and \
isinstance(dict_body, list) and \
len(dict_body) > item:
dict_body = dict_body[item]
continue
return None
return dict_body

5
pytchat/tool/mining/block.py
View File

@@ -38,6 +38,9 @@ class Block:
during_split : bool :
whether this block is in the process of during_split.
while True, this block is excluded from duplicate split procedure.
seektime : float :
the last position of this block(seconds) already fetched.
"""
__slots__ = ['first','last','end','continuation','chat_data','remaining',
@@ -45,7 +48,7 @@ class Block:
def __init__(self, first = 0, last = 0, end = 0,
continuation = '', chat_data = [], is_last = False,
during_split = False,seektime = None):
during_split = False, seektime = None):
self.first = first
self.last = last
self.end = end

9
pytchat/tool/mining/dlworker.py
View File

@@ -34,7 +34,6 @@ class DownloadWorker:
self.video_id:str = video_id
self.parent_block:Block = None
async def run(self, session):
while self.block.continuation:
patch = await self.fetch(
@@ -44,11 +43,3 @@ class DownloadWorker:
self.block.done = True
def fd(name,mes,src,patch,end):
def offset(chats):
if len(chats)==0:
return None,None
return parser.get_offset(chats[0]),parser.get_offset(chats[-1])
with open("v://tlog.csv",encoding="utf-8",mode="a") as f:
f.write(f"WORKER,{name},mes,{mes},edge,{offset(src)[1]},first,{offset(patch)[0]},last,{offset(patch)[1]},end,{end}\n")

3
pytchat/tool/mining/parser.py
View File

@@ -35,10 +35,7 @@ def parse(jsn):
metadata = cont.get('liveChatReplayContinuationData')
if metadata:
continuation = metadata.get("continuation")
#print(continuation)
actions = contents['liveChatContinuation'].get('actions')
# print(list(actions[0]['replayChatItemAction']["actions"][0].values()
# )[0]['item'].get("liveChatPaidMessageRenderer"))
if continuation:
return continuation, [action["replayChatItemAction"]["actions"][0]
for action in actions