ramforth/pytchat-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change structure of default processor

Browse Source
This commit is contained in:
taizan-hokouto
2020-10-24 19:12:00 +09:00
parent 4905b1e4d8
commit 3200c5654f
9 changed files with 257 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
pytchat/processors/default/custom_encoder.py Normal file
View File

@@ -0,0 +1,11 @@
import json
from .renderer.base import Author
from .renderer.paidmessage import Colors
from .renderer.paidsticker import Colors2
class CustomEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, Author) or isinstance(obj, Colors) or isinstance(obj, Colors2):
return vars(obj)
return json.JSONEncoder.default(self, obj)

156
pytchat/processors/default/processor.py
View File

@@ -1,5 +1,7 @@
import asyncio import asyncio
import json
import time import time
from .custom_encoder import CustomEncoder
from .renderer.textmessage import LiveChatTextMessageRenderer from .renderer.textmessage import LiveChatTextMessageRenderer
from .renderer.paidmessage import LiveChatPaidMessageRenderer from .renderer.paidmessage import LiveChatPaidMessageRenderer
from .renderer.paidsticker import LiveChatPaidStickerRenderer from .renderer.paidsticker import LiveChatPaidStickerRenderer
@@ -11,25 +13,121 @@ from ... import config
logger = config.logger(__name__) logger = config.logger(__name__)
class Chat:
def json(self) -> str:
return json.dumps(vars(self), ensure_ascii=False, cls=CustomEncoder)
class Chatdata: class Chatdata:
def __init__(self, chatlist: list, timeout: float):
def __init__(self, chatlist: list, timeout: float, abs_diff):
self.items = chatlist self.items = chatlist
self.interval = timeout self.interval = timeout
self.abs_diff = abs_diff
self.itemcount = 0
self.before_timestamp = -2**64
def tick(self): def tick(self):
if self.interval == 0: '''DEPRECATE
Use sync_items()
'''
if len(self.items) < 1:
time.sleep(1) time.sleep(1)
return return
time.sleep(self.interval / len(self.items)) if self.itemcount == 0:
self.starttime = time.time()
if len(self.items) == 1:
total_itemcount = 1
else:
total_itemcount = len(self.items) - 1
next_chattime = (self.items[0].timestamp + (self.items[-1].timestamp - self.items[0].timestamp) / total_itemcount * self.itemcount) / 1000
tobe_disptime = self.abs_diff + next_chattime
wait_sec = tobe_disptime - time.time()
self.itemcount += 1
if wait_sec < 0:
wait_sec = 0
time.sleep(wait_sec)
async def tick_async(self): async def tick_async(self):
if self.interval == 0: '''DEPRECATE
Use async_items()
'''
if len(self.items) < 1:
await asyncio.sleep(1) await asyncio.sleep(1)
return return
await asyncio.sleep(self.interval / len(self.items)) if self.itemcount == 0:
self.starttime = time.time()
if len(self.items) == 1:
total_itemcount = 1
else:
total_itemcount = len(self.items) - 1
next_chattime = (self.items[0].timestamp + (self.items[-1].timestamp - self.items[0].timestamp) / total_itemcount * self.itemcount) / 1000
tobe_disptime = self.abs_diff + next_chattime
wait_sec = tobe_disptime - time.time()
self.itemcount += 1
if wait_sec < 0:
wait_sec = 0
await asyncio.sleep(wait_sec)
def sync_items(self):
starttime = time.time()
if len(self.items) > 0:
last_chattime = self.items[-1].timestamp / 1000
tobe_disptime = self.abs_diff + last_chattime
wait_total_sec = max(tobe_disptime - time.time(), 0)
if len(self.items) > 1:
wait_sec = wait_total_sec / len(self.items)
elif len(self.items) == 1:
wait_sec = 0
for c in self.items:
if wait_sec < 0:
wait_sec = 0
time.sleep(wait_sec)
yield c
stop_interval = time.time() - starttime
if stop_interval < 1:
time.sleep(1 - stop_interval)
async def async_items(self):
starttime = time.time()
if len(self.items) > 0:
last_chattime = self.items[-1].timestamp / 1000
tobe_disptime = self.abs_diff + last_chattime
wait_total_sec = max(tobe_disptime - time.time(), 0)
if len(self.items) > 1:
wait_sec = wait_total_sec / len(self.items)
elif len(self.items) == 1:
wait_sec = 0
for c in self.items:
if wait_sec < 0:
wait_sec = 0
await asyncio.sleep(wait_sec)
yield c
stop_interval = time.time() - starttime
if stop_interval < 1:
await asyncio.sleep(1 - stop_interval)
def json(self) -> str:
return json.dumps([vars(a) for a in self.items], ensure_ascii=False, cls=CustomEncoder)
class DefaultProcessor(ChatProcessor): class DefaultProcessor(ChatProcessor):
def __init__(self):
self.first = True
self.abs_diff = 0
self.renderers = {
"liveChatTextMessageRenderer": LiveChatTextMessageRenderer(),
"liveChatPaidMessageRenderer": LiveChatPaidMessageRenderer(),
"liveChatPaidStickerRenderer": LiveChatPaidStickerRenderer(),
"liveChatLegacyPaidMessageRenderer": LiveChatLegacyPaidMessageRenderer(),
"liveChatMembershipItemRenderer": LiveChatMembershipItemRenderer()
}
def process(self, chat_components: list): def process(self, chat_components: list):
chatlist = [] chatlist = []
@@ -46,43 +144,35 @@ class DefaultProcessor(ChatProcessor):
continue continue
if action.get('addChatItemAction') is None: if action.get('addChatItemAction') is None:
continue continue
if action['addChatItemAction'].get('item') is None: item = action['addChatItemAction'].get('item')
if item is None:
continue continue
chat = self._parse(item)
chat = self._parse(action)
if chat: if chat:
chatlist.append(chat) chatlist.append(chat)
return Chatdata(chatlist, float(timeout))
def _parse(self, sitem): if self.first and chatlist:
action = sitem.get("addChatItemAction") self.abs_diff = time.time() - chatlist[0].timestamp / 1000 + 2
if action: self.first = False
item = action.get("item")
if item is None: chatdata = Chatdata(chatlist, float(timeout), self.abs_diff)
return None
return chatdata
def _parse(self, item):
try: try:
renderer = self._get_renderer(item) key = list(item.keys())[0]
renderer = self.renderers.get(key)
if renderer is None: if renderer is None:
return None return None
renderer.setitem(item.get(key), Chat())
renderer.settype()
renderer.get_snippet() renderer.get_snippet()
renderer.get_authordetails() renderer.get_authordetails()
rendered_chatobj = renderer.get_chatobj()
renderer.clear()
except (KeyError, TypeError) as e: except (KeyError, TypeError) as e:
logger.error(f"{str(type(e))}-{str(e)} sitem:{str(sitem)}") logger.error(f"{str(type(e))}-{str(e)} item:{str(item)}")
return None return None
return renderer
def _get_renderer(self, item): return rendered_chatobj
if item.get("liveChatTextMessageRenderer"):
renderer = LiveChatTextMessageRenderer(item)
elif item.get("liveChatPaidMessageRenderer"):
renderer = LiveChatPaidMessageRenderer(item)
elif item.get("liveChatPaidStickerRenderer"):
renderer = LiveChatPaidStickerRenderer(item)
elif item.get("liveChatLegacyPaidMessageRenderer"):
renderer = LiveChatLegacyPaidMessageRenderer(item)
elif item.get("liveChatMembershipItemRenderer"):
renderer = LiveChatMembershipItemRenderer(item)
else:
renderer = None
return renderer

131
pytchat/processors/default/renderer/base.py
View File

@@ -6,89 +6,96 @@ class Author:
class BaseRenderer: class BaseRenderer:
def __init__(self, item, chattype): def setitem(self, item, chat):
self.renderer = list(item.values())[0] self.item = item
self.chattype = chattype self.chat = chat
self.author = Author() self.chat.author = Author()
def settype(self):
pass
def get_snippet(self): def get_snippet(self):
self.type = self.chattype self.chat.id = self.item.get('id')
self.id = self.renderer.get('id') timestampUsec = int(self.item.get("timestampUsec", 0))
timestampUsec = int(self.renderer.get("timestampUsec", 0)) self.chat.timestamp = int(timestampUsec / 1000)
self.timestamp = int(timestampUsec / 1000) tst = self.item.get("timestampText")
tst = self.renderer.get("timestampText")
if tst: if tst:
self.elapsedTime = tst.get("simpleText") self.chat.elapsedTime = tst.get("simpleText")
else: else:
self.elapsedTime = "" self.chat.elapsedTime = ""
self.datetime = self.get_datetime(timestampUsec) self.chat.datetime = self.get_datetime(timestampUsec)
self.message, self.messageEx = self.get_message(self.renderer) self.chat.message, self.chat.messageEx = self.get_message(self.item)
self.id = self.renderer.get('id') self.chat.id = self.item.get('id')
self.amountValue = 0.0 self.chat.amountValue = 0.0
self.amountString = "" self.chat.amountString = ""
self.currency = "" self.chat.currency = ""
self.bgColor = 0 self.chat.bgColor = 0
def get_authordetails(self): def get_authordetails(self):
self.author.badgeUrl = "" self.chat.author.badgeUrl = ""
(self.author.isVerified, (self.chat.author.isVerified,
self.author.isChatOwner, self.chat.author.isChatOwner,
self.author.isChatSponsor, self.chat.author.isChatSponsor,
self.author.isChatModerator) = ( self.chat.author.isChatModerator) = (
self.get_badges(self.renderer) self.get_badges(self.item)
) )
self.author.channelId = self.renderer.get("authorExternalChannelId") self.chat.author.channelId = self.item.get("authorExternalChannelId")
self.author.channelUrl = "http://www.youtube.com/channel/" + self.author.channelId self.chat.author.channelUrl = "http://www.youtube.com/channel/" + self.chat.author.channelId
self.author.name = self.renderer["authorName"]["simpleText"] self.chat.author.name = self.item["authorName"]["simpleText"]
self.author.imageUrl = self.renderer["authorPhoto"]["thumbnails"][1]["url"] self.chat.author.imageUrl = self.item["authorPhoto"]["thumbnails"][1]["url"]
def get_message(self, renderer): def get_message(self, item):
message = '' message = ''
message_ex = [] message_ex = []
if renderer.get("message"): runs = item.get("message", {}).get("runs", {})
runs = renderer["message"].get("runs") for r in runs:
if runs: if not hasattr(r, "get"):
for r in runs: continue
if r: if r.get('emoji'):
if r.get('emoji'): message += r['emoji'].get('shortcuts', [''])[0]
message += r['emoji'].get('shortcuts', [''])[0] message_ex.append({
message_ex.append({ 'id': r['emoji'].get('emojiId').split('/')[-1],
'id': r['emoji'].get('emojiId').split('/')[-1], 'txt': r['emoji'].get('shortcuts', [''])[0],
'txt': r['emoji'].get('shortcuts', [''])[0], 'url': r['emoji']['image']['thumbnails'][0].get('url')
'url': r['emoji']['image']['thumbnails'][0].get('url') })
}) else:
else: message += r.get('text', '')
message += r.get('text', '') message_ex.append(r.get('text', ''))
message_ex.append(r.get('text', ''))
return message, message_ex return message, message_ex
def get_badges(self, renderer): def get_badges(self, renderer):
self.author.type = '' self.chat.author.type = ''
isVerified = False isVerified = False
isChatOwner = False isChatOwner = False
isChatSponsor = False isChatSponsor = False
isChatModerator = False isChatModerator = False
badges = renderer.get("authorBadges") badges = renderer.get("authorBadges", {})
if badges: for badge in badges:
for badge in badges: if badge["liveChatAuthorBadgeRenderer"].get("icon"):
if badge["liveChatAuthorBadgeRenderer"].get("icon"): author_type = badge["liveChatAuthorBadgeRenderer"]["icon"]["iconType"]
author_type = badge["liveChatAuthorBadgeRenderer"]["icon"]["iconType"] self.chat.author.type = author_type
self.author.type = author_type if author_type == 'VERIFIED':
if author_type == 'VERIFIED': isVerified = True
isVerified = True if author_type == 'OWNER':
if author_type == 'OWNER': isChatOwner = True
isChatOwner = True if author_type == 'MODERATOR':
if author_type == 'MODERATOR': isChatModerator = True
isChatModerator = True if badge["liveChatAuthorBadgeRenderer"].get("customThumbnail"):
if badge["liveChatAuthorBadgeRenderer"].get("customThumbnail"): isChatSponsor = True
isChatSponsor = True self.chat.author.type = 'MEMBER'
self.author.type = 'MEMBER' self.get_badgeurl(badge)
self.get_badgeurl(badge)
return isVerified, isChatOwner, isChatSponsor, isChatModerator return isVerified, isChatOwner, isChatSponsor, isChatModerator
def get_badgeurl(self, badge): def get_badgeurl(self, badge):
self.author.badgeUrl = badge["liveChatAuthorBadgeRenderer"]["customThumbnail"]["thumbnails"][0]["url"] self.chat.author.badgeUrl = badge["liveChatAuthorBadgeRenderer"]["customThumbnail"]["thumbnails"][0]["url"]
def get_datetime(self, timestamp): def get_datetime(self, timestamp):
dt = datetime.fromtimestamp(timestamp / 1000000) dt = datetime.fromtimestamp(timestamp / 1000000)
return dt.strftime('%Y-%m-%d %H:%M:%S') return dt.strftime('%Y-%m-%d %H:%M:%S')
def get_chatobj(self):
return self.chat
def clear(self):
self.item = None
self.chat = None

12
pytchat/processors/default/renderer/legacypaid.py
View File

@@ -2,14 +2,14 @@ from .base import BaseRenderer
class LiveChatLegacyPaidMessageRenderer(BaseRenderer): class LiveChatLegacyPaidMessageRenderer(BaseRenderer):
def __init__(self, item): def settype(self):
super().__init__(item, "newSponsor") self.chat.type = "newSponsor"
def get_authordetails(self): def get_authordetails(self):
super().get_authordetails() super().get_authordetails()
self.author.isChatSponsor = True self.chat.author.isChatSponsor = True
def get_message(self, renderer): def get_message(self, item):
message = (renderer["eventText"]["runs"][0]["text"] message = (item["eventText"]["runs"][0]["text"]
) + ' / ' + (renderer["detailText"]["simpleText"]) ) + ' / ' + (item["detailText"]["simpleText"])
return message, [message] return message, [message]

15
pytchat/processors/default/renderer/membership.py
View File

@@ -2,14 +2,17 @@ from .base import BaseRenderer
class LiveChatMembershipItemRenderer(BaseRenderer): class LiveChatMembershipItemRenderer(BaseRenderer):
def __init__(self, item): def settype(self):
super().__init__(item, "newSponsor") self.chat.type = "newSponsor"
def get_authordetails(self): def get_authordetails(self):
super().get_authordetails() super().get_authordetails()
self.author.isChatSponsor = True self.chat.author.isChatSponsor = True
def get_message(self, renderer): def get_message(self, item):
message = ''.join([mes.get("text", "") try:
for mes in renderer["headerSubtext"]["runs"]]) message = ''.join([mes.get("text", "")
for mes in item["headerSubtext"]["runs"]])
except KeyError:
return "Welcome New Member!", ["Welcome New Member!"]
return message, [message] return message, [message]

33
pytchat/processors/default/renderer/paidmessage.py
View File

@@ -9,23 +9,23 @@ class Colors:
class LiveChatPaidMessageRenderer(BaseRenderer): class LiveChatPaidMessageRenderer(BaseRenderer):
def __init__(self, item): def settype(self):
super().__init__(item, "superChat") self.chat.type = "superChat"
def get_snippet(self): def get_snippet(self):
super().get_snippet() super().get_snippet()
amountDisplayString, symbol, amount = ( amountDisplayString, symbol, amount = (
self.get_amountdata(self.renderer) self.get_amountdata(self.item)
) )
self.amountValue = amount self.chat.amountValue = amount
self.amountString = amountDisplayString self.chat.amountString = amountDisplayString
self.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get( self.chat.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get(
symbol) else symbol symbol) else symbol
self.bgColor = self.renderer.get("bodyBackgroundColor", 0) self.chat.bgColor = self.item.get("bodyBackgroundColor", 0)
self.colors = self.get_colors() self.chat.colors = self.get_colors()
def get_amountdata(self, renderer): def get_amountdata(self, item):
amountDisplayString = renderer["purchaseAmountText"]["simpleText"] amountDisplayString = item["purchaseAmountText"]["simpleText"]
m = superchat_regex.search(amountDisplayString) m = superchat_regex.search(amountDisplayString)
if m: if m:
symbol = m.group(1) symbol = m.group(1)
@@ -36,11 +36,12 @@ class LiveChatPaidMessageRenderer(BaseRenderer):
return amountDisplayString, symbol, amount return amountDisplayString, symbol, amount
def get_colors(self): def get_colors(self):
item = self.item
colors = Colors() colors = Colors()
colors.headerBackgroundColor = self.renderer.get("headerBackgroundColor", 0) colors.headerBackgroundColor = item.get("headerBackgroundColor", 0)
colors.headerTextColor = self.renderer.get("headerTextColor", 0) colors.headerTextColor = item.get("headerTextColor", 0)
colors.bodyBackgroundColor = self.renderer.get("bodyBackgroundColor", 0) colors.bodyBackgroundColor = item.get("bodyBackgroundColor", 0)
colors.bodyTextColor = self.renderer.get("bodyTextColor", 0) colors.bodyTextColor = item.get("bodyTextColor", 0)
colors.timestampColor = self.renderer.get("timestampColor", 0) colors.timestampColor = item.get("timestampColor", 0)
colors.authorNameTextColor = self.renderer.get("authorNameTextColor", 0) colors.authorNameTextColor = item.get("authorNameTextColor", 0)
return colors return colors

37
pytchat/processors/default/renderer/paidsticker.py
View File

@@ -4,30 +4,30 @@ from .base import BaseRenderer
superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$") superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
class Colors: class Colors2:
pass pass
class LiveChatPaidStickerRenderer(BaseRenderer): class LiveChatPaidStickerRenderer(BaseRenderer):
def __init__(self, item): def settype(self):
super().__init__(item, "superSticker") self.chat.type = "superSticker"
def get_snippet(self): def get_snippet(self):
super().get_snippet() super().get_snippet()
amountDisplayString, symbol, amount = ( amountDisplayString, symbol, amount = (
self.get_amountdata(self.renderer) self.get_amountdata(self.item)
) )
self.amountValue = amount self.chat.amountValue = amount
self.amountString = amountDisplayString self.chat.amountString = amountDisplayString
self.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get( self.chat.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get(
symbol) else symbol symbol) else symbol
self.bgColor = self.renderer.get("backgroundColor", 0) self.chat.bgColor = self.item.get("backgroundColor", 0)
self.sticker = "".join(("https:", self.chat.sticker = "".join(("https:",
self.renderer["sticker"]["thumbnails"][0]["url"])) self.item["sticker"]["thumbnails"][0]["url"]))
self.colors = self.get_colors() self.chat.colors = self.get_colors()
def get_amountdata(self, renderer): def get_amountdata(self, item):
amountDisplayString = renderer["purchaseAmountText"]["simpleText"] amountDisplayString = item["purchaseAmountText"]["simpleText"]
m = superchat_regex.search(amountDisplayString) m = superchat_regex.search(amountDisplayString)
if m: if m:
symbol = m.group(1) symbol = m.group(1)
@@ -38,9 +38,10 @@ class LiveChatPaidStickerRenderer(BaseRenderer):
return amountDisplayString, symbol, amount return amountDisplayString, symbol, amount
def get_colors(self): def get_colors(self):
colors = Colors() item = self.item
colors.moneyChipBackgroundColor = self.renderer.get("moneyChipBackgroundColor", 0) colors = Colors2()
colors.moneyChipTextColor = self.renderer.get("moneyChipTextColor", 0) colors.moneyChipBackgroundColor = item.get("moneyChipBackgroundColor", 0)
colors.backgroundColor = self.renderer.get("backgroundColor", 0) colors.moneyChipTextColor = item.get("moneyChipTextColor", 0)
colors.authorNameTextColor = self.renderer.get("authorNameTextColor", 0) colors.backgroundColor = item.get("backgroundColor", 0)
colors.authorNameTextColor = item.get("authorNameTextColor", 0)
return colors return colors

4
pytchat/processors/default/renderer/textmessage.py
View File

@@ -2,5 +2,5 @@ from .base import BaseRenderer
class LiveChatTextMessageRenderer(BaseRenderer): class LiveChatTextMessageRenderer(BaseRenderer):
def __init__(self, item): def settype(self):
super().__init__(item, "textMessage") self.chat.type = "textMessage"

10
tests/test_default_processor.py
View File

@@ -17,7 +17,6 @@ def test_textmessage(mocker):
} }
ret = processor.process([data]).items[0] ret = processor.process([data]).items[0]
assert ret.chattype == "textMessage"
assert ret.id == "dummy_id" assert ret.id == "dummy_id"
assert ret.message == "dummy_message" assert ret.message == "dummy_message"
assert ret.timestamp == 1570678496000 assert ret.timestamp == 1570678496000
@@ -47,7 +46,6 @@ def test_textmessage_replay_member(mocker):
} }
ret = processor.process([data]).items[0] ret = processor.process([data]).items[0]
assert ret.chattype == "textMessage"
assert ret.type == "textMessage" assert ret.type == "textMessage"
assert ret.id == "dummy_id" assert ret.id == "dummy_id"
assert ret.message == "dummy_message" assert ret.message == "dummy_message"
@@ -80,8 +78,6 @@ def test_superchat(mocker):
} }
ret = processor.process([data]).items[0] ret = processor.process([data]).items[0]
print(json.dumps(chatdata, ensure_ascii=False))
assert ret.chattype == "superChat"
assert ret.type == "superChat" assert ret.type == "superChat"
assert ret.id == "dummy_id" assert ret.id == "dummy_id"
assert ret.message == "dummy_message" assert ret.message == "dummy_message"
@@ -124,8 +120,6 @@ def test_supersticker(mocker):
} }
ret = processor.process([data]).items[0] ret = processor.process([data]).items[0]
print(json.dumps(chatdata, ensure_ascii=False))
assert ret.chattype == "superSticker"
assert ret.type == "superSticker" assert ret.type == "superSticker"
assert ret.id == "dummy_id" assert ret.id == "dummy_id"
assert ret.message == "" assert ret.message == ""
@@ -167,8 +161,6 @@ def test_sponsor(mocker):
} }
ret = processor.process([data]).items[0] ret = processor.process([data]).items[0]
print(json.dumps(chatdata, ensure_ascii=False))
assert ret.chattype == "newSponsor"
assert ret.type == "newSponsor" assert ret.type == "newSponsor"
assert ret.id == "dummy_id" assert ret.id == "dummy_id"
assert ret.message == "新規メンバー" assert ret.message == "新規メンバー"
@@ -202,8 +194,6 @@ def test_sponsor_legacy(mocker):
} }
ret = processor.process([data]).items[0] ret = processor.process([data]).items[0]
print(json.dumps(chatdata, ensure_ascii=False))
assert ret.chattype == "newSponsor"
assert ret.type == "newSponsor" assert ret.type == "newSponsor"
assert ret.id == "dummy_id" assert ret.id == "dummy_id"
assert ret.message == "新規メンバー / ようこそ、author_name！" assert ret.message == "新規メンバー / ようこそ、author_name！"