ramforth/pytchat-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change structure of default processor

Browse Source
This commit is contained in:
taizan-hokouto
2020-10-24 19:12:00 +09:00
parent 4905b1e4d8
commit 3200c5654f
9 changed files with 257 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
pytchat/processors/default/custom_encoder.py Normal file
View File

@@ -0,0 +1,11 @@
import json
from .renderer.base import Author
from .renderer.paidmessage import Colors
from .renderer.paidsticker import Colors2
class CustomEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, Author) or isinstance(obj, Colors) or isinstance(obj, Colors2):
return vars(obj)
return json.JSONEncoder.default(self, obj)

158
pytchat/processors/default/processor.py
View File

@@ -1,5 +1,7 @@
import asyncio
import json
import time
from .custom_encoder import CustomEncoder
from .renderer.textmessage import LiveChatTextMessageRenderer
from .renderer.paidmessage import LiveChatPaidMessageRenderer
from .renderer.paidsticker import LiveChatPaidStickerRenderer
@@ -11,25 +13,121 @@ from ... import config
logger = config.logger(__name__)
class Chat:
def json(self) -> str:
return json.dumps(vars(self), ensure_ascii=False, cls=CustomEncoder)
class Chatdata:
def __init__(self, chatlist: list, timeout: float):
def __init__(self, chatlist: list, timeout: float, abs_diff):
self.items = chatlist
self.interval = timeout
self.abs_diff = abs_diff
self.itemcount = 0
self.before_timestamp = -2**64
def tick(self):
if self.interval == 0:
'''DEPRECATE
Use sync_items()
'''
if len(self.items) < 1:
time.sleep(1)
return
time.sleep(self.interval / len(self.items))
if self.itemcount == 0:
self.starttime = time.time()
if len(self.items) == 1:
total_itemcount = 1
else:
total_itemcount = len(self.items) - 1
next_chattime = (self.items[0].timestamp + (self.items[-1].timestamp - self.items[0].timestamp) / total_itemcount * self.itemcount) / 1000
tobe_disptime = self.abs_diff + next_chattime
wait_sec = tobe_disptime - time.time()
self.itemcount += 1
if wait_sec < 0:
wait_sec = 0
time.sleep(wait_sec)
async def tick_async(self):
if self.interval == 0:
'''DEPRECATE
Use async_items()
'''
if len(self.items) < 1:
await asyncio.sleep(1)
return
await asyncio.sleep(self.interval / len(self.items))
if self.itemcount == 0:
self.starttime = time.time()
if len(self.items) == 1:
total_itemcount = 1
else:
total_itemcount = len(self.items) - 1
next_chattime = (self.items[0].timestamp + (self.items[-1].timestamp - self.items[0].timestamp) / total_itemcount * self.itemcount) / 1000
tobe_disptime = self.abs_diff + next_chattime
wait_sec = tobe_disptime - time.time()
self.itemcount += 1
if wait_sec < 0:
wait_sec = 0
await asyncio.sleep(wait_sec)
def sync_items(self):
starttime = time.time()
if len(self.items) > 0:
last_chattime = self.items[-1].timestamp / 1000
tobe_disptime = self.abs_diff + last_chattime
wait_total_sec = max(tobe_disptime - time.time(), 0)
if len(self.items) > 1:
wait_sec = wait_total_sec / len(self.items)
elif len(self.items) == 1:
wait_sec = 0
for c in self.items:
if wait_sec < 0:
wait_sec = 0
time.sleep(wait_sec)
yield c
stop_interval = time.time() - starttime
if stop_interval < 1:
time.sleep(1 - stop_interval)
async def async_items(self):
starttime = time.time()
if len(self.items) > 0:
last_chattime = self.items[-1].timestamp / 1000
tobe_disptime = self.abs_diff + last_chattime
wait_total_sec = max(tobe_disptime - time.time(), 0)
if len(self.items) > 1:
wait_sec = wait_total_sec / len(self.items)
elif len(self.items) == 1:
wait_sec = 0
for c in self.items:
if wait_sec < 0:
wait_sec = 0
await asyncio.sleep(wait_sec)
yield c
stop_interval = time.time() - starttime
if stop_interval < 1:
await asyncio.sleep(1 - stop_interval)
def json(self) -> str:
return json.dumps([vars(a) for a in self.items], ensure_ascii=False, cls=CustomEncoder)
class DefaultProcessor(ChatProcessor):
def __init__(self):
self.first = True
self.abs_diff = 0
self.renderers = {
"liveChatTextMessageRenderer": LiveChatTextMessageRenderer(),
"liveChatPaidMessageRenderer": LiveChatPaidMessageRenderer(),
"liveChatPaidStickerRenderer": LiveChatPaidStickerRenderer(),
"liveChatLegacyPaidMessageRenderer": LiveChatLegacyPaidMessageRenderer(),
"liveChatMembershipItemRenderer": LiveChatMembershipItemRenderer()
}
def process(self, chat_components: list):
chatlist = []
@@ -46,43 +144,35 @@ class DefaultProcessor(ChatProcessor):
continue
if action.get('addChatItemAction') is None:
continue
if action['addChatItemAction'].get('item') is None:
item = action['addChatItemAction'].get('item')
if item is None:
continue
chat = self._parse(action)
chat = self._parse(item)
if chat:
chatlist.append(chat)
return Chatdata(chatlist, float(timeout))
if self.first and chatlist:
self.abs_diff = time.time() - chatlist[0].timestamp / 1000 + 2
self.first = False
def _parse(self, sitem):
action = sitem.get("addChatItemAction")
if action:
item = action.get("item")
if item is None:
return None
chatdata = Chatdata(chatlist, float(timeout), self.abs_diff)
return chatdata
def _parse(self, item):
try:
renderer = self._get_renderer(item)
key = list(item.keys())[0]
renderer = self.renderers.get(key)
if renderer is None:
return None
renderer.setitem(item.get(key), Chat())
renderer.settype()
renderer.get_snippet()
renderer.get_authordetails()
rendered_chatobj = renderer.get_chatobj()
renderer.clear()
except (KeyError, TypeError) as e:
logger.error(f"{str(type(e))}-{str(e)} sitem:{str(sitem)}")
logger.error(f"{str(type(e))}-{str(e)} item:{str(item)}")
return None
return renderer
def _get_renderer(self, item):
if item.get("liveChatTextMessageRenderer"):
renderer = LiveChatTextMessageRenderer(item)
elif item.get("liveChatPaidMessageRenderer"):
renderer = LiveChatPaidMessageRenderer(item)
elif item.get("liveChatPaidStickerRenderer"):
renderer = LiveChatPaidStickerRenderer(item)
elif item.get("liveChatLegacyPaidMessageRenderer"):
renderer = LiveChatLegacyPaidMessageRenderer(item)
elif item.get("liveChatMembershipItemRenderer"):
renderer = LiveChatMembershipItemRenderer(item)
else:
renderer = None
return renderer
return rendered_chatobj

131
pytchat/processors/default/renderer/base.py
View File

@@ -6,89 +6,96 @@ class Author:
class BaseRenderer:
def __init__(self, item, chattype):
self.renderer = list(item.values())[0]
self.chattype = chattype
self.author = Author()
def setitem(self, item, chat):
self.item = item
self.chat = chat
self.chat.author = Author()
def settype(self):
pass
def get_snippet(self):
self.type = self.chattype
self.id = self.renderer.get('id')
timestampUsec = int(self.renderer.get("timestampUsec", 0))
self.timestamp = int(timestampUsec / 1000)
tst = self.renderer.get("timestampText")
self.chat.id = self.item.get('id')
timestampUsec = int(self.item.get("timestampUsec", 0))
self.chat.timestamp = int(timestampUsec / 1000)
tst = self.item.get("timestampText")
if tst:
self.elapsedTime = tst.get("simpleText")
self.chat.elapsedTime = tst.get("simpleText")
else:
self.elapsedTime = ""
self.datetime = self.get_datetime(timestampUsec)
self.message, self.messageEx = self.get_message(self.renderer)
self.id = self.renderer.get('id')
self.amountValue = 0.0
self.amountString = ""
self.currency = ""
self.bgColor = 0
self.chat.elapsedTime = ""
self.chat.datetime = self.get_datetime(timestampUsec)
self.chat.message, self.chat.messageEx = self.get_message(self.item)
self.chat.id = self.item.get('id')
self.chat.amountValue = 0.0
self.chat.amountString = ""
self.chat.currency = ""
self.chat.bgColor = 0
def get_authordetails(self):
self.author.badgeUrl = ""
(self.author.isVerified,
self.author.isChatOwner,
self.author.isChatSponsor,
self.author.isChatModerator) = (
self.get_badges(self.renderer)
self.chat.author.badgeUrl = ""
(self.chat.author.isVerified,
self.chat.author.isChatOwner,
self.chat.author.isChatSponsor,
self.chat.author.isChatModerator) = (
self.get_badges(self.item)
)
self.author.channelId = self.renderer.get("authorExternalChannelId")
self.author.channelUrl = "http://www.youtube.com/channel/" + self.author.channelId
self.author.name = self.renderer["authorName"]["simpleText"]
self.author.imageUrl = self.renderer["authorPhoto"]["thumbnails"][1]["url"]
self.chat.author.channelId = self.item.get("authorExternalChannelId")
self.chat.author.channelUrl = "http://www.youtube.com/channel/" + self.chat.author.channelId
self.chat.author.name = self.item["authorName"]["simpleText"]
self.chat.author.imageUrl = self.item["authorPhoto"]["thumbnails"][1]["url"]
def get_message(self, renderer):
def get_message(self, item):
message = ''
message_ex = []
if renderer.get("message"):
runs = renderer["message"].get("runs")
if runs:
for r in runs:
if r:
if r.get('emoji'):
message += r['emoji'].get('shortcuts', [''])[0]
message_ex.append({
'id': r['emoji'].get('emojiId').split('/')[-1],
'txt': r['emoji'].get('shortcuts', [''])[0],
'url': r['emoji']['image']['thumbnails'][0].get('url')
})
else:
message += r.get('text', '')
message_ex.append(r.get('text', ''))
runs = item.get("message", {}).get("runs", {})
for r in runs:
if not hasattr(r, "get"):
continue
if r.get('emoji'):
message += r['emoji'].get('shortcuts', [''])[0]
message_ex.append({
'id': r['emoji'].get('emojiId').split('/')[-1],
'txt': r['emoji'].get('shortcuts', [''])[0],
'url': r['emoji']['image']['thumbnails'][0].get('url')
})
else:
message += r.get('text', '')
message_ex.append(r.get('text', ''))
return message, message_ex
def get_badges(self, renderer):
self.author.type = ''
self.chat.author.type = ''
isVerified = False
isChatOwner = False
isChatSponsor = False
isChatModerator = False
badges = renderer.get("authorBadges")
if badges:
for badge in badges:
if badge["liveChatAuthorBadgeRenderer"].get("icon"):
author_type = badge["liveChatAuthorBadgeRenderer"]["icon"]["iconType"]
self.author.type = author_type
if author_type == 'VERIFIED':
isVerified = True
if author_type == 'OWNER':
isChatOwner = True
if author_type == 'MODERATOR':
isChatModerator = True
if badge["liveChatAuthorBadgeRenderer"].get("customThumbnail"):
isChatSponsor = True
self.author.type = 'MEMBER'
self.get_badgeurl(badge)
badges = renderer.get("authorBadges", {})
for badge in badges:
if badge["liveChatAuthorBadgeRenderer"].get("icon"):
author_type = badge["liveChatAuthorBadgeRenderer"]["icon"]["iconType"]
self.chat.author.type = author_type
if author_type == 'VERIFIED':
isVerified = True
if author_type == 'OWNER':
isChatOwner = True
if author_type == 'MODERATOR':
isChatModerator = True
if badge["liveChatAuthorBadgeRenderer"].get("customThumbnail"):
isChatSponsor = True
self.chat.author.type = 'MEMBER'
self.get_badgeurl(badge)
return isVerified, isChatOwner, isChatSponsor, isChatModerator
def get_badgeurl(self, badge):
self.author.badgeUrl = badge["liveChatAuthorBadgeRenderer"]["customThumbnail"]["thumbnails"][0]["url"]
self.chat.author.badgeUrl = badge["liveChatAuthorBadgeRenderer"]["customThumbnail"]["thumbnails"][0]["url"]
def get_datetime(self, timestamp):
dt = datetime.fromtimestamp(timestamp / 1000000)
return dt.strftime('%Y-%m-%d %H:%M:%S')
def get_chatobj(self):
return self.chat
def clear(self):
self.item = None
self.chat = None

12
pytchat/processors/default/renderer/legacypaid.py
View File

@@ -2,14 +2,14 @@ from .base import BaseRenderer
class LiveChatLegacyPaidMessageRenderer(BaseRenderer):
def __init__(self, item):
super().__init__(item, "newSponsor")
def settype(self):
self.chat.type = "newSponsor"
def get_authordetails(self):
super().get_authordetails()
self.author.isChatSponsor = True
self.chat.author.isChatSponsor = True
def get_message(self, renderer):
message = (renderer["eventText"]["runs"][0]["text"]
) + ' / ' + (renderer["detailText"]["simpleText"])
def get_message(self, item):
message = (item["eventText"]["runs"][0]["text"]
) + ' / ' + (item["detailText"]["simpleText"])
return message, [message]

15
pytchat/processors/default/renderer/membership.py
View File

@@ -2,14 +2,17 @@ from .base import BaseRenderer
class LiveChatMembershipItemRenderer(BaseRenderer):
def __init__(self, item):
super().__init__(item, "newSponsor")
def settype(self):
self.chat.type = "newSponsor"
def get_authordetails(self):
super().get_authordetails()
self.author.isChatSponsor = True
self.chat.author.isChatSponsor = True
def get_message(self, renderer):
message = ''.join([mes.get("text", "")
for mes in renderer["headerSubtext"]["runs"]])
def get_message(self, item):
try:
message = ''.join([mes.get("text", "")
for mes in item["headerSubtext"]["runs"]])
except KeyError:
return "Welcome New Member!", ["Welcome New Member!"]
return message, [message]

33
pytchat/processors/default/renderer/paidmessage.py
View File

@@ -9,23 +9,23 @@ class Colors:
class LiveChatPaidMessageRenderer(BaseRenderer):
def __init__(self, item):
super().__init__(item, "superChat")
def settype(self):
self.chat.type = "superChat"
def get_snippet(self):
super().get_snippet()
amountDisplayString, symbol, amount = (
self.get_amountdata(self.renderer)
self.get_amountdata(self.item)
)
self.amountValue = amount
self.amountString = amountDisplayString
self.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get(
self.chat.amountValue = amount
self.chat.amountString = amountDisplayString
self.chat.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get(
symbol) else symbol
self.bgColor = self.renderer.get("bodyBackgroundColor", 0)
self.colors = self.get_colors()
self.chat.bgColor = self.item.get("bodyBackgroundColor", 0)
self.chat.colors = self.get_colors()
def get_amountdata(self, renderer):
amountDisplayString = renderer["purchaseAmountText"]["simpleText"]
def get_amountdata(self, item):
amountDisplayString = item["purchaseAmountText"]["simpleText"]
m = superchat_regex.search(amountDisplayString)
if m:
symbol = m.group(1)
@@ -36,11 +36,12 @@ class LiveChatPaidMessageRenderer(BaseRenderer):
return amountDisplayString, symbol, amount
def get_colors(self):
item = self.item
colors = Colors()
colors.headerBackgroundColor = self.renderer.get("headerBackgroundColor", 0)
colors.headerTextColor = self.renderer.get("headerTextColor", 0)
colors.bodyBackgroundColor = self.renderer.get("bodyBackgroundColor", 0)
colors.bodyTextColor = self.renderer.get("bodyTextColor", 0)
colors.timestampColor = self.renderer.get("timestampColor", 0)
colors.authorNameTextColor = self.renderer.get("authorNameTextColor", 0)
colors.headerBackgroundColor = item.get("headerBackgroundColor", 0)
colors.headerTextColor = item.get("headerTextColor", 0)
colors.bodyBackgroundColor = item.get("bodyBackgroundColor", 0)
colors.bodyTextColor = item.get("bodyTextColor", 0)
colors.timestampColor = item.get("timestampColor", 0)
colors.authorNameTextColor = item.get("authorNameTextColor", 0)
return colors

37
pytchat/processors/default/renderer/paidsticker.py
View File

@@ -4,30 +4,30 @@ from .base import BaseRenderer
superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
class Colors:
class Colors2:
pass
class LiveChatPaidStickerRenderer(BaseRenderer):
def __init__(self, item):
super().__init__(item, "superSticker")
def settype(self):
self.chat.type = "superSticker"
def get_snippet(self):
super().get_snippet()
amountDisplayString, symbol, amount = (
self.get_amountdata(self.renderer)
self.get_amountdata(self.item)
)
self.amountValue = amount
self.amountString = amountDisplayString
self.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get(
self.chat.amountValue = amount
self.chat.amountString = amountDisplayString
self.chat.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get(
symbol) else symbol
self.bgColor = self.renderer.get("backgroundColor", 0)
self.sticker = "".join(("https:",
self.renderer["sticker"]["thumbnails"][0]["url"]))
self.colors = self.get_colors()
self.chat.bgColor = self.item.get("backgroundColor", 0)
self.chat.sticker = "".join(("https:",
self.item["sticker"]["thumbnails"][0]["url"]))
self.chat.colors = self.get_colors()
def get_amountdata(self, renderer):
amountDisplayString = renderer["purchaseAmountText"]["simpleText"]
def get_amountdata(self, item):
amountDisplayString = item["purchaseAmountText"]["simpleText"]
m = superchat_regex.search(amountDisplayString)
if m:
symbol = m.group(1)
@@ -38,9 +38,10 @@ class LiveChatPaidStickerRenderer(BaseRenderer):
return amountDisplayString, symbol, amount
def get_colors(self):
colors = Colors()
colors.moneyChipBackgroundColor = self.renderer.get("moneyChipBackgroundColor", 0)
colors.moneyChipTextColor = self.renderer.get("moneyChipTextColor", 0)
colors.backgroundColor = self.renderer.get("backgroundColor", 0)
colors.authorNameTextColor = self.renderer.get("authorNameTextColor", 0)
item = self.item
colors = Colors2()
colors.moneyChipBackgroundColor = item.get("moneyChipBackgroundColor", 0)
colors.moneyChipTextColor = item.get("moneyChipTextColor", 0)
colors.backgroundColor = item.get("backgroundColor", 0)
colors.authorNameTextColor = item.get("authorNameTextColor", 0)
return colors

4
pytchat/processors/default/renderer/textmessage.py
View File

@@ -2,5 +2,5 @@ from .base import BaseRenderer
class LiveChatTextMessageRenderer(BaseRenderer):
def __init__(self, item):
super().__init__(item, "textMessage")
def settype(self):
self.chat.type = "textMessage"

10
tests/test_default_processor.py
View File

@@ -17,7 +17,6 @@ def test_textmessage(mocker):
}
ret = processor.process([data]).items[0]
assert ret.chattype == "textMessage"
assert ret.id == "dummy_id"
assert ret.message == "dummy_message"
assert ret.timestamp == 1570678496000
@@ -47,7 +46,6 @@ def test_textmessage_replay_member(mocker):
}
ret = processor.process([data]).items[0]
assert ret.chattype == "textMessage"
assert ret.type == "textMessage"
assert ret.id == "dummy_id"
assert ret.message == "dummy_message"
@@ -80,8 +78,6 @@ def test_superchat(mocker):
}
ret = processor.process([data]).items[0]
print(json.dumps(chatdata, ensure_ascii=False))
assert ret.chattype == "superChat"
assert ret.type == "superChat"
assert ret.id == "dummy_id"
assert ret.message == "dummy_message"
@@ -124,8 +120,6 @@ def test_supersticker(mocker):
}
ret = processor.process([data]).items[0]
print(json.dumps(chatdata, ensure_ascii=False))
assert ret.chattype == "superSticker"
assert ret.type == "superSticker"
assert ret.id == "dummy_id"
assert ret.message == ""
@@ -167,8 +161,6 @@ def test_sponsor(mocker):
}
ret = processor.process([data]).items[0]
print(json.dumps(chatdata, ensure_ascii=False))
assert ret.chattype == "newSponsor"
assert ret.type == "newSponsor"
assert ret.id == "dummy_id"
assert ret.message == "新規メンバー"
@@ -202,8 +194,6 @@ def test_sponsor_legacy(mocker):
}
ret = processor.process([data]).items[0]
print(json.dumps(chatdata, ensure_ascii=False))
assert ret.chattype == "newSponsor"
assert ret.type == "newSponsor"
assert ret.id == "dummy_id"
assert ret.message == "新規メンバー / ようこそ、author_name！"