diff --git a/pytchat/processors/default/custom_encoder.py b/pytchat/processors/default/custom_encoder.py new file mode 100644 index 0000000..1916d1b --- /dev/null +++ b/pytchat/processors/default/custom_encoder.py @@ -0,0 +1,11 @@ +import json +from .renderer.base import Author +from .renderer.paidmessage import Colors +from .renderer.paidsticker import Colors2 + + +class CustomEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, Author) or isinstance(obj, Colors) or isinstance(obj, Colors2): + return vars(obj) + return json.JSONEncoder.default(self, obj) diff --git a/pytchat/processors/default/processor.py b/pytchat/processors/default/processor.py index c4f8f47..b83a57b 100644 --- a/pytchat/processors/default/processor.py +++ b/pytchat/processors/default/processor.py @@ -1,5 +1,7 @@ import asyncio +import json import time +from .custom_encoder import CustomEncoder from .renderer.textmessage import LiveChatTextMessageRenderer from .renderer.paidmessage import LiveChatPaidMessageRenderer from .renderer.paidsticker import LiveChatPaidStickerRenderer @@ -11,25 +13,121 @@ from ... import config logger = config.logger(__name__) +class Chat: + def json(self) -> str: + return json.dumps(vars(self), ensure_ascii=False, cls=CustomEncoder) + + class Chatdata: - def __init__(self, chatlist: list, timeout: float): + + def __init__(self, chatlist: list, timeout: float, abs_diff): self.items = chatlist self.interval = timeout + self.abs_diff = abs_diff + self.itemcount = 0 + self.before_timestamp = -2**64 def tick(self): - if self.interval == 0: + '''DEPRECATE + Use sync_items() + ''' + if len(self.items) < 1: time.sleep(1) return - time.sleep(self.interval / len(self.items)) + if self.itemcount == 0: + self.starttime = time.time() + if len(self.items) == 1: + total_itemcount = 1 + else: + total_itemcount = len(self.items) - 1 + next_chattime = (self.items[0].timestamp + (self.items[-1].timestamp - self.items[0].timestamp) / total_itemcount * self.itemcount) / 1000 + tobe_disptime = self.abs_diff + next_chattime + wait_sec = tobe_disptime - time.time() + self.itemcount += 1 + + if wait_sec < 0: + wait_sec = 0 + + time.sleep(wait_sec) async def tick_async(self): - if self.interval == 0: + '''DEPRECATE + Use async_items() + ''' + if len(self.items) < 1: await asyncio.sleep(1) return - await asyncio.sleep(self.interval / len(self.items)) + if self.itemcount == 0: + self.starttime = time.time() + if len(self.items) == 1: + total_itemcount = 1 + else: + total_itemcount = len(self.items) - 1 + next_chattime = (self.items[0].timestamp + (self.items[-1].timestamp - self.items[0].timestamp) / total_itemcount * self.itemcount) / 1000 + tobe_disptime = self.abs_diff + next_chattime + wait_sec = tobe_disptime - time.time() + self.itemcount += 1 + + if wait_sec < 0: + wait_sec = 0 + + await asyncio.sleep(wait_sec) + + def sync_items(self): + starttime = time.time() + if len(self.items) > 0: + last_chattime = self.items[-1].timestamp / 1000 + tobe_disptime = self.abs_diff + last_chattime + wait_total_sec = max(tobe_disptime - time.time(), 0) + if len(self.items) > 1: + wait_sec = wait_total_sec / len(self.items) + elif len(self.items) == 1: + wait_sec = 0 + for c in self.items: + if wait_sec < 0: + wait_sec = 0 + time.sleep(wait_sec) + yield c + stop_interval = time.time() - starttime + if stop_interval < 1: + time.sleep(1 - stop_interval) + + async def async_items(self): + starttime = time.time() + if len(self.items) > 0: + last_chattime = self.items[-1].timestamp / 1000 + tobe_disptime = self.abs_diff + last_chattime + wait_total_sec = max(tobe_disptime - time.time(), 0) + if len(self.items) > 1: + wait_sec = wait_total_sec / len(self.items) + elif len(self.items) == 1: + wait_sec = 0 + for c in self.items: + if wait_sec < 0: + wait_sec = 0 + await asyncio.sleep(wait_sec) + yield c + + stop_interval = time.time() - starttime + if stop_interval < 1: + await asyncio.sleep(1 - stop_interval) + + def json(self) -> str: + return json.dumps([vars(a) for a in self.items], ensure_ascii=False, cls=CustomEncoder) class DefaultProcessor(ChatProcessor): + def __init__(self): + self.first = True + self.abs_diff = 0 + self.renderers = { + "liveChatTextMessageRenderer": LiveChatTextMessageRenderer(), + "liveChatPaidMessageRenderer": LiveChatPaidMessageRenderer(), + "liveChatPaidStickerRenderer": LiveChatPaidStickerRenderer(), + "liveChatLegacyPaidMessageRenderer": LiveChatLegacyPaidMessageRenderer(), + "liveChatMembershipItemRenderer": LiveChatMembershipItemRenderer() + } + def process(self, chat_components: list): chatlist = [] @@ -46,43 +144,35 @@ class DefaultProcessor(ChatProcessor): continue if action.get('addChatItemAction') is None: continue - if action['addChatItemAction'].get('item') is None: + item = action['addChatItemAction'].get('item') + if item is None: continue - - chat = self._parse(action) + chat = self._parse(item) if chat: chatlist.append(chat) - return Chatdata(chatlist, float(timeout)) + + if self.first and chatlist: + self.abs_diff = time.time() - chatlist[0].timestamp / 1000 + 2 + self.first = False - def _parse(self, sitem): - action = sitem.get("addChatItemAction") - if action: - item = action.get("item") - if item is None: - return None + chatdata = Chatdata(chatlist, float(timeout), self.abs_diff) + + return chatdata + + def _parse(self, item): try: - renderer = self._get_renderer(item) + key = list(item.keys())[0] + renderer = self.renderers.get(key) if renderer is None: return None - + renderer.setitem(item.get(key), Chat()) + renderer.settype() renderer.get_snippet() renderer.get_authordetails() + rendered_chatobj = renderer.get_chatobj() + renderer.clear() except (KeyError, TypeError) as e: - logger.error(f"{str(type(e))}-{str(e)} sitem:{str(sitem)}") + logger.error(f"{str(type(e))}-{str(e)} item:{str(item)}") return None - return renderer - - def _get_renderer(self, item): - if item.get("liveChatTextMessageRenderer"): - renderer = LiveChatTextMessageRenderer(item) - elif item.get("liveChatPaidMessageRenderer"): - renderer = LiveChatPaidMessageRenderer(item) - elif item.get("liveChatPaidStickerRenderer"): - renderer = LiveChatPaidStickerRenderer(item) - elif item.get("liveChatLegacyPaidMessageRenderer"): - renderer = LiveChatLegacyPaidMessageRenderer(item) - elif item.get("liveChatMembershipItemRenderer"): - renderer = LiveChatMembershipItemRenderer(item) - else: - renderer = None - return renderer + + return rendered_chatobj diff --git a/pytchat/processors/default/renderer/base.py b/pytchat/processors/default/renderer/base.py index 64fbecc..d6826c9 100644 --- a/pytchat/processors/default/renderer/base.py +++ b/pytchat/processors/default/renderer/base.py @@ -6,89 +6,96 @@ class Author: class BaseRenderer: - def __init__(self, item, chattype): - self.renderer = list(item.values())[0] - self.chattype = chattype - self.author = Author() + def setitem(self, item, chat): + self.item = item + self.chat = chat + self.chat.author = Author() + + def settype(self): + pass def get_snippet(self): - self.type = self.chattype - self.id = self.renderer.get('id') - timestampUsec = int(self.renderer.get("timestampUsec", 0)) - self.timestamp = int(timestampUsec / 1000) - tst = self.renderer.get("timestampText") + self.chat.id = self.item.get('id') + timestampUsec = int(self.item.get("timestampUsec", 0)) + self.chat.timestamp = int(timestampUsec / 1000) + tst = self.item.get("timestampText") if tst: - self.elapsedTime = tst.get("simpleText") + self.chat.elapsedTime = tst.get("simpleText") else: - self.elapsedTime = "" - self.datetime = self.get_datetime(timestampUsec) - self.message, self.messageEx = self.get_message(self.renderer) - self.id = self.renderer.get('id') - self.amountValue = 0.0 - self.amountString = "" - self.currency = "" - self.bgColor = 0 + self.chat.elapsedTime = "" + self.chat.datetime = self.get_datetime(timestampUsec) + self.chat.message, self.chat.messageEx = self.get_message(self.item) + self.chat.id = self.item.get('id') + self.chat.amountValue = 0.0 + self.chat.amountString = "" + self.chat.currency = "" + self.chat.bgColor = 0 def get_authordetails(self): - self.author.badgeUrl = "" - (self.author.isVerified, - self.author.isChatOwner, - self.author.isChatSponsor, - self.author.isChatModerator) = ( - self.get_badges(self.renderer) + self.chat.author.badgeUrl = "" + (self.chat.author.isVerified, + self.chat.author.isChatOwner, + self.chat.author.isChatSponsor, + self.chat.author.isChatModerator) = ( + self.get_badges(self.item) ) - self.author.channelId = self.renderer.get("authorExternalChannelId") - self.author.channelUrl = "http://www.youtube.com/channel/" + self.author.channelId - self.author.name = self.renderer["authorName"]["simpleText"] - self.author.imageUrl = self.renderer["authorPhoto"]["thumbnails"][1]["url"] + self.chat.author.channelId = self.item.get("authorExternalChannelId") + self.chat.author.channelUrl = "http://www.youtube.com/channel/" + self.chat.author.channelId + self.chat.author.name = self.item["authorName"]["simpleText"] + self.chat.author.imageUrl = self.item["authorPhoto"]["thumbnails"][1]["url"] - def get_message(self, renderer): + def get_message(self, item): message = '' message_ex = [] - if renderer.get("message"): - runs = renderer["message"].get("runs") - if runs: - for r in runs: - if r: - if r.get('emoji'): - message += r['emoji'].get('shortcuts', [''])[0] - message_ex.append({ - 'id': r['emoji'].get('emojiId').split('/')[-1], - 'txt': r['emoji'].get('shortcuts', [''])[0], - 'url': r['emoji']['image']['thumbnails'][0].get('url') - }) - else: - message += r.get('text', '') - message_ex.append(r.get('text', '')) + runs = item.get("message", {}).get("runs", {}) + for r in runs: + if not hasattr(r, "get"): + continue + if r.get('emoji'): + message += r['emoji'].get('shortcuts', [''])[0] + message_ex.append({ + 'id': r['emoji'].get('emojiId').split('/')[-1], + 'txt': r['emoji'].get('shortcuts', [''])[0], + 'url': r['emoji']['image']['thumbnails'][0].get('url') + }) + else: + message += r.get('text', '') + message_ex.append(r.get('text', '')) return message, message_ex def get_badges(self, renderer): - self.author.type = '' + self.chat.author.type = '' isVerified = False isChatOwner = False isChatSponsor = False isChatModerator = False - badges = renderer.get("authorBadges") - if badges: - for badge in badges: - if badge["liveChatAuthorBadgeRenderer"].get("icon"): - author_type = badge["liveChatAuthorBadgeRenderer"]["icon"]["iconType"] - self.author.type = author_type - if author_type == 'VERIFIED': - isVerified = True - if author_type == 'OWNER': - isChatOwner = True - if author_type == 'MODERATOR': - isChatModerator = True - if badge["liveChatAuthorBadgeRenderer"].get("customThumbnail"): - isChatSponsor = True - self.author.type = 'MEMBER' - self.get_badgeurl(badge) + badges = renderer.get("authorBadges", {}) + for badge in badges: + if badge["liveChatAuthorBadgeRenderer"].get("icon"): + author_type = badge["liveChatAuthorBadgeRenderer"]["icon"]["iconType"] + self.chat.author.type = author_type + if author_type == 'VERIFIED': + isVerified = True + if author_type == 'OWNER': + isChatOwner = True + if author_type == 'MODERATOR': + isChatModerator = True + if badge["liveChatAuthorBadgeRenderer"].get("customThumbnail"): + isChatSponsor = True + self.chat.author.type = 'MEMBER' + self.get_badgeurl(badge) return isVerified, isChatOwner, isChatSponsor, isChatModerator def get_badgeurl(self, badge): - self.author.badgeUrl = badge["liveChatAuthorBadgeRenderer"]["customThumbnail"]["thumbnails"][0]["url"] + self.chat.author.badgeUrl = badge["liveChatAuthorBadgeRenderer"]["customThumbnail"]["thumbnails"][0]["url"] def get_datetime(self, timestamp): dt = datetime.fromtimestamp(timestamp / 1000000) return dt.strftime('%Y-%m-%d %H:%M:%S') + + def get_chatobj(self): + return self.chat + + def clear(self): + self.item = None + self.chat = None diff --git a/pytchat/processors/default/renderer/legacypaid.py b/pytchat/processors/default/renderer/legacypaid.py index dc6b2e4..f3b31c4 100644 --- a/pytchat/processors/default/renderer/legacypaid.py +++ b/pytchat/processors/default/renderer/legacypaid.py @@ -2,14 +2,14 @@ from .base import BaseRenderer class LiveChatLegacyPaidMessageRenderer(BaseRenderer): - def __init__(self, item): - super().__init__(item, "newSponsor") + def settype(self): + self.chat.type = "newSponsor" def get_authordetails(self): super().get_authordetails() - self.author.isChatSponsor = True + self.chat.author.isChatSponsor = True - def get_message(self, renderer): - message = (renderer["eventText"]["runs"][0]["text"] - ) + ' / ' + (renderer["detailText"]["simpleText"]) + def get_message(self, item): + message = (item["eventText"]["runs"][0]["text"] + ) + ' / ' + (item["detailText"]["simpleText"]) return message, [message] diff --git a/pytchat/processors/default/renderer/membership.py b/pytchat/processors/default/renderer/membership.py index 7a7d100..f198abf 100644 --- a/pytchat/processors/default/renderer/membership.py +++ b/pytchat/processors/default/renderer/membership.py @@ -2,14 +2,17 @@ from .base import BaseRenderer class LiveChatMembershipItemRenderer(BaseRenderer): - def __init__(self, item): - super().__init__(item, "newSponsor") + def settype(self): + self.chat.type = "newSponsor" def get_authordetails(self): super().get_authordetails() - self.author.isChatSponsor = True + self.chat.author.isChatSponsor = True - def get_message(self, renderer): - message = ''.join([mes.get("text", "") - for mes in renderer["headerSubtext"]["runs"]]) + def get_message(self, item): + try: + message = ''.join([mes.get("text", "") + for mes in item["headerSubtext"]["runs"]]) + except KeyError: + return "Welcome New Member!", ["Welcome New Member!"] return message, [message] diff --git a/pytchat/processors/default/renderer/paidmessage.py b/pytchat/processors/default/renderer/paidmessage.py index 70bd055..19ae001 100644 --- a/pytchat/processors/default/renderer/paidmessage.py +++ b/pytchat/processors/default/renderer/paidmessage.py @@ -9,23 +9,23 @@ class Colors: class LiveChatPaidMessageRenderer(BaseRenderer): - def __init__(self, item): - super().__init__(item, "superChat") + def settype(self): + self.chat.type = "superChat" def get_snippet(self): super().get_snippet() amountDisplayString, symbol, amount = ( - self.get_amountdata(self.renderer) + self.get_amountdata(self.item) ) - self.amountValue = amount - self.amountString = amountDisplayString - self.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get( + self.chat.amountValue = amount + self.chat.amountString = amountDisplayString + self.chat.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get( symbol) else symbol - self.bgColor = self.renderer.get("bodyBackgroundColor", 0) - self.colors = self.get_colors() + self.chat.bgColor = self.item.get("bodyBackgroundColor", 0) + self.chat.colors = self.get_colors() - def get_amountdata(self, renderer): - amountDisplayString = renderer["purchaseAmountText"]["simpleText"] + def get_amountdata(self, item): + amountDisplayString = item["purchaseAmountText"]["simpleText"] m = superchat_regex.search(amountDisplayString) if m: symbol = m.group(1) @@ -36,11 +36,12 @@ class LiveChatPaidMessageRenderer(BaseRenderer): return amountDisplayString, symbol, amount def get_colors(self): + item = self.item colors = Colors() - colors.headerBackgroundColor = self.renderer.get("headerBackgroundColor", 0) - colors.headerTextColor = self.renderer.get("headerTextColor", 0) - colors.bodyBackgroundColor = self.renderer.get("bodyBackgroundColor", 0) - colors.bodyTextColor = self.renderer.get("bodyTextColor", 0) - colors.timestampColor = self.renderer.get("timestampColor", 0) - colors.authorNameTextColor = self.renderer.get("authorNameTextColor", 0) + colors.headerBackgroundColor = item.get("headerBackgroundColor", 0) + colors.headerTextColor = item.get("headerTextColor", 0) + colors.bodyBackgroundColor = item.get("bodyBackgroundColor", 0) + colors.bodyTextColor = item.get("bodyTextColor", 0) + colors.timestampColor = item.get("timestampColor", 0) + colors.authorNameTextColor = item.get("authorNameTextColor", 0) return colors diff --git a/pytchat/processors/default/renderer/paidsticker.py b/pytchat/processors/default/renderer/paidsticker.py index 6a52a7e..cfa4fa1 100644 --- a/pytchat/processors/default/renderer/paidsticker.py +++ b/pytchat/processors/default/renderer/paidsticker.py @@ -4,30 +4,30 @@ from .base import BaseRenderer superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$") -class Colors: +class Colors2: pass class LiveChatPaidStickerRenderer(BaseRenderer): - def __init__(self, item): - super().__init__(item, "superSticker") + def settype(self): + self.chat.type = "superSticker" def get_snippet(self): super().get_snippet() amountDisplayString, symbol, amount = ( - self.get_amountdata(self.renderer) + self.get_amountdata(self.item) ) - self.amountValue = amount - self.amountString = amountDisplayString - self.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get( + self.chat.amountValue = amount + self.chat.amountString = amountDisplayString + self.chat.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get( symbol) else symbol - self.bgColor = self.renderer.get("backgroundColor", 0) - self.sticker = "".join(("https:", - self.renderer["sticker"]["thumbnails"][0]["url"])) - self.colors = self.get_colors() + self.chat.bgColor = self.item.get("backgroundColor", 0) + self.chat.sticker = "".join(("https:", + self.item["sticker"]["thumbnails"][0]["url"])) + self.chat.colors = self.get_colors() - def get_amountdata(self, renderer): - amountDisplayString = renderer["purchaseAmountText"]["simpleText"] + def get_amountdata(self, item): + amountDisplayString = item["purchaseAmountText"]["simpleText"] m = superchat_regex.search(amountDisplayString) if m: symbol = m.group(1) @@ -38,9 +38,10 @@ class LiveChatPaidStickerRenderer(BaseRenderer): return amountDisplayString, symbol, amount def get_colors(self): - colors = Colors() - colors.moneyChipBackgroundColor = self.renderer.get("moneyChipBackgroundColor", 0) - colors.moneyChipTextColor = self.renderer.get("moneyChipTextColor", 0) - colors.backgroundColor = self.renderer.get("backgroundColor", 0) - colors.authorNameTextColor = self.renderer.get("authorNameTextColor", 0) + item = self.item + colors = Colors2() + colors.moneyChipBackgroundColor = item.get("moneyChipBackgroundColor", 0) + colors.moneyChipTextColor = item.get("moneyChipTextColor", 0) + colors.backgroundColor = item.get("backgroundColor", 0) + colors.authorNameTextColor = item.get("authorNameTextColor", 0) return colors diff --git a/pytchat/processors/default/renderer/textmessage.py b/pytchat/processors/default/renderer/textmessage.py index 475a70d..608cfef 100644 --- a/pytchat/processors/default/renderer/textmessage.py +++ b/pytchat/processors/default/renderer/textmessage.py @@ -2,5 +2,5 @@ from .base import BaseRenderer class LiveChatTextMessageRenderer(BaseRenderer): - def __init__(self, item): - super().__init__(item, "textMessage") + def settype(self): + self.chat.type = "textMessage" diff --git a/tests/test_default_processor.py b/tests/test_default_processor.py index 5ab0c34..7ed3ec3 100644 --- a/tests/test_default_processor.py +++ b/tests/test_default_processor.py @@ -17,7 +17,6 @@ def test_textmessage(mocker): } ret = processor.process([data]).items[0] - assert ret.chattype == "textMessage" assert ret.id == "dummy_id" assert ret.message == "dummy_message" assert ret.timestamp == 1570678496000 @@ -47,7 +46,6 @@ def test_textmessage_replay_member(mocker): } ret = processor.process([data]).items[0] - assert ret.chattype == "textMessage" assert ret.type == "textMessage" assert ret.id == "dummy_id" assert ret.message == "dummy_message" @@ -80,8 +78,6 @@ def test_superchat(mocker): } ret = processor.process([data]).items[0] - print(json.dumps(chatdata, ensure_ascii=False)) - assert ret.chattype == "superChat" assert ret.type == "superChat" assert ret.id == "dummy_id" assert ret.message == "dummy_message" @@ -124,8 +120,6 @@ def test_supersticker(mocker): } ret = processor.process([data]).items[0] - print(json.dumps(chatdata, ensure_ascii=False)) - assert ret.chattype == "superSticker" assert ret.type == "superSticker" assert ret.id == "dummy_id" assert ret.message == "" @@ -167,8 +161,6 @@ def test_sponsor(mocker): } ret = processor.process([data]).items[0] - print(json.dumps(chatdata, ensure_ascii=False)) - assert ret.chattype == "newSponsor" assert ret.type == "newSponsor" assert ret.id == "dummy_id" assert ret.message == "新規メンバー" @@ -202,8 +194,6 @@ def test_sponsor_legacy(mocker): } ret = processor.process([data]).items[0] - print(json.dumps(chatdata, ensure_ascii=False)) - assert ret.chattype == "newSponsor" assert ret.type == "newSponsor" assert ret.id == "dummy_id" assert ret.message == "新規メンバー / ようこそ、author_name！"