Merge branch 'develop'

README.md

@@ -13,7 +13,7 @@ Other features:
 + Quick fetching of initial chat data by generating continuation params
 instead of web scraping.
 
-より詳細な説明は [wiki](https://github.com/taizan-hokuto/pytchat/wiki) をご参照ください。
+For more detailed information, see [wiki](https://github.com/taizan-hokuto/pytchat/wiki).
 
 ## Install
 ```python
@@ -40,11 +40,11 @@ while chat.is_alive():
 from pytchat import LiveChat
 import time
 
-chat = LiveChat("G1w62uEMZ74", callback = func)
-while chat.is_alive():
-    #other background operation here.
-    time.sleep(3)
-
+def main()
+    chat = LiveChat("G1w62uEMZ74", callback = func)
+    #other background operation.
+   
+#callback function is automatically called periodically.
 def func(data):
     for c in data.items:
         print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}")
@@ -58,10 +58,9 @@ import asyncio
 
 async def main():
     chat = LiveChatAsync("G1w62uEMZ74", callback = func)
-    while chat.is_alive():
-        #other background operation here.
-        await asyncio.sleep(3)
+    #other background operation.
 
+#callback function is automatically called periodically.
 async def func(data):
     for c in data.items:
         print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}")
@@ -96,10 +95,9 @@ import asyncio
 
 async def main():
     chat = ReplayChatAsync("G1w62uEMZ74", seektime = 1000, callback = func)
-    while chat.is_alive():
         #other background operation here.
-        await asyncio.sleep(3)
 
+#callback function is automatically called periodically.
 async def func(data):
     for count in range(0,len(data.items)):
         c= data.items[count]
@@ -107,15 +105,15 @@ async def func(data):
             tick=data.items[count+1].timestamp -data.items[count].timestamp
         else:
             tick=0
-        print(f"<{c.timestampText}> [{c.author.name}]-{c.message} {c.amountString}")
+        print(f"<{c.elapsedTime}> [{c.author.name}]-{c.message} {c.amountString}")
         await asyncio.sleep(tick/1000)
 
 loop = asyncio.get_event_loop()
 loop.run_until_complete(main())
 ```
 
-## Chatdata Structure of Default Processor
-Structure of each item which got from items() function.
+## Structure of Default Processor
+Each item can be got with items() function.
 <table>
   <tr>
     <th>name</th>
@@ -137,6 +135,11 @@ Structure of each item which got from items() function.
     <td>str</td>
     <td>emojis are represented by ":(shortcut text):"</td>
   </tr>
+  <tr>
+    <td>messageEx</td>
+    <td>str</td>
+    <td>list of message texts and emoji URLs.</td>
+  </tr>
   <tr>
     <td>timestamp</td>
     <td>int</td>
@@ -147,9 +150,9 @@ Structure of each item which got from items() function.
     <td>str</td>
     <td>ex. "2019-10-10 12:34:56"</td>
   </tr>
-    <td>timestampText</td>
+    <td>elapsedTime</td>
     <td>str</td>
-    <td>elapsed time. (ex. "1:02:27")</td>
+    <td>elapsed time. (ex. "1:02:27") *Replay Only.</td>
   </tr>
   <tr>
     <td>amountValue</td>
@@ -193,7 +196,7 @@ Structure of author object.
   <tr>
     <td>channelId</td>
     <td>str</td>
-    <td></td>
+    <td>*chatter's channel ID. NOT broadcasting video's channel ID.</td>
   </tr>
   <tr>
     <td>channelUrl</td>

pytchat/__init__.py

@@ -2,7 +2,7 @@
 pytchat is a python library for fetching youtube live chat without using yt api, Selenium, or BeautifulSoup.
 """
 __copyright__    = 'Copyright (C) 2019 taizan-hokuto'
-__version__      = '0.0.3.0'
+__version__      = '0.0.3.4'
 __license__      = 'MIT'
 __author__       = 'taizan-hokuto'
 __author_email__ = '55448286+taizan-hokuto@users.noreply.github.com'

pytchat/config/__init__.py

@@ -1,4 +1,4 @@
 import logging
-LOGGER_MODE = logging.DEBUG
+LOGGER_MODE = logging.ERROR
 headers = {
         'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'}

pytchat/core_async/livechat.py

@@ -3,7 +3,6 @@ import datetime
 import json
 import random
 import signal
-import threading
 import time
 import traceback
 import urllib.parse
@@ -123,7 +122,7 @@ class LiveChatAsync:
 
     async def _startlisten(self):
         """最初のcontinuationパラメータを取得し、
-        _listenループを開始する
+        _listenループのタスクを作成し開始する
         """
         initial_continuation = await self._get_initial_continuation()
         if initial_continuation is None:
@@ -286,13 +285,4 @@ class LiveChatAsync:
         logger.debug(f"残っているタスクを終了しています")
         await asyncio.gather(*tasks,return_exceptions=True)
         loop = asyncio.get_event_loop()
-        loop.stop()
-
-
-
-  
-    
-
-
-
-
+        loop.stop()

pytchat/core_async/replaychat.py

@@ -21,14 +21,19 @@ logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
 MAX_RETRY = 10
 headers = config.headers
 
+
+
 class ReplayChatAsync:
-    ''' aiohttpを利用してYouTubeのライブ配信のチャットデータを取得する
+    '''asyncio(aiohttp)を利用してYouTubeのチャットデータを取得する。
 
     Parameter
     ---------
     video_id : str
         動画ID
 
+    seektime : int
+        リプレイするチャットデータの開始時間（秒）
+
     processor : ChatProcessor
         チャットデータを加工するオブジェクト
 
@@ -46,6 +51,9 @@ class ReplayChatAsync:
     done_callback : func
         listener終了時に呼び出すコールバック。
 
+    exception_handler : func
+        例外を処理する関数
+
     direct_mode : bool
         Trueの場合、bufferを使わずにcallbackを呼ぶ。
         Trueの場合、callbackの設定が必須
@@ -53,26 +61,23 @@ class ReplayChatAsync:
 
     Attributes
     ---------
-    _executor : ThreadPoolExecutor
-        チャットデータ取得ループ（_listen）用のスレッド
-
     _is_alive : bool
-        チャット取得を終了したか
+        チャット取得を停止するためのフラグ
     '''
 
     _setup_finished = False
 
     def __init__(self, video_id,
-                seektime =0,
+                seektime = 0,
                 processor = DefaultProcessor(),
-                buffer = Buffer(maxsize = 20),
+                buffer = None,
                 interruptable = True,
                 callback = None,
                 done_callback = None,
                 exception_handler = None,
                 direct_mode = False):
         self.video_id  = video_id
-        self.seektime= seektime
+        self.seektime = seektime
         self.processor = processor
         self._buffer = buffer
         self._callback = callback
@@ -151,8 +156,8 @@ class ReplayChatAsync:
 
     async def _listen(self, continuation):
         ''' continuationに紐付いたチャットデータを取得し
-        にチャットデータを格納、
-        次のcontinuaitonを取得してループする
+        Bufferにチャットデータを格納、
+        次のcontinuaitonを取得してループする。
 
         Parameter
         ---------
@@ -163,10 +168,10 @@ class ReplayChatAsync:
             async with aiohttp.ClientSession() as session:
                 while(continuation and self._is_alive):
                     if self._pauser.empty():
-                        #pauseが呼ばれて_pauserが空状態のときは一時停止する
+                        #pause
                         await self._pauser.get()
-                        #resumeが呼ばれて_pauserにitemが入ったら再開する
-                        #直後に_pauserにitem(None)を入れてブロックを防ぐ
+                        #resume
+                        #prohibit from blocking by putting None into _pauser.
                         self._pauser.put_nowait(None)
                     livechat_json = (await
                       self._get_livechat_json(continuation, session, headers)
@@ -186,11 +191,10 @@ class ReplayChatAsync:
                     else:
                         await self._buffer.put(chat_component)
                     diff_time = timeout - (time.time()-time_mark)
-                    if diff_time < 0 : diff_time=0
                     await asyncio.sleep(diff_time)       
                     continuation = metadata.get('continuation')  
         except ChatParseException as e:
-            logger.error(f"{str(e)}（動画ID:\"{self.video_id}\"）")
+            logger.info(f"{str(e)}（video_id:\"{self.video_id}\"）")
             return            
         except (TypeError , json.JSONDecodeError) :
             logger.error(f"{traceback.format_exc(limit = -1)}")

pytchat/core_multithread/livechat.py

@@ -57,7 +57,7 @@ class LiveChat:
         チャットデータ取得ループ（_listen）用のスレッド
 
     _is_alive : bool
-        チャット取得を終了したか
+        チャット取得を停止するためのフラグ
     '''
 
     _setup_finished = False
@@ -142,7 +142,7 @@ class LiveChat:
 
     def _listen(self, continuation):
         ''' continuationに紐付いたチャットデータを取得し
-        にチャットデータを格納、
+        BUfferにチャットデータを格納、
         次のcontinuaitonを取得してループする
 
         Parameter
@@ -157,7 +157,6 @@ class LiveChat:
                       self._get_livechat_json(continuation, session, headers)
                     )
                     metadata, chatdata =  self._parser.parse( livechat_json )
-                    #チャットデータを含むコンポーネントを組み立ててbufferに投入する
                     timeout = metadata['timeoutMs']/1000
                     chat_component = {
                         "video_id" : self.video_id,
@@ -171,16 +170,12 @@ class LiveChat:
                             )
                     else:
                         self._buffer.put(chat_component)
-                    #次のchatを取得するまでsleepする
                     diff_time = timeout - (time.time()-time_mark)
                     if diff_time < 0 : diff_time=0
                     time.sleep(diff_time)        
-                    #次のチャットデータのcontinuationパラメータを取り出す。
                     continuation = metadata.get('continuation')  
-    
-                    #whileループ先頭に戻る
         except ChatParseException as e:
-            logger.error(f"{str(e)}（動画ID:\"{self.video_id}\"）")
+            logger.info(f"{str(e)}（video_id:\"{self.video_id}\"）")
             return            
         except (TypeError , json.JSONDecodeError) :
             logger.error(f"{traceback.format_exc(limit = -1)}")

pytchat/core_multithread/replaychat.py

@@ -30,6 +30,9 @@ class ReplayChat:
     video_id : str
         動画ID
 
+    seektime : int
+        リプレイするチャットデータの開始時間（秒）
+
     processor : ChatProcessor
         チャットデータを加工するオブジェクト
 
@@ -65,7 +68,7 @@ class ReplayChat:
     #チャット監視中のListenerのリスト
     _listeners= []
     def __init__(self, video_id,
-                seektime =0,
+                seektime = 0,
                 processor = DefaultProcessor(),
                 buffer = Buffer(maxsize = 20),
                 interruptable = True,
@@ -74,7 +77,7 @@ class ReplayChat:
                 direct_mode = False
                 ):
         self.video_id  = video_id
-        self.seektime= seektime
+        self.seektime = seektime
         self.processor = processor
         self._buffer = buffer
         self._callback = callback
@@ -159,16 +162,15 @@ class ReplayChat:
             with requests.Session() as session:
                 while(continuation and self._is_alive):
                     if self._pauser.empty():
-                        #pauseが呼ばれて_pauserが空状態のときは一時停止する
+                        #pause
                         self._pauser.get()
-                        #resumeが呼ばれて_pauserにitemが入ったら再開する
-                        #直後に_pauserにitem(None)を入れてブロックを防ぐ
+                        #resume
+                        #prohibit from blocking by putting None into _pauser.
                         self._pauser.put_nowait(None)
                     livechat_json = (
                       self._get_livechat_json(continuation, session, headers)
                     )
                     metadata, chatdata =  self._parser.parse( livechat_json )
-                    #チャットデータを含むコンポーネントを組み立ててbufferに投入する
                     timeout = metadata['timeoutMs']/1000
                     chat_component = {
                         "video_id" : self.video_id,

pytchat/mylogger.py

@@ -17,7 +17,7 @@ def get_logger(modname,mode=logging.DEBUG):
     logger.addHandler(handler1)
     #create handler2 for recording log file
     if mode <= logging.DEBUG:
-        handler2 = logging.FileHandler(filename="log.txt")
+        handler2 = logging.FileHandler(filename="log.txt", encoding='utf-8')
         handler2.setLevel(logging.ERROR)
         handler2.setFormatter(my_formatter)
 

pytchat/parser/replay.py

@@ -36,9 +36,7 @@ class Parser:
             raise NoContentsException('チャットデータを取得できませんでした。')
         interval = self.get_interval(actions)
         metadata.setdefault("timeoutMs",interval)
-        chatdata = []
-        for action in actions:
-            chatdata.append(action["replayChatItemAction"]["actions"][0])
+        chatdata = [action["replayChatItemAction"]["actions"][0] for action in actions]
         return metadata, chatdata
 
     def get_interval(self, actions: list):

pytchat/processors/default/processor.py

@@ -4,7 +4,9 @@ from .renderer.textmessage import LiveChatTextMessageRenderer
 from .renderer.paidmessage import LiveChatPaidMessageRenderer
 from .renderer.paidsticker import LiveChatPaidStickerRenderer
 from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
-
+from ... import config
+from ... import mylogger
+logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
 
 class Chatdata:
     def __init__(self,chatlist:list, timeout:float):
@@ -40,32 +42,31 @@ class DefaultProcessor:
                     if action.get('addChatItemAction') is None: continue
                     if action['addChatItemAction'].get('item') is None: continue
 
-                    chat = self.parse(action)
+                    chat = self._parse(action)
                     if chat:
                         chatlist.append(chat)
         return Chatdata(chatlist, float(timeout))
   
 
-    def parse(self, sitem):
+    def _parse(self, sitem):
 
         action = sitem.get("addChatItemAction")
         if action:
             item = action.get("item")
         if item is None: return None
         try:
-            renderer = self.get_renderer(item)
+            renderer = self._get_renderer(item)
             if renderer == None:
                 return None
 
             renderer.get_snippet()
             renderer.get_authordetails()
         except (KeyError,TypeError,AttributeError) as e:
-            print(f"------{str(type(e))}-{str(e)}----------")
-            print(sitem)
+            logger.error(f"{str(type(e))}-{str(e)} sitem:{str(sitem)}")
             return None
         return renderer        
 
-    def get_renderer(self, item):
+    def _get_renderer(self, item):
         if item.get("liveChatTextMessageRenderer"):
             renderer = LiveChatTextMessageRenderer(item)
         elif item.get("liveChatPaidMessageRenderer"):

pytchat/processors/default/renderer/base.py

@@ -15,11 +15,12 @@ class BaseRenderer:
         self.timestamp = int(timestampUsec/1000)
         tst = self.renderer.get("timestampText")
         if tst:
-            self.timestampText = tst.get("simpleText")
+            self.elapsedTime = tst.get("simpleText")
         else:
-            self.timestampText = ""
+            self.elapsedTime = ""
         self.datetime = self.get_datetime(timestampUsec)
         self.message = self.get_message(self.renderer)
+        self.messageEx = self.get_message_ex(self.renderer)
         self.id =  self.renderer.get('id')
         self.amountValue= 0.0
         self.amountString = ""
@@ -54,6 +55,19 @@ class BaseRenderer:
                             message += r.get('text','')
         return message
 
+    def get_message_ex(self,renderer):
+        message = []
+        if renderer.get("message"):
+            runs=renderer["message"].get("runs")
+            if runs:
+                for r in runs:
+                    if r:
+                        if r.get('emoji'):
+                            message.append(r['emoji']['image']['thumbnails'][1].get('url'))
+                        else:
+                            message.append(r.get('text',''))
+        return message
+
     def get_badges(self,renderer):
         isVerified = False
         isChatOwner = False

pytchat/processors/default/renderer/paidsticker.py

@@ -1,12 +1,42 @@
 import re
 from . import currency
-from .paidmessage import LiveChatPaidMessageRenderer
+from .base import BaseRenderer
+superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
 
-class LiveChatPaidStickerRenderer(LiveChatPaidMessageRenderer):
+class LiveChatPaidStickerRenderer(BaseRenderer):
     def __init__(self, item):
         super().__init__(item, "superSticker")
 
 
+    def get_snippet(self):
+        super().get_snippet()
+
+        self.author.name = self.renderer["authorName"]["simpleText"]
+
+        amountDisplayString, symbol, amount =(
+            self.get_amountdata(self.renderer)
+        )
+        self.message = ""
+        self.amountValue = amount
+        self.amountString = amountDisplayString
+        self.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol
+        self.bgColor = self.renderer.get("moneyChipBackgroundColor", 0)
+        self.sticker = "https:"+self.renderer["sticker"]["thumbnails"][0]["url"]
+            
+
+
+    def get_amountdata(self,renderer):
+        amountDisplayString = renderer["purchaseAmountText"]["simpleText"]
+        m = superchat_regex.search(amountDisplayString)
+        if m:
+            symbol = m.group(1)
+            amount = float(m.group(2).replace(',',''))
+        else:
+            symbol = ""
+            amount = 0.0
+        return amountDisplayString, symbol, amount
+
+