Merge branch 'develop'

/ Eliminate version from requirements.txt

MANIFEST.in

@@ -1 +1,3 @@
 include requirements.txt
+include requirements_test.txt
+

README.md

@@ -1,11 +1,10 @@
-
 pytchat
 =======
 
 pytchat is a python library for fetching youtube live chat.
 
 ## Description
-pytchat is a python library for fetching youtube live chat.
+pytchat is a python library for fetching youtube live chat
 without using youtube api, Selenium or BeautifulSoup.
 
 Other features:
@@ -14,13 +13,18 @@ Other features:
 + Quick fetching of initial chat data by generating continuation params
 instead of web scraping.
 
+より詳細な説明は [wiki](https://github.com/taizan-hokuto/pytchat/wiki) をご参照ください。
+
 ## Install
-```
+```python
 pip install pytchat
 ```
+## Demo
+![demo](https://taizan-hokuto.github.io/statics/demo.gif "demo")
 
 ## Examples
-```
+### on-demand mode
+```python
 from pytchat import LiveChat
 
 chat = LiveChat("G1w62uEMZ74")
@@ -31,12 +35,14 @@ while chat.is_alive():
         data.tick()
 ```
 
-callback mode
+### callback mode
-```
+```python
 from pytchat import LiveChat
+import time
 
 chat = LiveChat("G1w62uEMZ74", callback = func)
 while chat.is_alive():
+    #other background operation here.
     time.sleep(3)
 
 def func(chatdata):
@@ -45,8 +51,8 @@ def func(chatdata):
         chat.tick()
 ```
 
-asyncio context:
+### asyncio context:
-```
+```python
 from pytchat import LiveChatAsync
 import asyncio
 
@@ -66,8 +72,8 @@ loop.run_until_complete(main())
 ```
 
 
-yt api compatible processor:
+### yt api compatible processor:
-```
+```python
 from pytchat import LiveChat, CompatibleProcessor
 
 chat = LiveChat("G1w62uEMZ74", 
@@ -113,6 +119,11 @@ Structure of each item which got from items() function.
     <td>int</td>
     <td>unixtime milliseconds</td>
   </tr>
+  <tr>
+    <td>datetime</td>
+    <td>str</td>
+    <td></td>
+  </tr>
   <tr>
     <td>amountValue</td>
     <td>float</td>
@@ -126,7 +137,12 @@ Structure of each item which got from items() function.
   <tr>
     <td>currency</td>
     <td>str</td>
-    <td>ex. "USD"</td>
+    <td>ISO 4217 currency codes (ex. "USD")</td>
+  </tr>
+  <tr>
+    <td>bgColor</td>
+    <td>int</td>
+    <td>RGB Int</td>
   </tr>
   <tr>
     <td>author</td>
@@ -188,10 +204,13 @@ Structure of author object.
     <td></td>
   </tr>
 </table>
+
 ## Licence
 
 [![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](LICENSE)
 
 ## Author
 
-[taizan-hokuto](https://github.com/taizan-hokuto)
+[taizan-hokuto](https://github.com/taizan-hokuto)
+
+[twitter:@taizan205](https://twitter.com/taizan205)

pytchat/__init__.py

@@ -2,17 +2,18 @@
 pytchat is a python library for fetching youtube live chat.
 """
 __copyright__    = 'Copyright (C) 2019 taizan-hokuto'
-__version__      = '0.0.1.5'
+__version__      = '0.0.2.3'
 __license__      = 'MIT'
 __author__       = 'taizan-hokuto'
 __author_email__ = '55448286+taizan-hokuto@users.noreply.github.com'
-__url__          = 'https://github.com/taizan-hokuto'
+__url__          = 'https://github.com/taizan-hokuto/pytchat'
 
 __all__ = ["core_async","core_multithread","processors"]
 
 from .api import (
     LiveChat,
     LiveChatAsync,
+    ChatProcessor,
     CompatibleProcessor, 
     SimpleDisplayProcessor,
     JsonfileArchiveProcessor

pytchat/api.py

@@ -1,5 +1,6 @@
 from .core_async.livechat import LiveChatAsync
 from .core_multithread.livechat import LiveChat
+from .processors.chat_processor import ChatProcessor
 from .processors.default.processor import DefaultProcessor
 from .processors.compatible.processor import CompatibleProcessor
 from .processors.simple_display_processor import SimpleDisplayProcessor

pytchat/core_async/buffer.py

@@ -2,7 +2,7 @@
 import asyncio
 class Buffer(asyncio.Queue):
     '''
-    チャットデータを格納するバッファの役割を持つLIFOキュー
+    チャットデータを格納するバッファの役割を持つFIFOキュー
 
     Parameter
     ---------

pytchat/core_async/listen_manager.py

@@ -1,184 +0,0 @@
-import asyncio
-from .listener import AsyncListener
-from .. import config
-from .. import mylogger
-import datetime
-import os
-import aiohttp
-import signal
-import threading
-from .buffer import Buffer
-from  concurrent.futures import CancelledError
-logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
-
-class ListenManager:
-
-    '''
-    動画IDまたは動画IDのリストを受け取り、
-    動画IDに対応したListenerを生成・保持する。
-
-    #Attributes
-    ----------
-    _listeners: dict  
-        ListenManegerがつかんでいるListener達のリスト.  
-        key:動画ID value:動画IDに対応するListener
-    _queue: Queue
-        動画IDを外部から受け渡しするためのキュー
-        _queueが空である間は、ノンブロッキングで他のタスクを実行
-        _queueに動画IDが投入されると、_dequeueメソッドで
-        直ちにListenerを生成し返す。
-    _event: threading.Event
-        キーボードのCTRL+Cを検知するためのEventオブジェクト
-    '''
-    def __init__(self,interruptable = True):
-        #チャット監視中の動画リスト
-        self._listeners={}
-        self._tasks = []
-        #外部からvideoを受け取るためのキュー
-        self._queue = asyncio.Queue()
-        self._event = threading.Event()
-        self._ready_queue()
-        self._is_alive = True
-        #キーボードのCtrl+cを押したとき、_hundler関数を呼び出すように設定
-        signal.signal(signal.SIGINT, (lambda a, b: self._handler(self._event, a, b)))
-
-    def is_alive(self)->bool:
-        '''
-        ListenManagerが稼働中であるか。
-            True->稼働中
-            False->Ctrl+Cが押されて終了
-        '''
-        logger.debug(f'check is_alive() :{self._is_alive}')
-        return self._is_alive
-    
-    def _handler(self, event, sig, handler):
-        '''
-        Ctrl+Cが押下されたとき、終了フラグをセットする。
-        '''
-        logger.debug('Ctrl+c pushed')
-        self._is_alive = False
-        logger.debug('terminating listeners.')
-        for listener in self._listeners.values():
-            listener.terminate()
-        logger.debug('end.')
-
-
-    def _ready_queue(self):
-        #loop = asyncio.get_event_loop()
-        self._tasks.append(
-            asyncio.create_task(self._dequeue())
-            )
-
-
-    async def set_video_ids(self,video_ids:list):
-        for video_id in video_ids:
-            if video_id:
-                await self._queue.put(video_id)
-
-
-    async def get_listener(self,video_id) -> AsyncListener:
-        return await self._create_listener(video_id)
-            
-    # async def getlivechat(self,video_id):
-    #     '''
-    #     指定された動画IDのチャットデータを返す
-
-    #     Parameter
-    #     ----------
-    #     video_id: str
-    #         動画ID
-
-    #     Return
-    #     ----------
-    #         引数で受け取った動画IDに対応する
-    #         Listenerオブジェクトへの参照
-            
-    #     '''
-    #     logger.debug('manager get/create listener')
-    #     listener = await self._create_listener(video_id)
-    #     '''
-    #     上が完了しないうちに、下が呼び出される
-    #     '''
-    #     if not listener._initialized:
-    #         await asyncio.sleep(2)
-    #         return []
-    #     if listener:
-    #         #listener._isfirstrun=False
-    #         return await listener.getlivechat()
-        
-
-
-    async def _dequeue(self):
-        '''
-        キューに入った動画IDを
-        Listener登録に回す。
-        
-        '''
-        while True:
-            video_id = await self._queue.get()
-            #listenerを登録、タスクとして実行する
-            logger.debug(f'deque got [{video_id}]')
-            await self._create_listener(video_id)
-
-    async def _create_listener(self, video_id) -> AsyncListener:
-        '''
-        Listenerを作成しチャット取得中リストに加え、
-        Listenerを返す
-        '''
-        if video_id is None or not isinstance(video_id, str):
-            raise TypeError('video_idは文字列でなければなりません')
-        if video_id in self._listeners:
-            return self._listeners[video_id]
-        else:
-            #listenerを登録する
-            listener = AsyncListener(video_id,interruptable = False,buffer = Buffer())
-            self._listeners.setdefault(video_id,listener)
-            #task = asyncio.ensure_future(listener.initialize())
-            #await asyncio.gather(listener.initialize())
-            #task.add_done_callback(self.finish)
-            #await listener.initialize()
-            #self._tasks.append(task)
-
-            return listener
-
- 
-    def finish(self,sender):
-        try:
-            if sender.result():
-                video_id = sender.result()[0]
-                message = sender.result()[1]
-            
-                #listener終了時のコールバック        
-                #sender.result()[]でデータを取得できる
-                logger.info(f'終了しました VIDEO_ID:[{video_id}] message:{message}')
-                #logger.info(f'終了しました')
-                if video_id in self._listeners:
-                    self._listeners.pop(video_id)
-        except CancelledError:
-            logger.debug('cancelled.')
-
-    def get_listeners(self):
-        return self._listeners
-
-    def shutdown(self):
-        '''
-        ListenManegerを終了する
-        '''
-        logger.debug("start shutdown")
-        self._is_alive =False
-        try:
-            #Listenerを停止する。
-            for listener in self._listeners.values():
-                listener.terminate()
-            #taskをキャンセルする。
-            for task in self._tasks:
-                if not task.done():
-                    #print(task)
-                    task.cancel()
-        except Exception as er:
-            logger.info(str(er),type(er))
-
-        logger.debug("finished.")
-
-    def get_tasks(self):
-        return self._tasks

pytchat/core_async/livechat.py

@@ -1,4 +1,4 @@
-import aiohttp, asyncio, async_timeout
+import aiohttp, asyncio
 import datetime
 import json
 import random

pytchat/core_multithread/buffer.py

@@ -3,7 +3,7 @@ import queue
 
 class Buffer(queue.Queue):
     '''
-    チャットデータを格納するバッファの役割を持つLIFOキュー
+    チャットデータを格納するバッファの役割を持つFIFOキュー
 
     Parameter
     ---------

pytchat/paramgen/liveparam.py

@@ -122,7 +122,7 @@ def _build(video_id, _ts1, _ts2, _ts3, _ts4, _ts5, topchatonly = False):
             )
 
     
-def _times():
+def _times(past_sec):
     
     def unixts_now():
         now = datetime.datetime.now(pytz.utc)
@@ -132,12 +132,18 @@ def _times():
     
     _ts1= n - random.uniform(0,1*3)    
     _ts2= n - random.uniform(0.01,0.99)    
-    _ts3= n - 60*60 + random.uniform(0,1)
+    _ts3= n - past_sec + random.uniform(0,1)
     _ts4= n - random.uniform(10*60,60*60)         
     _ts5= n - random.uniform(0.01,0.99)
     return list(map(lambda x:int(x*1000000),[_ts1,_ts2,_ts3,_ts4,_ts5]))
 
 
-def getparam(video_id):
+def getparam(video_id,past_sec = 60):
-    return _build(video_id,*_times())
+    '''
+    Parameter
+    ---------
+    past_sec : int
+        seconds to load past chat data
+    '''
+    return _build(video_id,*_times(past_sec))
 

pytchat/processors/chat_processor.py

@@ -5,11 +5,12 @@ class ChatProcessor:
     '''
     def process(self, chat_components: list):
         '''
-        チャットデータの加工を表すインターフェース
+        チャットデータの加工を表すインターフェース。
-        Listenerから呼び出される。        
+        LiveChatオブジェクトから呼び出される。 
+               
         Parameter
         ----------
-        chat_components: list<component>
+        chat_components: [LIST:component]
             component : dict {
                 "video_id" : str
                     動画ID

pytchat/processors/compatible/parser.py

@@ -1,43 +0,0 @@
-
-from .renderer.textmessage import LiveChatTextMessageRenderer
-from .renderer.paidmessage import LiveChatPaidMessageRenderer
-from .renderer.paidsticker import LiveChatPaidStickerRenderer
-from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
-
-def parse(sitem):
-
-    action = sitem.get("addChatItemAction")
-    if action:
-        item = action.get("item")
-    if item is None: return None
-    rd={}
-    try:
-        renderer = get_renderer(item)
-        if renderer == None:
-            return None
-
-        rd["kind"] = "youtube#liveChatMessage"
-        rd["etag"] = ""
-        rd["id"] = 'LCC.' + renderer.get_id()
-        rd["snippet"]       = renderer.get_snippet()
-        rd["authorDetails"] = renderer.get_authordetails()
-    except (KeyError,TypeError,AttributeError) as e:
-        print(f"------{str(type(e))}-{str(e)}----------")
-        print(sitem)
-        return None
-    
-    return rd        
-
-def get_renderer(item):
-    if item.get("liveChatTextMessageRenderer"):
-        renderer = LiveChatTextMessageRenderer(item)
-    elif item.get("liveChatPaidMessageRenderer"):
-        renderer = LiveChatPaidMessageRenderer(item)
-    elif item.get( "liveChatPaidStickerRenderer"):
-        renderer = LiveChatPaidStickerRenderer(item)
-    elif item.get("liveChatLegacyPaidMessageRenderer"):
-        renderer = LiveChatLegacyPaidMessageRenderer(item)
-    else:
-        renderer = None
-    return renderer
-

pytchat/processors/compatible/processor.py

@@ -1,11 +1,15 @@
-from . import parser
-import json
-import os
-import traceback
 import datetime
 import time
-class CompatibleProcessor():
+from .renderer.textmessage import LiveChatTextMessageRenderer
-    
+from .renderer.paidmessage import LiveChatPaidMessageRenderer
+from .renderer.paidsticker import LiveChatPaidStickerRenderer
+from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
+from ... import mylogger
+from ... import config
+logger = mylogger.get_logger(__name__,mode=config.LOGGER_MODE)
+
+class CompatibleProcessor:
+
     def process(self, chat_components: list):
 
         chatlist = []
@@ -26,7 +30,7 @@ class CompatibleProcessor():
                     if action.get('addChatItemAction') is None: continue
                     if action['addChatItemAction'].get('item') is None: continue
 
-                    chat = parser.parse(action)
+                    chat = self.parse(action)
                     if chat:
                         chatlist.append(chat)
         ret["pollingIntervalMillis"] = int(timeout*1000)
@@ -36,4 +40,42 @@ class CompatibleProcessor():
         }
         ret["items"] = chatlist
 
-        return ret
+        return ret
+
+    def parse(self, sitem):
+
+        action = sitem.get("addChatItemAction")
+        if action:
+            item = action.get("item")
+        if item is None: return None
+        rd={}
+        try:
+            renderer = self.get_renderer(item)
+            if renderer == None:
+                return None
+
+            rd["kind"] = "youtube#liveChatMessage"
+            rd["etag"] = ""
+            rd["id"] = 'LCC.' + renderer.get_id()
+            rd["snippet"]       = renderer.get_snippet()
+            rd["authorDetails"] = renderer.get_authordetails()
+        except (KeyError,TypeError,AttributeError) as e:
+            logger.error(f"Error: {str(type(e))}-{str(e)}")
+            logger.error(f"item: {sitem}")
+            return None
+        
+        return rd        
+
+    def get_renderer(self, item):
+        if item.get("liveChatTextMessageRenderer"):
+            renderer = LiveChatTextMessageRenderer(item)
+        elif item.get("liveChatPaidMessageRenderer"):
+            renderer = LiveChatPaidMessageRenderer(item)
+        elif item.get( "liveChatPaidStickerRenderer"):
+            renderer = LiveChatPaidStickerRenderer(item)
+        elif item.get("liveChatLegacyPaidMessageRenderer"):
+            renderer = LiveChatLegacyPaidMessageRenderer(item)
+        else:
+            renderer = None
+        return renderer
+

pytchat/processors/default/parser.py

@@ -1,39 +0,0 @@
-
-from .renderer.textmessage import LiveChatTextMessageRenderer
-from .renderer.paidmessage import LiveChatPaidMessageRenderer
-from .renderer.paidsticker import LiveChatPaidStickerRenderer
-from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
-
-def parse(sitem):
-
-    action = sitem.get("addChatItemAction")
-    if action:
-        item = action.get("item")
-    if item is None: return None
-    try:
-        renderer = get_renderer(item)
-        if renderer == None:
-            return None
-
-        renderer.get_snippet()
-        renderer.get_authordetails()
-    except (KeyError,TypeError,AttributeError) as e:
-        print(f"------{str(type(e))}-{str(e)}----------")
-        print(sitem)
-        return None
-    
-    return renderer        
-
-def get_renderer(item):
-    if item.get("liveChatTextMessageRenderer"):
-        renderer = LiveChatTextMessageRenderer(item)
-    elif item.get("liveChatPaidMessageRenderer"):
-        renderer = LiveChatPaidMessageRenderer(item)
-    elif item.get( "liveChatPaidStickerRenderer"):
-        renderer = LiveChatPaidStickerRenderer(item)
-    elif item.get("liveChatLegacyPaidMessageRenderer"):
-        renderer = LiveChatLegacyPaidMessageRenderer(item)
-    else:
-        renderer = None
-    return renderer
-

pytchat/processors/default/processor.py

@@ -1,6 +1,9 @@
-from . import parser
 import asyncio
 import time
+from .renderer.textmessage import LiveChatTextMessageRenderer
+from .renderer.paidmessage import LiveChatPaidMessageRenderer
+from .renderer.paidsticker import LiveChatPaidStickerRenderer
+from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
 
 
 class Chatdata:
@@ -16,7 +19,7 @@ class Chatdata:
 
     async def tick_async(self):
         if self.interval == 0:
-            await asyncio.sleep(3)
+            await asyncio.sleep(0.5)
             return
         await asyncio.sleep(self.interval/len(self.items))
 
@@ -37,8 +40,40 @@ class DefaultProcessor:
                     if action.get('addChatItemAction') is None: continue
                     if action['addChatItemAction'].get('item') is None: continue
 
-                    chat = parser.parse(action)
+                    chat = self.parse(action)
                     if chat:
                         chatlist.append(chat)
         return Chatdata(chatlist, float(timeout))
   
+
+    def parse(self, sitem):
+
+        action = sitem.get("addChatItemAction")
+        if action:
+            item = action.get("item")
+        if item is None: return None
+        try:
+            renderer = self.get_renderer(item)
+            if renderer == None:
+                return None
+
+            renderer.get_snippet()
+            renderer.get_authordetails()
+        except (KeyError,TypeError,AttributeError) as e:
+            print(f"------{str(type(e))}-{str(e)}----------")
+            print(sitem)
+            return None
+        return renderer        
+
+    def get_renderer(self, item):
+        if item.get("liveChatTextMessageRenderer"):
+            renderer = LiveChatTextMessageRenderer(item)
+        elif item.get("liveChatPaidMessageRenderer"):
+            renderer = LiveChatPaidMessageRenderer(item)
+        elif item.get( "liveChatPaidStickerRenderer"):
+            renderer = LiveChatPaidStickerRenderer(item)
+        elif item.get("liveChatLegacyPaidMessageRenderer"):
+            renderer = LiveChatLegacyPaidMessageRenderer(item)
+        else:
+            renderer = None
+        return renderer

pytchat/processors/default/renderer/currency.py

@@ -23,7 +23,7 @@ symbols = {
     "PLN\xa0": {"fxtext": "PLN", "jptext": "ポーランド・ズロチ"},
     "R$": {"fxtext": "BRL", "jptext": "ブラジル・レアル"},
     "RUB\xa0": {"fxtext": "RUB", "jptext": "ロシア・ルーブル"},
-    "SEK\xa0": {"fxtext": "SEK", "jptext": "スウェーデン・クローネ"},
+    "SEK\xa0": {"fxtext": "SEK", "jptext": "スウェーデン・クローナ"},
     "£": {"fxtext": "GBP", "jptext": "英・ポンド"},
     "₩": {"fxtext": "KRW", "jptext": "韓国・ウォン"},
     "€": {"fxtext": "EUR", "jptext": "欧・ユーロ"},

pytchat/processors/simple_display_processor.py

@@ -31,7 +31,7 @@ class SimpleDisplayProcessor(ChatProcessor):
                     purchase_amount_text = ''
                 else:
                     root = ( action['addChatItemAction']['item'].get('liveChatPaidMessageRenderer') or 
-                            action['addChatItemAction']['item'].get('liveChatPaidMessageRenderer') )
+                            action['addChatItemAction']['item'].get('liveChatPaidStickerRenderer') )
                     if root:
                         author_name = root['authorName']['simpleText']
                         message = self._parse_message(root.get('message'))

requirements.txt

@@ -1,9 +1,4 @@
-aiohttp==3.6.0
+aiohttp
-aioresponses==0.6.0
+pytz
-mock==3.0.5
+requests
-mocker==1.1.1
+urllib3
-pytest==5.1.2
-pytest-mock==1.10.4
-pytz==2019.2
-requests==2.22.0
-urllib3==1.25.3

requirements_test.txt

@@ -0,0 +1,5 @@
+aioresponses
+mock
+mocker
+pytest
+pytest-mock

setup.py

@@ -1,14 +1,18 @@
-from setuptools import setup, find_packages
+from setuptools import setup, find_packages, Command
 from codecs import open
-from os import path
+from os import path, system
 import re
 
 package_name = "pytchat"
 
 root_dir = path.abspath(path.dirname(__file__))
 
-def _requires_from_file(filename):
+def _requirements():
-    return open(filename).read().splitlines()
+    return [name.rstrip() for name in open(path.join(root_dir, 'requirements.txt')).readlines()]
+
+def _test_requirements():
+    return [name.rstrip() for name in open(path.join(root_dir, 'requirements_test.txt')).readlines()]
+
 
 with open(path.join(root_dir, package_name, '__init__.py')) as f:
     init_text = f.read()
@@ -27,18 +31,20 @@ assert url
 with open('README.md', encoding='utf-8') as f:
     long_description = f.read()
 
+
+
 setup(
     name=package_name,
-    packages=[package_name],
+    packages=find_packages(),
-
     version=version,
-
     url=url,
     author=author,
     author_email=author_email,
     long_description=long_description,
     long_description_content_type='text/markdown',
     license=license,
+    install_requires=_requirements(),
+    tests_require=_test_requirements(),
     description="a python library for fetching youtube live chat.",
     classifiers=[
         'Natural Language :: Japanese',
@@ -52,5 +58,5 @@ setup(
         'License :: OSI Approved :: MIT License',
     ],
     keywords='youtube livechat asyncio',
-    install_requires=_requires_from_file('requirements.txt')
+
 )