Fix MANIFEST.in

: make it possible to select whether to get only top chat.

MANIFEST.in

@@ -1,7 +1,5 @@
 include requirements.txt
 include requirements_test.txt
-prune testrun*.py
-prune log.txt
-prune quote.txt
-prune .gitignore
-prun tests
+include README.MD
+global-exclude tests/*
+global-exclude pytchat/testrun*.py

pytchat/__init__.py

@@ -2,7 +2,7 @@
 pytchat is a python library for fetching youtube live chat without using yt api, Selenium, or BeautifulSoup.
 """
 __copyright__    = 'Copyright (C) 2019 taizan-hokuto'
-__version__      = '0.0.4.4'
+__version__      = '0.0.4.9'
 __license__      = 'MIT'
 __author__       = 'taizan-hokuto'
 __author_email__ = '55448286+taizan-hokuto@users.noreply.github.com'

pytchat/core_async/livechat.py

@@ -64,6 +64,9 @@ class LiveChatAsync:
         Trueの場合、ライブチャットが取得できる場合であっても
         強制的にアーカイブ済みチャットを取得する。
     
+    topchat_only : bool
+        Trueの場合、上位チャットのみ取得する。
+          
     Attributes
     ---------
     _is_alive : bool
@@ -81,7 +84,8 @@ class LiveChatAsync:
                 done_callback = None,
                 exception_handler = None,
                 direct_mode = False,
-                force_replay = False
+                force_replay = False,
+                topchat_only = False
                 ): 
         self.video_id  = video_id
         self.seektime = seektime
@@ -102,6 +106,7 @@ class LiveChatAsync:
         self._setup()
         self._first_fetch = True
         self._fetch_url = "live_chat/get_live_chat?continuation="
+        self._topchat_only = topchat_only
         if not LiveChatAsync._setup_finished:
             LiveChatAsync._setup_finished = True
             if exception_handler == None:
@@ -200,7 +205,8 @@ class LiveChatAsync:
             '''
             self._pauser.put_nowait(None)
             if not self._is_replay:
-                continuation = liveparam.getparam(self.video_id,3)
+                continuation = liveparam.getparam(
+                    self.video_id, 3, self._topchat_only)
         return continuation
 
     async def _get_contents(self, continuation, session, headers):
@@ -220,9 +226,9 @@ class LiveChatAsync:
             if contents is None or self._is_replay:
                 '''Try to fetch archive chat data.'''
                 self._parser.is_replay = True
-                self._fetch_url = ("live_chat_replay/"  
-                    "get_live_chat_replay?continuation=")
-                continuation = arcparam.getparam(self.video_id, self.seektime)
+                self._fetch_url = "live_chat_replay/get_live_chat_replay?continuation="
+                continuation = arcparam.getparam(
+                    self.video_id, self.seektime, self._topchat_only)
                 livechat_json = (await  self._get_livechat_json(
                     continuation, session, headers))
                 contents = self._parser.get_contents(livechat_json)

pytchat/core_multithread/livechat.py

@@ -60,6 +60,9 @@ class LiveChat:
         Trueの場合、ライブチャットが取得できる場合であっても
         強制的にアーカイブ済みチャットを取得する。
 
+    topchat_only : bool
+        Trueの場合、上位チャットのみ取得する。
+        
     Attributes
     ---------
     _executor : ThreadPoolExecutor
@@ -80,7 +83,8 @@ class LiveChat:
                 callback = None,
                 done_callback = None,
                 direct_mode = False,
-                force_replay = False
+                force_replay = False,
+                topchat_only  = False
                 ):
         self.video_id  = video_id
         self.seektime = seektime
@@ -101,7 +105,7 @@ class LiveChat:
         self._setup()
         self._first_fetch = True
         self._fetch_url = "live_chat/get_live_chat?continuation="
-
+        self._topchat_only = topchat_only
         if not LiveChat._setup_finished:
             LiveChat._setup_finished = True
             if interruptable:
@@ -214,8 +218,7 @@ class LiveChat:
             if contents is None or self._is_replay:
                 '''Try to fetch archive chat data.'''
                 self._parser.is_replay = True
-                self._fetch_url = ("live_chat_replay/"  
-                    "get_live_chat_replay?continuation=")
+                self._fetch_url = "live_chat_replay/get_live_chat_replay?continuation="
                 continuation = arcparam.getparam(self.video_id, self.seektime)
                 livechat_json = ( self._get_livechat_json(
                     continuation, session, headers))
@@ -230,8 +233,7 @@ class LiveChat:
         continuation = urllib.parse.quote(continuation)
         livechat_json = None
         status_code = 0
-        url =(
-           f"https://www.youtube.com/{self._fetch_url}{continuation}&pbj=1")
+        url =f"https://www.youtube.com/{self._fetch_url}{continuation}&pbj=1"
         for _ in range(MAX_RETRY + 1):
             with session.get(url ,headers = headers) as resp:
                 try:

pytchat/mylogger.py

@@ -1,31 +0,0 @@
-import logging
-import datetime
-
-
-def get_logger(modname,mode=logging.DEBUG):
-    logger = logging.getLogger(modname)
-    if mode == None:
-        logger.addHandler(logging.NullHandler())
-        return logger
-    logger.setLevel(mode)
-    #create handler1 for showing info
-    handler1 = logging.StreamHandler()
-    my_formatter  = MyFormatter()
-    handler1.setFormatter(my_formatter)
-
-    handler1.setLevel(mode) 
-    logger.addHandler(handler1)
-    #create handler2 for recording log file
-    if mode <= logging.DEBUG:
-        handler2 = logging.FileHandler(filename="log.txt", encoding='utf-8')
-        handler2.setLevel(logging.ERROR)
-        handler2.setFormatter(my_formatter)
-
-
-        logger.addHandler(handler2)
-    return logger
-
-class MyFormatter(logging.Formatter):
-    def format(self, record):
-        s =(datetime.datetime.fromtimestamp(record.created)).strftime("%m-%d %H:%M:%S")+'| '+ (record.module).ljust(15)+(' { '+record.funcName).ljust(20) +":"+str(record.lineno).rjust(4)+'} - '+record.getMessage()
-        return s

pytchat/paramgen/arcparam.py

@@ -52,8 +52,8 @@ def _nval(val):
     buf += val.to_bytes(1,'big')
     return buf
 
-def _build(video_id, seektime, topchatonly = False):
-    switch_01 = b'\x04' if topchatonly else b'\x01'
+def _build(video_id, seektime, topchat_only):
+    switch_01 = b'\x04' if topchat_only else b'\x01'
     if seektime < 0:
         times =_nval(0)
         switch = b'\x04'       
@@ -102,12 +102,14 @@ def _build(video_id, seektime, topchatonly = False):
                 ).decode()
             )
 
-def getparam(video_id, seektime = 0):
+def getparam(video_id, seektime = 0, topchat_only = False):
     '''
     Parameter
     ---------
     seektime : int
         unit:seconds
         start position of fetching chat data.
+    topchat_only : bool
+        if True, fetch only 'top chat'
     '''
-    return _build(video_id, seektime)
+    return _build(video_id, seektime, topchat_only)

pytchat/paramgen/liveparam.py

@@ -66,9 +66,9 @@ def _nval(val):
     buf += val.to_bytes(1,'big')
     return buf
 
-def _build(video_id, _ts1, _ts2, _ts3, _ts4, _ts5, topchatonly = False):
+def _build(video_id, _ts1, _ts2, _ts3, _ts4, _ts5, topchat_only):
     #_short_type2
-    switch_01 = b'\x04' if topchatonly else b'\x01'
+    switch_01 = b'\x04' if topchat_only else b'\x01'
     parity = _tzparity(video_id, _ts1^_ts2^_ts3^_ts4^_ts5)
 
     header_magic= b'\xD2\x87\xCC\xC8\x03'
@@ -155,12 +155,14 @@ def _times(past_sec):
     return list(map(lambda x:int(x*1000000),[_ts1,_ts2,_ts3,_ts4,_ts5]))
 
 
-def getparam(video_id,past_sec = 0):
+def getparam(video_id, past_sec = 0, topchat_only = False):
     '''
     Parameter
     ---------
     past_sec : int
         seconds to load past chat data
+    topchat_only : bool
+        if True, fetch only 'top chat'
     '''
-    return _build(video_id,*_times(past_sec))
+    return _build(video_id,*_times(past_sec),topchat_only)
 

pytchat/parser/live.py

@@ -26,7 +26,8 @@ class Parser:
         if jsn is None: 
             raise ChatParseException('Called with none JSON object.')
         if jsn['response']['responseContext'].get('errors'):
-            raise ResponseContextError('The video_id would be wrong, or video is deleted or private.')
+            raise ResponseContextError('The video_id would be wrong,'
+                'or video is deleted or private.')
         contents=jsn['response'].get('continuationContents')
         return contents
 
@@ -77,7 +78,8 @@ class Parser:
             metadata.setdefault("timeoutMs",interval)
             """Archived chat has different structures than live chat, 
             so make it the same format."""
-            chatdata = [action["replayChatItemAction"]["actions"][0] for action in actions]
+            chatdata = [action["replayChatItemAction"]["actions"][0]
+                for action in actions]
         else:
             metadata.setdefault('timeoutMs', 10000)
             chatdata = actions

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages, Command
 #from codecs import open as open_c
-from os import path, system, remove, rename
+from os import path, system, remove, rename, removedirs
 import re
 
 package_name = "pytchat"
@@ -13,6 +13,15 @@ def _requirements():
 def _test_requirements():
     return [name.rstrip() for name in open(path.join(root_dir, 'requirements_test.txt')).readlines()]
 
+txt= ''
+with open('README.MD', 'r', encoding='utf-8') as f:
+    txt = f.read()
+
+with open('README1.MD', 'w', encoding='utf-8', newline='\n') as f:
+    f.write(txt)
+    
+remove("README.MD")
+rename("README1.MD","README.MD")
 
 with open(path.join(root_dir, package_name, '__init__.py')) as f:
     init_text = f.read()
@@ -30,14 +39,7 @@ assert url
 
 
 
-with open('README.MD', 'r', encoding='utf-8') as f:
-    txt = f.read()
 
-with open('README1.MD', 'w', encoding='utf-8', newline='\n') as f:
-    f.write(txt)
-    
-remove("README.MD")
-rename("README1.MD","README.MD")
 with open('README.md', encoding='utf-8') as f:
     long_description = f.read()
 
@@ -45,7 +47,7 @@ with open('README.md', encoding='utf-8') as f:
 
 setup(
     name=package_name,
-    packages=find_packages(),
+    packages=find_packages(exclude=['*log.txt','*tests']),
     version=version,
     url=url,
     author=author,
@@ -54,7 +56,7 @@ setup(
     long_description_content_type='text/markdown',
     license=license,
     install_requires=_requirements(),
-    tests_require=_test_requirements(),
+    #tests_require=_test_requirements(),
     description="a python library for fetching youtube live chat.",
     classifiers=[
         'Natural Language :: Japanese',

tests/test_liveparam.py

@@ -4,6 +4,6 @@ from pytchat.paramgen import liveparam
 def test_liveparam_0(mocker):
     _ts1= 1546268400
     param = liveparam._build("01234567890",
-    *([_ts1*1000000 for i in range(5)]))
+    *([_ts1*1000000 for i in range(5)]), topchat_only=False)
     test_param="0ofMyAPiARp8Q2c4S0RRb0xNREV5TXpRMU5qYzRPVEFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5TURFeU16UTFOamM0T1RBbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiAuNbVqsrfAjAAOABAAkorCAEQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgLjW1arK3wJYA1CAuNbVqsrfAliAuNbVqsrfAmgBggEECAEQAIgBAKABgLjW1arK3wI%3D"
     assert test_param == param