Merge branch 'release/v0.5.0'

Fix for python3.9

.github/workflows/run_test.yml

@@ -0,0 +1,27 @@
+name: Run All UnitTest
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: [3.7, 3.8, 3.9]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt -r requirements_test.txt
+      - name: Test with pytest
+        run: |
+          export PYTHONPATH=./
+          pytest --verbose --color=yes

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 taizan-hokuto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Pipfile

@@ -0,0 +1,13 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+httpx = {extras = ["http2"], version = "0.16.1"}
+
+[dev-packages]
+pytest-mock = "*"
+pytest-httpx = "*"
+wheel = "*"
+twine = "*"

Pipfile.lock

@@ -0,0 +1,375 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "e1eb34f14c75998519a90838b283ccd23bd168afa8e4837f956c5c4df66376f9"
+        },
+        "pipfile-spec": 6,
+        "requires": {},
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "certifi": {
+            "hashes": [
+                "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd",
+                "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4"
+            ],
+            "version": "==2020.11.8"
+        },
+        "h11": {
+            "hashes": [
+                "sha256:3c6c61d69c6f13d41f1b80ab0322f1872702a3ba26e12aa864c928f6a43fbaab",
+                "sha256:ab6c335e1b6ef34b205d5ca3e228c9299cc7218b049819ec84a388c2525e5d87"
+            ],
+            "version": "==0.11.0"
+        },
+        "h2": {
+            "hashes": [
+                "sha256:61e0f6601fa709f35cdb730863b4e5ec7ad449792add80d1410d4174ed139af5",
+                "sha256:875f41ebd6f2c44781259005b157faed1a5031df3ae5aa7bcb4628a6c0782f14"
+            ],
+            "version": "==3.2.0"
+        },
+        "hpack": {
+            "hashes": [
+                "sha256:0edd79eda27a53ba5be2dfabf3b15780928a0dff6eb0c60a3d6767720e970c89",
+                "sha256:8eec9c1f4bfae3408a3f30500261f7e6a65912dc138526ea054f9ad98892e9d2"
+            ],
+            "version": "==3.0.0"
+        },
+        "httpcore": {
+            "hashes": [
+                "sha256:420700af11db658c782f7e8fda34f9dcd95e3ee93944dd97d78cb70247e0cd06",
+                "sha256:dd1d762d4f7c2702149d06be2597c35fb154c5eff9789a8c5823fbcf4d2978d6"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.12.2"
+        },
+        "httpx": {
+            "extras": [
+                "http2"
+            ],
+            "hashes": [
+                "sha256:126424c279c842738805974687e0518a94c7ae8d140cd65b9c4f77ac46ffa537",
+                "sha256:9cffb8ba31fac6536f2c8cde30df859013f59e4bcc5b8d43901cb3654a8e0a5b"
+            ],
+            "index": "pypi",
+            "version": "==0.16.1"
+        },
+        "hyperframe": {
+            "hashes": [
+                "sha256:5187962cb16dcc078f23cb5a4b110098d546c3f41ff2d4038a9896893bbd0b40",
+                "sha256:a9f5c17f2cc3c719b917c4f33ed1c61bd1f8dfac4b1bd23b7c80b3400971b41f"
+            ],
+            "version": "==5.2.0"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
+                "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
+            ],
+            "version": "==2.10"
+        },
+        "rfc3986": {
+            "extras": [
+                "idna2008"
+            ],
+            "hashes": [
+                "sha256:112398da31a3344dc25dbf477d8df6cb34f9278a94fee2625d89e4514be8bb9d",
+                "sha256:af9147e9aceda37c91a05f4deb128d4b4b49d6b199775fd2d2927768abdc8f50"
+            ],
+            "version": "==1.4.0"
+        },
+        "sniffio": {
+            "hashes": [
+                "sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663",
+                "sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==1.2.0"
+        }
+    },
+    "develop": {
+        "atomicwrites": {
+            "hashes": [
+                "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197",
+                "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"
+            ],
+            "markers": "sys_platform == 'win32'",
+            "version": "==1.4.0"
+        },
+        "attrs": {
+            "hashes": [
+                "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6",
+                "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==20.3.0"
+        },
+        "bleach": {
+            "hashes": [
+                "sha256:52b5919b81842b1854196eaae5ca29679a2f2e378905c346d3ca8227c2c66080",
+                "sha256:9f8ccbeb6183c6e6cddea37592dfb0167485c1e3b13b3363bc325aa8bda3adbd"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==3.2.1"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd",
+                "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4"
+            ],
+            "version": "==2020.11.8"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+            ],
+            "version": "==3.0.4"
+        },
+        "colorama": {
+            "hashes": [
+                "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b",
+                "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"
+            ],
+            "markers": "sys_platform == 'win32'",
+            "version": "==0.4.4"
+        },
+        "docutils": {
+            "hashes": [
+                "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af",
+                "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==0.16"
+        },
+        "h11": {
+            "hashes": [
+                "sha256:3c6c61d69c6f13d41f1b80ab0322f1872702a3ba26e12aa864c928f6a43fbaab",
+                "sha256:ab6c335e1b6ef34b205d5ca3e228c9299cc7218b049819ec84a388c2525e5d87"
+            ],
+            "version": "==0.11.0"
+        },
+        "httpcore": {
+            "hashes": [
+                "sha256:420700af11db658c782f7e8fda34f9dcd95e3ee93944dd97d78cb70247e0cd06",
+                "sha256:dd1d762d4f7c2702149d06be2597c35fb154c5eff9789a8c5823fbcf4d2978d6"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.12.2"
+        },
+        "httpx": {
+            "extras": [
+                "http2"
+            ],
+            "hashes": [
+                "sha256:126424c279c842738805974687e0518a94c7ae8d140cd65b9c4f77ac46ffa537",
+                "sha256:9cffb8ba31fac6536f2c8cde30df859013f59e4bcc5b8d43901cb3654a8e0a5b"
+            ],
+            "index": "pypi",
+            "version": "==0.16.1"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
+                "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
+            ],
+            "version": "==2.10"
+        },
+        "iniconfig": {
+            "hashes": [
+                "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
+                "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"
+            ],
+            "version": "==1.1.1"
+        },
+        "keyring": {
+            "hashes": [
+                "sha256:12de23258a95f3b13e5b167f7a641a878e91eab8ef16fafc077720a95e6115bb",
+                "sha256:207bd66f2a9881c835dad653da04e196c678bf104f8252141d2d3c4f31051579"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==21.5.0"
+        },
+        "packaging": {
+            "hashes": [
+                "sha256:05af3bb85d320377db281cf254ab050e1a7ebcbf5410685a9a407e18a1f81236",
+                "sha256:eb41423378682dadb7166144a4926e443093863024de508ca5c9737d6bc08376"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==20.7"
+        },
+        "pkginfo": {
+            "hashes": [
+                "sha256:a6a4ac943b496745cec21f14f021bbd869d5e9b4f6ec06918cffea5a2f4b9193",
+                "sha256:ce14d7296c673dc4c61c759a0b6c14bae34e34eb819c0017bb6ca5b7292c56e9"
+            ],
+            "version": "==1.6.1"
+        },
+        "pluggy": {
+            "hashes": [
+                "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
+                "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==0.13.1"
+        },
+        "py": {
+            "hashes": [
+                "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2",
+                "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.9.0"
+        },
+        "pygments": {
+            "hashes": [
+                "sha256:381985fcc551eb9d37c52088a32914e00517e57f4a21609f48141ba08e193fa0",
+                "sha256:88a0bbcd659fcb9573703957c6b9cff9fab7295e6e76db54c9d00ae42df32773"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==2.7.2"
+        },
+        "pyparsing": {
+            "hashes": [
+                "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
+                "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
+            ],
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.4.7"
+        },
+        "pytest": {
+            "hashes": [
+                "sha256:4288fed0d9153d9646bfcdf0c0428197dba1ecb27a33bb6e031d002fa88653fe",
+                "sha256:c0a7e94a8cdbc5422a51ccdad8e6f1024795939cc89159a0ae7f0b316ad3823e"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==6.1.2"
+        },
+        "pytest-httpx": {
+            "hashes": [
+                "sha256:0a7c56e559b23efbf857054cd74de60a7c540694a162423f89c70da6ad358d8e",
+                "sha256:d32e8f6fb7e028f0313f5f5a2d463c8673eb43fd11a9bfe8527299717a7764c4"
+            ],
+            "index": "pypi",
+            "version": "==0.10.1"
+        },
+        "pytest-mock": {
+            "hashes": [
+                "sha256:024e405ad382646318c4281948aadf6fe1135632bea9cc67366ea0c4098ef5f2",
+                "sha256:a4d6d37329e4a893e77d9ffa89e838dd2b45d5dc099984cf03c703ac8411bb82"
+            ],
+            "index": "pypi",
+            "version": "==3.3.1"
+        },
+        "pywin32-ctypes": {
+            "hashes": [
+                "sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942",
+                "sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98"
+            ],
+            "markers": "sys_platform == 'win32'",
+            "version": "==0.2.0"
+        },
+        "readme-renderer": {
+            "hashes": [
+                "sha256:267854ac3b1530633c2394ead828afcd060fc273217c42ac36b6be9c42cd9a9d",
+                "sha256:6b7e5aa59210a40de72eb79931491eaf46fefca2952b9181268bd7c7c65c260a"
+            ],
+            "version": "==28.0"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:7f1a0b932f4a60a1a65caa4263921bb7d9ee911957e0ae4a23a6dd08185ad5f8",
+                "sha256:e786fa28d8c9154e6a4de5d46a1d921b8749f8b74e28bde23768e5e16eece998"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==2.25.0"
+        },
+        "requests-toolbelt": {
+            "hashes": [
+                "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f",
+                "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0"
+            ],
+            "version": "==0.9.1"
+        },
+        "rfc3986": {
+            "extras": [
+                "idna2008"
+            ],
+            "hashes": [
+                "sha256:112398da31a3344dc25dbf477d8df6cb34f9278a94fee2625d89e4514be8bb9d",
+                "sha256:af9147e9aceda37c91a05f4deb128d4b4b49d6b199775fd2d2927768abdc8f50"
+            ],
+            "version": "==1.4.0"
+        },
+        "six": {
+            "hashes": [
+                "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
+                "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.15.0"
+        },
+        "sniffio": {
+            "hashes": [
+                "sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663",
+                "sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==1.2.0"
+        },
+        "toml": {
+            "hashes": [
+                "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
+                "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
+            ],
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==0.10.2"
+        },
+        "tqdm": {
+            "hashes": [
+                "sha256:5c0d04e06ccc0da1bd3fa5ae4550effcce42fcad947b4a6cafa77bdc9b09ff22",
+                "sha256:9e7b8ab0ecbdbf0595adadd5f0ebbb9e69010e0bd48bbb0c15e550bf2a5292df"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==4.54.0"
+        },
+        "twine": {
+            "hashes": [
+                "sha256:34352fd52ec3b9d29837e6072d5a2a7c6fe4290e97bba46bb8d478b5c598f7ab",
+                "sha256:ba9ff477b8d6de0c89dd450e70b2185da190514e91c42cc62f96850025c10472"
+            ],
+            "index": "pypi",
+            "version": "==3.2.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",
+                "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
+            "version": "==1.26.2"
+        },
+        "webencodings": {
+            "hashes": [
+                "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78",
+                "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"
+            ],
+            "version": "==0.5.1"
+        },
+        "wheel": {
+            "hashes": [
+                "sha256:906864fb722c0ab5f2f9c35b2c65e3af3c009402c108a709c0aca27bc2c9187b",
+                "sha256:aaef9b8c36db72f8bf7f1e54f85f875c4d466819940863ca0b3f3f77f0a1646f"
+            ],
+            "index": "pypi",
+            "version": "==0.36.1"
+        }
+    }
+}

README.md

@@ -3,141 +3,65 @@ pytchat
 
 pytchat is a python library for fetching youtube live chat.
  
+
+<br><br><br>
 ## Description
 pytchat is a python library for fetching youtube live chat
-without using youtube api, Selenium or BeautifulSoup.
-
-pytchatはAPIを使わずにYouTubeチャットを取得するための軽量pythonライブラリです。
+without using Selenium or BeautifulSoup.
 
 Other features:
-+ Customizable chat data processors including youtube api compatible one.
++ Customizable [chat data processors](https://github.com/taizan-hokuto/pytchat/wiki/ChatProcessor) including youtube api compatible one.
 + Available on asyncio context. 
 + Quick fetching of initial chat data by generating continuation params
 instead of web scraping.
 
 For more detailed information, see [wiki](https://github.com/taizan-hokuto/pytchat/wiki). <br>
-より詳細な解説は[wiki](https://github.com/taizan-hokuto/pytchat/wiki/Home_jp)を参照してください。
+[wiki (Japanese)](https://github.com/taizan-hokuto/pytchat/wiki/Home_jp)
 
 ## Install
 ```python
 pip install pytchat
 ```
-## Demo
-![demo](https://taizan-hokuto.github.io/statics/demo.gif "demo")
-
 ## Examples
-### on-demand mode
+
+
+### Fetch chat data (see [wiki](https://github.com/taizan-hokuto/pytchat/wiki/PytchatCore))
 ```python
-from pytchat import LiveChat
-livechat = LiveChat(video_id = "Zvp1pJpie4I")
-
-while livechat.is_alive():
-  try:
-    chatdata = livechat.get()
-    for c in chatdata.items:
-        print(f"{c.datetime} [{c.author.name}]- {c.message}")
-        chatdata.tick()
-  except KeyboardInterrupt:
-    livechat.terminate()
-    break
-```
-
-### callback mode
-```python
-from pytchat import LiveChat
-import time
-
-def main():
-  livechat = LiveChat(video_id = "Zvp1pJpie4I", callback = disp)
-  while livechat.is_alive():
-    #other background operation.
-    time.sleep(1)
-  livechat.terminate()
-
-#callback function (automatically called)
-def disp(chatdata):
-    for c in chatdata.items:
-        print(f"{c.datetime} [{c.author.name}]- {c.message}")
-        chatdata.tick()
-
-if __name__ == '__main__':
-  main()
-
-```
-
-### asyncio context:
-```python
-from pytchat import LiveChatAsync
-from concurrent.futures import CancelledError
-import asyncio
-
-async def main():
-  livechat = LiveChatAsync("Zvp1pJpie4I", callback = func)
-  while livechat.is_alive():
-    #other background operation.
-    await asyncio.sleep(3)
-
-#callback function is automatically called.
-async def func(chatdata):
-  for c in chatdata.items:
-    print(f"{c.datetime} [{c.author.name}]-{c.message} {c.amountString}")
-    await chatdata.tick_async()
-
-if __name__ == '__main__':
-  try:
-    loop = asyncio.get_event_loop()
-    loop.run_until_complete(main())
-  except CancelledError:
-    pass
-```
-
-
-### youtube api compatible processor:
-```python
-from pytchat import LiveChat, CompatibleProcessor
-import time
-
-chat = LiveChat("Zvp1pJpie4I", 
-  processor = CompatibleProcessor() )
-
+import pytchat
+chat = pytchat.create(video_id="uIx8l2xlYVY")
 while chat.is_alive():
-  try:
-    data = chat.get()
-    polling = data['pollingIntervalMillis']/1000
-    for c in data['items']:
-      if c.get('snippet'):
-        print(f"[{c['authorDetails']['displayName']}]"
-              f"-{c['snippet']['displayMessage']}")
-        time.sleep(polling/len(data['items']))
-  except KeyboardInterrupt:
-    chat.terminate()
+    for c in chat.get().sync_items():
+        print(f"{c.datetime} [{c.author.name}]- {c.message}")
 ```
-### replay: 
-If specified video is not live,
-automatically try to fetch archived chat data.
 
+
+### Output JSON format string (feature of [DefaultProcessor](https://github.com/taizan-hokuto/pytchat/wiki/DefaultProcessor))
 ```python
-from pytchat import LiveChat
+import pytchat
+import time
 
-def main():
-  #seektime (seconds): start position of chat.
-  chat = LiveChat("ojes5ULOqhc", seektime = 60*30)
-  print('Replay from 30:00')
-  try:
-    while chat.is_alive():
-      data = chat.get()
-      for c in data.items:
-        print(f"{c.elapsedTime} [{c.author.name}]-{c.message} {c.amountString}")
-        data.tick()
-  except KeyboardInterrupt:
-    chat.terminate()
-
-if __name__ == '__main__':
-  main()
+chat = pytchat.create(video_id="uIx8l2xlYVY")
+while chat.is_alive():
+    print(chat.get().json())
+    time.sleep(5)
+    '''
+    # Each chat item can also be output in JSON format.
+    for c in chat.get().items:
+        print(c.json())
+    '''     
 ```
 
+
+### other
++ Fetch chat with a buffer ([LiveChat](https://github.com/taizan-hokuto/pytchat/wiki/LiveChat))
+
++ Use with asyncio ([LiveChatAsync](https://github.com/taizan-hokuto/pytchat/wiki/LiveChatAsync))
+
++ YT API compatible chat processor ([CompatibleProcessor](https://github.com/taizan-hokuto/pytchat/wiki/CompatibleProcessor))
+
+
 ## Structure of Default Processor
-Each item can be got with `items` function.
+Each item can be got with `sync_items()` function.
 <table>
   <tr>
     <th>name</th>
@@ -162,7 +86,7 @@ Each item can be got with `items` function.
   <tr>
     <td>messageEx</td>
     <td>str</td>
-    <td>list of message texts and emoji URLs.</td>
+    <td>list of message texts and emoji dicts(id, txt, url).</td>
   </tr>
   <tr>
     <td>timestamp</td>
@@ -263,8 +187,4 @@ Structure of author object.
 
 [![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](LICENSE)
 
-## Author
 
-[taizan-hokuto](https://github.com/taizan-hokuto)
-
-[twitter:@taizan205](https://twitter.com/taizan205)

pytchat/__init__.py

@@ -1,14 +1,29 @@
 """
-pytchat is a python library for fetching youtube live chat without using yt api, Selenium, or BeautifulSoup.
+pytchat is a lightweight python library to browse youtube livechat without Selenium or BeautifulSoup.
 """
-__copyright__    = 'Copyright (C) 2019 taizan-hokuto'
-__version__      = '0.0.6.1'
+__copyright__    = 'Copyright (C) 2019, 2020 taizan-hokuto'
+__version__      = '0.5.0'
 __license__      = 'MIT'
 __author__       = 'taizan-hokuto'
 __author_email__ = '55448286+taizan-hokuto@users.noreply.github.com'
 __url__          = 'https://github.com/taizan-hokuto/pytchat'
 
-__all__ = ["core_async","core_multithread","processors"]
+
+from .exceptions import (
+    ChatParseException,
+    ResponseContextError,
+    NoContents,
+    NoContinuation,
+    IllegalFunctionCall,
+    InvalidVideoIdException,
+    UnknownConnectionError,
+    RetryExceedMaxCount,
+    ChatDataFinished,
+    ReceivedUnknownContinuation,
+    FailedExtractContinuation,
+    VideoInfoParseError,
+    PatternUnmatchError
+)
 
 from .api import (
     config,
@@ -18,10 +33,12 @@ from .api import (
     CompatibleProcessor,
     DummyProcessor,
     DefaultProcessor,
-    Extractor, 
+    HTMLArchiver,
+    TSVArchiver,
     JsonfileArchiver,
     SimpleDisplayProcessor,
     SpeedCalculator,
     SuperchatCalculator,
-    VideoInfo
+    create
 )
+# flake8: noqa

pytchat/api.py

@@ -1,13 +1,34 @@
 from . import config
+from .core import create
 from .core_multithread.livechat import LiveChat
 from .core_async.livechat import LiveChatAsync
 from .processors.chat_processor import ChatProcessor
 from .processors.compatible.processor import CompatibleProcessor
 from .processors.default.processor import DefaultProcessor
 from .processors.dummy_processor import DummyProcessor
+from .processors.html_archiver import HTMLArchiver
+from .processors.tsv_archiver import TSVArchiver
 from .processors.jsonfile_archiver import JsonfileArchiver
 from .processors.simple_display_processor import SimpleDisplayProcessor
 from .processors.speed.calculator import SpeedCalculator
 from .processors.superchat.calculator import SuperchatCalculator
-from .tool.extract.extractor import Extractor
-from .tool.videoinfo import VideoInfo
+
+
+__all__ = [
+    config,
+    LiveChat,
+    LiveChatAsync,
+    ChatProcessor,
+    CompatibleProcessor,
+    DummyProcessor,
+    DefaultProcessor,
+    HTMLArchiver,
+    TSVArchiver,
+    JsonfileArchiver,
+    SimpleDisplayProcessor,
+    SpeedCalculator,
+    SuperchatCalculator,
+    create
+]
+
+# flake8: noqa

pytchat/config/__init__.py

@@ -1,11 +1,14 @@
-import logging
+import logging  # noqa
 from . import mylogger
-
+from base64 import a85decode as dc
 headers = {
-        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'}
+    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36 Edg/86.0.622.63,gzip(gfe)',
+}
 
-def logger(module_name: str, loglevel = None):
-    module_logger = mylogger.get_logger(module_name, loglevel = loglevel)
+_sml = dc(b"BQS?8F#ks-GB\\6`H#IhIF^eo7@rH3;H#IhIF^eor06T''Ch\\'(?XmbXF>%9<FC/iuG%G#jBOQ!ICLqcS5tQB2;gCZ)?UdXC;f$GR3)MM2<(0>O7mh!,G@+K5?SO9T@okV").decode()
+_smr = dc(b"BQS?8F#ks-GB\\6`H#IhIF^eo7@rH3;H#IhIF^eor06T''Ch\\'(?XmbXF>%9<FC/iuG%G#jBOQ!iEb03+@<k(QAU-F)8U=fDGsP557S5F7CiNH7;)D3N77^*B6YU@\\?WfBr0emZX=#^").decode()
+
+
+def logger(module_name: str, loglevel=None):
+    module_logger = mylogger.get_logger(module_name, loglevel=loglevel)
     return module_logger
-
-

pytchat/config/mylogger.py

@@ -1,32 +1,38 @@
-from logging import NullHandler, getLogger, StreamHandler, FileHandler, Formatter
+from logging import NullHandler, getLogger, StreamHandler, FileHandler
 import logging
-import datetime
+from datetime import datetime
 
 
-def get_logger(modname,loglevel=logging.DEBUG):
+def get_logger(modname, loglevel=logging.DEBUG):
     logger = getLogger(modname)
-    if loglevel == None:
+    if loglevel is None:
         logger.addHandler(NullHandler())
         return logger
     logger.setLevel(loglevel)
-    #create handler1 for showing info
+    # create handler1 for showing info
     handler1 = StreamHandler()
-    my_formatter  = MyFormatter()
+    my_formatter = MyFormatter()
     handler1.setFormatter(my_formatter)
 
     handler1.setLevel(loglevel)
     logger.addHandler(handler1)
-    #create handler2 for recording log file
+    # create handler2 for recording log file
     if loglevel <= logging.DEBUG:
         handler2 = FileHandler(filename="log.txt", encoding='utf-8')
         handler2.setLevel(logging.ERROR)
         handler2.setFormatter(my_formatter)
 
-
         logger.addHandler(handler2)
     return logger
 
+
 class MyFormatter(logging.Formatter):
     def format(self, record):
-        s =(datetime.datetime.fromtimestamp(record.created)).strftime("%m-%d %H:%M:%S")+'| '+ (record.module).ljust(15)+(' { '+record.funcName).ljust(20) +":"+str(record.lineno).rjust(4)+'} - '+record.getMessage()
-        return s
+        timestamp = (
+            datetime.fromtimestamp(record.created)).strftime("%m-%d %H:%M:%S")
+        module = (record.module).ljust(15)
+        funcname = (record.funcName).ljust(18)
+        lineno = str(record.lineno).rjust(4)
+        message = record.getMessage()
+
+        return timestamp + '| ' + module + ' { ' + funcname + ':' + lineno + '} - ' + message

pytchat/core/__init__.py

@@ -0,0 +1,7 @@
+from .pytchat import PytchatCore
+from .. util import extract_video_id
+
+
+def create(video_id: str, **kwargs):
+    _vid = extract_video_id(video_id)
+    return PytchatCore(_vid, **kwargs)

pytchat/core/pytchat.py

@@ -0,0 +1,208 @@
+import httpx
+import json
+import signal
+import time
+import traceback
+from ..parser.live import Parser
+from .. import config
+from .. import exceptions
+from ..paramgen import liveparam, arcparam
+from ..processors.default.processor import DefaultProcessor
+from ..processors.combinator import Combinator
+from .. import util
+
+headers = config.headers
+MAX_RETRY = 10
+
+
+class PytchatCore:
+    '''
+
+    Parameter
+    ---------
+    video_id : str
+
+    seektime : int
+        start position of fetching chat (seconds).
+        This option is valid for archived chat only.
+        If negative value, chat data posted before the start of the broadcast
+        will be retrieved as well.
+
+    processor : ChatProcessor
+
+    interruptable : bool
+        Allows keyboard interrupts.
+        Set this parameter to False if your own threading program causes
+        the problem.
+
+    force_replay : bool
+        force to fetch archived chat data, even if specified video is live.
+
+    topchat_only : bool
+        If True, get only top chat.
+
+    hold_exception : bool [default:True]
+        If True, when exceptions occur, the exception is held internally,
+        and can be raised by raise_for_status().
+
+    Attributes
+    ---------
+    _is_alive : bool
+        Flag to stop getting chat.
+    '''
+
+    def __init__(self, video_id,
+                 seektime=-1,
+                 processor=DefaultProcessor(),
+                 interruptable=True,
+                 force_replay=False,
+                 topchat_only=False,
+                 hold_exception=True,
+                 logger=config.logger(__name__),
+                 ):
+        self._video_id = util.extract_video_id(video_id)
+        self.seektime = seektime
+        if isinstance(processor, tuple):
+            self.processor = Combinator(processor)
+        else:
+            self.processor = processor
+        self._is_alive = True
+        self._is_replay = force_replay
+        self._hold_exception = hold_exception
+        self._exception_holder = None
+        self._parser = Parser(
+            is_replay=self._is_replay,
+            exception_holder=self._exception_holder
+        )
+        self._first_fetch = True
+        self._fetch_url = config._sml
+        self._topchat_only = topchat_only
+        self._dat = ''
+        self._last_offset_ms = 0
+        self._logger = logger
+        if interruptable:
+            signal.signal(signal.SIGINT, lambda a, b: self.terminate())
+        self._setup()
+
+    def _setup(self):
+        time.sleep(0.1)  # sleep shortly to prohibit skipping fetching data
+        """Fetch first continuation parameter,
+        create and start _listen loop.
+        """
+        self.continuation = liveparam.getparam(self._video_id, 3)
+        
+    def _get_chat_component(self):
+        
+        ''' Fetch chat data and store them into buffer,
+        get next continuaiton parameter and loop.
+
+        Parameter
+        ---------
+        continuation : str
+            parameter for next chat data
+        '''
+        try:
+            with httpx.Client(http2=True) as client:
+                if self.continuation and self._is_alive:
+                    contents = self._get_contents(self.continuation, client, headers)
+                    metadata, chatdata = self._parser.parse(contents)
+                    timeout = metadata['timeoutMs'] / 1000
+                    chat_component = {
+                        "video_id": self._video_id,
+                        "timeout": timeout,
+                        "chatdata": chatdata
+                    }
+                    self.continuation = metadata.get('continuation')
+                    self._last_offset_ms = metadata.get('last_offset_ms', 0)
+                    return chat_component
+        except exceptions.ChatParseException as e:
+            self._logger.debug(f"[{self._video_id}]{str(e)}")
+            self._raise_exception(e)
+        except Exception as e:
+            self._logger.error(f"{traceback.format_exc(limit=-1)}")
+            self._raise_exception(e)
+
+    def _get_contents(self, continuation, client, headers):
+        '''Get 'continuationContents' from livechat json.
+           If contents is None at first fetching,
+           try to fetch archive chat data.
+
+          Return:
+          -------
+            'continuationContents' which includes metadata & chat data.
+        '''
+        livechat_json = (
+            self._get_livechat_json(continuation, client, replay=self._is_replay, offset_ms=self._last_offset_ms)
+        )
+        contents, dat = self._parser.get_contents(livechat_json)
+        if self._dat == '' and dat:
+            self._dat = dat
+        if self._first_fetch:
+            if contents is None or self._is_replay:
+                '''Try to fetch archive chat data.'''
+                self._parser.is_replay = True
+                self._fetch_url = config._smr
+                continuation = arcparam.getparam(
+                    self._video_id, self.seektime, self._topchat_only)
+                livechat_json = (self._get_livechat_json(continuation, client, replay=True, offset_ms=self.seektime * 1000))
+                reload_continuation = self._parser.reload_continuation(
+                    self._parser.get_contents(livechat_json)[0])
+                if reload_continuation:
+                    livechat_json = (self._get_livechat_json(
+                        reload_continuation, client, headers))
+                contents, _ = self._parser.get_contents(livechat_json)
+                self._is_replay = True
+            self._first_fetch = False
+        return contents
+
+    def _get_livechat_json(self, continuation, client, replay: bool, offset_ms: int = 0):
+        '''
+        Get json which includes chat data.
+        '''
+        livechat_json = None
+        err = None
+        if offset_ms < 0:
+            offset_ms = 0
+        param = util.get_param(continuation, dat=self._dat, replay=replay, offsetms=offset_ms)
+        for _ in range(MAX_RETRY + 1):
+            with httpx.Client(http2=True) as client:
+                try:
+                    response = client.post(self._fetch_url, json=param)
+                    livechat_json = json.loads(response.text)
+                    break
+                except (json.JSONDecodeError, httpx.ConnectTimeout, httpx.ReadTimeout, httpx.ConnectError) as e:
+                    err = e
+                    time.sleep(2)
+                    continue
+        else:
+            self._logger.error(f"[{self._video_id}]"
+                               f"Exceeded retry count. Last error: {str(err)}")
+            self._raise_exception(exceptions.RetryExceedMaxCount())
+        return livechat_json
+    
+    def get(self):
+        if self.is_alive():
+            chat_component = self._get_chat_component()
+            return self.processor.process([chat_component])
+        else:
+            return []
+
+    def is_replay(self):
+        return self._is_replay
+
+    def is_alive(self):
+        return self._is_alive
+
+    def terminate(self):
+        self._is_alive = False
+        self.processor.finalize()
+
+    def raise_for_status(self):
+        if self._exception_holder is not None:
+            raise self._exception_holder
+
+    def _raise_exception(self, exception: Exception = None):
+        self.terminate()
+        if self._hold_exception is False:
+            raise exception
+        self._exception_holder = exception

pytchat/core_async/buffer.py

@@ -1,25 +1,35 @@
 
 import asyncio
+
+
 class Buffer(asyncio.Queue):
     '''
-    チャットデータを格納するバッファの役割を持つFIFOキュー
+    Buffer for storing chat data.
 
     Parameter
     ---------
     maxsize : int
-        格納するチャットブロックの最大個数。0の場合は無限。
-        最大値を超える場合は古いチャットブロックから破棄される。
+        Maximum number of chat blocks to be stored.
+        If it exceeds the maximum, the oldest chat block will be discarded.
     '''
-    def __init__(self,maxsize = 0):
+
+    def __init__(self, maxsize=0):
         super().__init__(maxsize)
 
-    async def put(self,item):
+    async def put(self, item):
         if item is None:
             return
         if super().full():
             super().get_nowait()
         await super().put(item)
 
+    def put_nowait(self, item):
+        if item is None:
+            return
+        if super().full():
+            super().get_nowait()
+        super().put_nowait(item)
+
     async def get(self):
         ret = []
         ret.append(await super().get())

pytchat/core_async/livechat.py

@@ -1,19 +1,18 @@
-import aiohttp, asyncio
-import datetime
+
+import asyncio
+import httpx
 import json
-import random
 import signal
 import time
 import traceback
-import urllib.parse
-from aiohttp.client_exceptions import ClientConnectorError
-from concurrent.futures import CancelledError
 from asyncio import Queue
+from concurrent.futures import CancelledError
 from .buffer import Buffer
 from ..parser.live import Parser
 from .. import config
-from ..exceptions  import ChatParseException,IllegalFunctionCall
-from ..paramgen    import liveparam, arcparam
+from .. import exceptions
+from .. import util
+from ..paramgen import liveparam, arcparam
 from ..processors.default.processor import DefaultProcessor
 from ..processors.combinator import Combinator
 
@@ -22,72 +21,69 @@ MAX_RETRY = 10
 
 
 class LiveChatAsync:
-    '''asyncio(aiohttp)を利用してYouTubeのライブ配信のチャットデータを取得する。
+    '''LiveChatAsync object fetches chat data and stores them
+    in a buffer with asyncio.
 
     Parameter
     ---------
     video_id : str
-        動画ID
 
     seektime : int
-        （ライブチャット取得時は無視）
-        取得開始するアーカイブ済みチャットの経過時間(秒）
-        マイナス値を指定した場合は、配信開始前のチャットも取得する。
+        start position of fetching chat (seconds).
+        This option is valid for archived chat only.
+        If negative value, chat data posted before the start of the broadcast
+        will be retrieved as well.
 
     processor : ChatProcessor
-        チャットデータを加工するオブジェクト
 
-    buffer : Buffer(maxsize:20[default])
-        チャットデータchat_componentを格納するバッファ。
-        maxsize : 格納できるchat_componentの個数
-        default値20個。1個で約5~10秒分。
+    buffer : Buffer
+        buffer of chat data fetched background.
 
     interruptable : bool
-        Ctrl+Cによる処理中断を行うかどうか。
+        Allows keyboard interrupts.
+        Set this parameter to False if your own threading program causes
+        the problem.
 
     callback : func
-        _listen()関数から一定間隔で自動的に呼びだす関数。
+        function called periodically from _listen().
 
     done_callback : func
-        listener終了時に呼び出すコールバック。
+        function called when listener ends.
 
     exception_handler : func
-        例外を処理する関数
 
     direct_mode : bool
-        Trueの場合、bufferを使わずにcallbackを呼ぶ。
-        Trueの場合、callbackの設定が必須
-        (設定していない場合IllegalFunctionCall例外を発生させる）
+        If True, invoke specified callback function without using buffer.
+        callback is required. If not, IllegalFunctionCall will be raised.
 
     force_replay : bool
-        Trueの場合、ライブチャットが取得できる場合であっても
-        強制的にアーカイブ済みチャットを取得する。
+        force to fetch archived chat data, even if specified video is live.
 
     topchat_only : bool
-        Trueの場合、上位チャットのみ取得する。
+        If True, get only top chat.
 
     Attributes
     ---------
     _is_alive : bool
-        チャット取得を停止するためのフラグ
+        Flag to stop getting chat.
     '''
 
     _setup_finished = False
 
     def __init__(self, video_id,
-                seektime = 0,
-                processor = DefaultProcessor(),
-                buffer = None,
-                interruptable = True,
-                callback = None,
-                done_callback = None,
-                exception_handler = None,
-                direct_mode = False,
-                force_replay = False,
-                topchat_only = False,
-                logger = config.logger(__name__),
-                ): 
-        self.video_id  = video_id
+                 seektime=-1,
+                 processor=DefaultProcessor(),
+                 buffer=None,
+                 interruptable=True,
+                 callback=None,
+                 done_callback=None,
+                 exception_handler=None,
+                 direct_mode=False,
+                 force_replay=False,
+                 topchat_only=False,
+                 logger=config.logger(__name__),
+                 ):
+        self._video_id = util.extract_video_id(video_id)
         self.seektime = seektime
         if isinstance(processor, tuple):
             self.processor = Combinator(processor)
@@ -98,59 +94,58 @@ class LiveChatAsync:
         self._done_callback = done_callback
         self._exception_handler = exception_handler
         self._direct_mode = direct_mode
-        self._is_alive   = True
+        self._is_alive = True
         self._is_replay = force_replay
-        self._parser = Parser(is_replay = self._is_replay)
+        self._parser = Parser(is_replay=self._is_replay)
         self._pauser = Queue()
         self._pauser.put_nowait(None)
-        self._setup()
         self._first_fetch = True
-        self._fetch_url = "live_chat/get_live_chat?continuation="
+        self._fetch_url = config._sml
         self._topchat_only = topchat_only
+        self._dat = ''
+        self._last_offset_ms = 0
         self._logger = logger
+        self.exception = None
         LiveChatAsync._logger = logger
 
-        if not LiveChatAsync._setup_finished:
-            LiveChatAsync._setup_finished = True
-            if exception_handler:
-                self._set_exception_handler(exception_handler)
-            if interruptable:
-                signal.signal(signal.SIGINT,  
-                    (lambda a, b:asyncio.create_task(
-                    LiveChatAsync.shutdown(None,signal.SIGINT,b))
-                ))
+        if exception_handler:
+            self._set_exception_handler(exception_handler)
+        if interruptable:
+            signal.signal(signal.SIGINT,
+                (lambda a, b: self._keyboard_interrupt()))
+        self._setup()
 
     def _setup(self):
-        #direct modeがTrueでcallback未設定の場合例外発生。
+        # An exception is raised when direct mode is true and no callback is set.
         if self._direct_mode:
             if self._callback is None:
-                raise IllegalFunctionCall(
+                raise exceptions.IllegalFunctionCall(
                     "When direct_mode=True, callback parameter is required.")
         else:
-            #direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成
+            # Create a default buffer if `direct_mode` is False and buffer is not set.
             if self._buffer is None:
-                self._buffer = Buffer(maxsize = 20)
-                #callbackが指定されている場合はcallbackを呼ぶループタスクを作成
+                self._buffer = Buffer(maxsize=20)
+                # Create a loop task to call callback if the `callback` param is specified.
             if self._callback is None:
                 pass
             else:
-                #callbackを呼ぶループタスクの開始
+                # Create a loop task to call callback if the `callback` param is specified.
                 loop = asyncio.get_event_loop()
                 loop.create_task(self._callback_loop(self._callback))
-        #_listenループタスクの開始
+        # Start a loop task for _listen()
         loop = asyncio.get_event_loop()
-        listen_task = loop.create_task(self._startlisten())
-        #add_done_callbackの登録
+        self.listen_task = loop.create_task(self._startlisten())
+        # Register add_done_callback
         if self._done_callback is None:
-            listen_task.add_done_callback(self.finish)
+            self.listen_task.add_done_callback(self._finish)
         else:
-            listen_task.add_done_callback(self._done_callback)
+            self.listen_task.add_done_callback(self._done_callback)
 
     async def _startlisten(self):
         """Fetch first continuation parameter,
         create and start _listen loop.
         """
-        initial_continuation = liveparam.getparam(self.video_id,3)
+        initial_continuation = liveparam.getparam(self._video_id, 3)
         await self._listen(initial_continuation)
 
     async def _listen(self, continuation):
@@ -163,39 +158,39 @@ class LiveChatAsync:
             parameter for next chat data
         '''
         try:
-            async with aiohttp.ClientSession() as session:
+            async with httpx.AsyncClient(http2=True) as client:
                 while(continuation and self._is_alive):
                     continuation = await self._check_pause(continuation)
-                    contents = await self._get_contents(
-                        continuation, session, headers)
-                    metadata, chatdata =  self._parser.parse(contents)
-
-                    timeout = metadata['timeoutMs']/1000
+                    contents = await self._get_contents(continuation, client, headers)
+                    metadata, chatdata = self._parser.parse(contents)
+                    timeout = metadata['timeoutMs'] / 1000
                     chat_component = {
-                        "video_id" : self.video_id,
-                        "timeout"  : timeout,
-                        "chatdata" : chatdata
+                        "video_id": self._video_id,
+                        "timeout": timeout,
+                        "chatdata": chatdata
                     }
-                    time_mark =time.time()
+                    time_mark = time.time()
                     if self._direct_mode:
-                        processed_chat = self.processor.process([chat_component])
-                        if isinstance(processed_chat,tuple):
+                        processed_chat = self.processor.process(
+                            [chat_component])
+                        if isinstance(processed_chat, tuple):
                             await self._callback(*processed_chat)
                         else:
                             await self._callback(processed_chat)
                     else:
                         await self._buffer.put(chat_component)
-                    diff_time = timeout - (time.time()-time_mark)
+                    diff_time = timeout - (time.time() - time_mark)
                     await asyncio.sleep(diff_time)
                     continuation = metadata.get('continuation')
-        except ChatParseException as e:
-            self._logger.debug(f"[{self.video_id}]{str(e)}")
-            return            
-        except (TypeError , json.JSONDecodeError) :
-            self._logger.error(f"{traceback.format_exc(limit = -1)}")
-            return
+                    self._last_offset_ms = metadata.get('last_offset_ms', 0)
+        except exceptions.ChatParseException as e:
+            self._logger.debug(f"[{self._video_id}]{str(e)}")
+            raise
+        except Exception:
+            self._logger.error(f"{traceback.format_exc(limit=-1)}")
+            raise
 
-        self._logger.debug(f"[{self.video_id}]finished fetching chat.")
+        self._logger.debug(f"[{self._video_id}] finished fetching chat.")
 
     async def _check_pause(self, continuation):
         if self._pauser.empty():
@@ -207,10 +202,10 @@ class LiveChatAsync:
             self._pauser.put_nowait(None)
             if not self._is_replay:
                 continuation = liveparam.getparam(
-                    self.video_id, 3, self._topchat_only)
+                    self._video_id, 3, self._topchat_only)
         return continuation
 
-    async def _get_contents(self, continuation, session, headers):
+    async def _get_contents(self, continuation, client, headers):
         '''Get 'continuationContents' from livechat json.
            If contents is None at first fetching,
            try to fetch archive chat data.
@@ -219,59 +214,61 @@ class LiveChatAsync:
           -------
             'continuationContents' which includes metadata & chatdata.
         '''
-        livechat_json = (await 
-            self._get_livechat_json(continuation, session, headers)
-        )
-        contents = self._parser.get_contents(livechat_json)
+        livechat_json = await self._get_livechat_json(continuation, client, replay=self._is_replay, offset_ms=self._last_offset_ms)
+        contents, dat = self._parser.get_contents(livechat_json)
+        if self._dat == '' and dat:
+            self._dat = dat
         if self._first_fetch:
             if contents is None or self._is_replay:
                 '''Try to fetch archive chat data.'''
                 self._parser.is_replay = True
-                self._fetch_url = "live_chat_replay/get_live_chat_replay?continuation="
+                self._fetch_url = config._smr
                 continuation = arcparam.getparam(
-                    self.video_id, self.seektime, self._topchat_only)
-                livechat_json = (await  self._get_livechat_json(
-                    continuation, session, headers))
+                    self._video_id, self.seektime, self._topchat_only)
+                livechat_json = (await self._get_livechat_json(
+                                 continuation, client, replay=True, offset_ms=self.seektime * 1000))
                 reload_continuation = self._parser.reload_continuation(
-                    self._parser.get_contents(livechat_json))
+                    self._parser.get_contents(livechat_json)[0])
                 if reload_continuation:
-                    livechat_json = (await  self._get_livechat_json(
-                        reload_continuation, session, headers))
-                contents = self._parser.get_contents(livechat_json)
+                    livechat_json = (await self._get_livechat_json(
+                        reload_continuation, client, headers))
+                contents, _ = self._parser.get_contents(livechat_json)
+                self._is_replay = True
             self._first_fetch = False
         return contents
 
-    async def _get_livechat_json(self, continuation, session, headers):
+    async def _get_livechat_json(self, continuation, client, replay: bool, offset_ms: int = 0):
         '''
         Get json which includes chat data.
         '''
-        continuation = urllib.parse.quote(continuation)
+        # continuation = urllib.parse.quote(continuation)
         livechat_json = None
-        status_code = 0
-        url =f"https://www.youtube.com/{self._fetch_url}{continuation}&pbj=1"
+        if offset_ms < 0:
+            offset_ms = 0
+        param = util.get_param(continuation, dat=self._dat, replay=replay, offsetms=offset_ms)
         for _ in range(MAX_RETRY + 1):
-            async with session.get(url ,headers = headers) as resp:
-                try:
-                    text = await resp.text()
-                    livechat_json = json.loads(text)
-                    break
-                except (ClientConnectorError,json.JSONDecodeError) :
-                    await asyncio.sleep(1)
-                    continue
+            try:
+                resp = await client.post(self._fetch_url, json=param)
+                livechat_json = resp.json()
+                break
+            except (json.JSONDecodeError, httpx.HTTPError):
+                await asyncio.sleep(2)
+                continue
         else:
-            self._logger.error(f"[{self.video_id}]"
-                    f"Exceeded retry count. status_code={status_code}")
-            return None
+            self._logger.error(f"[{self._video_id}]"
+                               f"Exceeded retry count.")
+            raise exceptions.RetryExceedMaxCount()
         return livechat_json
 
-    async def _callback_loop(self,callback):
-        """ コンストラクタでcallbackを指定している場合、バックグラウンドで
-        callbackに指定された関数に一定間隔でチャットデータを投げる。        
+    async def _callback_loop(self, callback):
+        """ If a callback is specified in the constructor,
+        it throws chat data at regular intervals to the
+        function specified in the callback in the backgroun
 
         Parameter
         ---------
         callback : func
-            加工済みのチャットデータを渡す先の関数。
+            function to which the processed chat data is passed.
         """
         while self.is_alive():
             items = await self._buffer.get()
@@ -282,17 +279,22 @@ class LiveChatAsync:
                 await self._callback(processed_chat)
 
     async def get(self):
-        """ bufferからデータを取り出し、processorに投げ、
-        加工済みのチャットデータを返す。
+        """
+        Retrieves data from the buffer,
+        throws it to the processor,
+        and returns the processed chat data.
 
         Returns
-             : Processorによって加工されたチャットデータ
+             : Chat data processed by the Processor
         """
         if self._callback is None:
-            items = await self._buffer.get()
-            return  self.processor.process(items)
-        raise IllegalFunctionCall(
-            "既にcallbackを登録済みのため、get()は実行できません。")
+            if self.is_alive():
+                items = await self._buffer.get()
+                return self.processor.process(items)
+            else:
+                return []
+        raise exceptions.IllegalFunctionCall(
+            "Callback parameter is already set, so get() cannot be performed.")
 
     def is_replay(self):
         return self._is_replay
@@ -312,36 +314,40 @@ class LiveChatAsync:
     def is_alive(self):
         return self._is_alive
 
-    def finish(self,sender):
-        '''Listener終了時のコールバック'''
+    def _finish(self, sender):
+        '''Called when the _listen() task finished.'''
         try:
-            self.terminate()
+            self._task_finished()
         except CancelledError:
-            self._logger.debug(f'[{self.video_id}]cancelled:{sender}')
+            self._logger.debug(f'[{self._video_id}] cancelled:{sender}')
 
     def terminate(self):
-        '''
-        Listenerを終了する。
-        '''
+        if self._pauser.empty():
+            self._pauser.put_nowait(None)
         self._is_alive = False
-        if self._direct_mode == False:
-            #bufferにダミーオブジェクトを入れてis_alive()を判定させる
-            self._buffer.put_nowait({'chatdata':'','timeout':0}) 
-        self._logger.info(f'[{self.video_id}]finished.')
+        self._buffer.put_nowait({})
+        self.processor.finalize()
+    
+    def _keyboard_interrupt(self):
+        self.exception = exceptions.ChatDataFinished()
+        self.terminate()
+
+    def _task_finished(self):
+        if self.is_alive():
+            self.terminate()
+        try:
+            self.listen_task.result()
+        except Exception as e:
+            self.exception = e
+            if not isinstance(e, exceptions.ChatParseException):
+                self._logger.error(f'Internal exception - {type(e)}{str(e)}')
+        self._logger.info(f'[{self._video_id}] finished.')
+
+    def raise_for_status(self):
+        if self.exception is not None:
+            raise self.exception
 
     @classmethod
     def _set_exception_handler(cls, handler):
         loop = asyncio.get_event_loop()
         loop.set_exception_handler(handler)
-    
-    @classmethod
-    async def shutdown(cls, event, sig = None, handler=None):
-        cls._logger.debug("shutdown...")
-        tasks = [t for t in asyncio.all_tasks() if t is not
-        asyncio.current_task()]
-        [task.cancel() for task in tasks]
-
-        cls._logger.debug(f"complete remaining tasks...")
-        await asyncio.gather(*tasks,return_exceptions=True)
-        loop = asyncio.get_event_loop()
-        loop.stop()

pytchat/core_async/replaychat.py

@@ -1,317 +0,0 @@
-import aiohttp, asyncio
-import datetime
-import json
-import random
-import signal
-import time
-import traceback
-import urllib.parse
-import warnings
-from aiohttp.client_exceptions import ClientConnectorError
-from concurrent.futures import CancelledError
-from asyncio import Queue
-from .buffer import Buffer
-from ..parser.replay import Parser
-from .. import config
-from ..exceptions  import ChatParseException,IllegalFunctionCall
-from ..paramgen    import arcparam
-from ..processors.default.processor import DefaultProcessor
-from ..processors.combinator import Combinator
-
-logger = config.logger(__name__)
-headers = config.headers
-MAX_RETRY = 10
-
-
-
-
-class ReplayChatAsync:
-    '''
-    ### -----------------------------------------------------------
-    ### [Warning] ReplayChatAsync is integrated into LiveChatAsync.
-    ### This class is deprecated and will be removed at v0.0.5.0.
-    ### ReplayChatAsyncはLiveChatAsyncに統合しました。
-    ### このクラスはv0.0.5.0で廃止予定です。
-    ### -----------------------------------------------------------
-
-    asyncio(aiohttp)を利用してYouTubeのチャットデータを取得する。
-
-    Parameter
-    ---------
-    video_id : str
-        動画ID
-
-    seektime : int
-        リプレイするチャットデータの開始時間（秒）
-
-    processor : ChatProcessor
-        チャットデータを加工するオブジェクト
-
-    buffer : Buffer(maxsize:20[default])
-        チャットデータchat_componentを格納するバッファ。
-        maxsize : 格納できるchat_componentの個数
-        default値20個。1個で約5~10秒分。
-
-    interruptable : bool
-        Ctrl+Cによる処理中断を行うかどうか。
-
-    callback : func
-        _listen()関数から一定間隔で自動的に呼びだす関数。
-
-    done_callback : func
-        listener終了時に呼び出すコールバック。
-
-    exception_handler : func
-        例外を処理する関数
-
-    direct_mode : bool
-        Trueの場合、bufferを使わずにcallbackを呼ぶ。
-        Trueの場合、callbackの設定が必須
-        (設定していない場合IllegalFunctionCall例外を発生させる）  
-
-    Attributes
-    ---------
-    _is_alive : bool
-        チャット取得を停止するためのフラグ
-    '''
-
-    _setup_finished = False
-
-    def __init__(self, video_id,
-                seektime = 0,
-                processor = DefaultProcessor(),
-                buffer = None,
-                interruptable = True,
-                callback = None,
-                done_callback = None,
-                exception_handler = None,
-                direct_mode = False):
-
-        warnings.warn(""
-            f"\n{'-'*60}\n[WARNING] ReplayChatAsync is integrated "
-            f"into LiveChatAsync.\n{' '*5} This is deprecated and will"
-            f" be removed at v0.0.5.0.\n{'-'*60}\n"
-        )
-        self.video_id  = video_id
-        self.seektime = seektime
-        if isinstance(processor, tuple):
-            self.processor = Combinator(processor)
-        else:
-            self.processor = processor
-        self._buffer = buffer
-        self._callback = callback
-        self._done_callback = done_callback
-        self._exception_handler = exception_handler
-        self._direct_mode = direct_mode
-        self._is_alive   = True
-        self._parser = Parser()
-        self._pauser = Queue()
-        self._pauser.put_nowait(None)
-        self._setup()
-
-        if not ReplayChatAsync._setup_finished:
-            ReplayChatAsync._setup_finished = True
-            if exception_handler == None:
-                self._set_exception_handler(self._handle_exception)
-            else:
-                self._set_exception_handler(exception_handler)
-            if interruptable:
-                signal.signal(signal.SIGINT,
-                  (lambda a, b:asyncio.create_task(  
-                  ReplayChatAsync.shutdown(None,signal.SIGINT,b))
-                ))
-
-    def _setup(self):
-        #direct modeがTrueでcallback未設定の場合例外発生。
-        if self._direct_mode:
-            if self._callback is None:
-                raise IllegalFunctionCall(
-                    "direct_mode=Trueの場合callbackの設定が必須です。")
-        else:
-            #direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成
-            if self._buffer is None:
-                self._buffer = Buffer(maxsize = 20)
-                #callbackが指定されている場合はcallbackを呼ぶループタスクを作成
-            if self._callback is None:
-                pass 
-            else:
-                #callbackを呼ぶループタスクの開始
-                loop = asyncio.get_event_loop()
-                loop.create_task(self._callback_loop(self._callback))
-        #_listenループタスクの開始
-        loop = asyncio.get_event_loop()
-        listen_task = loop.create_task(self._startlisten())
-        #add_done_callbackの登録
-        if self._done_callback is None:
-            listen_task.add_done_callback(self.finish)
-        else:
-            listen_task.add_done_callback(self._done_callback)
-
-    async def _startlisten(self):
-        """最初のcontinuationパラメータを取得し、
-        _listenループのタスクを作成し開始する
-        """
-        initial_continuation = arcparam.getparam(self.video_id, self.seektime)
-        await self._listen(initial_continuation)
-   
-    async def _listen(self, continuation):
-        ''' continuationに紐付いたチャットデータを取得し
-        Bufferにチャットデータを格納、
-        次のcontinuaitonを取得してループする。
-
-        Parameter
-        ---------
-        continuation : str
-            次のチャットデータ取得に必要なパラメータ
-        '''
-        try:
-            async with aiohttp.ClientSession() as session:
-                while(continuation and self._is_alive):
-                    if self._pauser.empty():
-                        '''pause'''
-                        await self._pauser.get()
-                        '''resume:
-                          prohibit from blocking by putting None into _pauser.
-                        '''
-                        self._pauser.put_nowait(None)
-                        #when replay, not reacquire continuation param
-                    livechat_json = (await
-                      self._get_livechat_json(continuation, session, headers)
-                    )
-                    metadata, chatdata =  self._parser.parse( livechat_json )
-                    timeout = metadata['timeoutMs']/1000
-                    chat_component = {
-                        "video_id" : self.video_id,
-                        "timeout"  : timeout,
-                        "chatdata" : chatdata
-                    }
-                    time_mark =time.time()
-                    if self._direct_mode:
-                        await self._callback(
-                            self.processor.process([chat_component])
-                            )
-                    else:
-                        await self._buffer.put(chat_component)
-                    diff_time = timeout - (time.time()-time_mark)
-                    await asyncio.sleep(diff_time)       
-                    continuation = metadata.get('continuation')  
-        except ChatParseException as e:
-            self.terminate()
-            logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）")
-            return            
-        except (TypeError , json.JSONDecodeError) :
-            self.terminate()
-            logger.error(f"{traceback.format_exc(limit = -1)}")
-            return
-        
-        logger.debug(f"[{self.video_id}]チャット取得を終了しました。")
-        self.terminate()
-
-    async def _get_livechat_json(self, continuation, session, headers):
-        '''
-        チャットデータが格納されたjsonデータを取得する。
-        '''
-        continuation = urllib.parse.quote(continuation)
-        livechat_json = None
-        status_code = 0
-        url =(
-            f"https://www.youtube.com/live_chat_replay/get_live_chat_replay?"
-            f"continuation={continuation}&pbj=1")
-        for _ in range(MAX_RETRY + 1):
-            async with session.get(url ,headers = headers) as resp:
-                try:
-                    text = await resp.text()
-                    status_code = resp.status
-                    livechat_json = json.loads(text)
-                    break
-                except (ClientConnectorError,json.JSONDecodeError) :
-                    await asyncio.sleep(1)
-                    continue
-        else:
-            logger.error(f"[{self.video_id}]"
-                    f"Exceeded retry count. status_code={status_code}")
-            return None
-        return livechat_json
-  
-    async def _callback_loop(self,callback):
-        """ コンストラクタでcallbackを指定している場合、バックグラウンドで
-        callbackに指定された関数に一定間隔でチャットデータを投げる。        
-        
-        Parameter
-        ---------
-        callback : func
-            加工済みのチャットデータを渡す先の関数。
-        """
-        while self.is_alive():
-            items = await self._buffer.get()
-            data = self.processor.process(items)
-            await callback(data)
-
-    async def get(self):
-        """ bufferからデータを取り出し、processorに投げ、
-        加工済みのチャットデータを返す。
-        
-        Returns
-             : Processorによって加工されたチャットデータ
-        """
-        if self._callback is None:
-            items = await self._buffer.get()
-            return  self.processor.process(items)
-        raise IllegalFunctionCall(
-            "既にcallbackを登録済みのため、get()は実行できません。")
-
-    def pause(self):
-        if self._callback is None:
-            return
-        if not self._pauser.empty():
-            self._pauser.get_nowait()
-
-    def resume(self):
-        if self._callback is None:
-            return
-        if self._pauser.empty():
-            self._pauser.put_nowait(None)
-        
-    def is_alive(self):
-        return self._is_alive
-
-    def finish(self,sender):
-        '''Listener終了時のコールバック'''
-        try: 
-            self.terminate()
-        except CancelledError:
-            logger.debug(f'[{self.video_id}]cancelled:{sender}')
-
-    def terminate(self):
-        '''
-        Listenerを終了する。
-        '''
-        self._is_alive = False
-        if self._direct_mode == False:
-            #bufferにダミーオブジェクトを入れてis_alive()を判定させる
-            self._buffer.put_nowait({'chatdata':'','timeout':1}) 
-        logger.info(f'[{self.video_id}]終了しました')
-  
-    @classmethod
-    def _set_exception_handler(cls, handler):
-        loop = asyncio.get_event_loop()
-        loop.set_exception_handler(handler)
-    
-    @classmethod
-    def _handle_exception(cls, loop, context):
-        if not isinstance(context["exception"],CancelledError):
-            logger.error(f"Caught exception: {context}")
-        loop= asyncio.get_event_loop()
-        loop.create_task(cls.shutdown(None,None,None))
-
-    @classmethod
-    async def shutdown(cls, event, sig = None, handler=None):
-        logger.debug("シャットダウンしています")
-        tasks = [t for t in asyncio.all_tasks() if t is not
-        asyncio.current_task()]
-        [task.cancel() for task in tasks]
-
-        logger.debug(f"残っているタスクを終了しています")
-        await asyncio.gather(*tasks,return_exceptions=True)
-        loop = asyncio.get_event_loop()
-        loop.stop()

pytchat/core_multithread/buffer.py

@@ -1,20 +1,22 @@
 
 import queue
 
+
 class Buffer(queue.Queue):
     '''
-    チャットデータを格納するバッファの役割を持つFIFOキュー
+    Buffer for storing chat data.
 
     Parameter
     ---------
-    max_size : int
-        格納するチャットブロックの最大個数。0の場合は無限。
-        最大値を超える場合は古いチャットブロックから破棄される。
+    maxsize : int
+        Maximum number of chat blocks to be stored.
+        If it exceeds the maximum, the oldest chat block will be discarded.
     '''
-    def __init__(self,maxsize = 0):
+
+    def __init__(self, maxsize=0):
         super().__init__(maxsize=maxsize)
 
-    def put(self,item):
+    def put(self, item):
         if item is None:
             return
         if super().full():
@@ -22,6 +24,13 @@ class Buffer(queue.Queue):
         else:
             super().put(item)
 
+    def put_nowait(self, item):
+        if item is None:
+            return
+        if super().full():
+            super().get_nowait()
+        else:
+            super().put_nowait(item)
 
     def get(self):
         ret = []

pytchat/core_multithread/livechat.py

@@ -1,18 +1,17 @@
-import requests
-import datetime
+import httpx
 import json
-import random
 import signal
 import time
 import traceback
-import urllib.parse
 from concurrent.futures import CancelledError, ThreadPoolExecutor
 from queue import Queue
+from threading import Event
 from .buffer import Buffer
 from ..parser.live import Parser
 from .. import config
-from ..exceptions  import ChatParseException,IllegalFunctionCall
-from ..paramgen    import liveparam, arcparam
+from .. import exceptions
+from .. import util
+from ..paramgen import liveparam, arcparam
 from ..processors.default.processor import DefaultProcessor
 from ..processors.combinator import Combinator
 
@@ -21,73 +20,70 @@ MAX_RETRY = 10
 
 
 class LiveChat:
-    ''' スレッドプールを利用してYouTubeのライブ配信のチャットデータを取得する
+    '''
+    LiveChat object fetches chat data and stores them
+    in a buffer with ThreadpoolExecutor.
 
     Parameter
     ---------
     video_id : str
-        動画ID
 
     seektime : int
-        （ライブチャット取得時は無視）
-        取得開始するアーカイブ済みチャットの経過時間(秒）
-        マイナス値を指定した場合は、配信開始前のチャットも取得する。
+        start position of fetching chat (seconds).
+        This option is valid for archived chat only.
+        If negative value, chat data posted before the start of the broadcast
+        will be retrieved as well.
 
     processor : ChatProcessor
-        チャットデータを加工するオブジェクト
 
-    buffer : Buffer(maxsize:20[default])
-        チャットデータchat_componentを格納するバッファ。
-        maxsize : 格納できるchat_componentの個数
-        default値20個。1個で約5~10秒分。
+    buffer : Buffer
+        buffer of chat data fetched background.
 
     interruptable : bool
-        Ctrl+Cによる処理中断を行うかどうか。
+        Allows keyboard interrupts.
+        Set this parameter to False if your own threading program causes
+        the problem.
 
     callback : func
-        _listen()関数から一定間隔で自動的に呼びだす関数。
+        function called periodically from _listen().
 
     done_callback : func
-        listener終了時に呼び出すコールバック。
+        function called when listener ends.
 
     direct_mode : bool
-        Trueの場合、bufferを使わずにcallbackを呼ぶ。
-        Trueの場合、callbackの設定が必須
-        (設定していない場合IllegalFunctionCall例外を発生させる）  
+        If True, invoke specified callback function without using buffer.
+        callback is required. If not, IllegalFunctionCall will be raised.
 
     force_replay : bool
-        Trueの場合、ライブチャットが取得できる場合であっても
-        強制的にアーカイブ済みチャットを取得する。
+        force to fetch archived chat data, even if specified video is live.
 
     topchat_only : bool
-        Trueの場合、上位チャットのみ取得する。
+        If True, get only top chat.
 
     Attributes
     ---------
     _executor : ThreadPoolExecutor
-        チャットデータ取得ループ（_listen）用のスレッド
+        This is used for _listen() loop.
 
     _is_alive : bool
-        チャット取得を停止するためのフラグ
+        Flag to stop getting chat.
     '''
 
     _setup_finished = False
-    #チャット監視中のListenerのリスト
-    _listeners = []
 
     def __init__(self, video_id,
-                seektime = 0,
-                processor = DefaultProcessor(),
-                buffer = None,
-                interruptable = True,
-                callback = None,
-                done_callback = None,
-                direct_mode = False,
-                force_replay = False,
-                topchat_only  = False,
-                logger = config.logger(__name__)
-                ):
-        self.video_id  = video_id
+                 seektime=-1,
+                 processor=DefaultProcessor(),
+                 buffer=None,
+                 interruptable=True,
+                 callback=None,
+                 done_callback=None,
+                 direct_mode=False,
+                 force_replay=False,
+                 topchat_only=False,
+                 logger=config.logger(__name__)
+                 ):
+        self._video_id = util.extract_video_id(video_id)
         self.seektime = seektime
         if isinstance(processor, tuple):
             self.processor = Combinator(processor)
@@ -98,55 +94,53 @@ class LiveChat:
         self._done_callback = done_callback
         self._executor = ThreadPoolExecutor(max_workers=2)
         self._direct_mode = direct_mode
-        self._is_alive   = True
+        self._is_alive = True
         self._is_replay = force_replay
-        self._parser = Parser(is_replay = self._is_replay)
+        self._parser = Parser(is_replay=self._is_replay)
         self._pauser = Queue()
         self._pauser.put_nowait(None)
-        self._setup()
         self._first_fetch = True
-        self._fetch_url = "live_chat/get_live_chat?continuation="
+        self._fetch_url = config._sml
         self._topchat_only = topchat_only
+        self._dat = ''
+        self._last_offset_ms = 0
+        self._event = Event()
         self._logger = logger
-        LiveChat._logger = logger
-        if not LiveChat._setup_finished:
-            LiveChat._setup_finished = True
-            if interruptable:
-                signal.signal(signal.SIGINT,  (lambda a, b:  
-                (LiveChat.shutdown(None,signal.SIGINT,b))
-                ))
-        LiveChat._listeners.append(self)
+        self.exception = None
+        if interruptable:
+            signal.signal(signal.SIGINT, lambda a, b: self.terminate())
+        self._setup()
 
     def _setup(self):
-        #direct modeがTrueでcallback未設定の場合例外発生。
+        # An exception is raised when direct mode is true and no callback is set.
         if self._direct_mode:
             if self._callback is None:
-                raise IllegalFunctionCall(
+                raise exceptions.IllegalFunctionCall(
                     "When direct_mode=True, callback parameter is required.")
         else:
-            #direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成
+            # Create a default buffer if `direct_mode` is False and buffer is not set.
             if self._buffer is None:
-                self._buffer = Buffer(maxsize = 20)
-                #callbackが指定されている場合はcallbackを呼ぶループタスクを作成
+                self._buffer = Buffer(maxsize=20)
+                # Create a loop task to call callback if the `callback` param is specified.
             if self._callback is None:
                 pass
             else:
-                #callbackを呼ぶループタスクの開始
-                self._executor.submit(self._callback_loop,self._callback)
-        #_listenループタスクの開始
-        listen_task = self._executor.submit(self._startlisten)
-        #add_done_callbackの登録
+                # Start a loop task calling callback function.
+                self._executor.submit(self._callback_loop, self._callback)
+        # Start a loop task for _listen()
+        self.listen_task = self._executor.submit(self._startlisten)
+        # Register add_done_callback
         if self._done_callback is None:
-            listen_task.add_done_callback(self.finish)
+            self.listen_task.add_done_callback(self._finish)
         else:
-            listen_task.add_done_callback(self._done_callback)
+            self.listen_task.add_done_callback(self._done_callback)
 
     def _startlisten(self):
-        time.sleep(0.1)  #sleep shortly to prohibit skipping fetching data
+        time.sleep(0.1)  # sleep shortly to prohibit skipping fetching data
         """Fetch first continuation parameter,
         create and start _listen loop.
         """
-        initial_continuation = liveparam.getparam(self.video_id,3)
+        initial_continuation = liveparam.getparam(self._video_id, 3)
         self._listen(initial_continuation)
 
     def _listen(self, continuation):
@@ -159,39 +153,39 @@ class LiveChat:
             parameter for next chat data
         '''
         try:
-            with requests.Session() as session:
+            with httpx.Client(http2=True) as client:
                 while(continuation and self._is_alive):
                     continuation = self._check_pause(continuation)
-                    contents = self._get_contents(
-                        continuation, session, headers)
-                    metadata, chatdata =  self._parser.parse(contents)
-
-                    timeout = metadata['timeoutMs']/1000
+                    contents = self._get_contents(continuation, client, headers)
+                    metadata, chatdata = self._parser.parse(contents)
+                    timeout = metadata['timeoutMs'] / 1000
                     chat_component = {
-                        "video_id" : self.video_id,
-                        "timeout"  : timeout,
-                        "chatdata" : chatdata
+                        "video_id": self._video_id,
+                        "timeout": timeout,
+                        "chatdata": chatdata
                     }
-                    time_mark =time.time()
+                    time_mark = time.time()
                     if self._direct_mode:
-                        processed_chat = self.processor.process([chat_component])
-                        if isinstance(processed_chat,tuple):
+                        processed_chat = self.processor.process(
+                            [chat_component])
+                        if isinstance(processed_chat, tuple):
                             self._callback(*processed_chat)
                         else:
                             self._callback(processed_chat)
                     else:
                         self._buffer.put(chat_component)
-                    diff_time = timeout - (time.time()-time_mark)
-                    time.sleep(diff_time if diff_time > 0 else 0)        
+                    diff_time = timeout - (time.time() - time_mark)
+                    self._event.wait(diff_time if diff_time > 0 else 0)
                     continuation = metadata.get('continuation')
-        except ChatParseException as e:
-            self._logger.debug(f"[{self.video_id}]{str(e)}")
-            return            
-        except (TypeError , json.JSONDecodeError) :
-            self._logger.error(f"{traceback.format_exc(limit = -1)}")
-            return
+                    self._last_offset_ms = metadata.get('last_offset_ms', 0)
+        except exceptions.ChatParseException as e:
+            self._logger.debug(f"[{self._video_id}]{str(e)}")
+            raise
+        except Exception:
+            self._logger.error(f"{traceback.format_exc(limit=-1)}")
+            raise
 
-        self._logger.debug(f"[{self.video_id}]finished fetching chat.")
+        self._logger.debug(f"[{self._video_id}] finished fetching chat.")
 
     def _check_pause(self, continuation):
         if self._pauser.empty():
@@ -202,10 +196,10 @@ class LiveChat:
             '''
             self._pauser.put_nowait(None)
             if not self._is_replay:
-                continuation = liveparam.getparam(self.video_id,3)
+                continuation = liveparam.getparam(self._video_id, 3)
         return continuation
 
-    def _get_contents(self, continuation, session, headers):
+    def _get_contents(self, continuation, client, headers):
         '''Get 'continuationContents' from livechat json.
            If contents is None at first fetching,
            try to fetch archive chat data.
@@ -214,59 +208,61 @@ class LiveChat:
           -------
             'continuationContents' which includes metadata & chat data.
         '''
-        livechat_json = ( 
-            self._get_livechat_json(continuation, session, headers)
-        )
-        contents = self._parser.get_contents(livechat_json)
+        livechat_json = self._get_livechat_json(continuation, client, headers)
+        contents, dat = self._parser.get_contents(livechat_json)
+        if self._dat == '' and dat:
+            self._dat = dat
         if self._first_fetch:
             if contents is None or self._is_replay:
                 '''Try to fetch archive chat data.'''
                 self._parser.is_replay = True
-                self._fetch_url = "live_chat_replay/get_live_chat_replay?continuation="
+                self._fetch_url = config._smr
                 continuation = arcparam.getparam(
-                    self.video_id, self.seektime, self._topchat_only)
-                livechat_json = ( self._get_livechat_json(
-                    continuation, session, headers))
+                    self._video_id, self.seektime, self._topchat_only)
+                livechat_json = (self._get_livechat_json(
+                                 continuation, client, replay=True, offset_ms=self.seektime * 1000))
                 reload_continuation = self._parser.reload_continuation(
-                    self._parser.get_contents(livechat_json))
+                    self._parser.get_contents(livechat_json)[0])
                 if reload_continuation:
                     livechat_json = (self._get_livechat_json(
-                        reload_continuation, session, headers))
-                contents = self._parser.get_contents(livechat_json)
+                        reload_continuation, client, headers))
+                contents, _ = self._parser.get_contents(livechat_json)
+                self._is_replay = True
             self._first_fetch = False
         return contents
 
-    def _get_livechat_json(self, continuation, session, headers):
+    def _get_livechat_json(self, continuation, client, replay: bool, offset_ms: int = 0):
         '''
         Get json which includes chat data.
         '''
-        continuation = urllib.parse.quote(continuation)
+        # continuation = urllib.parse.quote(continuation)
         livechat_json = None
-        status_code = 0
-        url =f"https://www.youtube.com/{self._fetch_url}{continuation}&pbj=1"
+        if offset_ms < 0:
+            offset_ms = 0
+        param = util.get_param(continuation, dat=self._dat, replay=replay, offsetms=offset_ms)
         for _ in range(MAX_RETRY + 1):
-            with session.get(url ,headers = headers) as resp:
-                try:
-                    text = resp.text
-                    livechat_json = json.loads(text)
-                    break
-                except json.JSONDecodeError :
-                    time.sleep(1)
-                    continue
+            try:
+                resp = client.post(self._fetch_url, json=param)
+                livechat_json = resp.json()
+                break
+            except (json.JSONDecodeError, httpx.HTTPError):
+                time.sleep(2)
+                continue
         else:
-            self._logger.error(f"[{self.video_id}]"
-                    f"Exceeded retry count. status_code={status_code}")
-            return None
+            self._logger.error(f"[{self._video_id}]"
+                               f"Exceeded retry count.")
+            raise exceptions.RetryExceedMaxCount()
         return livechat_json
 
-    def _callback_loop(self,callback):
-        """ コンストラクタでcallbackを指定している場合、バックグラウンドで
-        callbackに指定された関数に一定間隔でチャットデータを投げる。        
+    def _callback_loop(self, callback):
+        """ If a callback is specified in the constructor,
+        it throws chat data at regular intervals to the
+        function specified in the callback in the backgroun
 
         Parameter
         ---------
         callback : func
-            加工済みのチャットデータを渡す先の関数。
+            function to which the processed chat data is passed.
         """
         while self.is_alive():
             items = self._buffer.get()
@@ -277,17 +273,22 @@ class LiveChat:
                 self._callback(processed_chat)
 
     def get(self):
-        """ bufferからデータを取り出し、processorに投げ、
-        加工済みのチャットデータを返す。
+        """
+        Retrieves data from the buffer,
+        throws it to the processor,
+        and returns the processed chat data.
 
         Returns
-             : Processorによって加工されたチャットデータ
+             : Chat data processed by the Processor
         """
         if self._callback is None:
-            items = self._buffer.get()
-            return  self.processor.process(items)
-        raise IllegalFunctionCall(
-            "既にcallbackを登録済みのため、get()は実行できません。")
+            if self.is_alive():
+                items = self._buffer.get()
+                return self.processor.process(items)
+            else:
+                return []
+        raise exceptions.IllegalFunctionCall(
+            "Callback parameter is already set, so get() cannot be performed.")
 
     def is_replay(self):
         return self._is_replay
@@ -307,25 +308,32 @@ class LiveChat:
     def is_alive(self):
         return self._is_alive
 
-    def finish(self,sender):
-        '''Listener終了時のコールバック'''
+    def _finish(self, sender):
+        '''Called when the _listen() task finished.'''
         try:
-            self.terminate()
+            self._task_finished()
         except CancelledError:
-            self._logger.debug(f'[{self.video_id}]cancelled:{sender}')
+            self._logger.debug(f'[{self._video_id}] cancelled:{sender}')
 
     def terminate(self):
-        '''
-        Listenerを終了する。
-        '''
+        if self._pauser.empty():
+            self._pauser.put_nowait(None)
         self._is_alive = False
-        if self._direct_mode == False:
-            #bufferにダミーオブジェクトを入れてis_alive()を判定させる
-            self._buffer.put({'chatdata':'','timeout':0}) 
-        self._logger.info(f'[{self.video_id}]finished.')
+        self._buffer.put({})
+        self._event.set()
+        self.processor.finalize()
 
-    @classmethod
-    def shutdown(cls, event, sig = None, handler=None):
-        cls._logger.debug("shutdown...")
-        for t in LiveChat._listeners:
-            t._is_alive = False
+    def _task_finished(self):
+        if self.is_alive():
+            self.terminate()
+        try:
+            self.listen_task.result()
+        except Exception as e:
+            self.exception = e
+            if not isinstance(e, exceptions.ChatParseException):
+                self._logger.error(f'Internal exception - {type(e)}{str(e)}')
+        self._logger.info(f'[{self._video_id}] finished.')
+
+    def raise_for_status(self):
+        if self.exception is not None:
+            raise self.exception

pytchat/core_multithread/replaychat.py

@@ -1,309 +0,0 @@
-import requests
-import datetime
-import json
-import random
-import signal
-import time
-import traceback
-import urllib.parse
-import warnings
-from concurrent.futures import CancelledError, ThreadPoolExecutor
-from queue import Queue
-from .buffer import Buffer
-from ..parser.replay import Parser
-from .. import config
-from ..exceptions  import ChatParseException,IllegalFunctionCall
-from ..paramgen    import arcparam
-from ..processors.default.processor import DefaultProcessor
-from ..processors.combinator import Combinator
-
-logger = config.logger(__name__)
-headers = config.headers
-MAX_RETRY = 10
-
-
-class ReplayChat:
-    '''
-    ### -----------------------------------------------------------
-    ### [Warning] ReplayChat is integrated into LiveChat.
-    ### This class is deprecated and will be removed at v0.0.5.0.
-    ### ReplayChatはLiveChatに統合しました。
-    ### このクラスはv0.0.5.0で廃止予定です。
-    ### -----------------------------------------------------------
-
-     スレッドプールを利用してYouTubeのライブ配信のチャットデータを取得する
-
-    Parameter
-    ---------
-    video_id : str
-        動画ID
-
-    seektime : int
-        リプレイするチャットデータの開始時間（秒）
-
-    processor : ChatProcessor
-        チャットデータを加工するオブジェクト
-
-    buffer : Buffer(maxsize:20[default])
-        チャットデータchat_componentを格納するバッファ。
-        maxsize : 格納できるchat_componentの個数
-        default値20個。1個で約5~10秒分。
-
-    interruptable : bool
-        Ctrl+Cによる処理中断を行うかどうか。
-
-    callback : func
-        _listen()関数から一定間隔で自動的に呼びだす関数。
-
-    done_callback : func
-        listener終了時に呼び出すコールバック。
-
-    direct_mode : bool
-        Trueの場合、bufferを使わずにcallbackを呼ぶ。
-        Trueの場合、callbackの設定が必須
-        (設定していない場合IllegalFunctionCall例外を発生させる）  
-
-    Attributes
-    ---------
-    _executor : ThreadPoolExecutor
-        チャットデータ取得ループ（_listen）用のスレッド
-
-    _is_alive : bool
-        チャット取得を停止するためのフラグ
-    '''
-
-    _setup_finished = False
-
-    #チャット監視中のListenerのリスト
-    _listeners= []
-
-    def __init__(self, video_id,
-                seektime = 0,
-                processor = DefaultProcessor(),
-                buffer = None,
-                interruptable = True,
-                callback = None,
-                done_callback = None,
-                direct_mode = False
-                ):
-
-        warnings.warn(""
-        f"\n{'-'*60}\n[WARNING] ReplayChat is integrated into LiveChat.\n"
-        f"{' '*5}This is deprecated and will be removed at v0.0.5.0.\n"
-        f"{'-'*60}\n"
-        )
-        self.video_id  = video_id
-        self.seektime = seektime
-        if isinstance(processor, tuple):
-            self.processor = Combinator(processor)
-        else:
-            self.processor = processor
-        self._buffer = buffer
-        self._callback = callback
-        self._done_callback = done_callback
-        self._executor = ThreadPoolExecutor(max_workers=2)
-        self._direct_mode = direct_mode
-        self._is_alive   = True
-        self._parser = Parser()
-        self._pauser = Queue()
-        self._pauser.put_nowait(None)
-       
-        self._setup()
-
-        if not ReplayChat._setup_finished:
-            ReplayChat._setup_finished = True
-            if interruptable:
-                signal.signal(signal.SIGINT,  (lambda a, b:  
-                (ReplayChat.shutdown(None,signal.SIGINT,b))
-                ))
-        ReplayChat._listeners.append(self)
-
-    def _setup(self):
-        #direct modeがTrueでcallback未設定の場合例外発生。
-        if self._direct_mode:
-            if self._callback is None:
-                raise IllegalFunctionCall(
-                    "direct_mode=Trueの場合callbackの設定が必須です。")
-        else:
-            #direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成
-            if self._buffer is None:
-                self._buffer = Buffer(maxsize = 20)
-                #callbackが指定されている場合はcallbackを呼ぶループタスクを作成
-            if self._callback is None:
-                pass 
-            else:
-                #callbackを呼ぶループタスクの開始
-                self._executor.submit(self._callback_loop,self._callback)
-        #_listenループタスクの開始
-        listen_task = self._executor.submit(self._startlisten)
-        #add_done_callbackの登録
-        if self._done_callback is None:
-            listen_task.add_done_callback(self.finish)
-        else:
-            listen_task.add_done_callback(self._done_callback)
-
-    def _startlisten(self):
-        """最初のcontinuationパラメータを取得し、
-        _listenループのタスクを作成し開始する
-        """
-        initial_continuation = self._get_initial_continuation()
-        if initial_continuation is None:
-            self.terminate()
-            logger.debug(f"[{self.video_id}]No initial continuation.")
-            return
-        self._listen(initial_continuation)
-   
-    def _get_initial_continuation(self):
-        ''' チャットデータ取得に必要な最初のcontinuationを取得する。'''
-        try:    
-            initial_continuation = arcparam.getparam(self.video_id,self.seektime)
-        except ChatParseException as e:
-            self.terminate()
-            logger.debug(f"[{self.video_id}]Error:{str(e)}")
-            return
-        except KeyError:
-            logger.debug(f"[{self.video_id}]KeyError:"
-                         f"{traceback.format_exc(limit = -1)}")
-            self.terminate()
-            return
-        return initial_continuation
-
-    def _listen(self, continuation):
-        ''' continuationに紐付いたチャットデータを取得し
-        BUfferにチャットデータを格納、
-        次のcontinuaitonを取得してループする
-
-        Parameter
-        ---------
-        continuation : str
-            次のチャットデータ取得に必要なパラメータ
-        '''
-        try:
-            with requests.Session() as session:
-                while(continuation and self._is_alive):
-                    if self._pauser.empty():
-                        #pause
-                        self._pauser.get()
-                        #resume
-                        #prohibit from blocking by putting None into _pauser.
-                        self._pauser.put_nowait(None)
-                    livechat_json = (
-                      self._get_livechat_json(continuation, session, headers)
-                    )
-                    metadata, chatdata =  self._parser.parse( livechat_json )
-                    timeout = metadata['timeoutMs']/1000
-                    chat_component = {
-                        "video_id" : self.video_id,
-                        "timeout"  : timeout,
-                        "chatdata" : chatdata
-                    }
-                    time_mark =time.time()
-                    if self._direct_mode:
-                        self._callback(
-                            self.processor.process([chat_component])
-                            )
-                    else:
-                        self._buffer.put(chat_component)
-                    diff_time = timeout - (time.time()-time_mark)
-                    if diff_time < 0 : diff_time=0
-                    time.sleep(diff_time)        
-                    continuation = metadata.get('continuation')  
-        except ChatParseException as e:
-            self.terminate()
-            logger.error(f"{str(e)}（video_id:\"{self.video_id}\"）")
-            return            
-        except (TypeError , json.JSONDecodeError) :
-            self.terminate()
-            logger.error(f"{traceback.format_exc(limit = -1)}")
-            return
-        
-        logger.debug(f"[{self.video_id}]チャット取得を終了しました。")
-
-    def _get_livechat_json(self, continuation, session, headers):
-        '''
-        チャットデータが格納されたjsonデータを取得する。
-        '''
-        continuation = urllib.parse.quote(continuation)
-        livechat_json = None
-        status_code = 0
-        url =(
-            f"https://www.youtube.com/live_chat_replay/get_live_chat_replay?"
-            f"continuation={continuation}&pbj=1")
-        for _ in range(MAX_RETRY + 1):
-            with session.get(url ,headers = headers) as resp:
-                try:
-                    text = resp.text
-                    status_code = resp.status_code
-                    livechat_json = json.loads(text)
-                    break
-                except json.JSONDecodeError :
-                    time.sleep(1)
-                    continue
-        else:
-            logger.error(f"[{self.video_id}]"
-                    f"Exceeded retry count. status_code={status_code}")
-            self.terminate()
-            return None
-        return livechat_json
-  
-    def _callback_loop(self,callback):
-        """ コンストラクタでcallbackを指定している場合、バックグラウンドで
-        callbackに指定された関数に一定間隔でチャットデータを投げる。        
-        
-        Parameter
-        ---------
-        callback : func
-            加工済みのチャットデータを渡す先の関数。
-        """
-        while self.is_alive():
-            items = self._buffer.get()
-            data = self.processor.process(items)
-            callback(data)
-
-    def get(self):
-        """ bufferからデータを取り出し、processorに投げ、
-        加工済みのチャットデータを返す。
-        
-        Returns
-             : Processorによって加工されたチャットデータ
-        """
-        if self._callback is None:
-            items = self._buffer.get()
-            return  self.processor.process(items)
-        raise IllegalFunctionCall(
-            "既にcallbackを登録済みのため、get()は実行できません。")
-
-    def pause(self):
-        if not self._pauser.empty():
-            self._pauser.get()
-
-    def resume(self):
-        if self._pauser.empty():
-            self._pauser.put_nowait(None)
-        
-
-    def is_alive(self):
-        return self._is_alive
-
-    def finish(self,sender):
-        '''Listener終了時のコールバック'''
-        try: 
-            self.terminate()
-        except RuntimeError:
-            logger.debug(f'[{self.video_id}]cancelled:{sender}')
-
-    def terminate(self):
-        '''
-        Listenerを終了する。
-        '''
-        self._is_alive = False
-        if self._direct_mode == False:
-            #bufferにダミーオブジェクトを入れてis_alive()を判定させる
-            self._buffer.put({'chatdata':'','timeout':1}) 
-        logger.info(f'[{self.video_id}]終了しました')
-  
-    @classmethod
-    def shutdown(cls, event, sig = None, handler=None):
-        logger.debug("シャットダウンしています")
-        for t in ReplayChat._listeners:
-            t._is_alive = False

pytchat/exceptions.py

@@ -1,46 +1,81 @@
 class ChatParseException(Exception):
     '''
-    チャットデータをパースするライブラリが投げる例外の基底クラス
+    Base exception thrown by the parser
     '''
     pass
 
-class NoYtinitialdataException(ChatParseException):
-    '''
-    配信ページ内にチャットデータurlが見つからないときに投げる例外
-    '''
-    pass
 
 class ResponseContextError(ChatParseException):
     '''
-    配信ページでチャットデータ無効の時に投げる例外
-    '''
-    pass
-
-class NoLivechatRendererException(ChatParseException):
-    '''
-    チャットデータのJSON中にlivechatRendererがない時に投げる例外
+    Thrown when chat data is invalid.
     '''
     pass
 
 
-class NoContentsException(ChatParseException):
+class NoContents(ChatParseException):
     '''
-    チャットデータのJSON中にContinuationContentsがない時に投げる例外
+    Thrown when ContinuationContents is missing in JSON.
     '''
     pass
 
-class NoContinuationsException(ChatParseException):
+
+class NoContinuation(ChatParseException):
     '''
-    チャットデータのContinuationContents中にcontinuationがない時に投げる例外
+    Thrown when continuation is missing in ContinuationContents.
     '''
     pass
 
+
 class IllegalFunctionCall(Exception):
     '''
-    set_callback()を実行済みにもかかわらず
-    get()を呼び出した場合の例外
+    Thrown when get() is called even though
+    set_callback() has been executed.
     '''
     pass
 
+
 class InvalidVideoIdException(Exception):
+    '''
+    Thrown when the video_id is not exist (VideoInfo).
+    '''
+    def __init__(self, doc):
+        self.msg = "InvalidVideoIdException"
+        self.doc = doc
+
+
+class UnknownConnectionError(Exception):
     pass
+
+
+class RetryExceedMaxCount(Exception):
+    '''
+    Thrown when the number of retries exceeds the maximum value.
+    '''
+    pass
+
+
+class ChatDataFinished(ChatParseException):
+    pass
+
+
+class ReceivedUnknownContinuation(ChatParseException):
+    pass
+
+
+class FailedExtractContinuation(ChatDataFinished):
+    pass
+
+
+class VideoInfoParseError(Exception):
+    '''
+    Base exception when parsing video info.
+    '''
+
+
+class PatternUnmatchError(VideoInfoParseError):
+    '''
+    Thrown when failed to parse video info with unmatched pattern.
+    '''
+    def __init__(self, doc=''):
+        self.msg = "PatternUnmatchError"
+        self.doc = doc

pytchat/paramgen/arcparam.py

@@ -1,107 +1,43 @@
+from . import enc
 from base64 import urlsafe_b64encode as b64enc
-from functools import reduce
-import math
-import random
-import urllib.parse
+from urllib.parse import quote
 
-'''
-Generate continuation parameter of youtube replay chat.
 
-Author: taizan-hokuto (2019) @taizan205
+def _header(video_id) -> str:
+    channel_id = '_' * 24
+    S1_3 = enc.rs(1, video_id)
+    S1_5 = enc.rs(1, channel_id) + enc.rs(2, video_id)
+    S1 = enc.rs(3, S1_3) + enc.rs(5, S1_5)
+    S3 = enc.rs(48687757, enc.rs(1, video_id))
+    header_replay = enc.rs(1, S1) + enc.rs(3, S3) + enc.nm(4, 1)
+    return b64enc(header_replay)
 
-ver 0.0.1 2019.10.05
-'''
 
-def _gen_vid(video_id):
-    """generate video_id parameter.
-    Parameter
-    ---------
-        video_id : str
-
-    Return
-    ---------
-        bytes : base64 encoded video_id parameter.
-    """
-    header_magic = b'\x0A\x0F\x1A\x0D\x0A'
-    header_id =  video_id.encode()
-    header_sep_1 = b'\x1A\x13\xEA\xA8\xDD\xB9\x01\x0D\x0A\x0B'
-    header_terminator = b'\x20\x01'
-
-    item = [
-        header_magic,
-        _nval(len(header_id)),
-        header_id,
-        header_sep_1,
-        header_id,
-        header_terminator
-    ]
-
-    return urllib.parse.quote(
-        b64enc(reduce(lambda x, y: x+y, item)).decode()
-    ).encode()
-
-def _nval(val):
-    """convert value to byte array"""
-    if val<0: raise ValueError
-    buf = b''
-    while val >> 7:
-        m = val & 0xFF | 0x80
-        buf += m.to_bytes(1,'big')
-        val >>= 7
-    buf += val.to_bytes(1,'big')
-    return buf
-
-def _build(video_id, seektime, topchat_only):
-    switch_01 = b'\x04' if topchat_only else b'\x01'
+def _build(video_id, seektime, topchat_only) -> str:
+    chattype = 4 if topchat_only else 1
+    fetch_before_start = 3
+    timestamp = 1000
     if seektime < 0:
-        times =_nval(0)
-        switch = b'\x04'       
+        fetch_before_start = 4
     elif seektime == 0:
-        times =_nval(1)
-        switch = b'\x03'       
+        timestamp = 1000
     else:
-        times =_nval(int(seektime*1000000))
-        switch = b'\x03'
-    parity = b'\x00'
+        timestamp = int(seektime * 1000000)
+    header = enc.rs(3, _header(video_id))
+    timestamp = enc.nm(5, timestamp)
+    s6 = enc.nm(6, 0)
+    s7 = enc.nm(7, 0)
+    s8 = enc.nm(8, 0)
+    s9 = enc.nm(9, fetch_before_start)
+    s10 = enc.rs(10, enc.nm(4, 0))
+    chattype = enc.rs(14, enc.nm(1, chattype))
+    s15 = enc.nm(15, 0)
+    entity = b''.join((header, timestamp, s6, s7, s8, s9, s10, chattype, s15))
+    continuation = enc.rs(156074452, entity)
+    return quote(b64enc(continuation).decode())
 
-    header_magic= b'\xA2\x9D\xB0\xD3\x04'
-    sep_0       = b'\x1A'
-    vid         = _gen_vid(video_id)
-    time_tag    = b'\x28'
-    timestamp1  = times
-    sep_1       = b'\x30\x00\x38\x00\x40\x00\x48'
-    sep_2       = b'\x52\x1C\x08\x00\x10\x00\x18\x00\x20\x00'
-    chkstr      = b'\x2A\x0E\x73\x74\x61\x74\x69\x63\x63\x68\x65\x63\x6B\x73\x75\x6D\x40'
-    sep_3       = b'\x00\x58\x03\x60'
-    sep_4       = b'\x68' + parity + b'\x72\x04\x08'
-    sep_5       = b'\x10' + parity + b'\x78\x00'
-    body = [
-        sep_0,
-        _nval(len(vid)),
-        vid,
-        time_tag,
-        timestamp1,
-        sep_1,
-        switch,
-        sep_2,
-        chkstr,
-        sep_3,
-        switch_01,
-        sep_4,
-        switch_01,
-        sep_5
-    ]
 
-    body = reduce(lambda x, y: x+y, body)
-    
-    return  urllib.parse.quote(
-                b64enc( header_magic +
-                        _nval(len(body)) +
-                        body
-                ).decode()
-            )
-
-def getparam(video_id, seektime = 0, topchat_only = False):
+def getparam(video_id, seektime=-1, topchat_only=False) -> str:
     '''
     Parameter
     ---------

pytchat/paramgen/arcparam_mining.py

@@ -1,133 +0,0 @@
-from base64 import urlsafe_b64encode as b64enc
-from functools import reduce
-import math
-import random
-import urllib.parse
-
-'''
-Generate continuation parameter of youtube replay chat.
-
-Author: taizan-hokuto (2019) @taizan205
-
-ver 0.0.1 2019.10.05
-'''
-
-def _gen_vid_long(video_id):
-    """generate video_id parameter.
-    Parameter
-    ---------
-        video_id : str
-
-    Return
-    ---------
-    byte[] : base64 encoded video_id parameter.
-    """
-    header_magic = b'\x0A\x0F\x1A\x0D\x0A'
-    header_id =  video_id.encode()
-    header_sep_1 = b'\x1A\x13\xEA\xA8\xDD\xB9\x01\x0D\x0A\x0B'
-    header_terminator = b'\x20\x01'
-
-    item = [
-        header_magic,
-        _nval(len(header_id)),
-        header_id,
-        header_sep_1,
-        header_id,
-        header_terminator
-    ]
-
-    return urllib.parse.quote(
-        b64enc(reduce(lambda x, y: x+y, item)).decode()
-    ).encode()
-
-def _gen_vid(video_id):
-    """generate video_id parameter.
-    Parameter
-    ---------
-        video_id : str
-
-    Return
-    ---------
-        bytes : base64 encoded video_id parameter.
-    """
-    header_magic = b'\x0A\x0F\x1A\x0D\x0A'
-    header_id =  video_id.encode()
-    header_terminator = b'\x20\x01'
-
-    item = [
-        header_magic,
-        _nval(len(header_id)),
-        header_id,
-        header_terminator
-    ]
-
-    return urllib.parse.quote(
-        b64enc(reduce(lambda x, y: x+y, item)).decode()
-    ).encode()
-
-def _nval(val):
-    """convert value to byte array"""
-    if val<0: raise ValueError
-    buf = b''
-    while val >> 7:
-        m = val & 0xFF | 0x80
-        buf += m.to_bytes(1,'big')
-        val >>= 7
-    buf += val.to_bytes(1,'big')
-    return buf
-
-def _build(video_id, seektime, topchat_only):
-    switch_01 = b'\x04' if topchat_only else b'\x01'
-    if seektime < 0:
-        raise ValueError("seektime must be greater than or equal to zero.")
-    if seektime == 0:
-        times = b''
-    else:
-        times =_nval(int(seektime*1000))
-    if seektime > 0:
-        _len_time = ( b'\x5A'
-                    + (len(times)+1).to_bytes(1,'big')
-                    + b'\x10')
-    else:
-        _len_time = b''
-    
-    header_magic = b'\xA2\x9D\xB0\xD3\x04'
-    sep_0        = b'\x1A'
-    vid          = _gen_vid(video_id)
-    _tag         = b'\x40\x01'
-    timestamp1   = times
-    sep_1        = b'\x60\x04\x72\x02\x08'
-    terminator   = b'\x78\x01'
-
-    body = [
-        sep_0,
-        _nval(len(vid)),
-        vid,
-        _tag,
-        _len_time,
-        timestamp1,
-        sep_1,
-        switch_01,
-        terminator
-    ]
-
-    body = reduce(lambda x, y: x+y, body)
-    
-    return  urllib.parse.quote(
-                b64enc( header_magic +
-                        _nval(len(body)) +
-                        body
-                ).decode()
-            )
-
-def getparam(video_id, seektime = 0.0, topchat_only = False):
-    '''
-    Parameter
-    ---------
-    seektime : int
-        unit:seconds
-        start position of fetching chat data.
-    topchat_only : bool
-        if True, fetch only 'top chat'
-    '''
-    return _build(video_id, seektime, topchat_only)

pytchat/paramgen/enc.py

@@ -0,0 +1,24 @@
+def vn(val):
+    if val < 0:
+        raise ValueError
+    buf = b''
+    while val >> 7:
+        m = val & 0xFF | 0x80
+        buf += m.to_bytes(1, 'big')
+        val >>= 7
+    buf += val.to_bytes(1, 'big')
+    return buf
+
+
+def tp(a, b, ary):
+    return vn((b << 3) | a) + ary
+
+
+def rs(a, ary):
+    if isinstance(ary, str):
+        ary = ary.encode()
+    return tp(2, a, vn(len(ary)) + ary)
+
+
+def nm(a, ary):
+    return tp(0, a, vn(ary))

pytchat/paramgen/liveparam.py

@@ -1,157 +1,59 @@
-from base64 import urlsafe_b64encode as b64enc
-from functools import reduce
-import time
 import random
-import urllib.parse
+import time
+from . import enc
+from base64 import urlsafe_b64encode as b64enc
+from urllib.parse import quote
 
-'''
-Generate continuation parameter of youtube live chat.
 
-Author: taizan-hokuto (2019) @taizan205
+def _header(video_id) -> str:
+    return b64enc(enc.rs(1, enc.rs(1, enc.rs(1, video_id))) + enc.nm(4, 1))
 
-ver 0.0.1 2019.10.05
-'''
-def _gen_vid(video_id):
-    """generate video_id parameter.
-    Parameter
-    ---------
-        video_id : str
 
-    Return
-    ---------
-    byte[] : base64 encoded video_id parameter.
-    """
-    header_magic = b'\x0A\x0F\x0A\x0D\x0A'
-    header_id =  video_id.encode()
-    header_sep_1 = b'\x1A'
-    header_sep_2 = b'\x43\xAA\xB9\xC1\xBD\x01\x3D\x0A'
-    header_suburl = ('https://www.youtube.com/live_chat?v='
-                    f'{video_id}&is_popout=1').encode()
-    header_terminator = b'\x20\x02'
+def _build(video_id, ts1, ts2, ts3, ts4, ts5, topchat_only) -> str:
+    chattype = 4 if topchat_only else 1
 
-    item = [
-        header_magic,
-        _nval(len(header_id)),
-        header_id,
-        header_sep_1,
-        header_sep_2,
-        _nval(len(header_suburl)),
-        header_suburl,
-        header_terminator
-    ]
+    b1 = enc.nm(1, 0)
+    b2 = enc.nm(2, 0)
+    b3 = enc.nm(3, 0)
+    b4 = enc.nm(4, 0)
+    b7 = enc.rs(7, '')
+    b8 = enc.nm(8, 0)
+    b9 = enc.rs(9, '')
+    timestamp2 = enc.nm(10, ts2)
+    b11 = enc.nm(11, 3)
+    b15 = enc.nm(15, 0)
 
-    return urllib.parse.quote(
-        b64enc(reduce(lambda x, y: x+y, item)).decode()
-    ).encode()
-
-def _tzparity(video_id,times):
-    t=0
-    for i,s in enumerate(video_id):
-        ss = ord(s)
-        if(ss % 2 == 0):
-            t += ss*(12-i)
-        else:
-            t ^= ss*i
-
-    return ((times^t) % 2).to_bytes(1,'big')
-
-def _nval(val):
-    """convert value to byte array"""
-    if val<0: raise ValueError
-    buf = b''
-    while val >> 7:
-        m = val & 0xFF | 0x80
-        buf += m.to_bytes(1,'big')
-        val >>= 7
-    buf += val.to_bytes(1,'big')
-    return buf
-
-def _build(video_id, _ts1, _ts2, _ts3, _ts4, _ts5, topchat_only):
-    #_short_type2
-    switch_01 = b'\x04' if topchat_only else b'\x01'
-    parity = _tzparity(video_id, _ts1^_ts2^_ts3^_ts4^_ts5)
-
-    header_magic= b'\xD2\x87\xCC\xC8\x03'
-    sep_0       = b'\x1A'
-    vid         = _gen_vid(video_id)
-    time_tag    = b'\x28'
-    timestamp1  = _nval(_ts1)
-    sep_1       = b'\x30\x00\x38\x00\x40\x02\x4A'
-    un_len      = b'\x2B'
-    sep_2       = b'\x08'+parity+b'\x10\x00\x18\x00\x20\x00'
-    chkstr      = b'\x2A\x0E\x73\x74\x61\x74\x69\x63\x63\x68\x65\x63\x6B\x73\x75\x6D'
-    sep_3       = b'\x3A\x00\x40\x00\x4A'
-    sep_4_len   = b'\x02'
-    sep_4       = b'\x08\x01'
-    ts_2_start  = b'\x50'
-    timestamp2  = _nval(_ts2)
-    ts_2_end    = b'\x58'
-    sep_5       = b'\x03'
-    ts_3_start  = b'\x50'
-    timestamp3  = _nval(_ts3)
-    ts_3_end    = b'\x58'
-    timestamp4  = _nval(_ts4)
-    sep_6       = b'\x68'
-    #switch
-    sep_7       = b'\x82\x01\x04\x08'
-    #switch
-    sep_8       = b'\x10\x00'
-    sep_9       = b'\x88\x01\x00\xA0\x01'
-    timestamp5  = _nval(_ts5)
-
-    body = [
-        sep_0,
-        _nval(len(vid)),
-        vid,
-        time_tag,
-        timestamp1,
-        sep_1,
-        un_len,
-        sep_2,
-        chkstr,
-        sep_3,
-        sep_4_len,
-        sep_4,
-        ts_2_start,
-        timestamp2,
-        ts_2_end,
-        sep_5,
-        ts_3_start,
-        timestamp3,
-        ts_3_end,
-        timestamp4,
-        sep_6,
-        switch_01,#
-        sep_7,
-        switch_01,#
-        sep_8,
-        sep_9,
-        timestamp5
-    ]
-
-    body = reduce(lambda x, y: x+y, body)
-    
-    return  urllib.parse.quote(
-                b64enc( header_magic +
-                        _nval(len(body)) +
-                        body
-                ).decode()
-            )
+    header = enc.rs(3, _header(video_id))
+    timestamp1 = enc.nm(5, ts1)
+    s6 = enc.nm(6, 0)
+    s7 = enc.nm(7, 0)
+    s8 = enc.nm(8, 1)
+    body = enc.rs(9, b''.join(
+        (b1, b2, b3, b4, b7, b8, b9, timestamp2, b11, b15)))
+    timestamp3 = enc.nm(10, ts3)
+    timestamp4 = enc.nm(11, ts4)
+    s13 = enc.nm(13, chattype)
+    chattype = enc.rs(16, enc.nm(1, chattype))
+    s17 = enc.nm(17, 0)
+    str19 = enc.rs(19, enc.nm(1, 0))
+    timestamp5 = enc.nm(20, ts5)
+    entity = b''.join((header, timestamp1, s6, s7, s8, body, timestamp3,
+                       timestamp4, s13, chattype, s17, str19, timestamp5))
+    continuation = enc.rs(119693434, entity)
+    return quote(b64enc(continuation).decode())
 
 
 def _times(past_sec):
-    
     n = int(time.time())
-    
-    _ts1= n - random.uniform(0,1*3)    
-    _ts2= n - random.uniform(0.01,0.99)    
-    _ts3= n - past_sec + random.uniform(0,1)
-    _ts4= n - random.uniform(10*60,60*60)         
-    _ts5= n - random.uniform(0.01,0.99)
-    return list(map(lambda x:int(x*1000000),[_ts1,_ts2,_ts3,_ts4,_ts5]))
+    _ts1 = n - random.uniform(0, 1 * 3)
+    _ts2 = n - random.uniform(0.01, 0.99)
+    _ts3 = n - past_sec + random.uniform(0, 1)
+    _ts4 = n - random.uniform(10 * 60, 60 * 60)
+    _ts5 = n - random.uniform(0.01, 0.99)
+    return list(map(lambda x: int(x * 1000000), [_ts1, _ts2, _ts3, _ts4, _ts5]))
 
 
-def getparam(video_id, past_sec = 0, topchat_only = False):
+def getparam(video_id, past_sec=0, topchat_only=False) -> str:
     '''
     Parameter
     ---------
@@ -160,5 +62,4 @@ def getparam(video_id, past_sec = 0, topchat_only = False):
     topchat_only : bool
         if True, fetch only 'top chat'
     '''
-    return _build(video_id,*_times(past_sec),topchat_only)
-
+    return _build(video_id, *_times(past_sec), topchat_only)

pytchat/parser/live.py

@@ -4,28 +4,36 @@ pytchat.parser.live
 Parser of live chat JSON.
 """
 
-import json
-from .. exceptions import ( 
-    ResponseContextError, 
-    NoContentsException, 
-    NoContinuationsException,
-    ChatParseException )
+from .. import exceptions
+
 
 class Parser:
+    '''
+    Parser of chat json.
     
-    __slots__ = ['is_replay']
+    Parameter
+    ----------
+    is_replay : bool
 
-    def __init__(self, is_replay): 
+    exception_holder : Object [default:Npne]
+        The object holding exceptions.
+        This is passed from the parent livechat object.
+    '''
+    __slots__ = ['is_replay', 'exception_holder']
+
+    def __init__(self, is_replay, exception_holder=None):
         self.is_replay = is_replay
+        self.exception_holder = exception_holder
 
     def get_contents(self, jsn):
         if jsn is None:
-            raise ChatParseException('Called with none JSON object.')
-        if jsn['response']['responseContext'].get('errors'):
-            raise ResponseContextError('The video_id would be wrong,'
-                'or video is deleted or private.')
-        contents=jsn['response'].get('continuationContents')
-        return contents
+            self.raise_exception(exceptions.IllegalFunctionCall('Called with none JSON object.'))
+        if jsn.get("error") or jsn.get("responseContext", {}).get("errors"):
+            raise exceptions.ResponseContextError(
+                'The video_id would be wrong, or video is deleted or private.')
+        contents = jsn.get('continuationContents')
+        visitor_data = jsn.get("responseContext", {}).get("visitorData")
+        return contents, visitor_data
 
     def parse(self, contents):
         """
@@ -46,60 +54,69 @@ class Parser:
 
         if contents is None:
             '''Broadcasting end or cannot fetch chat stream'''
-            raise NoContentsException('Chat data stream is empty.')
+            self.raise_exception(exceptions.NoContents('Chat data stream is empty.'))
 
         cont = contents['liveChatContinuation']['continuations'][0]
         if cont is None:
-            raise NoContinuationsException('No Continuation')
-        metadata = (cont.get('invalidationContinuationData')  or
-                    cont.get('timedContinuationData')         or
-                    cont.get('reloadContinuationData')        or
-                    cont.get('liveChatReplayContinuationData')
+            self.raise_exception(exceptions.NoContinuation('No Continuation'))
+        metadata = (cont.get('invalidationContinuationData')
+                    or cont.get('timedContinuationData')
+                    or cont.get('reloadContinuationData')
+                    or cont.get('liveChatReplayContinuationData')
                     )
         if metadata is None:
             if cont.get("playerSeekContinuationData"):
-                raise ChatParseException('Finished chat data')
+                self.raise_exception(exceptions.ChatDataFinished('Finished chat data'))
             unknown = list(cont.keys())[0]
             if unknown:
-                raise ChatParseException(f"Received unknown continuation type:{unknown}")
+                self.raise_exception(exceptions.ReceivedUnknownContinuation(
+                    f"Received unknown continuation type:{unknown}"))
             else:
-                raise ChatParseException('Cannot extract continuation data')
+                self.raise_exception(exceptions.FailedExtractContinuation('Cannot extract continuation data'))
         return self._create_data(metadata, contents)
 
     def reload_continuation(self, contents):
         """
-        When `seektime = 0` or seektime is abbreviated , 
+        When `seektime == 0` or seektime is abbreviated ,
         check if fetched chat json has no chat data.
         If so, try to fetch playerSeekContinuationData.
         This function must be run only first fetching.
         """
+        if contents is None:
+            '''Broadcasting end or cannot fetch chat stream'''
+            self.raise_exception(exceptions.NoContents('Chat data stream is empty.'))
         cont = contents['liveChatContinuation']['continuations'][0]
+
         if cont.get("liveChatReplayContinuationData"):
-            #chat data exist.
+            # chat data exist.
             return None
-        #chat data do not exist, get playerSeekContinuationData.
+        # chat data do not exist, get playerSeekContinuationData.
         init_cont = cont.get("playerSeekContinuationData")
         if init_cont:
             return init_cont.get("continuation")
-        raise ChatParseException('Finished chat data')
+        self.raise_exception(exceptions.ChatDataFinished('Finished chat data'))
 
     def _create_data(self, metadata, contents):
         actions = contents['liveChatContinuation'].get('actions')
         if self.is_replay:
-            interval = self._get_interval(actions)
-            metadata.setdefault("timeoutMs",interval)
+            last_offset_ms = self._get_lastoffset(actions)
+            metadata.setdefault("timeoutMs", 5000)
+            metadata.setdefault("last_offset_ms", last_offset_ms)
             """Archived chat has different structures than live chat,
             so make it the same format."""
             chatdata = [action["replayChatItemAction"]["actions"][0]
-                for action in actions]
+                        for action in actions]
         else:
-            metadata.setdefault('timeoutMs', 10000)
+            metadata.setdefault('timeoutMs', 5000)
             chatdata = actions
         return metadata, chatdata
 
-    def _get_interval(self, actions: list):
-        if actions is None:
-            return 0
-        start = int(actions[0]["replayChatItemAction"]["videoOffsetTimeMsec"])
-        last = int(actions[-1]["replayChatItemAction"]["videoOffsetTimeMsec"])
-        return (last - start)
+    def _get_lastoffset(self, actions: list):
+        if actions:
+            return int(actions[-1]["replayChatItemAction"]["videoOffsetTimeMsec"])
+        return 0
+
+    def raise_exception(self, exception):
+        if self.exception_holder is None:
+            raise exception
+        self.exception_holder = exception

pytchat/parser/replay.py

@@ -1,76 +0,0 @@
-import json
-from .. import config
-from .. exceptions import ( 
-    ResponseContextError, 
-    NoContentsException, 
-    NoContinuationsException )
-
-
-logger = config.logger(__name__)
-
-class Parser:
-    def parse(self, jsn):
-        """
-        このparse関数はReplayChat._listen() 関数から定期的に呼び出される。
-        引数jsnはYoutubeから取得したアーカイブ済みチャットデータの生JSONであり、
-        このparse関数によって与えられたJSONを以下に分割して返す。
-         + timeout (次のチャットデータ取得までのインターバル)
-         + chat data（チャットデータ本体）
-         + continuation （次のチャットデータ取得に必要となるパラメータ）.
-        
-        ライブ配信のチャットとアーカイブ済み動画のチャットは構造が若干異なっているが、
-        ライブチャットと同じデータ形式に変換することにより、
-        同じprocessorでライブとリプレイどちらでも利用できるようにしている。
-
-        Parameter
-        ----------
-        + jsn : dict
-            + Youtubeから取得したチャットデータのJSONオブジェクト。
-              （pythonの辞書形式に変換済みの状態で渡される）
-
-        Returns
-        -------
-        + metadata : dict
-            + チャットデータに付随するメタデータ。timeout、 動画ID、continuationパラメータで構成される。           
-        + chatdata : list[dict]
-            + チャットデータ本体のリスト。
-        """
-        if jsn is None: 
-            return {'timeoutMs':0,'continuation':None},[]
-        if jsn['response']['responseContext'].get('errors'):
-            raise ResponseContextError('動画に接続できません。'
-        '動画IDが間違っているか、動画が削除／非公開の可能性があります。')
-        contents=jsn['response'].get('continuationContents')
-        #配信が終了した場合、もしくはチャットデータが取得できない場合
-        if contents is None:
-            raise NoContentsException('チャットデータを取得できませんでした。')
-
-        cont = contents['liveChatContinuation']['continuations'][0]
-        if cont is None:
-            raise NoContinuationsException('Continuationがありません。')
-        metadata = cont.get('liveChatReplayContinuationData')
-        if metadata is None:
-            unknown = list(cont.keys())[0]
-            if unknown != "playerSeekContinuationData":
-                logger.debug(f"Received unknown continuation type:{unknown}")
-                metadata = cont.get(unknown)
-        actions = contents['liveChatContinuation'].get('actions')
-        if actions is None:
-            #後続のチャットデータなし
-            return {"continuation":None,"timeout":0,"chatdata":[]}
-        interval = self.get_interval(actions)
-        metadata.setdefault("timeoutMs",interval)
-        """アーカイブ済みチャットはライブチャットと構造が異なっているため、以下の行により
-        ライブチャットと同じ形式にそろえる"""
-        chatdata = [action["replayChatItemAction"]["actions"][0] for action in actions]
-        return metadata, chatdata
-
-    def get_interval(self, actions: list):
-        if actions is None:
-            return 0
-        start = int(actions[0]["replayChatItemAction"]["videoOffsetTimeMsec"])
-        last = int(actions[-1]["replayChatItemAction"]["videoOffsetTimeMsec"])
-        return (last - start)
-
-
-

pytchat/processors/chat_processor.py

@@ -3,6 +3,7 @@ class ChatProcessor:
     Abstract class that processes chat data.
     Receive chat data (actions) from Listener.
     '''
+
     def process(self, chat_components: list):
         '''
         Interface that represents processing of chat data.
@@ -21,7 +22,9 @@ class ChatProcessor:
         '''
         pass
 
-
-
-
-
+    def finalize(self, *args, **kwargs):
+        '''
+        Interface for finalizing the process.
+        Called when chat fetching finished.
+        '''
+        pass

pytchat/processors/combinator.py

@@ -1,5 +1,6 @@
 from .chat_processor import ChatProcessor
 
+
 class Combinator(ChatProcessor):
     '''
     Combinator combines multiple chat processors.
@@ -34,6 +35,8 @@ class Combinator(ChatProcessor):
             Tuple of chat data processed by each chat processor.
         '''
         return tuple(processor.process(chat_components)
-            for processor in self.processors)
-            
+                     for processor in self.processors)
 
+    def finalize(self, *args, **kwargs):
+        [processor.finalize(*args, **kwargs)
+                     for processor in self.processors]

pytchat/processors/compatible/processor.py

@@ -1,20 +1,20 @@
-import datetime
-import time
 from .renderer.textmessage import LiveChatTextMessageRenderer
 from .renderer.paidmessage import LiveChatPaidMessageRenderer
 from .renderer.paidsticker import LiveChatPaidStickerRenderer
 from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
+from .renderer.membership import LiveChatMembershipItemRenderer
 from .. chat_processor import ChatProcessor
 from ... import config
 logger = config.logger(__name__)
 
+
 class CompatibleProcessor(ChatProcessor):
 
     def process(self, chat_components: list):
 
         chatlist = []
         timeout = 0
-        ret={}
+        ret = {}
         ret["kind"] = "youtube#liveChatMessageListResponse"
         ret["etag"] = ""
         ret["nextPageToken"] = ""
@@ -24,19 +24,23 @@ class CompatibleProcessor(ChatProcessor):
                 timeout += chat_component.get('timeout', 0)
                 chatdata = chat_component.get('chatdata')
 
-                if chatdata is None: break
+                if chatdata is None:
+                    break
                 for action in chatdata:
-                    if action is None: continue
-                    if action.get('addChatItemAction') is None: continue
-                    if action['addChatItemAction'].get('item') is None: continue
+                    if action is None:
+                        continue
+                    if action.get('addChatItemAction') is None:
+                        continue
+                    if action['addChatItemAction'].get('item') is None:
+                        continue
 
                     chat = self.parse(action)
                     if chat:
                         chatlist.append(chat)
-        ret["pollingIntervalMillis"] = int(timeout*1000)
-        ret["pageInfo"]={
-            "totalResults":len(chatlist),
-            "resultsPerPage":len(chatlist),
+        ret["pollingIntervalMillis"] = int(timeout * 1000)
+        ret["pageInfo"] = {
+            "totalResults": len(chatlist),
+            "resultsPerPage": len(chatlist),
         }
         ret["items"] = chatlist
 
@@ -47,19 +51,20 @@ class CompatibleProcessor(ChatProcessor):
         action = sitem.get("addChatItemAction")
         if action:
             item = action.get("item")
-        if item is None: return None
-        rd={}
+        if item is None:
+            return None
+        rd = {}
         try:
             renderer = self.get_renderer(item)
-            if renderer == None:
+            if renderer is None:
                 return None
 
             rd["kind"] = "youtube#liveChatMessage"
             rd["etag"] = ""
             rd["id"] = 'LCC.' + renderer.get_id()
-            rd["snippet"]       = renderer.get_snippet()
+            rd["snippet"] = renderer.get_snippet()
             rd["authorDetails"] = renderer.get_authordetails()
-        except (KeyError,TypeError,AttributeError) as e:
+        except (KeyError, TypeError, AttributeError) as e:
             logger.error(f"Error: {str(type(e))}-{str(e)}")
             logger.error(f"item: {sitem}")
             return None
@@ -71,11 +76,12 @@ class CompatibleProcessor(ChatProcessor):
             renderer = LiveChatTextMessageRenderer(item)
         elif item.get("liveChatPaidMessageRenderer"):
             renderer = LiveChatPaidMessageRenderer(item)
-        elif item.get( "liveChatPaidStickerRenderer"):
+        elif item.get("liveChatPaidStickerRenderer"):
             renderer = LiveChatPaidStickerRenderer(item)
         elif item.get("liveChatLegacyPaidMessageRenderer"):
             renderer = LiveChatLegacyPaidMessageRenderer(item)
+        elif item.get("liveChatMembershipItemRenderer"):
+            renderer = LiveChatMembershipItemRenderer(item)
         else:
             renderer = None
         return renderer
-

pytchat/processors/compatible/renderer/base.py

@@ -1,83 +1,82 @@
-import datetime, pytz
+from datetime import datetime, timedelta, timezone
+
+TZ_UTC = timezone(timedelta(0), 'UTC')
+
 
 class BaseRenderer:
     def __init__(self, item, chattype):
         self.renderer = list(item.values())[0]
         self.chattype = chattype
 
-
     def get_snippet(self):
 
         message = self.get_message(self.renderer)
 
         return {
-            "type"              : self.chattype,
-            "liveChatId"        : "",
-            "authorChannelId"   : self.renderer.get("authorExternalChannelId"),
-            "publishedAt"       : self.get_publishedat(self.renderer.get("timestampUsec",0)),
-            "hasDisplayContent" : True,
-            "displayMessage"    : message,
+            "type": self.chattype,
+            "liveChatId": "",
+            "authorChannelId": self.renderer.get("authorExternalChannelId"),
+            "publishedAt": self.get_publishedat(self.renderer.get("timestampUsec", 0)),
+            "hasDisplayContent": True,
+            "displayMessage": message,
             "textMessageDetails": {
-                "messageText"   : message  
+                "messageText": message
             }
         }
 
-
     def get_authordetails(self):
         authorExternalChannelId = self.renderer.get("authorExternalChannelId")
-        #parse subscriber type
+        # parse subscriber type
         isVerified, isChatOwner, isChatSponsor, isChatModerator = (
             self.get_badges(self.renderer)
         )
-        return  {
-            "channelId"         : authorExternalChannelId,
-            "channelUrl"        : "http://www.youtube.com/channel/"+authorExternalChannelId,
-            "displayName"       : self.renderer["authorName"]["simpleText"],
-            "profileImageUrl"   : self.renderer["authorPhoto"]["thumbnails"][1]["url"] ,
-            "isVerified"        : isVerified,
-            "isChatOwner"       : isChatOwner,
-            "isChatSponsor"     : isChatSponsor,
-            "isChatModerator"   : isChatModerator
+        return {
+            "channelId": authorExternalChannelId,
+            "channelUrl": "http://www.youtube.com/channel/" + authorExternalChannelId,
+            "displayName": self.renderer["authorName"]["simpleText"],
+            "profileImageUrl": self.renderer["authorPhoto"]["thumbnails"][1]["url"],
+            "isVerified": isVerified,
+            "isChatOwner": isChatOwner,
+            "isChatSponsor": isChatSponsor,
+            "isChatModerator": isChatModerator
         }
 
-
-    def get_message(self,renderer):
+    def get_message(self, renderer):
         message = ''
         if renderer.get("message"):
-            runs=renderer["message"].get("runs")
+            runs = renderer["message"].get("runs")
             if runs:
                 for r in runs:
                     if r:
                         if r.get('emoji'):
-                            message += r['emoji'].get('shortcuts',[''])[0]
+                            message += r['emoji'].get('shortcuts', [''])[0]
                         else:
-                            message += r.get('text','')
+                            message += r.get('text', '')
         return message
 
-    def get_badges(self,renderer):
+    def get_badges(self, renderer):
         isVerified = False
         isChatOwner = False
         isChatSponsor = False
         isChatModerator = False
-        badges=renderer.get("authorBadges")
+        badges = renderer.get("authorBadges")
         if badges:
             for badge in badges:
-                author_type  = badge["liveChatAuthorBadgeRenderer"]["accessibility"]["accessibilityData"]["label"]
-                if author_type == '確認済み':
+                author_type = badge["liveChatAuthorBadgeRenderer"]["accessibility"]["accessibilityData"]["label"]
+                if author_type == 'VERIFIED' or author_type == '確認済み':
                     isVerified = True
-                if author_type == '所有者':
+                if author_type == 'OWNER' or author_type == '所有者':
                     isChatOwner = True
-                if 'メンバー' in author_type:
+                if 'メンバー' in author_type or 'MEMBER' in author_type:
                     isChatSponsor = True
-                if author_type == 'モデレーター':
+                if author_type == 'MODERATOR' or author_type == 'モデレーター':
                     isChatModerator = True
         return isVerified, isChatOwner, isChatSponsor, isChatModerator
 
     def get_id(self):
         return self.renderer.get('id')
 
-    def get_publishedat(self,timestamp):
-        dt = datetime.datetime.fromtimestamp(int(timestamp)/1000000)
-        return dt.astimezone(pytz.utc).isoformat(
-            timespec='milliseconds').replace('+00:00','Z')
-        
+    def get_publishedat(self, timestamp):
+        dt = datetime.fromtimestamp(int(timestamp) / 1000000)
+        return dt.astimezone(TZ_UTC).isoformat(
+            timespec='milliseconds').replace('+00:00', 'Z')

pytchat/processors/compatible/renderer/currency.py

@@ -1,12 +1,12 @@
 '''
-YouTubeスーパーチャットで使用される通貨の記号とレート検索用の略号の
-対応表
+Table of symbols for the currencies used in YouTube Superchat.
+
 Key：
-    YouTubeスーパーチャットで使用される通貨の記号
-    （アルファベットで終わる場合、0xA0(&npsp)が付く）
+     Currency symbols used in YouTube Super Chat
+     If it ends with an alphabet, it will be followed by 0xA0(&npsp).
 Value:
-    fxtext: 3文字の通貨略称
-    jptest: 日本語テキスト
+    fxtext: ISO 4217 currency code
+    jptest: japanese text
 '''
 symbols = {
     "$": {"fxtext": "USD", "jptext": "米・ドル"},

pytchat/processors/compatible/renderer/legacypaid.py

@@ -1,4 +1,6 @@
 from .base import BaseRenderer
+
+
 class LiveChatLegacyPaidMessageRenderer(BaseRenderer):
     def __init__(self, item):
         super().__init__(item, "newSponsorEvent")
@@ -8,36 +10,33 @@ class LiveChatLegacyPaidMessageRenderer(BaseRenderer):
         message = self.get_message(self.renderer)
 
         return {
-            "type"              : self.chattype,
-            "liveChatId"        : "",
-            "authorChannelId"   : self.renderer.get("authorExternalChannelId"),
-            "publishedAt"       : self.get_publishedat(self.renderer.get("timestampUsec",0)),
-            "hasDisplayContent" : True,
-            "displayMessage"    : message,
+            "type": self.chattype,
+            "liveChatId": "",
+            "authorChannelId": self.renderer.get("authorExternalChannelId"),
+            "publishedAt": self.get_publishedat(self.renderer.get("timestampUsec", 0)),
+            "hasDisplayContent": True,
+            "displayMessage": message,
 
         }
 
     def get_authordetails(self):
         authorExternalChannelId = self.renderer.get("authorExternalChannelId")
-        #parse subscriber type
+        # parse subscriber type
         isVerified, isChatOwner, _, isChatModerator = (
             self.get_badges(self.renderer)
         )
-        return  {
-            "channelId"         : authorExternalChannelId,
-            "channelUrl"        : "http://www.youtube.com/channel/"+authorExternalChannelId,
-            "displayName"       : self.renderer["authorName"]["simpleText"],
-            "profileImageUrl"   : self.renderer["authorPhoto"]["thumbnails"][1]["url"] ,
-            "isVerified"        : isVerified,
-            "isChatOwner"       : isChatOwner,
-            "isChatSponsor"     : True,
-            "isChatModerator"   : isChatModerator
+        return {
+            "channelId": authorExternalChannelId,
+            "channelUrl": "http://www.youtube.com/channel/" + authorExternalChannelId,
+            "displayName": self.renderer["authorName"]["simpleText"],
+            "profileImageUrl": self.renderer["authorPhoto"]["thumbnails"][1]["url"],
+            "isVerified": isVerified,
+            "isChatOwner": isChatOwner,
+            "isChatSponsor": True,
+            "isChatModerator": isChatModerator
         }
 
-
-    def get_message(self,renderer):
+    def get_message(self, renderer):
         message = (renderer["eventText"]["runs"][0]["text"]
-            )+' / '+(renderer["detailText"]["simpleText"])
+                   ) + ' / ' + (renderer["detailText"]["simpleText"])
         return message
-
-

pytchat/processors/compatible/renderer/membership.py

@@ -0,0 +1,40 @@
+from .base import BaseRenderer
+
+
+class LiveChatMembershipItemRenderer(BaseRenderer):
+    def __init__(self, item):
+        super().__init__(item, "newSponsorEvent")
+
+    def get_snippet(self):
+        message = self.get_message(self.renderer)
+        return {
+            "type": self.chattype,
+            "liveChatId": "",
+            "authorChannelId": self.renderer.get("authorExternalChannelId"),
+            "publishedAt": self.get_publishedat(self.renderer.get("timestampUsec", 0)),
+            "hasDisplayContent": True,
+            "displayMessage": message,
+
+        }
+
+    def get_authordetails(self):
+        authorExternalChannelId = self.renderer.get("authorExternalChannelId")
+        # parse subscriber type
+        isVerified, isChatOwner, _, isChatModerator = (
+            self.get_badges(self.renderer)
+        )
+        return {
+            "channelId": authorExternalChannelId,
+            "channelUrl": "http://www.youtube.com/channel/" + authorExternalChannelId,
+            "displayName": self.renderer["authorName"]["simpleText"],
+            "profileImageUrl": self.renderer["authorPhoto"]["thumbnails"][1]["url"],
+            "isVerified": isVerified,
+            "isChatOwner": isChatOwner,
+            "isChatSponsor": True,
+            "isChatModerator": isChatModerator
+        }
+
+    def get_message(self, renderer):
+        message = ''.join([mes.get("text", "")
+                           for mes in renderer["headerSubtext"]["runs"]])
+        return message, [message]

pytchat/processors/compatible/renderer/paidmessage.py

@@ -3,6 +3,7 @@ from . import currency
 from .base import BaseRenderer
 superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
 
+
 class LiveChatPaidMessageRenderer(BaseRenderer):
     def __init__(self, item):
         super().__init__(item, "superChatEvent")
@@ -10,31 +11,31 @@ class LiveChatPaidMessageRenderer(BaseRenderer):
     def get_snippet(self):
         authorName = self.renderer["authorName"]["simpleText"]
         message = self.get_message(self.renderer)
-        amountDisplayString, symbol, amountMicros =(
+        amountDisplayString, symbol, amountMicros = (
             self.get_amountdata(self.renderer)
         )
         return {
-            "type"              : self.chattype,
-            "liveChatId"        : "",
-            "authorChannelId"   : self.renderer.get("authorExternalChannelId"),
-            "publishedAt"       : self.get_publishedat(self.renderer.get("timestampUsec",0)),
-            "hasDisplayContent" : True,
-            "displayMessage"    : amountDisplayString+" from "+authorName+': \"'+ message+'\"',
-            "superChatDetails"  : {
-                "amountMicros"          : amountMicros,
-                "currency"              : currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol,
-                "amountDisplayString"   : amountDisplayString,
-                "tier"                  : 0,
-                "backgroundColor" : self.renderer.get("bodyBackgroundColor", 0)
+            "type": self.chattype,
+            "liveChatId": "",
+            "authorChannelId": self.renderer.get("authorExternalChannelId"),
+            "publishedAt": self.get_publishedat(self.renderer.get("timestampUsec", 0)),
+            "hasDisplayContent": True,
+            "displayMessage": amountDisplayString + " from " + authorName + ': \"' + message + '\"',
+            "superChatDetails": {
+                "amountMicros": amountMicros,
+                "currency": currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol,
+                "amountDisplayString": amountDisplayString,
+                "tier": 0,
+                "backgroundColor": self.renderer.get("bodyBackgroundColor", 0)
             }
         }
 
-    def get_amountdata(self,renderer):
+    def get_amountdata(self, renderer):
         amountDisplayString = renderer["purchaseAmountText"]["simpleText"]
         m = superchat_regex.search(amountDisplayString)
         if m:
             symbol = m.group(1)
-            amountMicros = int(float(m.group(2).replace(',',''))*1000000)
+            amountMicros = int(float(m.group(2).replace(',', '')) * 1000000)
         else:
             symbol = ""
             amountMicros = 0

pytchat/processors/compatible/renderer/paidsticker.py

@@ -3,46 +3,45 @@ from . import currency
 from .base import BaseRenderer
 superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
 
+
 class LiveChatPaidStickerRenderer(BaseRenderer):
     def __init__(self, item):
         super().__init__(item, "superStickerEvent")
 
     def get_snippet(self):
         authorName = self.renderer["authorName"]["simpleText"]
-        amountDisplayString, symbol, amountMicros =(
+        amountDisplayString, symbol, amountMicros = (
             self.get_amountdata(self.renderer)
         )
 
         return {
-            "type"              : self.chattype,
-            "liveChatId"        : "",
-            "authorChannelId"   : self.renderer.get("authorExternalChannelId"),
-            "publishedAt"       : self.get_publishedat(self.renderer.get("timestampUsec",0)),
-            "hasDisplayContent" : True,
-            "displayMessage"    : "Super Sticker " + amountDisplayString + " from "+authorName,
-            "superStickerDetails"  : {
-                "superStickerMetaData" : {
+            "type": self.chattype,
+            "liveChatId": "",
+            "authorChannelId": self.renderer.get("authorExternalChannelId"),
+            "publishedAt": self.get_publishedat(self.renderer.get("timestampUsec", 0)),
+            "hasDisplayContent": True,
+            "displayMessage": "Super Sticker " + amountDisplayString + " from " + authorName,
+            "superStickerDetails": {
+                "superStickerMetaData": {
                     "stickerId": "",
                     "altText": "",
                     "language": ""
                 },
-                "amountMicros"          : amountMicros,
-                "currency"              : currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol,
-                "amountDisplayString"   : amountDisplayString,
-                "tier"                  : 0,
-                "backgroundColor" : self.renderer.get("bodyBackgroundColor", 0)
+                "amountMicros": amountMicros,
+                "currency": currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol,
+                "amountDisplayString": amountDisplayString,
+                "tier": 0,
+                "backgroundColor": self.renderer.get("bodyBackgroundColor", 0)
             }
         }
 
-    def get_amountdata(self,renderer):
+    def get_amountdata(self, renderer):
         amountDisplayString = renderer["purchaseAmountText"]["simpleText"]
         m = superchat_regex.search(amountDisplayString)
         if m:
             symbol = m.group(1)
-            amountMicros = int(float(m.group(2).replace(',',''))*1000000)
+            amountMicros = int(float(m.group(2).replace(',', '')) * 1000000)
         else:
             symbol = ""
             amountMicros = 0
         return amountDisplayString, symbol, amountMicros
- 
-

pytchat/processors/compatible/renderer/textmessage.py

@@ -1,4 +1,6 @@
 from .base import BaseRenderer
+
+
 class LiveChatTextMessageRenderer(BaseRenderer):
     def __init__(self, item):
         super().__init__(item, "textMessageEvent")

pytchat/processors/default/custom_encoder.py

@@ -0,0 +1,11 @@
+import json
+from .renderer.base import Author
+from .renderer.paidmessage import Colors
+from .renderer.paidsticker import Colors2
+
+
+class CustomEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, Author) or isinstance(obj, Colors) or isinstance(obj, Colors2):
+            return vars(obj)
+        return json.JSONEncoder.default(self, obj)

pytchat/processors/default/processor.py

@@ -1,31 +1,132 @@
 import asyncio
+import json
 import time
+from .custom_encoder import CustomEncoder
 from .renderer.textmessage import LiveChatTextMessageRenderer
 from .renderer.paidmessage import LiveChatPaidMessageRenderer
 from .renderer.paidsticker import LiveChatPaidStickerRenderer
 from .renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
+from .renderer.membership import LiveChatMembershipItemRenderer
 from .. chat_processor import ChatProcessor
 from ... import config
+
 logger = config.logger(__name__)
 
+
+class Chat:
+    def json(self) -> str:
+        return json.dumps(vars(self), ensure_ascii=False, cls=CustomEncoder)
+
+
 class Chatdata:
-    def __init__(self,chatlist:list, timeout:float):
+
+    def __init__(self, chatlist: list, timeout: float, abs_diff):
         self.items = chatlist
         self.interval = timeout
+        self.abs_diff = abs_diff
+        self.itemcount = 0
 
     def tick(self):
-        if self.interval == 0:
+        '''DEPRECATE
+            Use sync_items()
+        '''
+        if len(self.items) < 1:
             time.sleep(1)
             return
-        time.sleep(self.interval/len(self.items))
+        if self.itemcount == 0:
+            self.starttime = time.time()
+        if len(self.items) == 1:
+            total_itemcount = 1
+        else:
+            total_itemcount = len(self.items) - 1
+        next_chattime = (self.items[0].timestamp + (self.items[-1].timestamp - self.items[0].timestamp) / total_itemcount * self.itemcount) / 1000
+        tobe_disptime = self.abs_diff + next_chattime
+        wait_sec = tobe_disptime - time.time()
+        self.itemcount += 1
+        
+        if wait_sec < 0:
+            wait_sec = 0
+       
+        time.sleep(wait_sec)
 
     async def tick_async(self):
-        if self.interval == 0:
+        '''DEPRECATE
+            Use async_items()
+        '''
+        if len(self.items) < 1:
             await asyncio.sleep(1)
             return
-        await asyncio.sleep(self.interval/len(self.items))
+        if self.itemcount == 0:
+            self.starttime = time.time()
+        if len(self.items) == 1:
+            total_itemcount = 1
+        else:
+            total_itemcount = len(self.items) - 1
+        next_chattime = (self.items[0].timestamp + (self.items[-1].timestamp - self.items[0].timestamp) / total_itemcount * self.itemcount) / 1000
+        tobe_disptime = self.abs_diff + next_chattime
+        wait_sec = tobe_disptime - time.time()
+        self.itemcount += 1
+        
+        if wait_sec < 0:
+            wait_sec = 0
+       
+        await asyncio.sleep(wait_sec)
+
+    def sync_items(self):
+        starttime = time.time()
+        if len(self.items) > 0:
+            last_chattime = self.items[-1].timestamp / 1000
+            tobe_disptime = self.abs_diff + last_chattime
+            wait_total_sec = max(tobe_disptime - time.time(), 0)
+            if len(self.items) > 1:
+                wait_sec = wait_total_sec / len(self.items)
+            elif len(self.items) == 1:
+                wait_sec = 0
+            for c in self.items:
+                if wait_sec < 0:
+                    wait_sec = 0
+                time.sleep(wait_sec)
+                yield c
+        stop_interval = time.time() - starttime
+        if stop_interval < 1:
+            time.sleep(1 - stop_interval)
+
+    async def async_items(self):
+        starttime = time.time()
+        if len(self.items) > 0:
+            last_chattime = self.items[-1].timestamp / 1000
+            tobe_disptime = self.abs_diff + last_chattime
+            wait_total_sec = max(tobe_disptime - time.time(), 0)
+            if len(self.items) > 1:
+                wait_sec = wait_total_sec / len(self.items)
+            elif len(self.items) == 1:
+                wait_sec = 0
+            for c in self.items:
+                if wait_sec < 0:
+                    wait_sec = 0
+                await asyncio.sleep(wait_sec)
+                yield c
+                
+        stop_interval = time.time() - starttime
+        if stop_interval < 1:
+            await asyncio.sleep(1 - stop_interval)
+
+    def json(self) -> str:
+        return ''.join(("[", ','.join((a.json() for a in self.items)), "]"))
+
 
 class DefaultProcessor(ChatProcessor):
+    def __init__(self):
+        self.first = True
+        self.abs_diff = 0
+        self.renderers = {
+            "liveChatTextMessageRenderer": LiveChatTextMessageRenderer(),
+            "liveChatPaidMessageRenderer": LiveChatPaidMessageRenderer(),
+            "liveChatPaidStickerRenderer": LiveChatPaidStickerRenderer(),
+            "liveChatLegacyPaidMessageRenderer": LiveChatLegacyPaidMessageRenderer(),
+            "liveChatMembershipItemRenderer": LiveChatMembershipItemRenderer()
+        }
+
     def process(self, chat_components: list):
 
         chatlist = []
@@ -33,47 +134,46 @@ class DefaultProcessor(ChatProcessor):
 
         if chat_components:
             for component in chat_components:
+                if component is None:
+                    continue
                 timeout += component.get('timeout', 0)
-                chatdata = component.get('chatdata')
-                if chatdata is None: continue
+                chatdata = component.get('chatdata')  # if from Extractor, chatdata is generator.
+                if chatdata is None:
+                    continue
                 for action in chatdata:
-                    if action is None: continue
-                    if action.get('addChatItemAction') is None: continue
-                    if action['addChatItemAction'].get('item') is None: continue
-
-                    chat = self._parse(action)
+                    if action is None:
+                        continue
+                    if action.get('addChatItemAction') is None:
+                        continue
+                    item = action['addChatItemAction'].get('item')
+                    if item is None:
+                        continue
+                    chat = self._parse(item)
                     if chat:
                         chatlist.append(chat)
-        return Chatdata(chatlist, float(timeout))
         
+        if self.first and chatlist:
+            self.abs_diff = time.time() - chatlist[0].timestamp / 1000
+            self.first = False
 
-    def _parse(self, sitem):
+        chatdata = Chatdata(chatlist, float(timeout), self.abs_diff)
 
-        action = sitem.get("addChatItemAction")
-        if action:
-            item = action.get("item")
-        if item is None: return None
+        return chatdata
+
+    def _parse(self, item):
         try:
-            renderer = self._get_renderer(item)
-            if renderer == None:
+            key = list(item.keys())[0]
+            renderer = self.renderers.get(key)
+            if renderer is None:
                 return None
-
+            renderer.setitem(item.get(key), Chat())
+            renderer.settype()
             renderer.get_snippet()
             renderer.get_authordetails()
-        except (KeyError,TypeError) as e:
-            logger.error(f"{str(type(e))}-{str(e)} sitem:{str(sitem)}")
+            rendered_chatobj = renderer.get_chatobj()
+            renderer.clear()
+        except (KeyError, TypeError) as e:
+            logger.error(f"{str(type(e))}-{str(e)} item:{str(item)}")
             return None
-        return renderer        
         
-    def _get_renderer(self, item):
-        if item.get("liveChatTextMessageRenderer"):
-            renderer = LiveChatTextMessageRenderer(item)
-        elif item.get("liveChatPaidMessageRenderer"):
-            renderer = LiveChatPaidMessageRenderer(item)
-        elif item.get( "liveChatPaidStickerRenderer"):
-            renderer = LiveChatPaidStickerRenderer(item)
-        elif item.get("liveChatLegacyPaidMessageRenderer"):
-            renderer = LiveChatLegacyPaidMessageRenderer(item)
-        else:
-            renderer = None
-        return renderer
+        return rendered_chatobj

pytchat/processors/default/renderer/base.py

@@ -1,90 +1,101 @@
 from datetime import datetime
+
+
 class Author:
     pass
+
+
 class BaseRenderer:
-    def __init__(self, item, chattype):
-        self.renderer = list(item.values())[0]
-        self.chattype = chattype
-        self.author = Author()
+    def setitem(self, item, chat):
+        self.item = item
+        self.chat = chat
+        self.chat.author = Author()
+
+    def settype(self):
+        pass
 
     def get_snippet(self):
-        self.type = self.chattype
-        self.id = self.renderer.get('id')
-        timestampUsec = int(self.renderer.get("timestampUsec",0))
-        self.timestamp = int(timestampUsec/1000)
-        tst = self.renderer.get("timestampText")
+        self.chat.id = self.item.get('id')
+        timestampUsec = int(self.item.get("timestampUsec", 0))
+        self.chat.timestamp = int(timestampUsec / 1000)
+        tst = self.item.get("timestampText")
         if tst:
-            self.elapsedTime = tst.get("simpleText")
+            self.chat.elapsedTime = tst.get("simpleText")
         else:
-            self.elapsedTime = ""
-        self.datetime = self.get_datetime(timestampUsec)
-        self.message ,self.messageEx = self.get_message(self.renderer)
-        self.id =  self.renderer.get('id')
-        self.amountValue= 0.0
-        self.amountString = ""
-        self.currency= ""
-        self.bgColor = 0
+            self.chat.elapsedTime = ""
+        self.chat.datetime = self.get_datetime(timestampUsec)
+        self.chat.message, self.chat.messageEx = self.get_message(self.item)
+        self.chat.id = self.item.get('id')
+        self.chat.amountValue = 0.0
+        self.chat.amountString = ""
+        self.chat.currency = ""
+        self.chat.bgColor = 0
 
     def get_authordetails(self):
-        self.author.badgeUrl = ""
-        (self.author.isVerified, 
-        self.author.isChatOwner, 
-        self.author.isChatSponsor, 
-        self.author.isChatModerator) = (
-            self.get_badges(self.renderer)
+        self.chat.author.badgeUrl = ""
+        (self.chat.author.isVerified,
+         self.chat.author.isChatOwner,
+         self.chat.author.isChatSponsor,
+         self.chat.author.isChatModerator) = (
+            self.get_badges(self.item)
         )
-        self.author.channelId = self.renderer.get("authorExternalChannelId")
-        self.author.channelUrl = "http://www.youtube.com/channel/"+self.author.channelId
-        self.author.name      = self.renderer["authorName"]["simpleText"]
-        self.author.imageUrl= self.renderer["authorPhoto"]["thumbnails"][1]["url"] 
+        self.chat.author.channelId = self.item.get("authorExternalChannelId")
+        self.chat.author.channelUrl = "http://www.youtube.com/channel/" + self.chat.author.channelId
+        self.chat.author.name = self.item["authorName"]["simpleText"]
+        self.chat.author.imageUrl = self.item["authorPhoto"]["thumbnails"][1]["url"]
 
-
-
-    def get_message(self,renderer):
+    def get_message(self, item):
         message = ''
         message_ex = []
-        if renderer.get("message"):
-            runs=renderer["message"].get("runs")
-            if runs:
-                for r in runs:
-                    if r:
-                        if r.get('emoji'):
-                            message += r['emoji'].get('shortcuts',[''])[0]
-                            message_ex.append(r['emoji']['image']['thumbnails'][1].get('url'))
-                        else:
-                            message += r.get('text','')
-                            message_ex.append(r.get('text',''))
+        runs = item.get("message", {}).get("runs", {})
+        for r in runs:
+            if not hasattr(r, "get"):
+                continue
+            if r.get('emoji'):
+                message += r['emoji'].get('shortcuts', [''])[0]
+                message_ex.append({
+                    'id': r['emoji'].get('emojiId').split('/')[-1],
+                    'txt': r['emoji'].get('shortcuts', [''])[0],
+                    'url': r['emoji']['image']['thumbnails'][0].get('url')
+                })
+            else:
+                message += r.get('text', '')
+                message_ex.append(r.get('text', ''))
         return message, message_ex
 
-
-
-    def get_badges(self,renderer):
+    def get_badges(self, renderer):
+        self.chat.author.type = ''
         isVerified = False
         isChatOwner = False
         isChatSponsor = False
         isChatModerator = False
-        badges=renderer.get("authorBadges")
-        if badges:
-            for badge in badges:
-                if badge["liveChatAuthorBadgeRenderer"].get("icon"):
-                    author_type  = badge["liveChatAuthorBadgeRenderer"]["icon"]["iconType"]
-                    if author_type == 'VERIFIED':
-                        isVerified = True
-                    if author_type == 'OWNER':
-                        isChatOwner = True
-                    if author_type == 'MODERATOR':
-                        isChatModerator = True
-                if badge["liveChatAuthorBadgeRenderer"].get("customThumbnail"):
-                    isChatSponsor = True
-                    self.get_badgeurl(badge)
+        badges = renderer.get("authorBadges", {})
+        for badge in badges:
+            if badge["liveChatAuthorBadgeRenderer"].get("icon"):
+                author_type = badge["liveChatAuthorBadgeRenderer"]["icon"]["iconType"]
+                self.chat.author.type = author_type
+                if author_type == 'VERIFIED':
+                    isVerified = True
+                if author_type == 'OWNER':
+                    isChatOwner = True
+                if author_type == 'MODERATOR':
+                    isChatModerator = True
+            if badge["liveChatAuthorBadgeRenderer"].get("customThumbnail"):
+                isChatSponsor = True
+                self.chat.author.type = 'MEMBER'
+                self.get_badgeurl(badge)
         return isVerified, isChatOwner, isChatSponsor, isChatModerator
 
+    def get_badgeurl(self, badge):
+        self.chat.author.badgeUrl = badge["liveChatAuthorBadgeRenderer"]["customThumbnail"]["thumbnails"][0]["url"]
 
-    def get_badgeurl(self,badge):
-        self.author.badgeUrl = badge["liveChatAuthorBadgeRenderer"]["customThumbnail"]["thumbnails"][0]["url"]
-
-
-
-    def get_datetime(self,timestamp):
-        dt = datetime.fromtimestamp(timestamp/1000000)
+    def get_datetime(self, timestamp):
+        dt = datetime.fromtimestamp(timestamp / 1000000)
         return dt.strftime('%Y-%m-%d %H:%M:%S')
+
+    def get_chatobj(self):
+        return self.chat
+
+    def clear(self):
+        self.item = None
+        self.chat = None

pytchat/processors/default/renderer/legacypaid.py

@@ -1,18 +1,15 @@
 from .base import BaseRenderer
+
+
 class LiveChatLegacyPaidMessageRenderer(BaseRenderer):
-    def __init__(self, item):
-        super().__init__(item, "newSponsor")
-
-        
+    def settype(self):
+        self.chat.type = "newSponsor"
 
     def get_authordetails(self):
         super().get_authordetails()
-        self.author.isChatSponsor =  True
-
-
-    def get_message(self,renderer):
-        message = (renderer["eventText"]["runs"][0]["text"]
-            )+' / '+(renderer["detailText"]["simpleText"])
-        return message
-
+        self.chat.author.isChatSponsor = True
 
+    def get_message(self, item):
+        message = (item["eventText"]["runs"][0]["text"]
+                   ) + ' / ' + (item["detailText"]["simpleText"])
+        return message, [message]

pytchat/processors/default/renderer/membership.py

@@ -0,0 +1,18 @@
+from .base import BaseRenderer
+
+
+class LiveChatMembershipItemRenderer(BaseRenderer):
+    def settype(self):
+        self.chat.type = "newSponsor"
+
+    def get_authordetails(self):
+        super().get_authordetails()
+        self.chat.author.isChatSponsor = True
+
+    def get_message(self, item):
+        try:
+            message = ''.join([mes.get("text", "")
+                           for mes in item["headerSubtext"]["runs"]])
+        except KeyError:
+            return "Welcome New Member!", ["Welcome New Member!"]
+        return message, [message]

pytchat/processors/default/renderer/paidmessage.py

@@ -3,30 +3,45 @@ from . import currency
 from .base import BaseRenderer
 superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
 
-class LiveChatPaidMessageRenderer(BaseRenderer):
-    def __init__(self, item):
-        super().__init__(item, "superChat")
 
+class Colors:
+    pass
+
+
+class LiveChatPaidMessageRenderer(BaseRenderer):
+    def settype(self):
+        self.chat.type = "superChat"
 
     def get_snippet(self):
         super().get_snippet()
-        amountDisplayString, symbol, amount =(
-            self.get_amountdata(self.renderer)
+        amountDisplayString, symbol, amount = (
+            self.get_amountdata(self.item)
         )
-        self.amountValue= amount
-        self.amountString = amountDisplayString
-        self.currency= currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol
-        self.bgColor= self.renderer.get("bodyBackgroundColor", 0)
+        self.chat.amountValue = amount
+        self.chat.amountString = amountDisplayString
+        self.chat.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get(
+            symbol) else symbol
+        self.chat.bgColor = self.item.get("bodyBackgroundColor", 0)
+        self.chat.colors = self.get_colors()
 
-
-
-    def get_amountdata(self,renderer):
-        amountDisplayString = renderer["purchaseAmountText"]["simpleText"]
+    def get_amountdata(self, item):
+        amountDisplayString = item["purchaseAmountText"]["simpleText"]
         m = superchat_regex.search(amountDisplayString)
         if m:
             symbol = m.group(1)
-            amount = float(m.group(2).replace(',',''))
+            amount = float(m.group(2).replace(',', ''))
         else:
             symbol = ""
             amount = 0.0
         return amountDisplayString, symbol, amount
+
+    def get_colors(self):
+        item = self.item
+        colors = Colors()
+        colors.headerBackgroundColor = item.get("headerBackgroundColor", 0)
+        colors.headerTextColor = item.get("headerTextColor", 0)
+        colors.bodyBackgroundColor = item.get("bodyBackgroundColor", 0)
+        colors.bodyTextColor = item.get("bodyTextColor", 0)
+        colors.timestampColor = item.get("timestampColor", 0)
+        colors.authorNameTextColor = item.get("authorNameTextColor", 0)
+        return colors

pytchat/processors/default/renderer/paidsticker.py

@@ -3,37 +3,45 @@ from . import currency
 from .base import BaseRenderer
 superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
 
-class LiveChatPaidStickerRenderer(BaseRenderer):
-    def __init__(self, item):
-        super().__init__(item, "superSticker")
 
+class Colors2:
+    pass
+
+
+class LiveChatPaidStickerRenderer(BaseRenderer):
+    def settype(self):
+        self.chat.type = "superSticker"
 
     def get_snippet(self):
         super().get_snippet()
-        amountDisplayString, symbol, amount =(
-            self.get_amountdata(self.renderer)
+        amountDisplayString, symbol, amount = (
+            self.get_amountdata(self.item)
         )
-        self.amountValue = amount
-        self.amountString = amountDisplayString
-        self.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol
-        self.bgColor = self.renderer.get("moneyChipBackgroundColor", 0)
-        self.sticker = "https:"+self.renderer["sticker"]["thumbnails"][0]["url"]
+        self.chat.amountValue = amount
+        self.chat.amountString = amountDisplayString
+        self.chat.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get(
+            symbol) else symbol
+        self.chat.bgColor = self.item.get("backgroundColor", 0)
+        self.chat.sticker = "".join(("https:",
+            self.item["sticker"]["thumbnails"][0]["url"]))
+        self.chat.colors = self.get_colors()
 
-
-
-    def get_amountdata(self,renderer):
-        amountDisplayString = renderer["purchaseAmountText"]["simpleText"]
+    def get_amountdata(self, item):
+        amountDisplayString = item["purchaseAmountText"]["simpleText"]
         m = superchat_regex.search(amountDisplayString)
         if m:
             symbol = m.group(1)
-            amount = float(m.group(2).replace(',',''))
+            amount = float(m.group(2).replace(',', ''))
         else:
             symbol = ""
             amount = 0.0
         return amountDisplayString, symbol, amount
 
-
- 
-
- 
-
+    def get_colors(self):
+        item = self.item
+        colors = Colors2()
+        colors.moneyChipBackgroundColor = item.get("moneyChipBackgroundColor", 0)
+        colors.moneyChipTextColor = item.get("moneyChipTextColor", 0)
+        colors.backgroundColor = item.get("backgroundColor", 0)
+        colors.authorNameTextColor = item.get("authorNameTextColor", 0)
+        return colors

pytchat/processors/default/renderer/textmessage.py

@@ -1,4 +1,6 @@
 from .base import BaseRenderer
+
+
 class LiveChatTextMessageRenderer(BaseRenderer):
-    def __init__(self, item):
-        super().__init__(item, "textMessage")
+    def settype(self):
+        self.chat.type = "textMessage"

pytchat/processors/dummy_processor.py

@@ -1,8 +1,10 @@
 from .chat_processor import ChatProcessor
 
+
 class DummyProcessor(ChatProcessor):
     '''
     Dummy processor just returns received chat_components directly.
     '''
+
     def process(self, chat_components: list):
         return chat_components

pytchat/processors/html_archiver.py

@@ -0,0 +1,170 @@
+import httpx
+import os
+import re
+import time
+from base64 import standard_b64encode
+from concurrent.futures import ThreadPoolExecutor
+from .chat_processor import ChatProcessor
+from .default.processor import DefaultProcessor
+from ..exceptions import UnknownConnectionError
+import tempfile
+
+PATTERN = re.compile(r"(.*)\(([0-9]+)\)$")
+
+fmt_headers = ['datetime', 'elapsed', 'authorName',
+               'message', 'superchat', 'type', 'authorChannel']
+
+HEADER_HTML = '''
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html;charset=UTF-8">
+'''
+
+TABLE_CSS = '''
+table.css {
+ border-collapse: collapse;
+}
+ 
+table.css thead{
+ border-collapse: collapse;
+ border: 1px solid #000
+}
+ 
+table.css tr td{
+ padding: 0.3em;
+ border: 1px solid #000
+}
+
+table.css th{
+ padding: 0.3em;
+ border: 1px solid #000
+}
+'''
+
+
+class HTMLArchiver(ChatProcessor):
+    '''
+    HTMLArchiver saves chat data as HTML table format.
+    '''
+    def __init__(self, save_path, callback=None):
+        super().__init__()
+        self.client = httpx.Client(http2=True)
+        self.save_path = self._checkpath(save_path)
+        self.processor = DefaultProcessor()
+        self.emoji_table = {}  # dict for custom emojis. key: emoji_id, value: base64 encoded image binary.
+        self.callback = callback
+        self.executor = ThreadPoolExecutor(max_workers=10)
+        self.tmp_fp = tempfile.NamedTemporaryFile(mode="a", encoding="utf-8", delete=False)
+        self.tmp_filename = self.tmp_fp.name
+        self.counter = 0
+
+    def _checkpath(self, filepath):
+        splitter = os.path.splitext(os.path.basename(filepath))
+        body = splitter[0]
+        extention = splitter[1]
+        newpath = filepath
+        counter = 1
+        while os.path.exists(newpath):
+            match = re.search(PATTERN, body)
+            if match:
+                counter = int(match[2]) + 1
+                num_with_bracket = f'({str(counter)})'
+                body = f'{match[1]}{num_with_bracket}'
+            else:
+                body = f'{body}({str(counter)})'
+            newpath = os.path.join(os.path.dirname(filepath), body + extention)
+        return newpath
+
+    def process(self, chat_components: list):
+        """
+        Returns
+        ----------
+        dict :
+            save_path : str :
+                Actual save path of file.
+            total_lines : int :
+                Count of total lines written to the file.
+        """
+        if chat_components is None or len(chat_components) == 0:
+            return self.save_path, self.counter
+        for c in self.processor.process(chat_components).items:
+            self.tmp_fp.write(
+                self._parse_html_line((
+                    c.datetime,
+                    c.elapsedTime,
+                    c.author.name,
+                    self._parse_message(c.messageEx),
+                    c.amountString,
+                    c.author.type,
+                    c.author.channelId)
+                )
+            )
+            if self.callback:
+                self.callback(None, 1)
+            self.counter += 1
+        return self.save_path, self.counter
+
+    def _parse_html_line(self, raw_line):
+        return ''.join(('<tr>',
+                        ''.join(''.join(('<td>', cell, '</td>')) for cell in raw_line),
+                        '</tr>\n'))
+
+    def _parse_table_header(self, raw_line):
+        return ''.join(('<thead><tr>',
+                        ''.join(''.join(('<th>', cell, '</th>')) for cell in raw_line),
+                        '</tr></thead>\n'))
+        
+    def _parse_message(self, message_items: list) -> str:
+        return ''.join(''.join(('<span class="', self._set_emoji_table(item), '" title="', item['txt'], '"></span>'))
+                       if type(item) is dict else item
+                       for item in message_items)
+
+    def _encode_img(self, url):
+        err = None
+        for _ in range(5):
+            try:
+                resp = self.client.get(url, timeout=30)
+                break
+            except httpx.HTTPError as e:
+                err = e
+                time.sleep(3)
+        else:
+            raise UnknownConnectionError(str(err))
+
+        return standard_b64encode(resp.content).decode()
+
+    def _set_emoji_table(self, item: dict):
+        emoji_id = ''.join(('Z', item['id'])) if 48 <= ord(item['id'][0]) <= 57 else item['id']
+        if emoji_id not in self.emoji_table:
+            self.emoji_table.setdefault(emoji_id, self.executor.submit(self._encode_img, item['url']))
+        return emoji_id
+
+    def _stylecode(self, name, code, width, height):
+        return ''.join((".", name, " { display: inline-block; background-image: url(data:image/png;base64,",
+                        code, "); background-repeat: no-repeat; width: ",
+                        str(width), "; height: ", str(height), ";}"))
+    
+    def _create_styles(self):
+        return '\n'.join(('<style type="text/css">',
+                          TABLE_CSS,
+                          '\n'.join(self._stylecode(key, self.emoji_table[key].result(), 24, 24)
+                                for key in self.emoji_table.keys()),
+                          '</style>\n'))
+    
+    def finalize(self):
+        if self.tmp_fp:
+            self.tmp_fp.flush()
+            self.tmp_fp = None
+        with open(self.save_path, mode='w', encoding='utf-8') as outfile:
+            # write header
+            outfile.writelines((
+                HEADER_HTML, self._create_styles(), '</head>\n',
+                '<body>\n', '<table class="css">\n',
+                self._parse_table_header(fmt_headers)))
+            # write body
+            fp = open(self.tmp_filename, mode="r", encoding="utf-8")
+            for line in fp:
+                outfile.write(line)
+            outfile.write('</table>\n</body>\n</html>')
+            fp.close()
+        os.remove(self.tmp_filename)

pytchat/processors/jsonfile_archiver.py

@@ -1,10 +1,10 @@
-import datetime
 import json
 import os
 import re
 from .chat_processor import ChatProcessor
 
-PATTERN  = re.compile(r"(.*)\(([0-9]+)\)$")
+PATTERN = re.compile(r"(.*)\(([0-9]+)\)$")
+
 
 class JsonfileArchiver(ChatProcessor):
     """
@@ -17,12 +17,13 @@ class JsonfileArchiver(ChatProcessor):
         it is automatically saved under a different name
         with suffix '(number)'
     """
-    def __init__(self,save_path):
+
+    def __init__(self, save_path):
         super().__init__()
         self.save_path = self._checkpath(save_path)
         self.line_counter = 0
 
-    def process(self,chat_components: list):
+    def process(self, chat_components: list):
         """
         Returns
         ----------
@@ -32,19 +33,23 @@ class JsonfileArchiver(ChatProcessor):
             total_lines : int :
                 count of total lines written to the file.
         """
-        if chat_components is None: return
-        with open(self.save_path, mode='a', encoding = 'utf-8') as f:
+        if chat_components is None:
+            return
+        with open(self.save_path, mode='a', encoding='utf-8') as f:
             for component in chat_components:
-                if component is None: continue
+                if component is None:
+                    continue
                 chatdata = component.get('chatdata')
-                if chatdata is None: continue
+                if chatdata is None:
+                    continue
                 for action in chatdata:
-                    if action is None: continue
-                    json_line = json.dumps(action, ensure_ascii = False)
-                    f.writelines(json_line+'\n')
-                    self.line_counter+=1
-            return { "save_path"  : self.save_path,
-                     "total_lines": self.line_counter }
+                    if action is None:
+                        continue
+                    json_line = json.dumps(action, ensure_ascii=False)
+                    f.writelines(json_line + '\n')
+                    self.line_counter += 1
+            return {"save_path": self.save_path,
+                    "total_lines": self.line_counter}
 
     def _checkpath(self, filepath):
         splitter = os.path.splitext(os.path.basename(filepath))
@@ -53,14 +58,12 @@ class JsonfileArchiver(ChatProcessor):
         newpath = filepath
         counter = 0
         while os.path.exists(newpath):
-            match = re.search(PATTERN,body)
+            match = re.search(PATTERN, body)
             if match:
-                counter=int(match[2])+1
+                counter = int(match[2]) + 1
                 num_with_bracket = f'({str(counter)})'
                 body = f'{match[1]}{num_with_bracket}'
             else:
                 body = f'{body}({str(counter)})'
-            newpath = os.path.join(os.path.dirname(filepath),body+extention)
+            newpath = os.path.join(os.path.dirname(filepath), body + extention)
         return newpath
-
-

pytchat/processors/simple_display_processor.py

@@ -1,10 +1,6 @@
-import json
-import os
-import traceback
-import datetime
-import time
 from .chat_processor import ChatProcessor
-##version 2
+
+
 class SimpleDisplayProcessor(ChatProcessor):
 
     def process(self, chat_components: list):
@@ -12,36 +8,42 @@ class SimpleDisplayProcessor(ChatProcessor):
         timeout = 0
 
         if chat_components is None:
-            return {"timeout":timeout, "chatlist":chatlist}
+            return {"timeout": timeout, "chatlist": chatlist}
         for component in chat_components:
             timeout += component.get('timeout', 0)
             chatdata = component.get('chatdata')
 
-            if chatdata is None:break
+            if chatdata is None:
+                break
             for action in chatdata:
-                if action is None:continue
-                if action.get('addChatItemAction') is None:continue
-                if action['addChatItemAction'].get('item') is None:continue
+                if action is None:
+                    continue
+                if action.get('addChatItemAction') is None:
+                    continue
+                if action['addChatItemAction'].get('item') is None:
+                    continue
 
-                root = action['addChatItemAction']['item'].get('liveChatTextMessageRenderer')
+                root = action['addChatItemAction']['item'].get(
+                    'liveChatTextMessageRenderer')
 
                 if root:
                     author_name = root['authorName']['simpleText']
                     message = self._parse_message(root.get('message'))
                     purchase_amount_text = ''
                 else:
-                    root = ( action['addChatItemAction']['item'].get('liveChatPaidMessageRenderer') or 
-                            action['addChatItemAction']['item'].get('liveChatPaidStickerRenderer') )
+                    root = (action['addChatItemAction']['item'].get('liveChatPaidMessageRenderer')
+                            or action['addChatItemAction']['item'].get('liveChatPaidStickerRenderer'))
                     if root:
                         author_name = root['authorName']['simpleText']
                         message = self._parse_message(root.get('message'))
                         purchase_amount_text = root['purchaseAmountText']['simpleText']
                     else:
                         continue
-                chatlist.append(f'[{author_name}]:  {message}  {purchase_amount_text}')
-        return {"timeout":timeout, "chatlist":chatlist}
+                chatlist.append(
+                    f'[{author_name}]:  {message}  {purchase_amount_text}')
+        return {"timeout": timeout, "chatlist": chatlist}
 
-    def _parse_message(self,message):
+    def _parse_message(self, message):
         if message is None:
             return ''
         if message.get('simpleText'):
@@ -51,11 +53,9 @@ class SimpleDisplayProcessor(ChatProcessor):
             tmp = ''
             for run in runs:
                 if run.get('emoji'):
-                    tmp+=(run['emoji']['shortcuts'][0])
+                    tmp += (run['emoji']['shortcuts'][0])
                 elif run.get('text'):
-                    tmp+=(run['text'])
+                    tmp += (run['text'])
             return tmp
         else:
             return ''
-
-

pytchat/processors/speed/calculator.py

@@ -5,6 +5,8 @@ Calculate speed of chat.
 """
 import time
 from .. chat_processor import ChatProcessor
+
+
 class RingQueue:
     """
     リング型キュー
@@ -50,17 +52,17 @@ class RingQueue:
         """
         if self.mergin:
             self.items.append(item)
-            self.last_pos = len(self.items)-1
-            if self.last_pos == self.capacity-1:
+            self.last_pos = len(self.items) - 1
+            if self.last_pos == self.capacity - 1:
                 self.mergin = False
             return
         self.last_pos += 1
-        if self.last_pos > self.capacity-1:
+        if self.last_pos > self.capacity - 1:
             self.last_pos = 0
         self.items[self.last_pos] = item
 
         self.first_pos += 1
-        if self.first_pos > self.capacity-1:
+        if self.first_pos > self.capacity - 1:
             self.first_pos = 0
 
     def get(self):
@@ -77,6 +79,7 @@ class RingQueue:
     def item_count(self):
         return len(self.items)
 
+
 class SpeedCalculator(ChatProcessor, RingQueue):
     """
     チャットの勢いを計算する。
@@ -91,7 +94,7 @@ class SpeedCalculator(ChatProcessor, RingQueue):
         RingQueueに格納するチャット勢い算出用データの最大数
     """
 
-    def __init__(self, capacity = 10):
+    def __init__(self, capacity=10):
         super().__init__(capacity)
         self.speed = 0
 
@@ -106,7 +109,6 @@ class SpeedCalculator(ChatProcessor, RingQueue):
             self.speed = self._calc_speed()
         return self.speed
 
-
     def _calc_speed(self):
         """
         RingQueue内のチャット勢い算出用データリストを元に、
@@ -117,13 +119,12 @@ class SpeedCalculator(ChatProcessor, RingQueue):
             チャット速度（１分間で換算したチャット数）
         """
         try:
-            #キュー内の総チャット数
+            # キュー内の総チャット数
             total = sum(item['chat_count'] for item in self.items)
-            #キュー内の最初と最後のチャットの時間差
-            duration = (self.items[self.last_pos]['endtime'] 
-                        - self.items[self.first_pos]['starttime'])
+            # キュー内の最初と最後のチャットの時間差
+            duration = (self.items[self.last_pos]['endtime'] - self.items[self.first_pos]['starttime'])
             if duration != 0:
-                return int(total*60/duration)
+                return int(total * 60 / duration)
             return 0
         except IndexError:
             return 0
@@ -143,61 +144,60 @@ class SpeedCalculator(ChatProcessor, RingQueue):
             '''
             チャットデータがない場合に空のデータをキューに投入する。
             '''
-            timestamp_now =  int(time.time())
+            timestamp_now = int(time.time())
             self.put({
-                'chat_count':0,
-                'starttime':int(timestamp_now),
-                'endtime':int(timestamp_now)
+                'chat_count': 0,
+                'starttime': int(timestamp_now),
+                'endtime': int(timestamp_now)
             })
 
-        def _get_timestamp(action :dict):
+        def _get_timestamp(action: dict):
             """
             チャットデータから時刻データを取り出す。
             """
             try:
                 item = action['addChatItemAction']['item']
                 timestamp = int(item[list(item.keys())[0]]['timestampUsec'])
-            except (KeyError,TypeError):
+            except (KeyError, TypeError):
                 return None
             return timestamp
 
-        if actions is None or len(actions)==0:
+        if actions is None or len(actions) == 0:
             _put_emptydata()
             return
 
-        #actions内の時刻データを持つチャットデータの数
-        counter=0        
-        #actions内の最初のチャットデータの時刻
-        starttime= None
-        #actions内の最後のチャットデータの時刻
-        endtime=None
+        # actions内の時刻データを持つチャットデータの数
+        counter = 0
+        # actions内の最初のチャットデータの時刻
+        starttime = None
+        # actions内の最後のチャットデータの時刻
+        endtime = None
 
         for action in actions:
-            #チャットデータからtimestampUsecを読み取る
+            # チャットデータからtimestampUsecを読み取る
             gettime = _get_timestamp(action)
 
-            #時刻のないデータだった場合は次の行のデータで読み取り試行
+            # 時刻のないデータだった場合は次の行のデータで読み取り試行
             if gettime is None:
                 continue
 
-            #最初に有効な時刻を持つデータのtimestampをstarttimeに設定
+            # 最初に有効な時刻を持つデータのtimestampをstarttimeに設定
             if starttime is None:
                 starttime = gettime
 
-            #最後のtimestampを設定(途中で時刻のないデータの場合もあるので上書きしていく)
+            # 最後のtimestampを設定(途中で時刻のないデータの場合もあるので上書きしていく)
             endtime = gettime
 
-            #チャットの数をインクリメント
+            # チャットの数をインクリメント
             counter += 1
 
-        #チャット速度用のデータをRingQueueに送る
+        # チャット速度用のデータをRingQueueに送る
         if starttime is None or endtime is None:
             _put_emptydata()
             return
 
         self.put({
-            'chat_count':counter,
-            'starttime':int(starttime/1000000),
-            'endtime':int(endtime/1000000)
+            'chat_count': counter,
+            'starttime': int(starttime / 1000000),
+            'endtime': int(endtime / 1000000)
         })
-

pytchat/processors/superchat/calculator.py

@@ -15,10 +15,12 @@ items_sticker = [
     'liveChatPaidStickerRenderer'
 ]
 
+
 class SuperchatCalculator(ChatProcessor):
     """
     Calculate the amount of SuperChat by currency.
     """
+
     def __init__(self):
         self.results = {}
 
@@ -34,14 +36,16 @@ class SuperchatCalculator(ChatProcessor):
             return self.results
         for component in chat_components:
             chatdata = component.get('chatdata')
-            if chatdata is None: continue
+            if chatdata is None:
+                continue
             for action in chatdata:
                 renderer = self._get_item(action, items_paid) or \
-                           self._get_item(action, items_sticker)
-                if renderer is None: continue
+                    self._get_item(action, items_sticker)
+                if renderer is None:
+                    continue
                 symbol, amount = self._parse(renderer)
-                self.results.setdefault(symbol,0)
-                self.results[symbol]+=amount
+                self.results.setdefault(symbol, 0)
+                self.results[symbol] += amount
         return self.results
 
     def _parse(self, renderer):
@@ -49,7 +53,7 @@ class SuperchatCalculator(ChatProcessor):
         m = superchat_regex.search(purchase_amount_text)
         if m:
             symbol = m.group(1)
-            amount = float(m.group(2).replace(',',''))
+            amount = float(m.group(2).replace(',', ''))
         else:
             symbol = ""
             amount = 0.0
@@ -69,6 +73,3 @@ class SuperchatCalculator(ChatProcessor):
                 continue
             return None
         return dict_body
-
-
-

pytchat/processors/tsv_archiver.py

@@ -0,0 +1,67 @@
+import csv
+import os
+import re
+from .chat_processor import ChatProcessor
+from .default.processor import DefaultProcessor
+
+PATTERN = re.compile(r"(.*)\(([0-9]+)\)$")
+fmt_headers = ['datetime', 'elapsed', 'authorName', 'message',
+               'superchatAmount', 'authorType', 'authorChannel']
+
+
+class TSVArchiver(ChatProcessor):
+    '''
+    TsvArchiver saves chat data as Tab Separated Values format text.
+    '''
+
+    def __init__(self, save_path):
+        super().__init__()
+        self.save_path = self._checkpath(save_path)
+        with open(self.save_path, mode='a', encoding='utf-8') as f:
+            writer = csv.writer(f, delimiter='\t')
+            writer.writerow(fmt_headers)
+        self.processor = DefaultProcessor()
+
+    def _checkpath(self, filepath):
+        splitter = os.path.splitext(os.path.basename(filepath))
+        body = splitter[0]
+        extention = splitter[1]
+        newpath = filepath
+        counter = 0
+        while os.path.exists(newpath):
+            match = re.search(PATTERN, body)
+            if match:
+                counter = int(match[2]) + 1
+                num_with_bracket = f'({str(counter)})'
+                body = f'{match[1]}{num_with_bracket}'
+            else:
+                body = f'{body}({str(counter)})'
+            newpath = os.path.join(os.path.dirname(filepath), body + extention)
+        return newpath
+
+    def process(self, chat_components: list):
+        """
+        Returns
+        ----------
+        dict :
+            save_path : str :
+                Actual save path of file.
+            total_lines : int :
+                count of total lines written to the file.
+        """
+        if chat_components is None or len(chat_components) == 0:
+            return
+
+        with open(self.save_path, mode='a', encoding='utf-8') as f:
+            writer = csv.writer(f, delimiter='\t')
+            chats = self.processor.process(chat_components).items
+            for c in chats:
+                writer.writerow([
+                    c.datetime,
+                    c.elapsedTime,
+                    c.author.name,
+                    c.message,
+                    c.amountString,
+                    c.author.type,
+                    c.author.channelId
+                ])

pytchat/tool/extract/asyncdl.py

@@ -1,131 +0,0 @@
-import aiohttp
-import asyncio
-import json
-from . import parser
-from . block import Block
-from . worker import ExtractWorker
-from . patch import Patch
-from ... import config 
-from ... paramgen import arcparam
-from concurrent.futures import CancelledError
-from urllib.parse import quote
-
-headers = config.headers
-REPLAY_URL = "https://www.youtube.com/live_chat_replay/" \
-             "get_live_chat_replay?continuation="
-
-def _split(start, end, count, min_interval_sec = 120):
-    """
-    Split section from `start` to `end` into `count` pieces,
-    and returns the beginning of each piece. 
-    The `count` is adjusted so that the length of each piece
-    is no smaller than `min_interval`.
-
-    Returns:
-    --------
-        List of the offset of each block's first chat data.
-    """
-    
-    if not (isinstance(start,int) or isinstance(start,float)) or \
-       not (isinstance(end,int) or isinstance(end,float)):
-        raise ValueError("start/end must be int or float")
-    if not isinstance(count,int):
-        raise ValueError("count must be int")
-    if start>end:
-        raise ValueError("end must be equal to or greater than start.")
-    if count<1:
-        raise ValueError("count must be equal to or greater than 1.")
-    if (end-start)/count < min_interval_sec:
-        count = int((end-start)/min_interval_sec) 
-        if count == 0 : count = 1
-    interval= (end-start)/count 
-    
-    if count == 1:
-        return [start]
-    return sorted( list(set( [int(start + interval*j)
-        for j in range(count) ])))
-
-def ready_blocks(video_id, duration, div, callback):
-    if div <= 0: raise ValueError
-
-    async def _get_blocks( video_id, duration, div, callback):
-        async with aiohttp.ClientSession() as session:
-            tasks = [_create_block(session, video_id,  seektime, callback)
-                for  seektime in _split(-1, duration, div)]
-            return await asyncio.gather(*tasks)
-
-    async def _create_block(session, video_id, seektime, callback):
-        continuation = arcparam.getparam(video_id, seektime = seektime)
-        url = f"{REPLAY_URL}{quote(continuation)}&pbj=1"
-        async with session.get(url, headers = headers) as resp:
-            text = await resp.text()
-        next_continuation, actions = parser.parse(json.loads(text))
-        if actions:
-            first = parser.get_offset(actions[0])
-            last = parser.get_offset(actions[-1])
-            if callback:
-                callback(actions,last-first)
-            return Block(
-                continuation = next_continuation,
-                chat_data = actions,
-                first = first,
-                last = last
-            )
-    """
-    fetch initial blocks.
-    """  
-    loop = asyncio.get_event_loop()
-    blocks = loop.run_until_complete(
-        _get_blocks(video_id, duration, div, callback))
-    return blocks
-
-def fetch_patch(callback, blocks, video_id):
-
-    async def _allocate_workers():
-        workers = [
-            ExtractWorker(
-                fetch = _fetch,  block = block,
-                blocks = blocks, video_id = video_id
-            )
-            for block in blocks
-        ]
-        async with aiohttp.ClientSession() as session:
-            tasks = [worker.run(session) for worker in workers]
-            return await asyncio.gather(*tasks)    
-
-    async def _fetch(continuation,session) -> Patch:
-        url = f"{REPLAY_URL}{quote(continuation)}&pbj=1"
-        async with session.get(url,headers = config.headers) as resp:
-            chat_json = await resp.text()
-        continuation, actions = parser.parse(json.loads(chat_json))
-        if actions:
-            last = parser.get_offset(actions[-1])
-            first = parser.get_offset(actions[0])
-            if callback:
-                callback(actions, last - first)
-            return Patch(actions, continuation, first, last)
-        return Patch(continuation = continuation)
-    """
-    allocate workers and assign blocks.
-    """   
-    loop = asyncio.get_event_loop()
-    try:
-        loop.run_until_complete(_allocate_workers())
-    except CancelledError:
-        pass
-
-async def _shutdown():
-    print("\nshutdown...")
-    tasks = [t for t in asyncio.all_tasks()
-         if t is not asyncio.current_task()]
-    for task in tasks:
-        task.cancel()
-        try:
-            await task
-        except asyncio.CancelledError:
-            pass
-
-def cancel():
-    loop = asyncio.get_event_loop()
-    loop.create_task(_shutdown())
-    

pytchat/tool/extract/block.py

@@ -1,57 +0,0 @@
-from . import parser
-class Block:
-    """Block object represents something like a box 
-    to join chunk of chatdata.
-
-    Parameter:
-    ---------
-    first : int :
-        videoOffsetTimeMs of the first chat_data 
-        (chat_data[0])
-        
-    last : int :
-        videoOffsetTimeMs of the last chat_data.
-        (chat_data[-1])
-
-        this value increases as fetching chatdata progresses.
-
-    end : int :
-        target videoOffsetTimeMs of last chat data for extract,
-        equals to first videoOffsetTimeMs of next block.
-        when extract worker reaches this offset, stop fetching.
-
-    continuation : str :
-        continuation param of last chat data.
-
-    chat_data : list 
-
-    done : bool :
-        whether this block has been fetched.
-    
-    remaining : int :
-        remaining data to extract.
-        equals end - last.
-    
-    is_last : bool :
-        whether this block is the last one in blocklist.
-
-    during_split : bool :
-        whether this block is in the process of during_split.
-        while True, this block is excluded from duplicate split procedure.
-    """
-    
-    __slots__ = ['first','last','end','continuation','chat_data','remaining',
-        'done','is_last','during_split']
-
-    def __init__(self, first = 0, last = 0, end = 0,
-                continuation = '', chat_data = [], is_last = False,
-                during_split = False):
-        self.first = first
-        self.last = last
-        self.end = end
-        self.continuation = continuation
-        self.chat_data = chat_data
-        self.done = False
-        self.remaining = self.end - self.last
-        self.is_last = is_last
-        self.during_split = during_split

pytchat/tool/extract/duplcheck.py

@@ -1,153 +0,0 @@
-from . import parser
-
-def check_duplicate(chatdata):
-    max_range = len(chatdata)-1 
-    tbl_offset = [None] * max_range
-    tbl_id = [None] * max_range
-    tbl_type = [None] * max_range
-
-    def create_table(chatdata, max_range):
-        for i in range(max_range):
-            tbl_offset[i] = parser.get_offset(chatdata[i])
-            tbl_id[i] = parser.get_id(chatdata[i]) 
-            tbl_type[i] = parser.get_type(chatdata[i])
-
-    def is_duplicate(i, j):
-        return ( 
-            tbl_offset[i] == tbl_offset[j]
-                and
-            tbl_id[i] == tbl_id[j]
-                and
-            tbl_type[i] == tbl_type[j]
-        )
-    print("creating table...")
-    create_table(chatdata,max_range)
-    print("searching duplicate data...")
-    return [{ "i":{
-                "index" : i, "id" : parser.get_id(chatdata[i]),
-                "offsetTime" : parser.get_offset(chatdata[i]),
-                "type" : parser.get_type(chatdata[i])
-                },
-            "j":{
-                "index" : j, "id" : parser.get_id(chatdata[j]),
-                "offsetTime" : parser.get_offset(chatdata[j]),
-                "type" : parser.get_type(chatdata[j])
-                }
-            }
-        for i in range(max_range) for j in range(i+1,max_range) 
-        if is_duplicate(i,j)]
-
-
-def check_duplicate_offset(chatdata):
-    max_range = len(chatdata)
-    tbl_offset = [None] * max_range
-    tbl_id = [None] * max_range
-    tbl_type = [None] * max_range
-
-    def create_table(chatdata, max_range):
-        for i in range(max_range):
-            tbl_offset[i] = parser.get_offset(chatdata[i])
-            tbl_id[i] = parser.get_id(chatdata[i]) 
-            tbl_type[i] = parser.get_type(chatdata[i])
-
-    def is_duplicate(i, j):
-        return ( 
-            tbl_offset[i] == tbl_offset[j]
-                 and
-            tbl_id[i] == tbl_id[j]
-        )
-
-    print("creating table...")
-    create_table(chatdata,max_range)
-    print("searching duplicate data...")
-
-    return [{
-                "index" : i, "id" : tbl_id[i],
-                "offsetTime" : tbl_offset[i],
-                "type:" : tbl_type[i]
-            }
-        for i in range(max_range-1)
-        if is_duplicate(i,i+1)]
-
-def remove_duplicate_head(blocks):
-    if len(blocks) == 1 : return blocks
-
-    def is_duplicate_head(index):
-
-        if len(blocks[index].chat_data) == 0:
-            return True
-        elif len(blocks[index+1].chat_data) == 0:
-            return False
-        
-        id_0 = parser.get_id(blocks[index].chat_data[0])
-        id_1 = parser.get_id(blocks[index+1].chat_data[0])
-        type_0 = parser.get_type(blocks[index].chat_data[0])
-        type_1 = parser.get_type(blocks[index+1].chat_data[0])
-        return (
-            blocks[index].first == blocks[index+1].first
-                and
-            id_0 == id_1
-                and
-            type_0 == type_1
-        )
-    ret = [blocks[i] for i in range(len(blocks)-1)
-        if (len(blocks[i].chat_data)>0 and 
-        not is_duplicate_head(i) )]
-    ret.append(blocks[-1])
-    return ret
-
-def remove_duplicate_tail(blocks):
-    if len(blocks) == 1 : return blocks    
-
-    def is_duplicate_tail(index):
-        if len(blocks[index].chat_data) == 0:
-            return True
-        elif len(blocks[index-1].chat_data) == 0:
-            return False
-        id_0 = parser.get_id(blocks[index-1].chat_data[-1])
-        id_1 = parser.get_id(blocks[index].chat_data[-1])
-        type_0 = parser.get_type(blocks[index-1].chat_data[-1])
-        type_1 = parser.get_type(blocks[index].chat_data[-1])
-        return (
-            blocks[index-1].last == blocks[index].last
-                and
-            id_0 == id_1
-                and
-            type_0 == type_1
-        )
-
-    ret = [blocks[i] for i in range(0,len(blocks))
-        if i == 0 or not  is_duplicate_tail(i) ]
-    return ret
-
-def remove_overlap(blocks):
-    """
-    Fix overlapped blocks after ready_blocks().
-    Align the last offset of each block to the first offset 
-    of next block (equals `end` offset of each block).
-    """
-    if len(blocks) == 1 : return blocks
-
-    for block in blocks:
-        if block.is_last:
-            break
-        if len(block.chat_data)==0:
-            continue
-        block_end = block.end
-        if block.last >= block_end:
-            for line in reversed(block.chat_data):
-                if parser.get_offset(line) < block_end:
-                    break
-                block.chat_data.pop()
-            block.last = parser.get_offset(line)
-            block.remaining=0
-            block.done=True
-            block.continuation = None
-    return blocks
-    
-        
-
-def _dump(blocks):
-    print(f"----------        first         last         end---")
-    for i,block in enumerate(blocks):
-        print(f"block[{i:3}]   {block.first:>10}   {block.last:>10}  {block.end:>10}")

pytchat/tool/extract/extractor.py

@@ -1,91 +0,0 @@
-from . import asyncdl
-from . import duplcheck 
-from . import parser
-from .. videoinfo import VideoInfo
-from ... import config
-from ... exceptions import InvalidVideoIdException
-
-logger = config.logger(__name__)
-headers=config.headers
-
-class Extractor:
-    def __init__(self, video_id, div = 1, callback = None, processor = None):
-        if not isinstance(div ,int) or div < 1:
-            raise ValueError('div must be positive integer.')
-        elif div > 10:
-            div = 10
-        self.video_id = video_id
-        self.div = div
-        self.callback = callback
-        self.processor = processor
-        self.duration = self._get_duration_of_video(video_id)
-        self.blocks = []
-
-    def _get_duration_of_video(self, video_id):
-        duration = 0
-        try:
-            duration = VideoInfo(video_id).get_duration()
-        except InvalidVideoIdException:
-            raise
-        return duration
-
-    def _ready_blocks(self):
-        blocks = asyncdl.ready_blocks(
-            self.video_id, self.duration, self.div, self.callback)
-        self.blocks = [block for block in blocks if block]
-        return self  
-
-    def _remove_duplicate_head(self):
-        self.blocks = duplcheck.remove_duplicate_head(self.blocks)
-        return self
-
-    def _set_block_end(self):
-        for i in range(len(self.blocks)-1):
-            self.blocks[i].end = self.blocks[i+1].first
-        self.blocks[-1].end = self.duration*1000
-        self.blocks[-1].is_last =True
-        return self
-
-    def _remove_overlap(self):
-        self.blocks = duplcheck.remove_overlap(self.blocks)
-        return self
-
-    def _download_blocks(self):
-        asyncdl.fetch_patch(self.callback, self.blocks, self.video_id)
-        return self
-
-    def _remove_duplicate_tail(self):
-        self.blocks = duplcheck.remove_duplicate_tail(self.blocks)
-        return self
-
-    def _combine(self):
-        ret = []
-        for block in self.blocks:
-            ret.extend(block.chat_data) 
-        return ret
-
-    def _execute_extract_operations(self):
-        return (
-            self._ready_blocks()
-                ._remove_duplicate_head()
-                ._set_block_end()
-                ._remove_overlap()
-                ._download_blocks()
-                ._remove_duplicate_tail()
-                ._combine()
-        )
-
-    def extract(self):
-        if self.duration == 0:
-            print("video is not archived.")
-            return []
-        data = self._execute_extract_operations()
-        if self.processor is None:
-           return data
-        return self.processor.process(
-            [{'video_id':None,'timeout':1,'chatdata' : (action
-            ["replayChatItemAction"]["actions"][0] for action in data)}]
-    )
-
-    def cancel(self):
-        asyncdl.cancel()

pytchat/tool/extract/parser.py

@@ -1,54 +0,0 @@
-import json
-from ... import config
-from ... exceptions import ( 
-    ResponseContextError, 
-    NoContentsException, 
-    NoContinuationsException )
-
-logger = config.logger(__name__)
-
-def parse(jsn):
-    """
-    Parse replay chat data.
-    Parameter:
-    ----------
-    jsn : dict
-        JSON of replay chat data.
-    Returns:
-    ------
-        continuation : str
-        actions : list
-
-    """
-    if jsn is None: 
-        raise ValueError("parameter JSON is None")
-    if jsn['response']['responseContext'].get('errors'):
-        raise ResponseContextError(
-    'video_id is invalid or private/deleted.')
-    contents=jsn['response'].get('continuationContents')
-    if contents is None:
-        raise NoContentsException('No chat data.')
-
-    cont = contents['liveChatContinuation']['continuations'][0]
-    if cont is None:
-        raise NoContinuationsException('No Continuation')
-    metadata = cont.get('liveChatReplayContinuationData')
-    if metadata:
-        continuation = metadata.get("continuation")
-        actions = contents['liveChatContinuation'].get('actions')
-        return continuation, actions
-    return None, []
-
-
-def get_offset(item):
-    return int(item['replayChatItemAction']["videoOffsetTimeMsec"])
-
-def get_id(item):
-    return list((list(item['replayChatItemAction']["actions"][0].values()
-                )[0])['item'].values())[0].get('id')
-
-def get_type(item):
-    return list((list(item['replayChatItemAction']["actions"][0].values()
-                )[0])['item'].keys())[0]
-
-

pytchat/tool/extract/patch.py

@@ -1,54 +0,0 @@
-from . import parser
-from . block import Block
-from typing import NamedTuple
-
-class Patch(NamedTuple):
-    """
-    Patch represents chunk of chat data
-    which is fetched by asyncdl.fetch_patch._fetch().
-    """
-    chats : list = []
-    continuation : str = None
-    first : int = None
-    last : int = None
-
-def fill(block:Block, patch:Patch):
-    block_end = block.end
-    if patch.last < block_end or block.is_last:
-        set_patch(block, patch)
-        return
-    for line in reversed(patch.chats):
-        line_offset = parser.get_offset(line)
-        if line_offset < block_end:
-            break
-        patch.chats.pop()
-    set_patch(block, patch._replace(
-        continuation = None,
-        last = line_offset
-        )
-    )
-    block.remaining=0
-    block.done=True
-
-
-def split(parent_block:Block, child_block:Block, patch:Patch):
-    parent_block.during_split = False
-    if patch.first <= parent_block.last:
-        ''' When patch overlaps with parent_block,
-        discard this block. '''
-        child_block.continuation = None
-        ''' Leave child_block.during_split == True 
-         to exclude from during_split sequence. '''
-        return    
-    child_block.during_split = False
-    child_block.first = patch.first
-    parent_block.end = patch.first
-    fill(child_block, patch)
-    
-
-def set_patch(block:Block, patch:Patch):
-    block.continuation = patch.continuation
-    block.chat_data.extend(patch.chats)
-    block.last = patch.last
-    block.remaining = block.end-block.last        
-

pytchat/tool/extract/worker.py

@@ -1,87 +0,0 @@
-from . import parser
-from . block import Block
-from . patch import Patch, fill, split
-from ... paramgen import arcparam
-
-class ExtractWorker:
-    """
-    ExtractWorker associates a download session with a block.
-
-    When the worker finishes fetching, the block
-    being fetched is splitted and assigned the free worker.
-
-    Parameter
-    ----------
-    fetch : func :
-        extract function of asyncdl
-
-    block : Block :
-        Block object that includes chat_data
-    
-    blocks : list :
-        List of Block(s)
-
-    video_id : str :
-
-    parent_block : Block :
-        the block from which current block is splitted 
-    """
-    __slots__ = ['block', 'fetch', 'blocks', 'video_id', 'parent_block']
-
-    def __init__(self, fetch, block, blocks, video_id ):
-        self.block = block
-        self.fetch = fetch
-        self.blocks = blocks
-        self.video_id = video_id
-        self.parent_block = None
-
-    async def run(self, session):
-        while self.block.continuation:
-            patch = await self.fetch(
-                self.block.continuation, session)
-            if patch.continuation is None:
-                """TODO : make the worker assigned to the last block
-                to work more than twice as possible.
-                """
-                break
-            if self.parent_block:
-                split(self.parent_block, self.block, patch)
-                self.parent_block = None
-            else:    
-                fill(self.block, patch)
-            if self.block.continuation is None:
-                """finished fetching this block """
-                self.block.done = True
-                self.block = _search_new_block(self)
-
-def _search_new_block(worker) -> Block:
-    index, undone_block = _get_undone_block(worker.blocks)
-    if undone_block is None:
-        return Block(continuation = None)
-    mean = (undone_block.last + undone_block.end)/2
-    continuation = arcparam.getparam(worker.video_id, seektime = mean/1000)
-    worker.parent_block = undone_block
-    worker.parent_block.during_split = True
-    new_block = Block(
-        end =  undone_block.end,
-        chat_data = [], 
-        continuation = continuation,
-        during_split = True,
-        is_last = worker.parent_block.is_last)
-    worker.blocks.insert(index+1, new_block)
-    return new_block
-
-def _get_undone_block(blocks) -> (int, Block):
-    min_interval_ms = 120000
-    max_remaining = 0
-    undone_block = None
-    index_undone_block = 0
-    for index, block in enumerate(blocks):
-        if block.done or block.during_split:
-            continue
-        remaining = block.remaining
-        if remaining > max_remaining and remaining > min_interval_ms:
-            index_undone_block = index
-            undone_block = block
-            max_remaining = remaining
-    return index_undone_block, undone_block

pytchat/tool/mining/asyncdl.py

@@ -1,141 +0,0 @@
-
-import aiohttp
-import asyncio
-import json
-from . import parser
-from . block import Block
-from . worker import ExtractWorker
-from . patch import Patch
-from ... import config 
-from ... paramgen import arcparam_mining as arcparam
-from concurrent.futures import CancelledError
-from urllib.parse import quote
-
-headers = config.headers
-REPLAY_URL = "https://www.youtube.com/live_chat_replay?continuation="
-INTERVAL = 1
-def _split(start, end, count, min_interval_sec = 120):
-    """
-    Split section from `start` to `end` into `count` pieces,
-    and returns the beginning of each piece. 
-    The `count` is adjusted so that the length of each piece
-    is no smaller than `min_interval`.
-
-    Returns:
-    --------
-        List of the offset of each block's first chat data.
-    """
-    
-    if not (isinstance(start,int) or isinstance(start,float)) or \
-       not (isinstance(end,int) or isinstance(end,float)):
-        raise ValueError("start/end must be int or float")
-    if not isinstance(count,int):
-        raise ValueError("count must be int")
-    if start>end:
-        raise ValueError("end must be equal to or greater than start.")
-    if count<1:
-        raise ValueError("count must be equal to or greater than 1.")
-    if (end-start)/count < min_interval_sec:
-        count = int((end-start)/min_interval_sec) 
-        if count == 0 : count = 1
-    interval= (end-start)/count 
-    
-    if count == 1:
-        return [start]
-    return sorted( list(set( [int(start + interval*j)
-        for j in range(count) ])))
-
-def ready_blocks(video_id, duration, div, callback):
-    if div <= 0: raise ValueError
-
-    async def _get_blocks( video_id, duration, div, callback):
-        async with aiohttp.ClientSession() as session:
-            tasks = [_create_block(session, video_id,  seektime, callback)
-                for  seektime in _split(0, duration, div)]
-            return await asyncio.gather(*tasks)
-
-    
-
-    async def _create_block(session, video_id, seektime, callback):
-        continuation = arcparam.getparam(video_id, seektime = seektime)
-        url=(f"{REPLAY_URL}{quote(continuation)}&playerOffsetMs="
-            f"{int(seektime*1000)}&hidden=false&pbj=1")
-        async with session.get(url, headers = headers) as resp:
-            chat_json = await resp.text()
-        if chat_json is None:
-            return
-        continuation, actions = parser.parse(json.loads(chat_json)[1])
-        first = seektime
-        seektime += INTERVAL
-        if callback:
-            callback(actions, INTERVAL)
-        return Block(
-            continuation = continuation,
-            chat_data = actions,
-            first = first,
-            last = seektime,
-            seektime = seektime
-        )
-    """
-    fetch initial blocks.
-    """  
-    loop = asyncio.get_event_loop()
-    blocks = loop.run_until_complete(
-        _get_blocks(video_id, duration, div, callback))
-    return blocks
-
-def fetch_patch(callback, blocks, video_id):
-
-    async def _allocate_workers():
-        workers = [
-            ExtractWorker(
-                fetch = _fetch,  block = block,
-                blocks = blocks, video_id = video_id
-            )
-            for block in blocks
-        ]
-        async with aiohttp.ClientSession() as session:
-            tasks = [worker.run(session) for worker in workers]
-            return await asyncio.gather(*tasks)    
-
-    async def _fetch(seektime,session) -> Patch:
-        continuation = arcparam.getparam(video_id, seektime = seektime)
-        url=(f"{REPLAY_URL}{quote(continuation)}&playerOffsetMs="
-            f"{int(seektime*1000)}&hidden=false&pbj=1")
-        async with session.get(url,headers = config.headers) as resp:
-            chat_json = await resp.text()
-        actions = []
-        try:
-            if chat_json is None:
-                return Patch()
-            continuation, actions = parser.parse(json.loads(chat_json)[1])
-        except json.JSONDecodeError:
-            pass
-        if callback:
-            callback(actions, INTERVAL)
-        return Patch(chats = actions, continuation = continuation, 
-            seektime = seektime, last = seektime)
-    """
-    allocate workers and assign blocks.
-    """   
-    loop = asyncio.get_event_loop()
-    try:
-        loop.run_until_complete(_allocate_workers())
-    except CancelledError:
-        pass
-
-async def _shutdown():
-    print("\nshutdown...")
-    tasks = [t for t in asyncio.all_tasks()
-         if t is not asyncio.current_task()]
-    for task in tasks:
-        task.cancel()
-        try:
-            await task
-        except asyncio.CancelledError:
-            pass
-
-def cancel():
-    loop = asyncio.get_event_loop()
-    loop.create_task(_shutdown())
-    

pytchat/tool/mining/block.py

@@ -1,62 +0,0 @@
-from . import parser
-class Block:
-    """Block object represents something like a box 
-    to join chunk of chatdata.
-
-    Parameter:
-    ---------
-    first : int :
-        videoOffsetTimeMs of the first chat_data 
-        (chat_data[0])
-        
-    last : int :
-        videoOffsetTimeMs of the last chat_data.
-        (chat_data[-1])
-
-        this value increases as fetching chatdata progresses.
-
-    end : int :
-        target videoOffsetTimeMs of last chat data for extract,
-        equals to first videoOffsetTimeMs of next block.
-        when extract worker reaches this offset, stop fetching.
-
-    continuation : str :
-        continuation param of last chat data.
-
-    chat_data : list 
-
-    done : bool :
-        whether this block has been fetched.
-    
-    remaining : int :
-        remaining data to extract.
-        equals end - last.
-    
-    is_last : bool :
-        whether this block is the last one in blocklist.
-
-    during_split : bool :
-        whether this block is in the process of during_split.
-        while True, this block is excluded from duplicate split procedure.
-    
-    seektime : float :
-        the last position of this block(seconds) already fetched.
-    """
-    
-    __slots__ = ['first','last','end','continuation','chat_data','remaining',
-        'done','is_last','during_split','seektime']
-
-    def __init__(self, first = 0, last = 0, end = 0,
-                continuation = '', chat_data = [], is_last = False,
-                during_split = False, seektime = None):
-        self.first = first
-        self.last = last
-        self.end = end
-        self.continuation = continuation
-        self.chat_data = chat_data
-        self.done = False
-        self.remaining = self.end - self.last
-        self.is_last = is_last
-        self.during_split = during_split
-        self.seektime = seektime
- 

pytchat/tool/mining/parser.py

@@ -1,67 +0,0 @@
-import json
-from ... import config
-from ... exceptions import ( 
-    ResponseContextError, 
-    NoContentsException, 
-    NoContinuationsException )
-
-logger = config.logger(__name__)
-
-def parse(jsn):
-    """
-    Parse replay chat data.
-    Parameter:
-    ----------
-    jsn : dict
-        JSON of replay chat data.
-    Returns:
-    ------
-        continuation : str
-        actions : list
-
-    """
-    if jsn is None: 
-        raise ValueError("parameter JSON is None")
-    if jsn['response']['responseContext'].get('errors'):
-        raise ResponseContextError(
-    'video_id is invalid or private/deleted.')
-    contents=jsn["response"].get('continuationContents')
-    if contents is None:
-        raise NoContentsException('No chat data.')
-
-    cont = contents['liveChatContinuation']['continuations'][0]
-    if cont is None:
-        raise NoContinuationsException('No Continuation')
-    metadata = cont.get('liveChatReplayContinuationData')
-    if metadata:
-        continuation = metadata.get("continuation")
-        actions = contents['liveChatContinuation'].get('actions')
-        if continuation:
-            return continuation, [action["replayChatItemAction"]["actions"][0] 
-                    for action in actions 
-                        if list(action['replayChatItemAction']["actions"][0].values()
-                    )[0]['item'].get("liveChatPaidMessageRenderer")
-                        or list(action['replayChatItemAction']["actions"][0].values()
-                    )[0]['item'].get("liveChatPaidStickerRenderer")
-                    ]
-    return None, []
-
-
-def get_offset(item):
-    return int(item['replayChatItemAction']["videoOffsetTimeMsec"])
-
-def get_id(item):
-    return list((list(item['replayChatItemAction']["actions"][0].values()
-                )[0])['item'].values())[0].get('id')
-
-def get_type(item):
-    return list((list(item['replayChatItemAction']["actions"][0].values()
-                )[0])['item'].keys())[0]
-import re                
-_REGEX_YTINIT = re.compile("window\\[\"ytInitialData\"\\]\\s*=\\s*({.+?});\\s+")
-def extract(text):
-
-    match = re.findall(_REGEX_YTINIT, str(text))
-    if match:
-        return match[0]
-    return None

pytchat/tool/mining/patch.py

@@ -1,27 +0,0 @@
-from . import parser
-from . block import Block
-from typing import NamedTuple
-
-class Patch(NamedTuple):
-    """
-    Patch represents chunk of chat data
-    which is fetched by asyncdl.fetch_patch._fetch().
-    """
-    chats : list = []
-    continuation : str = None
-    seektime : float = None
-    first : int = None
-    last : int = None
-
-def fill(block:Block, patch:Patch):
-    if patch.last < block.end:
-        set_patch(block, patch)
-        return
-    block.continuation = None
-
-def set_patch(block:Block, patch:Patch):
-    block.continuation = patch.continuation
-    block.chat_data.extend(patch.chats)
-    block.last = patch.seektime
-    block.seektime = patch.seektime       
-

pytchat/tool/mining/superchat_miner.py

@@ -1,72 +0,0 @@
-from . import asyncdl
-from . import parser
-from .. videoinfo import VideoInfo
-from ... import config
-from ... exceptions import InvalidVideoIdException
-logger = config.logger(__name__)
-headers=config.headers
-
-class SuperChatMiner:
-    def __init__(self, video_id, duration, div, callback):
-        if not isinstance(div ,int) or div < 1:
-            raise ValueError('div must be positive integer.')
-        elif div > 10:
-            div = 10
-        if not isinstance(duration ,int) or duration < 1:
-            raise ValueError('duration must be positive integer.')
-        self.video_id = video_id
-        self.duration = duration
-        self.div = div
-        self.callback = callback
-        self.blocks = []
-
-    def _ready_blocks(self):
-        blocks = asyncdl.ready_blocks(
-            self.video_id, self.duration, self.div, self.callback)
-        self.blocks = [block for block in blocks if block is not None]
-        return self  
-
-    def _set_block_end(self):
-        for i in range(len(self.blocks)-1):
-            self.blocks[i].end = self.blocks[i+1].first
-        self.blocks[-1].end = self.duration
-        self.blocks[-1].is_last =True
-        return self
-
-    def _download_blocks(self):
-        asyncdl.fetch_patch(self.callback, self.blocks, self.video_id)
-        return self
-
-    def _combine(self):
-        ret = []
-        for block in self.blocks:
-            ret.extend(block.chat_data) 
-        return ret
-
-    def extract(self):
-        return (
-            self._ready_blocks()
-                ._set_block_end()
-                ._download_blocks()
-                ._combine()
-        )
-
-def extract(video_id, div = 1, callback = None, processor = None):
-    duration = 0
-    try:
-        duration = VideoInfo(video_id).get_duration()
-    except InvalidVideoIdException:
-        raise
-    if duration == 0:
-        print("video is live.")
-        return []
-    data = SuperChatMiner(video_id, duration, div, callback).extract()
-    if processor is None:
-        return data
-    return processor.process(
-        [{'video_id':None,'timeout':1,'chatdata' : (action
-        for action in data)}]
-    )
-
-def cancel():
-    asyncdl.cancel()

pytchat/tool/mining/worker.py

@@ -1,45 +0,0 @@
-from . import parser
-from . block import Block
-from . patch import Patch, fill
-from ... paramgen import arcparam
-INTERVAL = 1
-class ExtractWorker:
-    """
-    ExtractWorker associates a download session with a block.
-
-    When the worker finishes fetching, the block
-    being fetched is splitted and assigned the free worker.
-
-    Parameter
-    ----------
-    fetch : func :
-        extract function of asyncdl
-
-    block : Block :
-        Block object that includes chat_data
-    
-    blocks : list :
-        List of Block(s)
-
-    video_id : str :
-
-    parent_block : Block :
-        the block from which current block is splitted 
-    """
-    __slots__ = ['block', 'fetch', 'blocks', 'video_id', 'parent_block']
-    def __init__(self, fetch, block, blocks, video_id ):
-        self.block:Block = block
-        self.fetch = fetch
-        self.blocks:list = blocks
-        self.video_id:str = video_id
-        self.parent_block:Block = None
-
-    async def run(self, session):
-        while self.block.continuation:
-            patch = await self.fetch(
-                self.block.seektime, session)
-            fill(self.block, patch)
-            self.block.seektime += INTERVAL
-        self.block.done = True    
-
-

pytchat/tool/videoinfo.py

@@ -1,148 +0,0 @@
-import json 
-import re
-import requests
-from .. import config
-from .. import util
-from ..exceptions import InvalidVideoIdException 
-
-headers = config.headers
-
-pattern = re.compile(r"yt\.setConfig\({'PLAYER_CONFIG': ({.*})}\);")
-
-item_channel_id =[
-    "videoDetails",
-    "embeddedPlayerOverlayVideoDetailsRenderer",
-    "channelThumbnailEndpoint",
-    "channelThumbnailEndpoint",
-    "urlEndpoint",
-    "urlEndpoint",
-    "url"
-]
-
-item_renderer = [
-    "embedPreview",
-    "thumbnailPreviewRenderer"
-]
-
-item_response = [
-    "args",
-    "embedded_player_response"
-]
-
-item_author_image =[
-    "videoDetails",
-    "embeddedPlayerOverlayVideoDetailsRenderer",
-    "channelThumbnail",
-    "thumbnails",
-    0,
-    "url"
-]
-
-item_thumbnail = [
-    "defaultThumbnail",
-    "thumbnails",
-    2,
-    "url"
-]
-
-item_channel_name = [
-    "videoDetails",
-    "embeddedPlayerOverlayVideoDetailsRenderer",
-    "expandedRenderer",
-    "embeddedPlayerOverlayVideoDetailsExpandedRenderer",
-    "title",
-    "runs",
-    0,
-    "text"
-]
-
-item_moving_thumbnail = [
-    "movingThumbnail",
-    "thumbnails",
-    0,
-    "url"
-]
-
-class VideoInfo:
-    '''
-    VideoInfo object retrieves YouTube video informations
-    from the video page.
-
-    Parameter
-    ---------
-    video_id : str
-
-    Exception
-    ---------
-    InvalidVideoIdException :
-        Occurs when video_id does not exist on YouTube.
-    '''
-    def __init__(self, video_id):
-        self.video_id = video_id
-        text = self._get_page_text(video_id)
-        self._parse(text)
-
-    def _get_page_text(self, video_id):
-        url = f"https://www.youtube.com/embed/{video_id}"
-        resp = requests.get(url, headers = headers)
-        resp.raise_for_status()
-        return resp.text
-
-    def _parse(self, text):
-        result = re.search(pattern, text)
-        res= json.loads(result.group(1))
-        response = self._get_item(res, item_response)
-        if response is None:
-            raise InvalidVideoIdException(
-                f"Specified video_id [{self.video_id}] is invalid.")
-        self._renderer = self._get_item(json.loads(response), item_renderer)
-        if self._renderer is None:
-            raise InvalidVideoIdException(
-                f"No renderer found in video_id: [{self.video_id}].")
-
-    def _get_item(self, dict_body, items: list):
-        for item in items:
-            if dict_body is None:
-                break
-            if isinstance(dict_body, dict):
-                dict_body = dict_body.get(item)
-                continue
-            if isinstance(item, int) and \
-               isinstance(dict_body, list) and \
-               len(dict_body) > item:
-                dict_body = dict_body[item]
-                continue
-            return None
-        return dict_body
-
-    def get_duration(self):
-        duration_seconds = self._renderer.get("videoDurationSeconds")
-        if duration_seconds:
-            '''Fetched value is string, so cast to integer.'''
-            return int(duration_seconds)
-        '''When key is not found, explicitly returns None.'''
-        return None
-
-    def get_title(self):
-        if self._renderer.get("title"):
-            return [''.join(run["text"]) 
-                for run in self._renderer["title"]["runs"]][0]
-        return None
-
-    def get_channel_id(self):
-        channel_url = self._get_item(self._renderer, item_channel_id)
-        if channel_url:
-            return channel_url[9:]
-        return None
-
-    def get_author_image(self):
-        return self._get_item(self._renderer, item_author_image) 
-
-    def get_thumbnail(self):
-        return self._get_item(self._renderer, item_thumbnail)
-
-    def get_channel_name(self):
-        return self._get_item(self._renderer, item_channel_name)
-    
-    def get_moving_thumbnail(self):
-        return self._get_item(self._renderer, item_moving_thumbnail)

pytchat/util/__init__.py

@@ -1,15 +1,94 @@
-import requests,json,datetime
+import datetime
+import httpx
+import json
+import os
+import re
 from .. import config
+from .. exceptions import InvalidVideoIdException
+
+PATTERN = re.compile(r"(.*)\(([0-9]+)\)$")
+
+PATTERN_YTURL = re.compile(r"((?<=(v|V)/)|(?<=be/)|(?<=(\?|\&)v=)|(?<=embed/))([\w-]+)")
+
+YT_VIDEO_ID_LENGTH = 11
+
+CLIENT_VERSION = ''.join(("2.", (datetime.datetime.today() - datetime.timedelta(days=1)).strftime("%Y%m%d"), ".01.00"))
+
+UA = config.headers["user-agent"]
+
 
 def extract(url):
-    _session = requests.Session()
+    _session = httpx.Client(http2=True)
     html = _session.get(url, headers=config.headers)
     with open(str(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')
-                        )+'test.json',mode ='w',encoding='utf-8') as f:
-        json.dump(html.json(),f,ensure_ascii=False)
+                  ) + 'test.json', mode='w', encoding='utf-8') as f:
+        json.dump(html.json(), f, ensure_ascii=False)
 
 
-def save(data,filename,extention):
-    with open(filename+"_"+(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')
-                            )+extention,mode ='w',encoding='utf-8') as f:
+def save(data, filename, extention) -> str:
+    save_filename = filename + "_" + \
+        (datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')) + extention
+    with open(save_filename, mode='w', encoding='utf-8') as f:
         f.writelines(data)
+    return save_filename
+
+
+def checkpath(filepath):
+    splitter = os.path.splitext(os.path.basename(filepath))
+    body = splitter[0]
+    extention = splitter[1]
+    newpath = filepath
+    counter = 1
+    while os.path.exists(newpath):
+        match = re.search(PATTERN, body)
+        if match:
+            counter = int(match[2]) + 1
+            num_with_bracket = f'({str(counter)})'
+            body = f'{match[1]}{num_with_bracket}'
+        else:
+            body = f'{body}({str(counter)})'
+        newpath = os.path.join(os.path.dirname(filepath), body + extention)
+    return newpath
+
+
+def get_param(continuation, replay=False, offsetms: int = 0, dat=''):
+    if offsetms < 0:
+        offsetms = 0
+    ret = {
+        "context": {
+            "client": {
+                "visitorData": dat,
+                "userAgent": UA,
+                "clientName": "WEB",
+                "clientVersion": CLIENT_VERSION,
+            },
+
+        },
+        "continuation": continuation,
+    }
+    if replay:
+        ret.setdefault("currentPlayerState", {
+                       "playerOffsetMs": str(int(offsetms))})
+    return ret
+
+
+def extract_video_id(url_or_id: str) -> str:
+    ret = ''
+    if '[' in url_or_id:
+        url_or_id = url_or_id.replace('[', '').replace(']', '')
+
+    if type(url_or_id) != str:
+        raise TypeError(f"{url_or_id}: URL or VideoID must be str, but {type(url_or_id)} is passed.")
+    if len(url_or_id) == YT_VIDEO_ID_LENGTH:
+        return url_or_id
+    match = re.search(PATTERN_YTURL, url_or_id)
+    if match is None:
+        raise InvalidVideoIdException(f"Invalid video id: {url_or_id}")
+    try:
+        ret = match.group(4)
+    except IndexError:
+        raise InvalidVideoIdException(f"Invalid video id: {url_or_id}")
+
+    if ret is None or len(ret) != YT_VIDEO_ID_LENGTH:
+        raise InvalidVideoIdException(f"Invalid video id: {url_or_id}")
+    return ret

requirements.txt

@@ -1,4 +1 @@
-aiohttp
-pytz
-requests
-urllib3
+httpx[http2]==0.16.1

requirements_test.txt

@@ -1,5 +1,2 @@
-aioresponses
-mock
-mocker
-pytest
 pytest-mock
+pytest-httpx

setup.py

@@ -44,29 +44,24 @@ with open('README.md', encoding='utf-8') as f:
 
 
 setup(
-    name=package_name,
-    packages=find_packages(exclude=['*log.txt','*tests','*testrun']),
-    version=version,
-    url=url,
     author=author,
     author_email=author_email,
-    long_description=long_description,
-    long_description_content_type='text/markdown',
-    license=license,
-    install_requires=_requirements(),
-    description="a python library for fetching youtube live chat.",
     classifiers=[
         'Natural Language :: Japanese',
         'Development Status :: 4 - Beta',
         'Programming Language :: Python',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.4',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',        
         'Programming Language :: Python :: 3.8',        
         'License :: OSI Approved :: MIT License',
     ],
+    description="a python library for fetching youtube live chat.",
+    install_requires=_requirements(),
     keywords='youtube livechat asyncio',
-
+    license=license,
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    name=package_name,
+    packages=find_packages(exclude=['*log.txt','*tests','*testrun']),
+    url=url,
+    version=version,
 )

tests/test_arcparam.py

@@ -1,28 +1,19 @@
-import pytest
-from pytchat.parser.live import Parser
+import json
+import httpx
 import pytchat.config as config
-import requests, json
 from pytchat.paramgen import arcparam
+from pytchat.parser.live import Parser
+
 
 def test_arcparam_0(mocker):
-    param = arcparam.getparam("01234567890",-1)
-    assert param == "op2w0wRyGjxDZzhhRFFvTE1ERXlNelExTmpjNE9UQWFFLXFvM2JrQkRRb0xNREV5TXpRMU5qYzRPVEFnQVElM0QlM0QoADAAOABAAEgEUhwIABAAGAAgACoOc3RhdGljY2hlY2tzdW1AAFgDYAFoAHIECAEQAHgA" 
+    param = arcparam.getparam("01234567890", -1)
+    assert param == "op2w0wSDARpsQ2pnYURRb0xNREV5TXpRMU5qYzRPVEFxSndvWVgxOWZYMTlmWDE5ZlgxOWZYMTlmWDE5ZlgxOWZYMTlmRWdzd01USXpORFUyTnpnNU1Cb1Q2cWpkdVFFTkNnc3dNVEl6TkRVMk56ZzVNQ0FCKOgHMAA4AEAASARSAiAAcgIIAXgA" 
+
 
 def test_arcparam_1(mocker):
-    param = arcparam.getparam("01234567890", seektime = 100000)
-    assert param == "op2w0wR3GjxDZzhhRFFvTE1ERXlNelExTmpjNE9UQWFFLXFvM2JrQkRRb0xNREV5TXpRMU5qYzRPVEFnQVElM0QlM0QogNDbw_QCMAA4AEAASANSHAgAEAAYACAAKg5zdGF0aWNjaGVja3N1bUAAWANgAWgAcgQIARAAeAA%3D"
-
-def test_arcparam_2(mocker):
-    param = arcparam.getparam("SsjCnHOk-Sk")
-    url=f"https://www.youtube.com/live_chat_replay/get_live_chat_replay?continuation={param}&pbj=1"
-    resp = requests.Session().get(url,headers = config.headers)
-    jsn = json.loads(resp.text)
-    parser = Parser(is_replay=True)
-    contents= parser.get_contents(jsn)
-    _ , chatdata = parser.parse(contents)
-    test_id = chatdata[0]["addChatItemAction"]["item"]["liveChatTextMessageRenderer"]["id"]
-    assert test_id == "CjoKGkNMYXBzZTdudHVVQ0Zjc0IxZ0FkTnFnQjVREhxDSnlBNHV2bnR1VUNGV0dnd2dvZDd3NE5aZy0w"
+    param = arcparam.getparam("01234567890", seektime=100000)
+    assert param == "op2w0wSHARpsQ2pnYURRb0xNREV5TXpRMU5qYzRPVEFxSndvWVgxOWZYMTlmWDE5ZlgxOWZYMTlmWDE5ZlgxOWZYMTlmRWdzd01USXpORFUyTnpnNU1Cb1Q2cWpkdVFFTkNnc3dNVEl6TkRVMk56ZzVNQ0FCKIDQ28P0AjAAOABAAEgDUgIgAHICCAF4AA%3D%3D"
 
 def test_arcparam_3(mocker):
     param = arcparam.getparam("01234567890")
-    assert param == "op2w0wRyGjxDZzhhRFFvTE1ERXlNelExTmpjNE9UQWFFLXFvM2JrQkRRb0xNREV5TXpRMU5qYzRPVEFnQVElM0QlM0QoATAAOABAAEgDUhwIABAAGAAgACoOc3RhdGljY2hlY2tzdW1AAFgDYAFoAHIECAEQAHgA"
+    assert param == "op2w0wSDARpsQ2pnYURRb0xNREV5TXpRMU5qYzRPVEFxSndvWVgxOWZYMTlmWDE5ZlgxOWZYMTlmWDE5ZlgxOWZYMTlmRWdzd01USXpORFUyTnpnNU1Cb1Q2cWpkdVFFTkNnc3dNVEl6TkRVMk56ZzVNQ0FCKOgHMAA4AEAASARSAiAAcgIIAXgA"

tests/test_arcparam_mining.py

@@ -1,40 +0,0 @@
-import pytest
-from pytchat.tool.mining import parser
-import pytchat.config as config
-import requests, json
-from pytchat.paramgen import arcparam_mining as arcparam
-
-def test_arcparam_e(mocker):
-    try:
-        arcparam.getparam("01234567890",-1)
-        assert False
-    except ValueError:
-        assert True
-
-    
-
-
-def test_arcparam_0(mocker):
-    param = arcparam.getparam("01234567890",0)
-
-    assert param =="op2w0wQsGiBDZzhhRFFvTE1ERXlNelExTmpjNE9UQWdBUSUzRCUzREABYARyAggBeAE%3D"
-
-
-def test_arcparam_1(mocker):
-    param = arcparam.getparam("01234567890", seektime = 100000)
-    print(param)
-    assert param == "op2w0wQzGiBDZzhhRFFvTE1ERXlNelExTmpjNE9UQWdBUSUzRCUzREABWgUQgMLXL2AEcgIIAXgB"
-
-def test_arcparam_2(mocker):
-    param = arcparam.getparam("PZz9NB0-Z64",1)
-    url=f"https://www.youtube.com/live_chat_replay?continuation={param}&playerOffsetMs=1000&pbj=1"
-    resp = requests.Session().get(url,headers = config.headers)
-    jsn = json.loads(resp.text)
-    _ , chatdata = parser.parse(jsn[1])
-    test_id = chatdata[0]["addChatItemAction"]["item"]["liveChatPaidMessageRenderer"]["id"]
-    print(test_id)
-    assert test_id == "ChwKGkNKSGE0YnFJeWVBQ0ZWcUF3Z0VkdGIwRm9R"
-
-def test_arcparam_3(mocker):
-    param = arcparam.getparam("01234567890")
-    assert param == "op2w0wQsGiBDZzhhRFFvTE1ERXlNelExTmpjNE9UQWdBUSUzRCUzREABYARyAggBeAE%3D"

tests/test_calculator_parse.py

@@ -6,50 +6,55 @@ parse = SuperchatCalculator()._parse
 
 
 def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
+    with open(path, mode='r', encoding='utf-8') as f:
         return f.read()
 
+
 def load_chatdata(filepath):
     parser = Parser(is_replay=True)
-    #print(json.loads(_open_file(filepath)))
-    contents = parser.get_contents( json.loads(_open_file(filepath)))
+    # print(json.loads(_open_file(filepath)))
+    contents = parser.get_contents(json.loads(_open_file(filepath)))[0]
     return parser.parse(contents)[1]
 
 
-
 def test_parse_1():
-    renderer ={"purchaseAmountText":{"simpleText":"￥2,000"}}
-    symbol ,amount = parse(renderer)
+    renderer = {"purchaseAmountText": {"simpleText": "￥2,000"}}
+    symbol, amount = parse(renderer)
     assert symbol == '￥'
     assert amount == 2000.0
 
+
 def test_parse_2():
-    renderer ={"purchaseAmountText":{"simpleText":"ABC\x0a200"}}
-    symbol ,amount = parse(renderer)
+    renderer = {"purchaseAmountText": {"simpleText": "ABC\x0a200"}}
+    symbol, amount = parse(renderer)
     assert symbol == 'ABC\x0a'
     assert amount == 200.0
 
+
 def test_process_0():
     """
     parse superchat data
     """
     chat_component = {
-        'video_id':'',
-        'timeout':10,
-        'chatdata':load_chatdata(r"tests\testdata\calculator\superchat_0.json")
+        'video_id': '',
+        'timeout': 10,
+        'chatdata': load_chatdata(r"tests/testdata/calculator/superchat_0.json")
     }
-    assert SuperchatCalculator().process([chat_component])=={'￥': 6800.0, '€': 2.0}
+    assert SuperchatCalculator().process([chat_component]) == {
+        '￥': 6800.0, '€': 2.0}
+
 
 def test_process_1():
     """
     parse no superchat data
     """
     chat_component = {
-        'video_id':'',
-        'timeout':10,
-        'chatdata':load_chatdata(r"tests\testdata\calculator\text_only.json")
+        'video_id': '',
+        'timeout': 10,
+        'chatdata': load_chatdata(r"tests/testdata/calculator/text_only.json")
     }
-    assert SuperchatCalculator().process([chat_component])=={}
+    assert SuperchatCalculator().process([chat_component]) == {}
+
 
 def test_process_2():
     """
@@ -57,12 +62,11 @@ def test_process_2():
     """
     try:
         chat_component = {
-            'video_id':'',
-            'timeout':10,
-            'chatdata':load_chatdata(r"tests\testdata\calculator\replay_end.json")
+            'video_id': '',
+            'timeout': 10,
+            'chatdata': load_chatdata(r"tests/testdata/calculator/replay_end.json")
         }
         assert False
         SuperchatCalculator().process([chat_component])
     except ChatParseException:
         assert True
-    

tests/test_compatible_processor.py

@@ -1,55 +1,49 @@
 import json
-import pytest
-import asyncio,aiohttp
 from pytchat.parser.live import Parser
 from pytchat.processors.compatible.processor import CompatibleProcessor
-from pytchat.exceptions import (
-    NoLivechatRendererException,NoYtinitialdataException,
-    ResponseContextError, NoContentsException)
-
-from pytchat.processors.compatible.renderer.textmessage import LiveChatTextMessageRenderer
-from pytchat.processors.compatible.renderer.paidmessage import LiveChatPaidMessageRenderer
-from pytchat.processors.compatible.renderer.paidsticker import LiveChatPaidStickerRenderer
-from pytchat.processors.compatible.renderer.legacypaid import LiveChatLegacyPaidMessageRenderer
 
 parser = Parser(is_replay=False)
 
+
 def test_textmessage(mocker):
     '''api互換processorのテスト：通常テキストメッセージ'''
     processor = CompatibleProcessor()
 
     _json = _open_file("tests/testdata/compatible/textmessage.json")
 
-    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json))[0])
     data = {
-        "video_id" : "",
-        "timeout" : 7,
-        "chatdata" : chatdata
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
     }
     ret = processor.process([data])
 
-    assert ret["kind"]== "youtube#liveChatMessageListResponse"
-    assert ret["pollingIntervalMillis"]==data["timeout"]*1000
+    assert ret["kind"] == "youtube#liveChatMessageListResponse"
+    assert ret["pollingIntervalMillis"] == data["timeout"] * 1000
     assert ret.keys() == {
-        "kind",   "etag",   "pageInfo",   "nextPageToken","pollingIntervalMillis","items"
+        "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
     }
     assert ret["pageInfo"].keys() == {
-        "totalResults",   "resultsPerPage"
+        "totalResults", "resultsPerPage"
     }
     assert ret["items"][0].keys() == {
-        "kind",   "etag",   "id",   "snippet", "authorDetails"
+        "kind", "etag", "id", "snippet", "authorDetails"
     }
     assert ret["items"][0]["snippet"].keys() == {
-        'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage', 'textMessageDetails'
+        'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage',
+        'textMessageDetails'
     }
     assert ret["items"][0]["authorDetails"].keys() == {
-        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator'
+        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor',
+        'isChatModerator'
     }
     assert ret["items"][0]["snippet"]["textMessageDetails"].keys() == {
         'messageText'
     }
     assert "LCC." in ret["items"][0]["id"]
-    assert ret["items"][0]["snippet"]["type"]=="textMessageEvent"
+    assert ret["items"][0]["snippet"]["type"] == "textMessageEvent"
+
 
 def test_newsponcer(mocker):
     '''api互換processorのテスト：メンバ新規登録'''
@@ -57,34 +51,72 @@ def test_newsponcer(mocker):
 
     _json = _open_file("tests/testdata/compatible/newSponsor.json")
 
-    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json))[0])
     data = {
-        "video_id" : "",
-        "timeout" : 7,
-        "chatdata" : chatdata
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
     }
     ret = processor.process([data])
 
-    assert ret["kind"]== "youtube#liveChatMessageListResponse"
-    assert ret["pollingIntervalMillis"]==data["timeout"]*1000
+    assert ret["kind"] == "youtube#liveChatMessageListResponse"
+    assert ret["pollingIntervalMillis"] == data["timeout"] * 1000
     assert ret.keys() == {
-        "kind",   "etag",   "pageInfo",   "nextPageToken","pollingIntervalMillis","items"
+        "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
     }
     assert ret["pageInfo"].keys() == {
-        "totalResults",   "resultsPerPage"
+        "totalResults", "resultsPerPage"
     }
     assert ret["items"][0].keys() == {
-        "kind",   "etag",   "id",   "snippet","authorDetails"
+        "kind", "etag", "id", "snippet", "authorDetails"
     }
     assert ret["items"][0]["snippet"].keys() == {
         'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage'
 
     }
     assert ret["items"][0]["authorDetails"].keys() == {
-        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator'
+        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor',
+        'isChatModerator'
     }
     assert "LCC." in ret["items"][0]["id"]
-    assert ret["items"][0]["snippet"]["type"]=="newSponsorEvent"
+    assert ret["items"][0]["snippet"]["type"] == "newSponsorEvent"
+
+
+def test_newsponcer_rev(mocker):
+    '''api互換processorのテスト：メンバ新規登録'''
+    processor = CompatibleProcessor()
+
+    _json = _open_file("tests/testdata/compatible/newSponsor_rev.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json))[0])
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+    ret = processor.process([data])
+
+    assert ret["kind"] == "youtube#liveChatMessageListResponse"
+    assert ret["pollingIntervalMillis"] == data["timeout"] * 1000
+    assert ret.keys() == {
+        "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
+    }
+    assert ret["pageInfo"].keys() == {
+        "totalResults", "resultsPerPage"
+    }
+    assert ret["items"][0].keys() == {
+        "kind", "etag", "id", "snippet", "authorDetails"
+    }
+    assert ret["items"][0]["snippet"].keys() == {
+        'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage'
+
+    }
+    assert ret["items"][0]["authorDetails"].keys() == {
+        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor',
+        'isChatModerator'
+    }
+    assert "LCC." in ret["items"][0]["id"]
+    assert ret["items"][0]["snippet"]["type"] == "newSponsorEvent"
 
 
 def test_superchat(mocker):
@@ -93,53 +125,56 @@ def test_superchat(mocker):
 
     _json = _open_file("tests/testdata/compatible/superchat.json")
 
-    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json))[0])
     data = {
-        "video_id" : "",
-        "timeout" : 7,
-        "chatdata" : chatdata
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
     }
     ret = processor.process([data])
 
-    assert ret["kind"]== "youtube#liveChatMessageListResponse"
-    assert ret["pollingIntervalMillis"]==data["timeout"]*1000
+    assert ret["kind"] == "youtube#liveChatMessageListResponse"
+    assert ret["pollingIntervalMillis"] == data["timeout"] * 1000
     assert ret.keys() == {
-        "kind",   "etag",   "pageInfo",   "nextPageToken","pollingIntervalMillis","items"
+        "kind", "etag", "pageInfo", "nextPageToken", "pollingIntervalMillis", "items"
     }
     assert ret["pageInfo"].keys() == {
-        "totalResults",   "resultsPerPage"
+        "totalResults", "resultsPerPage"
     }
     assert ret["items"][0].keys() == {
-        "kind",   "etag",   "id",   "snippet", "authorDetails"
+        "kind", "etag", "id", "snippet", "authorDetails"
     }
     assert ret["items"][0]["snippet"].keys() == {
-        'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage', 'superChatDetails'
+        'type', 'liveChatId', 'authorChannelId', 'publishedAt', 'hasDisplayContent', 'displayMessage',
+        'superChatDetails'
     }
     assert ret["items"][0]["authorDetails"].keys() == {
-        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor', 'isChatModerator'
+        'channelId', 'channelUrl', 'displayName', 'profileImageUrl', 'isVerified', 'isChatOwner', 'isChatSponsor',
+        'isChatModerator'
     }
     assert ret["items"][0]["snippet"]["superChatDetails"].keys() == {
         'amountMicros', 'currency', 'amountDisplayString', 'tier', 'backgroundColor'
     }
     assert "LCC." in ret["items"][0]["id"]
-    assert ret["items"][0]["snippet"]["type"]=="superChatEvent"
+    assert ret["items"][0]["snippet"]["type"] == "superChatEvent"
+
 
 def test_unregistered_currency(mocker):
     processor = CompatibleProcessor()
 
     _json = _open_file("tests/testdata/unregistered_currency.json")
 
-    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json))[0])
 
     data = {
-        "video_id" : "",
-        "timeout" : 7,
-        "chatdata" : chatdata
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
     }
     ret = processor.process([data])
     assert ret["items"][0]["snippet"]["superChatDetails"]["currency"] == "[UNREGISTERD]"
 
 
 def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
+    with open(path, mode='r', encoding='utf-8') as f:
         return f.read()

tests/test_default_processor.py

@@ -0,0 +1,227 @@
+import json
+from datetime import datetime
+from pytchat.parser.live import Parser
+from pytchat.processors.default.processor import DefaultProcessor
+
+
+TEST_TIMETSTAMP = 1570678496000000
+
+
+def get_local_datetime(timestamp):
+    dt = datetime.fromtimestamp(timestamp / 1000000)
+    return dt.strftime('%Y-%m-%d %H:%M:%S')
+
+
+def test_textmessage(mocker):
+    '''text message'''
+    processor = DefaultProcessor()
+    parser = Parser(is_replay=False)
+    _json = _open_file("tests/testdata/default/textmessage.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json))[0])
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+
+    ret = processor.process([data]).items[0]
+    assert ret.id == "dummy_id"
+    assert ret.message == "dummy_message"
+    assert ret.timestamp == 1570678496000
+    assert ret.datetime == get_local_datetime(TEST_TIMETSTAMP)
+    assert ret.author.name == "author_name"
+    assert ret.author.channelId == "author_channel_id"
+    assert ret.author.channelUrl == "http://www.youtube.com/channel/author_channel_id"
+    assert ret.author.imageUrl == "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg"
+    assert ret.author.badgeUrl == ""
+    assert ret.author.isVerified is False
+    assert ret.author.isChatOwner is False
+    assert ret.author.isChatSponsor is False
+    assert ret.author.isChatModerator is False
+
+
+def test_textmessage_replay_member(mocker):
+    '''text message replay member'''
+    processor = DefaultProcessor()
+    parser = Parser(is_replay=True)
+    _json = _open_file("tests/testdata/default/replay_member_text.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json))[0])
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+    
+    ret = processor.process([data]).items[0]
+    assert ret.type == "textMessage"
+    assert ret.id == "dummy_id"
+    assert ret.message == "dummy_message"
+    assert ret.messageEx == ["dummy_message"]
+    assert ret.timestamp == 1570678496000
+    assert ret.datetime == get_local_datetime(TEST_TIMETSTAMP)
+    assert ret.elapsedTime == "1:23:45"
+    assert ret.author.name == "author_name"
+    assert ret.author.channelId == "author_channel_id"
+    assert ret.author.channelUrl == "http://www.youtube.com/channel/author_channel_id"
+    assert ret.author.imageUrl == "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg"
+    assert ret.author.badgeUrl == "https://yt3.ggpht.com/X=s16-c-k"
+    assert ret.author.isVerified is False
+    assert ret.author.isChatOwner is False
+    assert ret.author.isChatSponsor is True
+    assert ret.author.isChatModerator is False
+
+
+def test_superchat(mocker):
+    '''superchat'''
+    processor = DefaultProcessor()
+    parser = Parser(is_replay=False)
+    _json = _open_file("tests/testdata/default/superchat.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json))[0])
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+    
+    ret = processor.process([data]).items[0]
+    assert ret.type == "superChat"
+    assert ret.id == "dummy_id"
+    assert ret.message == "dummy_message"
+    assert ret.messageEx == ["dummy_message"]
+    assert ret.timestamp == 1570678496000
+    assert ret.datetime == get_local_datetime(TEST_TIMETSTAMP)
+    assert ret.elapsedTime == ""
+    assert ret.amountValue == 800
+    assert ret.amountString == "￥800"
+    assert ret.currency == "JPY"
+    assert ret.bgColor == 4280150454
+    assert ret.author.name == "author_name"
+    assert ret.author.channelId == "author_channel_id"
+    assert ret.author.channelUrl == "http://www.youtube.com/channel/author_channel_id"
+    assert ret.author.imageUrl == "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg"
+    assert ret.author.badgeUrl == ""
+    assert ret.author.isVerified is False
+    assert ret.author.isChatOwner is False
+    assert ret.author.isChatSponsor is False
+    assert ret.author.isChatModerator is False
+    assert ret.colors.headerBackgroundColor == 4278239141
+    assert ret.colors.headerTextColor == 4278190080
+    assert ret.colors.bodyBackgroundColor == 4280150454
+    assert ret.colors.bodyTextColor == 4278190080
+    assert ret.colors.authorNameTextColor == 2315255808
+    assert ret.colors.timestampColor == 2147483648
+
+
+def test_supersticker(mocker):
+    '''supersticker'''
+    processor = DefaultProcessor()
+    parser = Parser(is_replay=False)
+    _json = _open_file("tests/testdata/default/supersticker.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json))[0])
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+   
+    ret = processor.process([data]).items[0]
+    assert ret.type == "superSticker"
+    assert ret.id == "dummy_id"
+    assert ret.message == ""
+    assert ret.messageEx == []
+    assert ret.timestamp == 1570678496000
+    assert ret.datetime == get_local_datetime(TEST_TIMETSTAMP)
+    assert ret.elapsedTime == ""
+    assert ret.amountValue == 200
+    assert ret.amountString == "￥200"
+    assert ret.currency == "JPY"
+    assert ret.bgColor == 4278237396
+    assert ret.sticker == "https://lh3.googleusercontent.com/param_s=s72-rp"
+    assert ret.author.name == "author_name"
+    assert ret.author.channelId == "author_channel_id"
+    assert ret.author.channelUrl == "http://www.youtube.com/channel/author_channel_id"
+    assert ret.author.imageUrl == "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg"
+    assert ret.author.badgeUrl == ""
+    assert ret.author.isVerified is False
+    assert ret.author.isChatOwner is False
+    assert ret.author.isChatSponsor is False
+    assert ret.author.isChatModerator is False
+    assert ret.colors.backgroundColor == 4278237396
+    assert ret.colors.moneyChipBackgroundColor == 4278248959
+    assert ret.colors.moneyChipTextColor == 4278190080
+    assert ret.colors.authorNameTextColor == 3003121664
+
+
+def test_sponsor(mocker):
+    '''sponsor(membership)'''
+    processor = DefaultProcessor()
+    parser = Parser(is_replay=False)
+    _json = _open_file("tests/testdata/default/newSponsor_current.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json))[0])
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+    
+    ret = processor.process([data]).items[0]
+    assert ret.type == "newSponsor"
+    assert ret.id == "dummy_id"
+    assert ret.message == "新規メンバー"
+    assert ret.messageEx == ["新規メンバー"]
+    assert ret.timestamp == 1570678496000
+    assert ret.datetime == get_local_datetime(TEST_TIMETSTAMP)
+    assert ret.elapsedTime == ""
+    assert ret.bgColor == 0
+    assert ret.author.name == "author_name"
+    assert ret.author.channelId == "author_channel_id"
+    assert ret.author.channelUrl == "http://www.youtube.com/channel/author_channel_id"
+    assert ret.author.imageUrl == "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg"
+    assert ret.author.badgeUrl == "https://yt3.ggpht.com/X=s32-c-k"
+    assert ret.author.isVerified is False
+    assert ret.author.isChatOwner is False
+    assert ret.author.isChatSponsor is True
+    assert ret.author.isChatModerator is False
+
+
+def test_sponsor_legacy(mocker):
+    '''lagacy sponsor(membership)'''
+    processor = DefaultProcessor()
+    parser = Parser(is_replay=False)
+    _json = _open_file("tests/testdata/default/newSponsor_lagacy.json")
+
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json))[0])
+    data = {
+        "video_id": "",
+        "timeout": 7,
+        "chatdata": chatdata
+    }
+    
+    ret = processor.process([data]).items[0]
+    assert ret.type == "newSponsor"
+    assert ret.id == "dummy_id"
+    assert ret.message == "新規メンバー / ようこそ、author_name！"
+    assert ret.messageEx == ["新規メンバー / ようこそ、author_name！"]
+    assert ret.timestamp == 1570678496000
+    assert ret.datetime == get_local_datetime(TEST_TIMETSTAMP)
+    assert ret.elapsedTime == ""
+    assert ret.bgColor == 0
+    assert ret.author.name == "author_name"
+    assert ret.author.channelId == "author_channel_id"
+    assert ret.author.channelUrl == "http://www.youtube.com/channel/author_channel_id"
+    assert ret.author.imageUrl == "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg"
+    assert ret.author.badgeUrl == ""
+    assert ret.author.isVerified is False
+    assert ret.author.isChatOwner is False
+    assert ret.author.isChatSponsor is True
+    assert ret.author.isChatModerator is False
+
+
+def _open_file(path):
+    with open(path, mode='r', encoding='utf-8') as f:
+        return f.read()

tests/test_extract_asyncdl.py

@@ -1,77 +0,0 @@
-import aiohttp
-import asyncio
-import json
-from pytchat.tool.extract import parser
-import sys
-import time
-from aioresponses import aioresponses
-from concurrent.futures import CancelledError
-from pytchat.tool.extract import asyncdl
-
-def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
-        return f.read()
-
-
-def test_asyncdl_split():
-
-    ret = asyncdl._split(0,1000,1)
-    assert ret == [0]
-
-    ret = asyncdl._split(1000,1000,10)
-    assert ret == [1000]
-
-    ret = asyncdl._split(0,1000,5)
-    assert ret == [0,200,400,600,800]
-
-    ret = asyncdl._split(10.5, 700.3, 5)
-    assert ret == [10, 148, 286, 424, 562]
-
-
-    ret = asyncdl._split(0,500,5)
-    assert ret == [0,125,250,375]
-    
-    ret = asyncdl._split(0,500,500)
-    assert ret == [0,125,250,375]
-    
-    ret = asyncdl._split(-1,1000,5)
-    assert ret == [-1, 199, 399, 599, 799]
-    
-    """invalid argument order"""
-    try:
-        ret = asyncdl._split(500,0,5)
-        assert False
-    except ValueError:
-        assert True
-
-    """invalid count"""
-    try:
-        ret = asyncdl._split(0,500,-1)
-        assert False
-    except ValueError:
-        assert True
-
-    try:
-        ret = asyncdl._split(0,500,0)
-        assert False
-    except ValueError:
-        assert True
-
-    """invalid argument type"""
-    try:
-        ret = asyncdl._split(0,5000,5.2)
-        assert False
-    except ValueError:
-        assert True
-
-    try:
-        ret = asyncdl._split(0,5000,"test")
-        assert False
-    except ValueError:
-        assert True
-
-    try:
-        ret = asyncdl._split([0,1],5000,5)
-        assert False
-    except ValueError:
-        assert True

tests/test_extract_duplcheck.py

@@ -1,128 +0,0 @@
-import aiohttp
-import asyncio
-import json
-import os, sys
-import time
-from pytchat.tool.extract import duplcheck
-from pytchat.tool.extract import parser
-from pytchat.tool.extract.block import Block
-from pytchat.tool.extract.duplcheck import _dump
-def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
-        return f.read()
-
-
-
-def test_overlap():
-    """
-    test overlap data 
-        operation : [0]  [2] [3]  [4] -> last :align to end
-                    [1] , [5] -> no change
-        
-    """
-
-    def load_chatdata(filename):
-        return parser.parse(
-            json.loads(_open_file("tests/testdata/extract_duplcheck/overlap/"+filename))
-        )[1]
-
-    blocks = (
-        Block(first = 0,    last= 12771, end=  9890,chat_data = load_chatdata("dp0-0.json")),     
-        Block(first = 9890, last= 15800, end= 20244,chat_data = load_chatdata("dp0-1.json")), 
-        Block(first = 20244,last= 45146, end= 32476,chat_data = load_chatdata("dp0-2.json")), 
-        Block(first = 32476,last= 50520, end= 41380,chat_data = load_chatdata("dp0-3.json")), 
-        Block(first = 41380,last= 62875, end= 52568,chat_data = load_chatdata("dp0-4.json")), 
-        Block(first = 52568,last= 62875, end= 54000,chat_data = load_chatdata("dp0-5.json"),is_last=True)
-    )
-    result = duplcheck.remove_overlap(blocks)
-    #dp0-0.json has item offset time is 9890 (equals block[0].end = block[1].first), 
-    #but must be aligne to the most close and smaller value:9779.
-    assert result[0].last == 9779
-    
-    assert result[1].last == 15800
-    
-    assert result[2].last == 32196
-    
-    assert result[3].last == 41116
-    
-    assert result[4].last == 52384
-    
-    #the last block must be always added to result.
-    assert result[5].last == 62875
-    
-def test_duplicate_head():
-
-    def load_chatdata(filename):
-        return parser.parse(
-            json.loads(_open_file("tests/testdata/extract_duplcheck/head/"+filename))
-        )[1]
-
-    """
-    test duplicate head data 
-        operation : [0] , [1]  -> discard [0]
-                    [1] , [2]  -> discard [1]
-                    [2] , [3]  -> append  [2]
-                    [3] , [4]  -> discard [3]
-                    [4] , [5]  -> append  [4]
-                    append [5]
-
-        result    : [2] , [4] , [5] 
-    """
-
-    #chat data offsets are ignored.
-    blocks = (
-        Block(first = 0, last = 2500, chat_data = load_chatdata("dp0-0.json")),     
-        Block(first = 0, last =38771, chat_data = load_chatdata("dp0-1.json")), 
-        Block(first = 0, last =45146, chat_data = load_chatdata("dp0-2.json")), 
-        Block(first = 20244, last =60520, chat_data = load_chatdata("dp0-3.json")), 
-        Block(first = 20244, last =62875, chat_data = load_chatdata("dp0-4.json")), 
-        Block(first = 52568, last =62875, chat_data = load_chatdata("dp0-5.json"))
-    )
-    _dump(blocks)
-    result = duplcheck.remove_duplicate_head(blocks)
-    
-    assert len(result) == 3
-    assert result[0].first == blocks[2].first
-    assert result[0].last  == blocks[2].last
-    assert result[1].first == blocks[4].first
-    assert result[1].last  == blocks[4].last
-    assert result[2].first == blocks[5].first
-    assert result[2].last  == blocks[5].last
-
-def test_duplicate_tail():
-    """
-    test duplicate tail data 
-        operation : append [0]
-                    [0] , [1]  -> discard [1]
-                    [1] , [2]  -> append  [2]
-                    [2] , [3]  -> discard [3]
-                    [3] , [4]  -> append  [4]
-                    [4] , [5]  -> discard [5]
-
-        result    : [0] , [2] , [4] 
-    """
-    def load_chatdata(filename):
-        return parser.parse(
-            json.loads(_open_file("tests/testdata/extract_duplcheck/head/"+filename))
-        )[1]
-    #chat data offsets are ignored.    
-    blocks = (
-        Block(first = 0,last = 2500, chat_data=load_chatdata("dp0-0.json")),     
-        Block(first = 1500,last = 2500, chat_data=load_chatdata("dp0-1.json")), 
-        Block(first = 10000,last = 45146, chat_data=load_chatdata("dp0-2.json")), 
-        Block(first = 20244,last = 45146, chat_data=load_chatdata("dp0-3.json")), 
-        Block(first = 20244,last = 62875, chat_data=load_chatdata("dp0-4.json")), 
-        Block(first = 52568,last = 62875, chat_data=load_chatdata("dp0-5.json"))
-    )
-
-    result = duplcheck.remove_duplicate_tail(blocks)
-    _dump(result)
-    assert len(result) == 3
-    assert result[0].first == blocks[0].first
-    assert result[0].last  == blocks[0].last
-    assert result[1].first == blocks[2].first
-    assert result[1].last  == blocks[2].last
-    assert result[2].first == blocks[4].first
-    assert result[2].last  == blocks[4].last
-
-

tests/test_extract_patch.py

@@ -1,238 +0,0 @@
-import aiohttp
-import asyncio
-import json
-import os, sys
-import time
-from aioresponses import aioresponses
-from pytchat.tool.extract import duplcheck
-from pytchat.tool.extract import parser
-from pytchat.tool.extract.block import Block
-from pytchat.tool.extract.patch import Patch, fill, split, set_patch
-from pytchat.tool.extract.duplcheck import _dump
-
-def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
-        return f.read()
-
-def load_chatdata(filename):
-        return parser.parse(
-            json.loads(_open_file("tests/testdata/fetch_patch/"+filename))
-        )[1]
-
-
-def test_split_0():
-    """
-    Normal case
-
-    ~~~~~~ before ~~~~~~
-
-     @parent_block  (# = already fetched)
-    
-     first    last                                     end
-       |########----------------------------------------|
-    
-
-     @child_block
-    
-     first = last = 0                                  end (=parent_end)
-       |                                                |
-    
-
-     @fetched patch
-                            |-- patch --|
-    
-     
-                            |
-                            |
-                            V 
-
-    ~~~~~~ after ~~~~~~
-    
-
-     @parent_block
-    
-     first    last         end (after split)   
-       |########------------|
-    
-     @child_block
-                          first       last            end            
-                            |###########---------------|
-    
-     @fetched patch
-                            |-- patch --|
-    """
-    parent = Block(first=0, last=4000, end=60000, continuation='parent', during_split=True)
-    child = Block(first=0, last=0, end=60000, continuation='mean', during_split=True)
-    patch = Patch(chats=load_chatdata('pt0-5.json'),
-        first=32500, last=34000, continuation='patch')
-    
-    split(parent,child,patch)
-
-    assert child.continuation == 'patch'
-    assert parent.last < child.first
-    assert parent.end == child.first
-    assert child.first < child.last
-    assert child.last < child.end
-    assert parent.during_split == False
-    assert child.during_split == False
-
-def test_split_1():
-    """patch.first <= parent_block.last
-
-    While awaiting at run()->asyncdl._fetch()
-    fetching parent_block proceeds, 
-    and parent.block.last exceeds patch.first.
-
-    In this case, fetched patch is all discarded,
-    and worker searches other processing block again. 
-
-    ~~~~~~ before ~~~~~~
-
-                          patch.first
-      first                 |    last                  end
-       |####################|#####|---------------------|
-                            ^
-     @child_block
-     first = last = 0                                  end (=parent_end)
-       |                                                |
-     
-     @fetched patch
-                            |-- patch --|
-    
-     
-                            |
-                            |
-                            V 
-    
-    ~~~~~~ after ~~~~~~
-
-     @parent_block
-     first                       last                  end
-       |###########################|--------------------|
-    
-     @child_block
-                                
-                            ..............　-> 　discard all data
-                   
-    """
-    parent = Block(first=0, last=33000, end=60000, continuation='parent', during_split=True)
-    child = Block(first=0, last=0, end=60000, continuation='mean', during_split=True)
-    patch = Patch(chats=load_chatdata('pt0-5.json'),
-        first=32500, last=34000, continuation='patch')
-    
-    split(parent,child,patch)
-
-    assert parent.last == 33000 #no change
-    assert parent.end == 60000 #no change
-    assert child.continuation is None
-    assert parent.during_split == False
-    assert child.during_split == True #exclude during_split sequence
-
-def test_split_2():
-    """child_block.end < patch.last:
-
-    Case the last offset of patch exceeds child_block.end.
-    In this case, remove overlapped data of patch.
-
-    ~~~~~~ before ~~~~~~
-
-     @parent_block  (# = already fetched)
-     first    last                           end (before split)
-       |########------------------------------|
-    
-     @child_block
-     first = last = 0                        end (=parent_end)
-       |                                      |
-    
-    continuation:succeed from patch
-    
-     @fetched patch
-                            |-------- patch --------|
-    
-     
-                            |
-                            |
-                            V 
-
-    ~~~~~~ after ~~~~~~
-
-     @parent_block
-     first    last         end (after split)   
-       |########------------|
-
-     @child_block                                  old patch.end            
-                          first            last=end |
-                            |#################|......   cut extra data.
-                                                    ^
-    continuation : None (extract complete)
-
-     @fetched patch                                 
-                            |-------- patch --------|
-    """
-    parent = Block(first=0, last=4000, end=33500, continuation='parent', during_split=True)
-    child = Block(first=0, last=0, end=33500, continuation='mean', during_split=True)
-    patch = Patch(chats=load_chatdata('pt0-5.json'),
-        first=32500, last=34000, continuation='patch')
-     
-    split(parent,child,patch)
-
-    assert child.continuation is None
-    assert parent.last < child.first
-    assert parent.end == child.first
-    assert child.first < child.last
-    assert child.last < child.end
-    assert child.continuation is None
-    assert parent.during_split == False
-    assert child.during_split == False
-
-def test_split_none():
-    """patch.last <= parent_block.last
-
-    While awaiting at run()->asyncdl._fetch()
-    fetching parent_block proceeds, 
-    and parent.block.last exceeds patch.first.
-
-    In this case, fetched patch is all discarded,
-    and worker searches other processing block again. 
-    
-    ~~~~~~ before ~~~~~~
-
-                          patch.first
-     first                  |   last                   end
-       |####################|###################|-------|
-                            ^
-     @child_block
-     first = last = 0                                  end (=parent_end)
-       |                                                |
-     
-     @fetched patch
-                            |-- patch --|
-                                      patch.last < parent_block.last                       .
-     
-                            |
-                            |
-                            V 
-    
-    ~~~~~~ after ~~~~~~
-
-     @parent_block
-     first                       last           end (before split)
-       |########################################|-------|
-
-     @child_block
-                                           
-                            ............    -> discard all data.
-
-    """
-    parent = Block(first=0, last=40000, end=60000, continuation='parent', during_split=True)
-    child = Block(first=0, last=0, end=60000, continuation='mean', during_split=True)
-    patch = Patch(chats=load_chatdata('pt0-5.json'),
-        first=32500, last=34000, continuation='patch')
-    
-    split(parent,child,patch)
-
-    assert parent.last == 40000 #no change
-    assert parent.end == 60000 #no change
-    assert child.continuation is None
-    assert parent.during_split == False
-    assert child.during_split == True #exclude during_split sequence

tests/test_extract_video_id.py

@@ -0,0 +1,55 @@
+from pytchat.util import extract_video_id
+from pytchat.exceptions import InvalidVideoIdException
+
+VALID_TEST_PATTERNS = (
+    ("ABC_EFG_IJK", "ABC_EFG_IJK"),
+    ("vid_test_be", "vid_test_be"),
+    ("https://www.youtube.com/watch?v=123_456_789", "123_456_789"),
+    ("https://www.youtube.com/watch?v=123_456_789&t=123s", "123_456_789"),
+    ("www.youtube.com/watch?v=123_456_789", "123_456_789"),
+    ("watch?v=123_456_789", "123_456_789"),
+    ("youtube.com/watch?v=123_456_789", "123_456_789"),
+    ("http://youtu.be/ABC_EFG_IJK", "ABC_EFG_IJK"),
+    ("youtu.be/ABC_EFG_IJK", "ABC_EFG_IJK"),
+    ("https://www.youtube.com/watch?v=ABC_EFG_IJK&list=XYZ_ABC_12345&start_radio=1&t=1", "ABC_EFG_IJK"),
+    ("https://www.youtube.com/embed/ABC_EFG_IJK", "ABC_EFG_IJK"),
+    ("www.youtube.com/embed/ABC_EFG_IJK", "ABC_EFG_IJK"),
+    ("youtube.com/embed/ABC_EFG_IJK", "ABC_EFG_IJK")
+)
+
+INVALID_TEST_PATTERNS = (
+    ("", ""),
+    ("0123456789", "0123456789"),  # less than 11 letters id
+    ("more_than_11_letter_string", "more_than_11_letter_string"),
+    ("https://www.youtube.com/watch?v=more_than_11_letter_string", "more_than_11_letter_string"),
+    ("https://www.youtube.com/channel/123_456_789", "123_456_789"),
+)
+
+TYPEERROR_TEST_PATTERNS = (
+    (100, 100),  # not string
+    (["123_456_789"], "123_456_789"),  # not string
+)
+
+
+def test_extract_valid_pattern():
+    for pattern in VALID_TEST_PATTERNS:
+        ret = extract_video_id(pattern[0])
+        assert ret == pattern[1]
+
+
+def test_extract_invalid_pattern():
+    for pattern in INVALID_TEST_PATTERNS:
+        try:
+            extract_video_id(pattern[0])
+            assert False
+        except InvalidVideoIdException:
+            assert True
+
+
+def test_extract_typeerror_pattern():
+    for pattern in TYPEERROR_TEST_PATTERNS:
+        try:
+            extract_video_id(pattern[0])
+            assert False
+        except TypeError:
+            assert True

tests/test_livechat.py

@@ -1,53 +1,48 @@
-import pytest
-from pytchat.parser.live import Parser
-
+import asyncio
 import json
-import asyncio,aiohttp
-
-from aioresponses import aioresponses
-from pytchat.core_async.livechat import LiveChatAsync
-from pytchat.exceptions import (
-    NoLivechatRendererException,NoYtinitialdataException,
-    ResponseContextError,NoContentsException)
-
-
+from pytest_httpx import HTTPXMock
+from concurrent.futures import CancelledError
 from pytchat.core_multithread.livechat import LiveChat
-import unittest
-from unittest import TestCase
+from pytchat.core_async.livechat import LiveChatAsync
+from pytchat.exceptions import ResponseContextError
+
 
 def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
+    with open(path, mode='r', encoding='utf-8') as f:
         return f.read()
 
-@aioresponses()
-def test_Async(*mock):
-    vid=''
-    _text = _open_file('tests/testdata/paramgen_firstread.json')
-    _text = json.loads(_text)
-    mock[0].get(f"https://www.youtube.com/live_chat?v={vid}&is_popout=1", status=200, body=_text)
+
+def add_response_file(httpx_mock: HTTPXMock, jsonfile_path: str):
+    testdata = json.loads(_open_file(jsonfile_path))
+    httpx_mock.add_response(json=testdata)
+
+
+def test_async(httpx_mock: HTTPXMock):
+    add_response_file(httpx_mock, 'tests/testdata/paramgen_firstread.json')
+
+    async def test_loop():
+        try:
+            chat = LiveChatAsync(video_id='__test_id__')
+            _ = await chat.get()
+            assert chat.is_alive()
+            chat.terminate()
+            assert not chat.is_alive()
+        except ResponseContextError:
+            assert False
+    loop = asyncio.get_event_loop()
     try:
-        chat = LiveChatAsync(video_id='')
+        loop.run_until_complete(test_loop())
+    except CancelledError:
+        assert True
+
+
+def test_multithread(httpx_mock: HTTPXMock):
+    add_response_file(httpx_mock, 'tests/testdata/paramgen_firstread.json')
+    try:
+        chat = LiveChat(video_id='__test_id__')
+        _ = chat.get()
         assert chat.is_alive()
         chat.terminate()
         assert not chat.is_alive()
     except ResponseContextError:
-        assert not chat.is_alive()
-
-def test_MultiThread(mocker):
-    _text = _open_file('tests/testdata/paramgen_firstread.json')
-    _text = json.loads(_text)
-    responseMock = mocker.Mock()
-    responseMock.status_code = 200
-    responseMock.text = _text
-    mocker.patch('requests.Session.get').return_value = responseMock
-    try:
-        chat = LiveChatAsync(video_id='')
-        assert chat.is_alive()
-        chat.terminate()
-        assert not chat.is_alive()
-    except ResponseContextError:
-        chat.terminate()
-        assert not chat.is_alive()
-
-
-
+        assert False

tests/test_livechat_2.py

@@ -1,125 +1,113 @@
-import asyncio, aiohttp
+import asyncio
 import json
-import pytest
-import re
-import requests
-import sys
-import time
-from aioresponses import aioresponses
+from pytest_httpx import HTTPXMock
 from concurrent.futures import CancelledError
-from unittest import TestCase
 from pytchat.core_multithread.livechat import LiveChat
 from pytchat.core_async.livechat import LiveChatAsync
-from pytchat.exceptions import (
-      NoLivechatRendererException,NoYtinitialdataException,
-      ResponseContextError,NoContentsException)
-from pytchat.parser.live import Parser
 from pytchat.processors.dummy_processor import DummyProcessor
 
+
 def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
+    with open(path, mode='r', encoding='utf-8') as f:
         return f.read()
 
-@aioresponses()
-def test_async_live_stream(*mock):
 
-    async def test_loop(*mock):
-        pattern = re.compile(r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$')
-        _text = _open_file('tests/testdata/test_stream.json')
-        mock[0].get(pattern, status=200, body=_text)
-        chat = LiveChatAsync(video_id='', processor = DummyProcessor())
+def add_response_file(httpx_mock: HTTPXMock, jsonfile_path: str):
+    testdata = json.loads(_open_file(jsonfile_path))
+    httpx_mock.add_response(json=testdata)
+
+
+def test_async_live_stream(httpx_mock: HTTPXMock):
+    add_response_file(httpx_mock, 'tests/testdata/test_stream.json')
+
+    async def test_loop():
+        chat = LiveChatAsync(video_id='__test_id__', processor=DummyProcessor())
         chats = await chat.get()
         rawdata = chats[0]["chatdata"]
-        #assert fetching livachat data
-        assert list(rawdata[0]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer"
-        assert list(rawdata[1]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer"
-        assert list(rawdata[2]["addChatItemAction"]["item"].keys())[0] == "liveChatPlaceholderItemRenderer"
-        assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerPaidMessageItemRenderer"
-        assert list(rawdata[4]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer"
-        assert list(rawdata[5]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidStickerRenderer"
-        assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerSponsorItemRenderer"                                
+        assert list(rawdata[0]["addChatItemAction"]["item"].keys())[
+            0] == "liveChatTextMessageRenderer"
+        assert list(rawdata[1]["addChatItemAction"]["item"].keys())[
+            0] == "liveChatTextMessageRenderer"
+        assert list(rawdata[2]["addChatItemAction"]["item"].keys())[
+            0] == "liveChatPlaceholderItemRenderer"
+        assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[
+            0] == "liveChatTickerPaidMessageItemRenderer"
+        assert list(rawdata[4]["addChatItemAction"]["item"].keys())[
+            0] == "liveChatPaidMessageRenderer"
+        assert list(rawdata[5]["addChatItemAction"]["item"].keys())[
+            0] == "liveChatPaidStickerRenderer"
+        assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[
+            0] == "liveChatTickerSponsorItemRenderer"
 
     loop = asyncio.get_event_loop()
     try:
-        loop.run_until_complete(test_loop(*mock))
+        loop.run_until_complete(test_loop())
     except CancelledError:
         assert True
 
-@aioresponses()        
-def test_async_replay_stream(*mock):
 
-    async def test_loop(*mock):
-        pattern_live = re.compile(r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$')
-        pattern_replay =  re.compile(r'^https://www.youtube.com/live_chat_replay/get_live_chat_replay\?continuation=.*$')
-        #empty livechat -> switch to fetch replaychat
-        _text_live = _open_file('tests/testdata/finished_live.json')
-        _text_replay = _open_file('tests/testdata/chatreplay.json')
-        mock[0].get(pattern_live, status=200, body=_text_live)
-        mock[0].get(pattern_replay, status=200, body=_text_replay)
+def test_async_replay_stream(httpx_mock: HTTPXMock):
+    add_response_file(httpx_mock, 'tests/testdata/finished_live.json')
+    add_response_file(httpx_mock, 'tests/testdata/chatreplay.json')
 
-        chat = LiveChatAsync(video_id='', processor = DummyProcessor())
-        chats = await chat.get()
-        rawdata = chats[0]["chatdata"]
-        #assert fetching replaychat data
-        assert list(rawdata[0]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer"
-        assert list(rawdata[14]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer"
-
-    loop = asyncio.get_event_loop()
-    try:
-        loop.run_until_complete(test_loop(*mock))
-    except CancelledError:
-        assert True
-
-@aioresponses()
-def test_async_force_replay(*mock):
-
-    async def test_loop(*mock):
-        pattern_live = re.compile(r'^https://www.youtube.com/live_chat/get_live_chat\?continuation=.*$')
-        pattern_replay =  re.compile(r'^https://www.youtube.com/live_chat_replay/get_live_chat_replay\?continuation=.*$')
-        #valid live data, but force_replay = True
-        _text_live = _open_file('tests/testdata/test_stream.json')
-        #valid replay data
-        _text_replay = _open_file('tests/testdata/chatreplay.json')
-        
-        mock[0].get(pattern_live, status=200, body=_text_live)
-        mock[0].get(pattern_replay, status=200, body=_text_replay)
-        #force replay
-        chat = LiveChatAsync(video_id='', processor = DummyProcessor(), force_replay = True)
+    async def test_loop():
+        chat = LiveChatAsync(video_id='__test_id__', processor=DummyProcessor())
         chats = await chat.get()
         rawdata = chats[0]["chatdata"]
         # assert fetching replaychat data
-        assert list(rawdata[14]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer"
-        # assert not mix livechat data
-        assert list(rawdata[2]["addChatItemAction"]["item"].keys())[0] != "liveChatPlaceholderItemRenderer"
+        assert list(rawdata[0]["addChatItemAction"]["item"].keys())[
+            0] == "liveChatTextMessageRenderer"
+        assert list(rawdata[14]["addChatItemAction"]["item"].keys())[
+            0] == "liveChatPaidMessageRenderer"
 
     loop = asyncio.get_event_loop()
     try:
-        loop.run_until_complete(test_loop(*mock))
+        loop.run_until_complete(test_loop())
     except CancelledError:
         assert True
 
-def test_multithread_live_stream(mocker):
 
-    _text = _open_file('tests/testdata/test_stream.json')
-    responseMock = mocker.Mock()
-    responseMock.status_code = 200
-    responseMock.text = _text
-    mocker.patch('requests.Session.get').return_value.__enter__.return_value = responseMock
+def test_async_force_replay(httpx_mock: HTTPXMock):
+    add_response_file(httpx_mock, 'tests/testdata/test_stream.json')
+    add_response_file(httpx_mock, 'tests/testdata/chatreplay.json')
 
-    chat = LiveChat(video_id='test_id', processor = DummyProcessor())
+    async def test_loop():
+        chat = LiveChatAsync(
+            video_id='__test_id__', processor=DummyProcessor(), force_replay=True)
+        chats = await chat.get()
+        rawdata = chats[0]["chatdata"]
+        # assert fetching replaychat data
+        assert list(rawdata[14]["addChatItemAction"]["item"].keys())[
+            0] == "liveChatPaidMessageRenderer"
+        # assert not mix livechat data
+        assert list(rawdata[2]["addChatItemAction"]["item"].keys())[
+            0] != "liveChatPlaceholderItemRenderer"
+
+    loop = asyncio.get_event_loop()
+    try:
+        loop.run_until_complete(test_loop())
+    except CancelledError:
+        assert True
+
+
+def test_multithread_live_stream(httpx_mock: HTTPXMock):
+    add_response_file(httpx_mock, 'tests/testdata/test_stream.json')
+    chat = LiveChat(video_id='__test_id__', processor=DummyProcessor())
     chats = chat.get()
     rawdata = chats[0]["chatdata"]
-    #assert fetching livachat data
-    assert list(rawdata[0]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer"
-    assert list(rawdata[1]["addChatItemAction"]["item"].keys())[0] == "liveChatTextMessageRenderer"
-    assert list(rawdata[2]["addChatItemAction"]["item"].keys())[0] == "liveChatPlaceholderItemRenderer"
-    assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerPaidMessageItemRenderer"
-    assert list(rawdata[4]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidMessageRenderer"
-    assert list(rawdata[5]["addChatItemAction"]["item"].keys())[0] == "liveChatPaidStickerRenderer"
-    assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[0] == "liveChatTickerSponsorItemRenderer"
+    # assert fetching livachat data
+    assert list(rawdata[0]["addChatItemAction"]["item"].keys())[
+        0] == "liveChatTextMessageRenderer"
+    assert list(rawdata[1]["addChatItemAction"]["item"].keys())[
+        0] == "liveChatTextMessageRenderer"
+    assert list(rawdata[2]["addChatItemAction"]["item"].keys())[
+        0] == "liveChatPlaceholderItemRenderer"
+    assert list(rawdata[3]["addLiveChatTickerItemAction"]["item"].keys())[
+        0] == "liveChatTickerPaidMessageItemRenderer"
+    assert list(rawdata[4]["addChatItemAction"]["item"].keys())[
+        0] == "liveChatPaidMessageRenderer"
+    assert list(rawdata[5]["addChatItemAction"]["item"].keys())[
+        0] == "liveChatPaidStickerRenderer"
+    assert list(rawdata[6]["addLiveChatTickerItemAction"]["item"].keys())[
+        0] == "liveChatTickerSponsorItemRenderer"
     chat.terminate()
-
-    
-
-
-

tests/test_liveparam.py

@@ -5,5 +5,5 @@ def test_liveparam_0(mocker):
     _ts1= 1546268400
     param = liveparam._build("01234567890",
     *([_ts1*1000000 for i in range(5)]), topchat_only=False)
-    test_param="0ofMyAPiARp8Q2c4S0RRb0xNREV5TXpRMU5qYzRPVEFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5TURFeU16UTFOamM0T1RBbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiAuNbVqsrfAjAAOABAAkorCAEQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgLjW1arK3wJYA1CAuNbVqsrfAliAuNbVqsrfAmgBggEECAEQAIgBAKABgLjW1arK3wI%3D"
+    test_param="0ofMyAN1GhxDZzhLRFFvTE1ERXlNelExTmpjNE9UQWdBUT09KIC41tWqyt8CMAA4AEABShsIABAAGAAgADoAQABKAFCAuNbVqsrfAlgDeABQgLjW1arK3wJYgLjW1arK3wJoAYIBAggBiAEAmgECCACgAYC41tWqyt8C"
     assert test_param == param

tests/test_parser.py

@@ -1,44 +1,34 @@
-import pytest
 from pytchat.parser.live import Parser
 import json
-import asyncio,aiohttp
-from aioresponses import aioresponses
-from pytchat.exceptions import (
-    NoLivechatRendererException,NoYtinitialdataException,
-    ResponseContextError, NoContentsException)
+from pytchat.exceptions import NoContents
+
+
+parser = Parser(is_replay=False)
 
 
 def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
+    with open(path, mode='r', encoding='utf-8') as f:
         return f.read()
-parser = Parser(is_replay = False)
 
-@aioresponses()
+
 def test_finishedlive(*mock):
-    '''配信が終了した動画を正しく処理できるか'''
-
     _text = _open_file('tests/testdata/finished_live.json')
     _text = json.loads(_text)
 
     try:
-        parser.parse(parser.get_contents(_text))
+        parser.parse(parser.get_contents(_text)[0])
         assert False
-    except NoContentsException:
+    except NoContents:
         assert True
 
-@aioresponses()
-def test_parsejson(*mock):
-    '''jsonを正常にパースできるか'''
 
+def test_parsejson(*mock):
     _text = _open_file('tests/testdata/paramgen_firstread.json')
     _text = json.loads(_text)
 
     try:
-        parser.parse(parser.get_contents(_text))
-        jsn = _text
-        timeout = jsn["response"]["continuationContents"]["liveChatContinuation"]["continuations"][0]["timedContinuationData"]["timeoutMs"]
-        continuation = jsn["response"]["continuationContents"]["liveChatContinuation"]["continuations"][0]["timedContinuationData"]["continuation"]
-        assert 5035 == timeout
-        assert "0ofMyAPiARp8Q2c4S0RRb0xhelJMZDBsWFQwdERkalFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5YXpSTGQwbFhUMHREZGpRbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiPz5-Os-PkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgJqXjrPj5AJYA1CRwciOs-PkAli3pNq1k-PkAmgBggEECAEQAIgBAKABjbfnjrPj5AI%3D" == continuation
-    except:
+        s, _ = parser.parse(parser.get_contents(_text)[0])
+        assert s['timeoutMs'] == 5035
+        assert s['continuation'] == "0ofMyAPiARp8Q2c4S0RRb0xhelJMZDBsWFQwdERkalFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5YXpSTGQwbFhUMHREZGpRbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiPz5-Os-PkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgJqXjrPj5AJYA1CRwciOs-PkAli3pNq1k-PkAmgBggEECAEQAIgBAKABjbfnjrPj5AI%3D"
+    except Exception:
         assert False

tests/test_speed_calculator.py

@@ -1,15 +1,9 @@
 import json
-import pytest
-import asyncio,aiohttp
 from pytchat.parser.live import Parser
-from pytchat.processors.compatible.processor import CompatibleProcessor
-from pytchat.exceptions import (
-    NoLivechatRendererException,NoYtinitialdataException,
-    ResponseContextError, NoContentsException)
-
 from pytchat.processors.speed.calculator import SpeedCalculator
 
-parser = Parser(is_replay =False)
+parser = Parser(is_replay=False)
+
 
 def test_speed_1(mocker):
     '''test speed calculation with normal json.
@@ -21,15 +15,16 @@ def test_speed_1(mocker):
 
     _json = _open_file("tests/testdata/speed/speedtest_normal.json")
 
-    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json))[0])
     data = {
-        "video_id" : "",
-        "timeout" : 10,
-        "chatdata" : chatdata
+        "video_id": "",
+        "timeout": 10,
+        "chatdata": chatdata
     }
     ret = processor.process([data])
     assert 30 == ret
 
+
 def test_speed_2(mocker):
     '''test speed calculation with no valid chat data.
     '''
@@ -37,14 +32,15 @@ def test_speed_2(mocker):
 
     _json = _open_file("tests/testdata/speed/speedtest_undefined.json")
 
-    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json))[0])
     data = {
-        "video_id" : "",
-        "timeout" : 10,
-        "chatdata" : chatdata
+        "video_id": "",
+        "timeout": 10,
+        "chatdata": chatdata
     }
     ret = processor.process([data])
-    assert 0 == ret
+    assert ret == 0
+
 
 def test_speed_3(mocker):
     '''test speed calculation with empty data.
@@ -53,16 +49,16 @@ def test_speed_3(mocker):
 
     _json = _open_file("tests/testdata/speed/speedtest_empty.json")
 
-    _, chatdata = parser.parse(parser.get_contents(json.loads(_json)))
+    _, chatdata = parser.parse(parser.get_contents(json.loads(_json))[0])
     data = {
-        "video_id" : "",
-        "timeout" : 10,
-        "chatdata" : chatdata
+        "video_id": "",
+        "timeout": 10,
+        "chatdata": chatdata
     }
     ret = processor.process([data])
-    assert 0 == ret
+    assert ret == 0
 
 
 def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
+    with open(path, mode='r', encoding='utf-8') as f:
         return f.read()

tests/test_videoinfo.py

@@ -1,62 +0,0 @@
-from pytchat.tool.videoinfo import VideoInfo
-from pytchat.exceptions import InvalidVideoIdException
-import pytest
-
-def _open_file(path):
-    with open(path,mode ='r',encoding = 'utf-8') as f:
-        return f.read()
-
-def _set_test_data(filepath, mocker):
-    _text = _open_file(filepath)
-    response_mock = mocker.Mock()
-    response_mock.status_code = 200
-    response_mock.text = _text
-    mocker.patch('requests.get').return_value = response_mock
-
-def test_archived_page(mocker):
-    _set_test_data('tests/testdata/videoinfo/archived_page.txt', mocker)
-    info = VideoInfo('test_id')
-    actual_thumbnail_url =  'https://i.ytimg.com/vi/fzI9FNjXQ0o/hqdefault.jpg'
-    assert info.video_id == 'test_id'
-    assert info.get_channel_name() == 'GitHub'
-    assert info.get_thumbnail() == actual_thumbnail_url
-    assert info.get_title() == 'GitHub Arctic Code Vault'
-    assert info.get_channel_id() == 'UC7c3Kb6jYCRj4JOHHZTxKsQ'
-    assert info.get_duration() == 148
-
-def test_live_page(mocker):
-    _set_test_data('tests/testdata/videoinfo/live_page.txt', mocker)
-    info = VideoInfo('test_id')
-    '''live page :duration = 0'''        
-    assert info.get_duration() == 0
-    assert info.video_id == 'test_id'
-    assert info.get_channel_name() == 'BGM channel'
-    assert info.get_thumbnail() == \
-        'https://i.ytimg.com/vi/fEvM-OUbaKs/hqdefault_live.jpg'
-    assert info.get_title() == (
-        'Coffee Jazz Music - Chill Out Lounge Jazz Music Radio'
-        ' - 24/7 Live Stream - Slow Jazz')
-    assert info.get_channel_id() == 'UCQINXHZqCU5i06HzxRkujfg'
-
-def test_invalid_video_id(mocker):
-    '''Test case invalid video_id is specified.'''
-    _set_test_data(
-        'tests/testdata/videoinfo/invalid_video_id_page.txt', mocker)
-    try:
-        _ = VideoInfo('test_id')
-        assert False
-    except InvalidVideoIdException:
-        assert True
-
-def test_no_info(mocker):
-    '''Test case the video page has renderer, but no info.'''
-    _set_test_data(
-        'tests/testdata/videoinfo/no_info_page.txt', mocker)
-    info = VideoInfo('test_id')
-    assert info.video_id == 'test_id'
-    assert info.get_channel_name() is None
-    assert info.get_thumbnail() is None
-    assert info.get_title() is None
-    assert info.get_channel_id() is None
-    assert info.get_duration() is None
-

tests/testdata/calculator/text_only.json

@@ -1,89 +1,87 @@
 {
- "response": {
   "responseContext": {
-   "webResponseContextExtensionData": ""
+    "webResponseContextExtensionData": ""
   },
   "continuationContents": {
-   "liveChatContinuation": {
-    "continuations": [
-     {
-      "invalidationContinuationData": {
-       "invalidationId": {
-        "objectSource": 1000,
-        "objectId": "___objectId___",
-        "topic": "chat~00000000000~0000000",
-        "subscribeToGcmTopics": true,
-        "protoCreationTimestampMs": "1577804400000"
-       },
-       "timeoutMs": 10000,
-       "continuation": "___continuation___"
-      }
-     }
-    ],
-    "actions": [
-     {
-      "replayChatItemAction": {
-       "actions": [
+    "liveChatContinuation": {
+      "continuations": [
         {
-         "addChatItemAction": {
-          "item": {
-           "liveChatTextMessageRenderer": {
-            "message": {
-             "runs": [
-              {
-               "text": "dummy_message"
-              }
-             ]
+          "invalidationContinuationData": {
+            "invalidationId": {
+              "objectSource": 1000,
+              "objectId": "___objectId___",
+              "topic": "chat~00000000000~0000000",
+              "subscribeToGcmTopics": true,
+              "protoCreationTimestampMs": "1577804400000"
             },
-            "authorName": {
-             "simpleText": "author_name"
-            },
-            "authorPhoto": {
-             "thumbnails": [
-              {
-               "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
-               "width": 32,
-               "height": 32
-              },
-              {
-               "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
-               "width": 64,
-               "height": 64
-              }
-             ]
-            },
-            "contextMenuEndpoint": {
-             "commandMetadata": {
-              "webCommandMetadata": {
-               "ignoreNavigation": true
-              }
-             },
-             "liveChatItemContextMenuEndpoint": {
-              "params": "___params___"
-             }
-            },
-            "id": "dummy_id",
-            "timestampUsec": 0,
-            "authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url",
-            "contextMenuAccessibility": {
-             "accessibilityData": {
-              "label": "コメントの操作"
-             }
-            },
-            "timestampText": {
-             "simpleText": "0:00"
-            }
-           }
-          },
-          "clientId": "dummy_client_id"
-         }
+            "timeoutMs": 10000,
+            "continuation": "___continuation___"
+          }
         }
-       ],
-       "videoOffsetTimeMsec": "10000"
-      }
-     }
-    ]
-   }
+      ],
+      "actions": [
+        {
+          "replayChatItemAction": {
+            "actions": [
+              {
+                "addChatItemAction": {
+                  "item": {
+                    "liveChatTextMessageRenderer": {
+                      "message": {
+                        "runs": [
+                          {
+                            "text": "dummy_message"
+                          }
+                        ]
+                      },
+                      "authorName": {
+                        "simpleText": "author_name"
+                      },
+                      "authorPhoto": {
+                        "thumbnails": [
+                          {
+                            "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                            "width": 32,
+                            "height": 32
+                          },
+                          {
+                            "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                            "width": 64,
+                            "height": 64
+                          }
+                        ]
+                      },
+                      "contextMenuEndpoint": {
+                        "commandMetadata": {
+                          "webCommandMetadata": {
+                            "ignoreNavigation": true
+                          }
+                        },
+                        "liveChatItemContextMenuEndpoint": {
+                          "params": "___params___"
+                        }
+                      },
+                      "id": "dummy_id",
+                      "timestampUsec": 0,
+                      "authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url",
+                      "contextMenuAccessibility": {
+                        "accessibilityData": {
+                          "label": "コメントの操作"
+                        }
+                      },
+                      "timestampText": {
+                        "simpleText": "0:00"
+                      }
+                    }
+                  },
+                  "clientId": "dummy_client_id"
+                }
+              }
+            ],
+            "videoOffsetTimeMsec": "10000"
+          }
+        }
+      ]
+    }
   }
- }
 }

tests/testdata/compatible/superchat.json

@@ -1,282 +1,281 @@
 {
-    "xsrf_token": "QUFFLUhqbWVqWVRTUjhBcGRkNWR5Q2F3VWhxcTd0RkF1UXxBQ3Jtc0trMThFVGVVNTFnWmZucnkwNlJJMEZ2bndUS1I3b2dpLUtTNE92dEgwX3Y0MkNZU2NXdGY1QTMtX09BUGphYUpDQlc1dFhiTm9jLS1sQXVCNHpsdllqcm1id0t1RFBCM3A1b2o3OGt0Yjd6TE5wcmxBbjNyQktjc1lWZ3hjM1RuYk83YkQ0VVN3MGUybjAwSE90SS1YNkxvMUV5YVE=",
-    "timing": {
-        "info": {
-            "st": 148
-        }
+  "xsrf_token": "QUFFLUhqbWVqWVRTUjhBcGRkNWR5Q2F3VWhxcTd0RkF1UXxBQ3Jtc0trMThFVGVVNTFnWmZucnkwNlJJMEZ2bndUS1I3b2dpLUtTNE92dEgwX3Y0MkNZU2NXdGY1QTMtX09BUGphYUpDQlc1dFhiTm9jLS1sQXVCNHpsdllqcm1id0t1RFBCM3A1b2o3OGt0Yjd6TE5wcmxBbjNyQktjc1lWZ3hjM1RuYk83YkQ0VVN3MGUybjAwSE90SS1YNkxvMUV5YVE=",
+  "timing": {
+    "info": {
+      "st": 148
+    }
+  },
+  "endpoint": {
+    "commandMetadata": {
+      "webCommandMetadata": {
+        "url": "/live_chat/get_live_chat?continuation=0ofMyAPiARp8Q2c4S0RRb0xPREJQZW5SS2FIVTNOemdhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5T0RCUGVuUkthSFUzTnpnbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCja2bHMpPvkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQwJKPoqT75AJYA1DXxuTMpPvkAliL_fvZoPvkAmgBggEECAEQAIgBAKABxM3xzKT75AI%253D"
+      }
     },
-    "endpoint": {
-        "commandMetadata": {
-            "webCommandMetadata": {
-                "url": "/live_chat/get_live_chat?continuation=0ofMyAPiARp8Q2c4S0RRb0xPREJQZW5SS2FIVTNOemdhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5T0RCUGVuUkthSFUzTnpnbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCja2bHMpPvkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQwJKPoqT75AJYA1DXxuTMpPvkAliL_fvZoPvkAmgBggEECAEQAIgBAKABxM3xzKT75AI%253D"
-            }
-        },
-        "urlEndpoint": {
-            "url": "/live_chat/get_live_chat?continuation=0ofMyAPiARp8Q2c4S0RRb0xPREJQZW5SS2FIVTNOemdhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5T0RCUGVuUkthSFUzTnpnbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCja2bHMpPvkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQwJKPoqT75AJYA1DXxuTMpPvkAliL_fvZoPvkAmgBggEECAEQAIgBAKABxM3xzKT75AI%253D"
+    "urlEndpoint": {
+      "url": "/live_chat/get_live_chat?continuation=0ofMyAPiARp8Q2c4S0RRb0xPREJQZW5SS2FIVTNOemdhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5T0RCUGVuUkthSFUzTnpnbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCja2bHMpPvkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQwJKPoqT75AJYA1DXxuTMpPvkAliL_fvZoPvkAmgBggEECAEQAIgBAKABxM3xzKT75AI%253D"
+    }
+  },
+  "url": "/live_chat/get_live_chat?continuation=0ofMyAPiARp8Q2c4S0RRb0xPREJQZW5SS2FIVTNOemdhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5T0RCUGVuUkthSFUzTnpnbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCja2bHMpPvkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQwJKPoqT75AJYA1DXxuTMpPvkAliL_fvZoPvkAmgBggEECAEQAIgBAKABxM3xzKT75AI%253D",
+  "csn": "n2STXd2iKZr2gAOt9qvgCg",
+
+  "responseContext": {
+    "serviceTrackingParams": [
+      {
+        "service": "CSI",
+        "params": [
+          {
+            "key": "GetLiveChat_rid",
+            "value": "0x9290108c05344647"
+          },
+          {
+            "key": "c",
+            "value": "WEB"
+          },
+          {
+            "key": "cver",
+            "value": "2.20191001.04.00"
+          },
+          {
+            "key": "yt_li",
+            "value": "0"
+          }
+        ]
+      },
+      {
+        "service": "GFEEDBACK",
+        "params": [
+          {
+            "key": "e",
+            "value": "23744176,23788875,23793834,23794620,23804281,23806159,23816483,23819244,23820768,23826780,23827354,23830392,23832125,23835020,23836965,23837741,23837772,23837993,23838235,23839362,23840155,23840217,23841118,23841454,23842630,23842662,23842883,23842986,23843289,23843534,23843767,23845644,9449243,9471239,9474360"
+          },
+          {
+            "key": "logged_in",
+            "value": "0"
+          }
+        ]
+      },
+      {
+        "service": "GUIDED_HELP",
+        "params": [
+          {
+            "key": "logged_in",
+            "value": "0"
+          }
+        ]
+      },
+      {
+        "service": "ECATCHER",
+        "params": [
+          {
+            "key": "client.name",
+            "value": "WEB"
+          },
+          {
+            "key": "client.version",
+            "value": "2.20191001"
+          },
+          {
+            "key": "innertube.build.changelist",
+            "value": "272006966"
+          },
+          {
+            "key": "innertube.build.experiments.source_version",
+            "value": "272166268"
+          },
+          {
+            "key": "innertube.build.label",
+            "value": "youtube.ytfe.innertube_20190930_5_RC0"
+          },
+          {
+            "key": "innertube.build.timestamp",
+            "value": "1569863426"
+          },
+          {
+            "key": "innertube.build.variants.checksum",
+            "value": "1a800c1a2396906f1cbb7f670d43b6f5"
+          },
+          {
+            "key": "innertube.run.job",
+            "value": "ytfe-innertube-replica-only.ytfe"
+          }
+        ]
+      }
+    ],
+    "webResponseContextExtensionData": {
+      "ytConfigData": {
+        "csn": "n2STXd2iKZr2gAOt9qvgCg",
+        "visitorData": "CgtPQm1xTmtvNm1Tcyifyc3sBQ%3D%3D"
+      }
+    }
+  },
+  "continuationContents": {
+    "liveChatContinuation": {
+      "continuations": [
+        {
+          "timedContinuationData": {
+            "timeoutMs": 8860,
+            "continuation": "0ofMyAPiARp8Q2c4S0RRb0xPREJQZW5SS2FIVTNOemdhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5T0RCUGVuUkthSFUzTnpnbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCid_ejQpPvkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQwJKPoqT75AJYA1DF-ovRpPvkAliL_fvZoPvkAmgBggEECAEQAIgBAKABtNic0aT75AI%3D"
+          }
         }
-    },
-    "url": "\/live_chat\/get_live_chat?continuation=0ofMyAPiARp8Q2c4S0RRb0xPREJQZW5SS2FIVTNOemdhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5T0RCUGVuUkthSFUzTnpnbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCja2bHMpPvkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQwJKPoqT75AJYA1DXxuTMpPvkAliL_fvZoPvkAmgBggEECAEQAIgBAKABxM3xzKT75AI%253D",
-    "csn": "n2STXd2iKZr2gAOt9qvgCg",
-    "response": {
-        "responseContext": {
-            "serviceTrackingParams": [
-                {
-                    "service": "CSI",
-                    "params": [
-                        {
-                            "key": "GetLiveChat_rid",
-                            "value": "0x9290108c05344647"
-                        },
-                        {
-                            "key": "c",
-                            "value": "WEB"
-                        },
-                        {
-                            "key": "cver",
-                            "value": "2.20191001.04.00"
-                        },
-                        {
-                            "key": "yt_li",
-                            "value": "0"
-                        }
-                    ]
+      ],
+      "actions": [
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatPaidMessageRenderer": {
+                "id": "ChwKGkNOblpoTXFrLS1RQ0ZRSU9ZQW9kclhrRXNn",
+                "timestampUsec": "1569940638420061",
+                "authorName": {
+                  "simpleText": "九十九 万"
                 },
-                {
-                    "service": "GFEEDBACK",
-                    "params": [
-                        {
-                            "key": "e",
-                            "value": "23744176,23788875,23793834,23794620,23804281,23806159,23816483,23819244,23820768,23826780,23827354,23830392,23832125,23835020,23836965,23837741,23837772,23837993,23838235,23839362,23840155,23840217,23841118,23841454,23842630,23842662,23842883,23842986,23843289,23843534,23843767,23845644,9449243,9471239,9474360"
-                        },
-                        {
-                            "key": "logged_in",
-                            "value": "0"
-                        }
-                    ]
-                },
-                {
-                    "service": "GUIDED_HELP",
-                    "params": [
-                        {
-                            "key": "logged_in",
-                            "value": "0"
-                        }
-                    ]
-                },
-                {
-                    "service": "ECATCHER",
-                    "params": [
-                        {
-                            "key": "client.name",
-                            "value": "WEB"
-                        },
-                        {
-                            "key": "client.version",
-                            "value": "2.20191001"
-                        },
-                        {
-                            "key": "innertube.build.changelist",
-                            "value": "272006966"
-                        },
-                        {
-                            "key": "innertube.build.experiments.source_version",
-                            "value": "272166268"
-                        },
-                        {
-                            "key": "innertube.build.label",
-                            "value": "youtube.ytfe.innertube_20190930_5_RC0"
-                        },
-                        {
-                            "key": "innertube.build.timestamp",
-                            "value": "1569863426"
-                        },
-                        {
-                            "key": "innertube.build.variants.checksum",
-                            "value": "1a800c1a2396906f1cbb7f670d43b6f5"
-                        },
-                        {
-                            "key": "innertube.run.job",
-                            "value": "ytfe-innertube-replica-only.ytfe"
-                        }
-                    ]
-                }
-            ],
-            "webResponseContextExtensionData": {
-                "ytConfigData": {
-                    "csn": "n2STXd2iKZr2gAOt9qvgCg",
-                    "visitorData": "CgtPQm1xTmtvNm1Tcyifyc3sBQ%3D%3D"
-                }
-            }
-        },
-        "continuationContents": {
-            "liveChatContinuation": {
-                "continuations": [
+                "authorPhoto": {
+                  "thumbnails": [
                     {
-                        "timedContinuationData": {
-                            "timeoutMs": 8860,
-                            "continuation": "0ofMyAPiARp8Q2c4S0RRb0xPREJQZW5SS2FIVTNOemdhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5T0RCUGVuUkthSFUzTnpnbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCid_ejQpPvkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQwJKPoqT75AJYA1DF-ovRpPvkAliL_fvZoPvkAmgBggEECAEQAIgBAKABtNic0aT75AI%3D"
-                        }
-                    }
-                ],
-                "actions": [
-                    {
-                        "addChatItemAction": {
-                            "item": {
-                                "liveChatPaidMessageRenderer": {
-                                    "id": "ChwKGkNOblpoTXFrLS1RQ0ZRSU9ZQW9kclhrRXNn",
-                                    "timestampUsec": "1569940638420061",
-                                    "authorName": {
-                                        "simpleText": "九十九 万"
-                                    },
-                                    "authorPhoto": {
-                                        "thumbnails": [
-                                            {
-                                                "url": "https://yt3.ggpht.com/-FoBvzoKhbZs/AAAAAAAAAAI/AAAAAAAAAAA/oyb-UCYFbfA/s32-c-k-no-mo-rj-c0xffffff/photo.jpg",
-                                                "width": 32,
-                                                "height": 32
-                                            },
-                                            {
-                                                "url": "https://yt3.ggpht.com/-FoBvzoKhbZs/AAAAAAAAAAI/AAAAAAAAAAA/oyb-UCYFbfA/s64-c-k-no-mo-rj-c0xffffff/photo.jpg",
-                                                "width": 64,
-                                                "height": 64
-                                            }
-                                        ]
-                                    },
-                                    "purchaseAmountText": {
-                                        "simpleText": "￥846"
-                                    },
-                                    "message": {
-                                        "runs": [
-                                            {
-                                                "text": "ボルガ博士お許しください代"
-                                            }
-                                        ]
-                                    },
-                                    "headerBackgroundColor": 4278239141,
-                                    "headerTextColor": 4278190080,
-                                    "bodyBackgroundColor": 4280150454,
-                                    "bodyTextColor": 4278190080,
-                                    "authorExternalChannelId": "UCoVtCMzg2vleAVsa7kw05AA",
-                                    "authorNameTextColor": 2315255808,
-                                    "contextMenuEndpoint": {
-                                        "commandMetadata": {
-                                            "webCommandMetadata": {
-                                                "ignoreNavigation": true
-                                            }
-                                        },
-                                        "liveChatItemContextMenuEndpoint": {
-                                            "params": "Q2g0S0hBb2FRMDV1V21oTmNXc3RMVkZEUmxGSlQxbEJiMlJ5V0d0RmMyY1FBQm80Q2cwS0N6Z3dUM3AwU21oMU56YzRLaWNLR0ZWRFNYbDBUbU52ZWpSd1YzcFlaa3hrWVRCRWIxVk1VUklMT0RCUGVuUkthSFUzTnpnZ0FpZ0JNaG9LR0ZWRGIxWjBRMDE2WnpKMmJHVkJWbk5oTjJ0M01EVkJRUSUzRCUzRA=="
-                                        }
-                                    },
-                                    "timestampColor": 2147483648,
-                                    "contextMenuAccessibility": {
-                                        "accessibilityData": {
-                                            "label": "コメントの操作"
-                                        }
-                                    }
-                                }
-                            }
-                        }
+                      "url": "https://yt3.ggpht.com/-FoBvzoKhbZs/AAAAAAAAAAI/AAAAAAAAAAA/oyb-UCYFbfA/s32-c-k-no-mo-rj-c0xffffff/photo.jpg",
+                      "width": 32,
+                      "height": 32
                     },
                     {
-                        "addLiveChatTickerItemAction": {
-                            "item": {
-                                "liveChatTickerPaidMessageItemRenderer": {
-                                    "id": "ChwKGkNOblpoTXFrLS1RQ0ZRSU9ZQW9kclhrRXNn",
-                                    "amount": {
-                                        "simpleText": "￥846"
-                                    },
-                                    "amountTextColor": 4278190080,
-                                    "startBackgroundColor": 4280150454,
-                                    "endBackgroundColor": 4278239141,
-                                    "authorPhoto": {
-                                        "thumbnails": [
-                                            {
-                                                "url": "https://yt3.ggpht.com/-FoBvzoKhbZs/AAAAAAAAAAI/AAAAAAAAAAA/oyb-UCYFbfA/s32-c-k-no-mo-rj-c0xffffff/photo.jpg",
-                                                "width": 32,
-                                                "height": 32
-                                            },
-                                            {
-                                                "url": "https://yt3.ggpht.com/-FoBvzoKhbZs/AAAAAAAAAAI/AAAAAAAAAAA/oyb-UCYFbfA/s64-c-k-no-mo-rj-c0xffffff/photo.jpg",
-                                                "width": 64,
-                                                "height": 64
-                                            }
-                                        ]
-                                    },
-                                    "durationSec": 120,
-                                    "showItemEndpoint": {
-                                        "commandMetadata": {
-                                            "webCommandMetadata": {
-                                                "ignoreNavigation": true
-                                            }
-                                        },
-                                        "showLiveChatItemEndpoint": {
-                                            "renderer": {
-                                                "liveChatPaidMessageRenderer": {
-                                                    "id": "ChwKGkNOblpoTXFrLS1RQ0ZRSU9ZQW9kclhrRXNn",
-                                                    "timestampUsec": "1569940638420061",
-                                                    "authorName": {
-                                                        "simpleText": "九十九 万"
-                                                    },
-                                                    "authorPhoto": {
-                                                        "thumbnails": [
-                                                            {
-                                                                "url": "https://yt3.ggpht.com/-FoBvzoKhbZs/AAAAAAAAAAI/AAAAAAAAAAA/oyb-UCYFbfA/s32-c-k-no-mo-rj-c0xffffff/photo.jpg",
-                                                                "width": 32,
-                                                                "height": 32
-                                                            },
-                                                            {
-                                                                "url": "https://yt3.ggpht.com/-FoBvzoKhbZs/AAAAAAAAAAI/AAAAAAAAAAA/oyb-UCYFbfA/s64-c-k-no-mo-rj-c0xffffff/photo.jpg",
-                                                                "width": 64,
-                                                                "height": 64
-                                                            }
-                                                        ]
-                                                    },
-                                                    "purchaseAmountText": {
-                                                        "simpleText": "￥846"
-                                                    },
-                                                    "message": {
-                                                        "runs": [
-                                                            {
-                                                                "text": "ボルガ博士お許しください代"
-                                                            }
-                                                        ]
-                                                    },
-                                                    "headerBackgroundColor": 4278239141,
-                                                    "headerTextColor": 4278190080,
-                                                    "bodyBackgroundColor": 4280150454,
-                                                    "bodyTextColor": 4278190080,
-                                                    "authorExternalChannelId": "UCoVtCMzg2vleAVsa7kw05AA",
-                                                    "authorNameTextColor": 2315255808,
-                                                    "contextMenuEndpoint": {
-                                                        "commandMetadata": {
-                                                            "webCommandMetadata": {
-                                                                "ignoreNavigation": true
-                                                            }
-                                                        },
-                                                        "liveChatItemContextMenuEndpoint": {
-                                                            "params": "Q2g0S0hBb2FRMDV1V21oTmNXc3RMVkZEUmxGSlQxbEJiMlJ5V0d0RmMyY1FBQm80Q2cwS0N6Z3dUM3AwU21oMU56YzRLaWNLR0ZWRFNYbDBUbU52ZWpSd1YzcFlaa3hrWVRCRWIxVk1VUklMT0RCUGVuUkthSFUzTnpnZ0FpZ0JNaG9LR0ZWRGIxWjBRMDE2WnpKMmJHVkJWbk5oTjJ0M01EVkJRUSUzRCUzRA=="
-                                                        }
-                                                    },
-                                                    "timestampColor": 2147483648,
-                                                    "contextMenuAccessibility": {
-                                                        "accessibilityData": {
-                                                            "label": "コメントの操作"
-                                                        }
-                                                    }
-                                                }
-                                            }
-                                        }
-                                    },
-                                    "authorExternalChannelId": "UCoVtCMzg2vleAVsa7kw05AA",
-                                    "fullDurationSec": 120
-                                }
-                            },
-                            "durationSec": "120"
-                        }
+                      "url": "https://yt3.ggpht.com/-FoBvzoKhbZs/AAAAAAAAAAI/AAAAAAAAAAA/oyb-UCYFbfA/s64-c-k-no-mo-rj-c0xffffff/photo.jpg",
+                      "width": 64,
+                      "height": 64
                     }
-                ]
+                  ]
+                },
+                "purchaseAmountText": {
+                  "simpleText": "￥846"
+                },
+                "message": {
+                  "runs": [
+                    {
+                      "text": "ボルガ博士お許しください代"
+                    }
+                  ]
+                },
+                "headerBackgroundColor": 4278239141,
+                "headerTextColor": 4278190080,
+                "bodyBackgroundColor": 4280150454,
+                "bodyTextColor": 4278190080,
+                "authorExternalChannelId": "UCoVtCMzg2vleAVsa7kw05AA",
+                "authorNameTextColor": 2315255808,
+                "contextMenuEndpoint": {
+                  "commandMetadata": {
+                    "webCommandMetadata": {
+                      "ignoreNavigation": true
+                    }
+                  },
+                  "liveChatItemContextMenuEndpoint": {
+                    "params": "Q2g0S0hBb2FRMDV1V21oTmNXc3RMVkZEUmxGSlQxbEJiMlJ5V0d0RmMyY1FBQm80Q2cwS0N6Z3dUM3AwU21oMU56YzRLaWNLR0ZWRFNYbDBUbU52ZWpSd1YzcFlaa3hrWVRCRWIxVk1VUklMT0RCUGVuUkthSFUzTnpnZ0FpZ0JNaG9LR0ZWRGIxWjBRMDE2WnpKMmJHVkJWbk5oTjJ0M01EVkJRUSUzRCUzRA=="
+                  }
+                },
+                "timestampColor": 2147483648,
+                "contextMenuAccessibility": {
+                  "accessibilityData": {
+                    "label": "コメントの操作"
+                  }
+                }
+              }
             }
+          }
+        },
+        {
+          "addLiveChatTickerItemAction": {
+            "item": {
+              "liveChatTickerPaidMessageItemRenderer": {
+                "id": "ChwKGkNOblpoTXFrLS1RQ0ZRSU9ZQW9kclhrRXNn",
+                "amount": {
+                  "simpleText": "￥846"
+                },
+                "amountTextColor": 4278190080,
+                "startBackgroundColor": 4280150454,
+                "endBackgroundColor": 4278239141,
+                "authorPhoto": {
+                  "thumbnails": [
+                    {
+                      "url": "https://yt3.ggpht.com/-FoBvzoKhbZs/AAAAAAAAAAI/AAAAAAAAAAA/oyb-UCYFbfA/s32-c-k-no-mo-rj-c0xffffff/photo.jpg",
+                      "width": 32,
+                      "height": 32
+                    },
+                    {
+                      "url": "https://yt3.ggpht.com/-FoBvzoKhbZs/AAAAAAAAAAI/AAAAAAAAAAA/oyb-UCYFbfA/s64-c-k-no-mo-rj-c0xffffff/photo.jpg",
+                      "width": 64,
+                      "height": 64
+                    }
+                  ]
+                },
+                "durationSec": 120,
+                "showItemEndpoint": {
+                  "commandMetadata": {
+                    "webCommandMetadata": {
+                      "ignoreNavigation": true
+                    }
+                  },
+                  "showLiveChatItemEndpoint": {
+                    "renderer": {
+                      "liveChatPaidMessageRenderer": {
+                        "id": "ChwKGkNOblpoTXFrLS1RQ0ZRSU9ZQW9kclhrRXNn",
+                        "timestampUsec": "1569940638420061",
+                        "authorName": {
+                          "simpleText": "九十九 万"
+                        },
+                        "authorPhoto": {
+                          "thumbnails": [
+                            {
+                              "url": "https://yt3.ggpht.com/-FoBvzoKhbZs/AAAAAAAAAAI/AAAAAAAAAAA/oyb-UCYFbfA/s32-c-k-no-mo-rj-c0xffffff/photo.jpg",
+                              "width": 32,
+                              "height": 32
+                            },
+                            {
+                              "url": "https://yt3.ggpht.com/-FoBvzoKhbZs/AAAAAAAAAAI/AAAAAAAAAAA/oyb-UCYFbfA/s64-c-k-no-mo-rj-c0xffffff/photo.jpg",
+                              "width": 64,
+                              "height": 64
+                            }
+                          ]
+                        },
+                        "purchaseAmountText": {
+                          "simpleText": "￥846"
+                        },
+                        "message": {
+                          "runs": [
+                            {
+                              "text": "ボルガ博士お許しください代"
+                            }
+                          ]
+                        },
+                        "headerBackgroundColor": 4278239141,
+                        "headerTextColor": 4278190080,
+                        "bodyBackgroundColor": 4280150454,
+                        "bodyTextColor": 4278190080,
+                        "authorExternalChannelId": "UCoVtCMzg2vleAVsa7kw05AA",
+                        "authorNameTextColor": 2315255808,
+                        "contextMenuEndpoint": {
+                          "commandMetadata": {
+                            "webCommandMetadata": {
+                              "ignoreNavigation": true
+                            }
+                          },
+                          "liveChatItemContextMenuEndpoint": {
+                            "params": "Q2g0S0hBb2FRMDV1V21oTmNXc3RMVkZEUmxGSlQxbEJiMlJ5V0d0RmMyY1FBQm80Q2cwS0N6Z3dUM3AwU21oMU56YzRLaWNLR0ZWRFNYbDBUbU52ZWpSd1YzcFlaa3hrWVRCRWIxVk1VUklMT0RCUGVuUkthSFUzTnpnZ0FpZ0JNaG9LR0ZWRGIxWjBRMDE2WnpKMmJHVkJWbk5oTjJ0M01EVkJRUSUzRCUzRA=="
+                          }
+                        },
+                        "timestampColor": 2147483648,
+                        "contextMenuAccessibility": {
+                          "accessibilityData": {
+                            "label": "コメントの操作"
+                          }
+                        }
+                      }
+                    }
+                  }
+                },
+                "authorExternalChannelId": "UCoVtCMzg2vleAVsa7kw05AA",
+                "fullDurationSec": 120
+              }
+            },
+            "durationSec": "120"
+          }
         }
+      ]
     }
+  }
 }

tests/testdata/compatible/supersticker.json

@@ -1,197 +1,195 @@
 {
-    "xsrf_token": "QUFFLUhqbWVqWVRTUjhBcGRkNWR5Q2F3VWhxcTd0RkF1UXxBQ3Jtc0trMThFVGVVNTFnWmZucnkwNlJJMEZ2bndUS1I3b2dpLUtTNE92dEgwX3Y0MkNZU2NXdGY1QTMtX09BUGphYUpDQlc1dFhiTm9jLS1sQXVCNHpsdllqcm1id0t1RFBCM3A1b2o3OGt0Yjd6TE5wcmxBbjNyQktjc1lWZ3hjM1RuYk83YkQ0VVN3MGUybjAwSE90SS1YNkxvMUV5YVE=",
-    "timing": {
-        "info": {
-            "st": 148
-        }
-    },
-    "endpoint": {
-        "commandMetadata": {
-            "webCommandMetadata": {
-                "url": "/live_chat/get_live_chat?continuation=0ofMyAPiARp8Q2c4S0RRb0xPREJQZW5SS2FIVTNOemdhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5T0RCUGVuUkthSFUzTnpnbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCja2bHMpPvkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQwJKPoqT75AJYA1DXxuTMpPvkAliL_fvZoPvkAmgBggEECAEQAIgBAKABxM3xzKT75AI%253D"
-            }
-        },
-        "urlEndpoint": {
-            "url": "/live_chat/get_live_chat?continuation=0ofMyAPiARp8Q2c4S0RRb0xPREJQZW5SS2FIVTNOemdhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5T0RCUGVuUkthSFUzTnpnbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCja2bHMpPvkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQwJKPoqT75AJYA1DXxuTMpPvkAliL_fvZoPvkAmgBggEECAEQAIgBAKABxM3xzKT75AI%253D"
-        }
-    },
-    "url": "\/live_chat\/get_live_chat?continuation=0ofMyAPiARp8Q2c4S0RRb0xPREJQZW5SS2FIVTNOemdhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5T0RCUGVuUkthSFUzTnpnbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCja2bHMpPvkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQwJKPoqT75AJYA1DXxuTMpPvkAliL_fvZoPvkAmgBggEECAEQAIgBAKABxM3xzKT75AI%253D",
-    "csn": "n2STXd2iKZr2gAOt9qvgCg",
-    "response": {
-        "responseContext": {
-            "serviceTrackingParams": [
-                {
-                    "service": "CSI",
-                    "params": [
-                        {
-                            "key": "GetLiveChat_rid",
-                            "value": "0x9290108c05344647"
-                        },
-                        {
-                            "key": "c",
-                            "value": "WEB"
-                        },
-                        {
-                            "key": "cver",
-                            "value": "2.20191001.04.00"
-                        },
-                        {
-                            "key": "yt_li",
-                            "value": "0"
-                        }
-                    ]
-                },
-                {
-                    "service": "GFEEDBACK",
-                    "params": [
-                        {
-                            "key": "e",
-                            "value": "23744176,23788875,23793834,23794620,23804281,23806159,23816483,23819244,23820768,23826780,23827354,23830392,23832125,23835020,23836965,23837741,23837772,23837993,23838235,23839362,23840155,23840217,23841118,23841454,23842630,23842662,23842883,23842986,23843289,23843534,23843767,23845644,9449243,9471239,9474360"
-                        },
-                        {
-                            "key": "logged_in",
-                            "value": "0"
-                        }
-                    ]
-                },
-                {
-                    "service": "GUIDED_HELP",
-                    "params": [
-                        {
-                            "key": "logged_in",
-                            "value": "0"
-                        }
-                    ]
-                },
-                {
-                    "service": "ECATCHER",
-                    "params": [
-                        {
-                            "key": "client.name",
-                            "value": "WEB"
-                        },
-                        {
-                            "key": "client.version",
-                            "value": "2.20191001"
-                        },
-                        {
-                            "key": "innertube.build.changelist",
-                            "value": "272006966"
-                        },
-                        {
-                            "key": "innertube.build.experiments.source_version",
-                            "value": "272166268"
-                        },
-                        {
-                            "key": "innertube.build.label",
-                            "value": "youtube.ytfe.innertube_20190930_5_RC0"
-                        },
-                        {
-                            "key": "innertube.build.timestamp",
-                            "value": "1569863426"
-                        },
-                        {
-                            "key": "innertube.build.variants.checksum",
-                            "value": "1a800c1a2396906f1cbb7f670d43b6f5"
-                        },
-                        {
-                            "key": "innertube.run.job",
-                            "value": "ytfe-innertube-replica-only.ytfe"
-                        }
-                    ]
-                }
-            ],
-            "webResponseContextExtensionData": {
-                "ytConfigData": {
-                    "csn": "n2STXd2iKZr2gAOt9qvgCg",
-                    "visitorData": "CgtPQm1xTmtvNm1Tcyifyc3sBQ%3D%3D"
-                }
-            }
-        },
-        "continuationContents": {
-            "liveChatContinuation": {
-                "continuations": [
-                    {
-                        "timedContinuationData": {
-                            "timeoutMs": 8860,
-                            "continuation": "0ofMyAPiARp8Q2c4S0RRb0xPREJQZW5SS2FIVTNOemdhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5T0RCUGVuUkthSFUzTnpnbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCid_ejQpPvkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQwJKPoqT75AJYA1DF-ovRpPvkAliL_fvZoPvkAmgBggEECAEQAIgBAKABtNic0aT75AI%3D"
-                        }
-                    }
-                ],
-                "actions": [
-                    {
-                        "addChatItemAction": {
-                            "item": {
-                                "liveChatPaidStickerRenderer": {
-                                    "id": "ChwKGkNQX2Qzb2pUcU9VQ0ZRdnVXQW9kaTNJS3NB",
-                                    "contextMenuEndpoint": {
-                                        "commandMetadata": {
-                                            "webCommandMetadata": {
-                                                "ignoreNavigation": true
-                                            }
-                                        },
-                                        "liveChatItemContextMenuEndpoint": {
-                                            "params": "Q2g0S0hBb2FRMUJmWkROdmFsUnhUMVZEUmxGMmRWZEJiMlJwTTBsTGMwRVFBQm80Q2cwS0N6VlVUSE42U0hNd2QxYzBLaWNLR0ZWRFdGSnNTVXN6UTNkZlZFcEpVVU0xYTFOS1NsRk5aeElMTlZSTWMzcEljekIzVnpRZ0FpZ0JNaG9LR0ZWRFRHOXJPVWQ0WVRGYU5rWTVWV3d5WVV0MlRFWkdadyUzRCUzRA=="
-                                        }
-                                    },
-                                    "contextMenuAccessibility": {
-                                        "accessibilityData": {
-                                            "label": "コメントの操作"
-                                        }
-                                    },
-                                    "timestampUsec": "1571499325098699",
-                                    "authorPhoto": {
-                                        "thumbnails": [
-                                            {
-                                                "url": "https: //yt3.ggpht.com/-xRQVNtDSO3w/AAAAAAAAAAI/AAAAAAAAAAA/Is9D9D7wwAE/s32-c-k-no-mo-rj-c0xffffff/photo.jpg",
-                                                "width": 32,
-                                                "height": 32
-                                            },
-                                            {
-                                                "url": "https://yt3.ggpht.com/-xRQVNtDSO3w/AAAAAAAAAAI/AAAAAAAAAAA/Is9D9D7wwAE/s64-c-k-no-mo-rj-c0xffffff/photo.jpg",
-                                                "width": 64,
-                                                "height": 64
-                                            }
-                                        ]
-                                    },
-                                    "authorName": {
-                                        "simpleText": "りお"
-                                    },
-                                    "authorExternalChannelId": "UCLok9Gxa1Z6F9Ul2aKvLFFg",
-                                    "sticker": {
-                                        "thumbnails": [
-                                            {
-                                                "url": "//lh3.googleusercontent.com/1aIk6vlk4gZ2ytc42j3WcIHYtWFWo2uVWVqbHFuxiGHO4XwyAS0u8vuu6VkiX5eR6uy9mfAupyP786_TbP0=s72-rwa",
-                                                "width": 72,
-                                                "height": 72
-                                            },
-                                            {
-                                                "url": "//lh3.googleusercontent.com/1aIk6vlk4gZ2ytc42j3WcIHYtWFWo2uVWVqbHFuxiGHO4XwyAS0u8vuu6VkiX5eR6uy9mfAupyP786_TbP0=s144-rwa",
-                                                "width": 144,
-                                                "height": 144
-                                            }
-                                        ],
-                                        "accessibility": {
-                                            "accessibilityData": {
-                                                "label": "気付いてもらえるように人差し指を上げたり下げたりしている柴犬"
-                                            }
-                                        }
-                                    },
-                                    "moneyChipBackgroundColor": 4278248959,
-                                    "moneyChipTextColor": 4278190080,
-                                    "purchaseAmountText": {
-                                        "simpleText": "￥200"
-                                    },
-                                    "stickerDisplayWidth": 72,
-                                    "stickerDisplayHeight": 72,
-                                    "backgroundColor": 4278237396,
-                                    "authorNameTextColor": 3003121664
-                                }
-                            }
-                        }
-                    }
-                ]
-            }
-        }
+  "xsrf_token": "QUFFLUhqbWVqWVRTUjhBcGRkNWR5Q2F3VWhxcTd0RkF1UXxBQ3Jtc0trMThFVGVVNTFnWmZucnkwNlJJMEZ2bndUS1I3b2dpLUtTNE92dEgwX3Y0MkNZU2NXdGY1QTMtX09BUGphYUpDQlc1dFhiTm9jLS1sQXVCNHpsdllqcm1id0t1RFBCM3A1b2o3OGt0Yjd6TE5wcmxBbjNyQktjc1lWZ3hjM1RuYk83YkQ0VVN3MGUybjAwSE90SS1YNkxvMUV5YVE=",
+  "timing": {
+    "info": {
+      "st": 148
     }
+  },
+  "endpoint": {
+    "commandMetadata": {
+      "webCommandMetadata": {
+        "url": "/live_chat/get_live_chat?continuation=0ofMyAPiARp8Q2c4S0RRb0xPREJQZW5SS2FIVTNOemdhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5T0RCUGVuUkthSFUzTnpnbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCja2bHMpPvkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQwJKPoqT75AJYA1DXxuTMpPvkAliL_fvZoPvkAmgBggEECAEQAIgBAKABxM3xzKT75AI%253D"
+      }
+    },
+    "urlEndpoint": {
+      "url": "/live_chat/get_live_chat?continuation=0ofMyAPiARp8Q2c4S0RRb0xPREJQZW5SS2FIVTNOemdhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5T0RCUGVuUkthSFUzTnpnbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCja2bHMpPvkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQwJKPoqT75AJYA1DXxuTMpPvkAliL_fvZoPvkAmgBggEECAEQAIgBAKABxM3xzKT75AI%253D"
+    }
+  },
+  "url": "/live_chat/get_live_chat?continuation=0ofMyAPiARp8Q2c4S0RRb0xPREJQZW5SS2FIVTNOemdhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5T0RCUGVuUkthSFUzTnpnbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCja2bHMpPvkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQwJKPoqT75AJYA1DXxuTMpPvkAliL_fvZoPvkAmgBggEECAEQAIgBAKABxM3xzKT75AI%253D",
+  "csn": "n2STXd2iKZr2gAOt9qvgCg",
+  "responseContext": {
+    "serviceTrackingParams": [
+      {
+        "service": "CSI",
+        "params": [
+          {
+            "key": "GetLiveChat_rid",
+            "value": "0x9290108c05344647"
+          },
+          {
+            "key": "c",
+            "value": "WEB"
+          },
+          {
+            "key": "cver",
+            "value": "2.20191001.04.00"
+          },
+          {
+            "key": "yt_li",
+            "value": "0"
+          }
+        ]
+      },
+      {
+        "service": "GFEEDBACK",
+        "params": [
+          {
+            "key": "e",
+            "value": "23744176,23788875,23793834,23794620,23804281,23806159,23816483,23819244,23820768,23826780,23827354,23830392,23832125,23835020,23836965,23837741,23837772,23837993,23838235,23839362,23840155,23840217,23841118,23841454,23842630,23842662,23842883,23842986,23843289,23843534,23843767,23845644,9449243,9471239,9474360"
+          },
+          {
+            "key": "logged_in",
+            "value": "0"
+          }
+        ]
+      },
+      {
+        "service": "GUIDED_HELP",
+        "params": [
+          {
+            "key": "logged_in",
+            "value": "0"
+          }
+        ]
+      },
+      {
+        "service": "ECATCHER",
+        "params": [
+          {
+            "key": "client.name",
+            "value": "WEB"
+          },
+          {
+            "key": "client.version",
+            "value": "2.20191001"
+          },
+          {
+            "key": "innertube.build.changelist",
+            "value": "272006966"
+          },
+          {
+            "key": "innertube.build.experiments.source_version",
+            "value": "272166268"
+          },
+          {
+            "key": "innertube.build.label",
+            "value": "youtube.ytfe.innertube_20190930_5_RC0"
+          },
+          {
+            "key": "innertube.build.timestamp",
+            "value": "1569863426"
+          },
+          {
+            "key": "innertube.build.variants.checksum",
+            "value": "1a800c1a2396906f1cbb7f670d43b6f5"
+          },
+          {
+            "key": "innertube.run.job",
+            "value": "ytfe-innertube-replica-only.ytfe"
+          }
+        ]
+      }
+    ],
+    "webResponseContextExtensionData": {
+      "ytConfigData": {
+        "csn": "n2STXd2iKZr2gAOt9qvgCg",
+        "visitorData": "CgtPQm1xTmtvNm1Tcyifyc3sBQ%3D%3D"
+      }
+    }
+  },
+  "continuationContents": {
+    "liveChatContinuation": {
+      "continuations": [
+        {
+          "timedContinuationData": {
+            "timeoutMs": 8860,
+            "continuation": "0ofMyAPiARp8Q2c4S0RRb0xPREJQZW5SS2FIVTNOemdhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5T0RCUGVuUkthSFUzTnpnbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCid_ejQpPvkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQwJKPoqT75AJYA1DF-ovRpPvkAliL_fvZoPvkAmgBggEECAEQAIgBAKABtNic0aT75AI%3D"
+          }
+        }
+      ],
+      "actions": [
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatPaidStickerRenderer": {
+                "id": "ChwKGkNQX2Qzb2pUcU9VQ0ZRdnVXQW9kaTNJS3NB",
+                "contextMenuEndpoint": {
+                  "commandMetadata": {
+                    "webCommandMetadata": {
+                      "ignoreNavigation": true
+                    }
+                  },
+                  "liveChatItemContextMenuEndpoint": {
+                    "params": "Q2g0S0hBb2FRMUJmWkROdmFsUnhUMVZEUmxGMmRWZEJiMlJwTTBsTGMwRVFBQm80Q2cwS0N6VlVUSE42U0hNd2QxYzBLaWNLR0ZWRFdGSnNTVXN6UTNkZlZFcEpVVU0xYTFOS1NsRk5aeElMTlZSTWMzcEljekIzVnpRZ0FpZ0JNaG9LR0ZWRFRHOXJPVWQ0WVRGYU5rWTVWV3d5WVV0MlRFWkdadyUzRCUzRA=="
+                  }
+                },
+                "contextMenuAccessibility": {
+                  "accessibilityData": {
+                    "label": "コメントの操作"
+                  }
+                },
+                "timestampUsec": "1571499325098699",
+                "authorPhoto": {
+                  "thumbnails": [
+                    {
+                      "url": "https: //yt3.ggpht.com/-xRQVNtDSO3w/AAAAAAAAAAI/AAAAAAAAAAA/Is9D9D7wwAE/s32-c-k-no-mo-rj-c0xffffff/photo.jpg",
+                      "width": 32,
+                      "height": 32
+                    },
+                    {
+                      "url": "https://yt3.ggpht.com/-xRQVNtDSO3w/AAAAAAAAAAI/AAAAAAAAAAA/Is9D9D7wwAE/s64-c-k-no-mo-rj-c0xffffff/photo.jpg",
+                      "width": 64,
+                      "height": 64
+                    }
+                  ]
+                },
+                "authorName": {
+                  "simpleText": "りお"
+                },
+                "authorExternalChannelId": "UCLok9Gxa1Z6F9Ul2aKvLFFg",
+                "sticker": {
+                  "thumbnails": [
+                    {
+                      "url": "//lh3.googleusercontent.com/1aIk6vlk4gZ2ytc42j3WcIHYtWFWo2uVWVqbHFuxiGHO4XwyAS0u8vuu6VkiX5eR6uy9mfAupyP786_TbP0=s72-rwa",
+                      "width": 72,
+                      "height": 72
+                    },
+                    {
+                      "url": "//lh3.googleusercontent.com/1aIk6vlk4gZ2ytc42j3WcIHYtWFWo2uVWVqbHFuxiGHO4XwyAS0u8vuu6VkiX5eR6uy9mfAupyP786_TbP0=s144-rwa",
+                      "width": 144,
+                      "height": 144
+                    }
+                  ],
+                  "accessibility": {
+                    "accessibilityData": {
+                      "label": "気付いてもらえるように人差し指を上げたり下げたりしている柴犬"
+                    }
+                  }
+                },
+                "moneyChipBackgroundColor": 4278248959,
+                "moneyChipTextColor": 4278190080,
+                "purchaseAmountText": {
+                  "simpleText": "￥200"
+                },
+                "stickerDisplayWidth": 72,
+                "stickerDisplayHeight": 72,
+                "backgroundColor": 4278237396,
+                "authorNameTextColor": 3003121664
+              }
+            }
+          }
+        }
+      ]
+    }
+  }
 }

tests/testdata/compatible/textmessage.json

@@ -1,177 +1,175 @@
 {
-    "response": {
-        "responseContext": {
-            "serviceTrackingParams": [
-                {
-                    "service": "CSI",
-                    "params": [
-                        {
-                            "key": "GetLiveChat_rid",
-                            "value": "0x3eff0db28fc39bbe"
-                        },
-                        {
-                            "key": "c",
-                            "value": "WEB"
-                        },
-                        {
-                            "key": "cver",
-                            "value": "2.20190920.05.01"
-                        },
-                        {
-                            "key": "yt_li",
-                            "value": "0"
-                        }
-                    ]
-                },
-                {
-                    "service": "GFEEDBACK",
-                    "params": [
-                        {
-                            "key": "e",
-                            "value": "23744176,23748146,23788851,23788875,23793834,23804281,23807353,23808952,23828082,23828243,23829333,23832544,23834418,23834656,23835020,23836434,23836965,23837742,23837772,23837993,23838301,23838576,23838576,23838742,23839360,23840216,23841655,23842986,23843288,23843533,23843743,23844780,24630231,9425362,9449243,9466592,9469037,9471235,9474358"
-                        },
-                        {
-                            "key": "logged_in",
-                            "value": "0"
-                        }
-                    ]
-                },
-                {
-                    "service": "GUIDED_HELP",
-                    "params": [
-                        {
-                            "key": "logged_in",
-                            "value": "0"
-                        }
-                    ]
-                },
-                {
-                    "service": "ECATCHER",
-                    "params": [
-                        {
-                            "key": "client.name",
-                            "value": "WEB"
-                        },
-                        {
-                            "key": "client.version",
-                            "value": "2.20190920"
-                        },
-                        {
-                            "key": "innertube.build.changelist",
-                            "value": "270169303"
-                        },
-                        {
-                            "key": "innertube.build.experiments.source_version",
-                            "value": "270377311"
-                        },
-                        {
-                            "key": "innertube.build.label",
-                            "value": "youtube.ytfe.innertube_20190919_5_RC1"
-                        },
-                        {
-                            "key": "innertube.build.timestamp",
-                            "value": "1568942548"
-                        },
-                        {
-                            "key": "innertube.build.variants.checksum",
-                            "value": "392d499f55b5e2c240adde58886a8143"
-                        },
-                        {
-                            "key": "innertube.run.job",
-                            "value": "ytfe-innertube-replica-only.ytfe"
-                        }
-                    ]
-                }
-            ],
-            "webResponseContextExtensionData": {
-                "ytConfigData": {
-                    "csn": "n96GXabRGouFlQTigY2YDg",
-                    "visitorData": "CgtKUldQeGJJRXhkcyifvZvsBQ%3D%3D"
-                }
-            }
-        },
-        "continuationContents": {
-            "liveChatContinuation": {
-                "continuations": [
-                    {
-                        "timedContinuationData": {
-                            "timeoutMs": 5041,
-                            "continuation": "0ofMyAPiARp8Q2c4S0RRb0xhelJMZDBsWFQwdERkalFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5YXpSTGQwbFhUMHREZGpRbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCj7hLmSs-PkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgJqXjrPj5AJYA1DsuuGSs-PkAli3pNq1k-PkAmgBggEECAEQAIgBAKAB7KDVk7Pj5AI%3D"
-                        }
-                    }
-                ],
-                "actions": [
-                    {
-                        "addChatItemAction": {
-                            "item": {
-                                "liveChatTextMessageRenderer": {
-                                    "message": {
-                                        "runs": [
-                                            {
-                                                "text": "text"
-                                            }
-                                        ]
-                                    },
-                                    "authorName": {
-                                        "simpleText": "name"
-                                    },
-                                    "authorPhoto": {
-                                        "thumbnails": [
-                                            {
-                                                "url": "https://yt3.ggpht.com/-8sLtPu1Hyw0/AAAAAAAAAAI/AAAAAAAAAAA/a_52bWnC0-s/s32-c-k-no-mo-rj-c0xffffff/photo.jpg",
-                                                "width": 32,
-                                                "height": 32
-                                            },
-                                            {
-                                                "url": "https://yt3.ggpht.com/-8sLtPu1Hyw0/AAAAAAAAAAI/AAAAAAAAAAA/a_52bWnC0-s/s64-c-k-no-mo-rj-c0xffffff/photo.jpg",
-                                                "width": 64,
-                                                "height": 64
-                                            }
-                                        ]
-                                    },
-                                    "contextMenuEndpoint": {
-                                        "commandMetadata": {
-                                            "webCommandMetadata": {
-                                                "ignoreNavigation": true
-                                            }
-                                        },
-                                        "liveChatItemContextMenuEndpoint": {
-                                            "params": "Q2pzS09Rb2FRMHRQTkhRMVEzbzBMVkZEUmxweGRrUlJiMlJZV0d0QlVHY1NHME5PYm5OdlVESm1OQzFSUTBaUmRsTlhRVzlrVVZGclJUTlJNeEFBR2pnS0RRb0xhelJMZDBsWFQwdERkalFxSndvWVZVTnZTWEZ1TVZvMWFYaERXbmRqTUVSWFNqZHlTME5uRWd0ck5FdDNTVmRQUzBOMk5DQUNLQUV5R2dvWVZVTnlOVGxXVlY5amRtWnlkVEF0YW1GeWNtUk1NMDEz"
-                                        }
-                                    },
-                                    "id": "CjkKGkNLTzR0NUN6NC1RQ0ZacXZEUW9kWFhrQVBnEhtDTm5zb1AyZjQtUUNGUXZTV0FvZFFRa0UzUTM%3D",
-                                    "timestampUsec": "1569119896722467",
-                                    "authorExternalChannelId": "UCr59VU_cvfru0-jarrdL3Mw",
-                                    "contextMenuAccessibility": {
-                                        "accessibilityData": {
-                                            "label": "コメントの操作"
-                                        }
-                                    }
-                                }
-                            },
-                            "clientId": "CNnsoP2f4-QCFQvSWAodQQkE3Q3"
-                        }
-                    }
-                ]
-            }
-        }
-    },
-    "endpoint": {
-        "commandMetadata": {
-            "webCommandMetadata": {
-                "url": "/live_chat/get_live_chat?continuation=0ofMyAPiARp8Q2c4S0RRb0xhelJMZDBsWFQwdERkalFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5YXpSTGQwbFhUMHREZGpRbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiPz5-Os-PkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgJqXjrPj5AJYA1CRwciOs-PkAli3pNq1k-PkAmgBggEECAEQAIgBAKABjbfnjrPj5AI%253D"
-            }
-        },
-        "urlEndpoint": {
-            "url": "/live_chat/get_live_chat?continuation=0ofMyAPiARp8Q2c4S0RRb0xhelJMZDBsWFQwdERkalFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5YXpSTGQwbFhUMHREZGpRbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiPz5-Os-PkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgJqXjrPj5AJYA1CRwciOs-PkAli3pNq1k-PkAmgBggEECAEQAIgBAKABjbfnjrPj5AI%253D"
-        }
-    },
-    "csn": "n96GXabRGouFlQTigY2YDg",
-    "xsrf_token": "QUFFLUhqbHNNWTF3NFJqc2h3cGE1NE9FWGdaWk5mRlVhUXxBQ3Jtc0tuTWhZNFcyTW1iZnA3ZnFTYUFudVFEUVE0cnFEOVBGcEU1MEh0Zlh4bll1amVmRl9OMkxZV3pKV1ZSbExBeDctTl95NGtBVnJZdlNxeS1KdWVNempEN2N6MHhaU1laV3hnVkZPeHp1OHVDTGVFSGUyOGduT0szbDV5N05LYUZTdzdoTDRwV1VJWndaVjdQVGRjNWVpR0YwUXgtZXc=",
-    "url": "\/live_chat\/get_live_chat?continuation=0ofMyAPiARp8Q2c4S0RRb0xhelJMZDBsWFQwdERkalFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5YXpSTGQwbFhUMHREZGpRbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiPz5-Os-PkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgJqXjrPj5AJYA1CRwciOs-PkAli3pNq1k-PkAmgBggEECAEQAIgBAKABjbfnjrPj5AI%253D",
-    "timing": {
-        "info": {
-            "st": 81
-        }
+  "responseContext": {
+    "serviceTrackingParams": [
+      {
+        "service": "CSI",
+        "params": [
+          {
+            "key": "GetLiveChat_rid",
+            "value": "0x3eff0db28fc39bbe"
+          },
+          {
+            "key": "c",
+            "value": "WEB"
+          },
+          {
+            "key": "cver",
+            "value": "2.20190920.05.01"
+          },
+          {
+            "key": "yt_li",
+            "value": "0"
+          }
+        ]
+      },
+      {
+        "service": "GFEEDBACK",
+        "params": [
+          {
+            "key": "e",
+            "value": "23744176,23748146,23788851,23788875,23793834,23804281,23807353,23808952,23828082,23828243,23829333,23832544,23834418,23834656,23835020,23836434,23836965,23837742,23837772,23837993,23838301,23838576,23838576,23838742,23839360,23840216,23841655,23842986,23843288,23843533,23843743,23844780,24630231,9425362,9449243,9466592,9469037,9471235,9474358"
+          },
+          {
+            "key": "logged_in",
+            "value": "0"
+          }
+        ]
+      },
+      {
+        "service": "GUIDED_HELP",
+        "params": [
+          {
+            "key": "logged_in",
+            "value": "0"
+          }
+        ]
+      },
+      {
+        "service": "ECATCHER",
+        "params": [
+          {
+            "key": "client.name",
+            "value": "WEB"
+          },
+          {
+            "key": "client.version",
+            "value": "2.20190920"
+          },
+          {
+            "key": "innertube.build.changelist",
+            "value": "270169303"
+          },
+          {
+            "key": "innertube.build.experiments.source_version",
+            "value": "270377311"
+          },
+          {
+            "key": "innertube.build.label",
+            "value": "youtube.ytfe.innertube_20190919_5_RC1"
+          },
+          {
+            "key": "innertube.build.timestamp",
+            "value": "1568942548"
+          },
+          {
+            "key": "innertube.build.variants.checksum",
+            "value": "392d499f55b5e2c240adde58886a8143"
+          },
+          {
+            "key": "innertube.run.job",
+            "value": "ytfe-innertube-replica-only.ytfe"
+          }
+        ]
+      }
+    ],
+    "webResponseContextExtensionData": {
+      "ytConfigData": {
+        "csn": "n96GXabRGouFlQTigY2YDg",
+        "visitorData": "CgtKUldQeGJJRXhkcyifvZvsBQ%3D%3D"
+      }
     }
+  },
+  "continuationContents": {
+    "liveChatContinuation": {
+      "continuations": [
+        {
+          "timedContinuationData": {
+            "timeoutMs": 5041,
+            "continuation": "0ofMyAPiARp8Q2c4S0RRb0xhelJMZDBsWFQwdERkalFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5YXpSTGQwbFhUMHREZGpRbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCj7hLmSs-PkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgJqXjrPj5AJYA1DsuuGSs-PkAli3pNq1k-PkAmgBggEECAEQAIgBAKAB7KDVk7Pj5AI%3D"
+          }
+        }
+      ],
+      "actions": [
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatTextMessageRenderer": {
+                "message": {
+                  "runs": [
+                    {
+                      "text": "text"
+                    }
+                  ]
+                },
+                "authorName": {
+                  "simpleText": "name"
+                },
+                "authorPhoto": {
+                  "thumbnails": [
+                    {
+                      "url": "https://yt3.ggpht.com/-8sLtPu1Hyw0/AAAAAAAAAAI/AAAAAAAAAAA/a_52bWnC0-s/s32-c-k-no-mo-rj-c0xffffff/photo.jpg",
+                      "width": 32,
+                      "height": 32
+                    },
+                    {
+                      "url": "https://yt3.ggpht.com/-8sLtPu1Hyw0/AAAAAAAAAAI/AAAAAAAAAAA/a_52bWnC0-s/s64-c-k-no-mo-rj-c0xffffff/photo.jpg",
+                      "width": 64,
+                      "height": 64
+                    }
+                  ]
+                },
+                "contextMenuEndpoint": {
+                  "commandMetadata": {
+                    "webCommandMetadata": {
+                      "ignoreNavigation": true
+                    }
+                  },
+                  "liveChatItemContextMenuEndpoint": {
+                    "params": "Q2pzS09Rb2FRMHRQTkhRMVEzbzBMVkZEUmxweGRrUlJiMlJZV0d0QlVHY1NHME5PYm5OdlVESm1OQzFSUTBaUmRsTlhRVzlrVVZGclJUTlJNeEFBR2pnS0RRb0xhelJMZDBsWFQwdERkalFxSndvWVZVTnZTWEZ1TVZvMWFYaERXbmRqTUVSWFNqZHlTME5uRWd0ck5FdDNTVmRQUzBOMk5DQUNLQUV5R2dvWVZVTnlOVGxXVlY5amRtWnlkVEF0YW1GeWNtUk1NMDEz"
+                  }
+                },
+                "id": "CjkKGkNLTzR0NUN6NC1RQ0ZacXZEUW9kWFhrQVBnEhtDTm5zb1AyZjQtUUNGUXZTV0FvZFFRa0UzUTM%3D",
+                "timestampUsec": "1569119896722467",
+                "authorExternalChannelId": "UCr59VU_cvfru0-jarrdL3Mw",
+                "contextMenuAccessibility": {
+                  "accessibilityData": {
+                    "label": "コメントの操作"
+                  }
+                }
+              }
+            },
+            "clientId": "CNnsoP2f4-QCFQvSWAodQQkE3Q3"
+          }
+        }
+      ]
+    }
+  },
+  "endpoint": {
+    "commandMetadata": {
+      "webCommandMetadata": {
+        "url": "/live_chat/get_live_chat?continuation=0ofMyAPiARp8Q2c4S0RRb0xhelJMZDBsWFQwdERkalFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5YXpSTGQwbFhUMHREZGpRbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiPz5-Os-PkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgJqXjrPj5AJYA1CRwciOs-PkAli3pNq1k-PkAmgBggEECAEQAIgBAKABjbfnjrPj5AI%253D"
+      }
+    },
+    "urlEndpoint": {
+      "url": "/live_chat/get_live_chat?continuation=0ofMyAPiARp8Q2c4S0RRb0xhelJMZDBsWFQwdERkalFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5YXpSTGQwbFhUMHREZGpRbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiPz5-Os-PkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgJqXjrPj5AJYA1CRwciOs-PkAli3pNq1k-PkAmgBggEECAEQAIgBAKABjbfnjrPj5AI%253D"
+    }
+  },
+  "csn": "n96GXabRGouFlQTigY2YDg",
+  "xsrf_token": "QUFFLUhqbHNNWTF3NFJqc2h3cGE1NE9FWGdaWk5mRlVhUXxBQ3Jtc0tuTWhZNFcyTW1iZnA3ZnFTYUFudVFEUVE0cnFEOVBGcEU1MEh0Zlh4bll1amVmRl9OMkxZV3pKV1ZSbExBeDctTl95NGtBVnJZdlNxeS1KdWVNempEN2N6MHhaU1laV3hnVkZPeHp1OHVDTGVFSGUyOGduT0szbDV5N05LYUZTdzdoTDRwV1VJWndaVjdQVGRjNWVpR0YwUXgtZXc=",
+  "url": "/live_chat/get_live_chat?continuation=0ofMyAPiARp8Q2c4S0RRb0xhelJMZDBsWFQwdERkalFhUTZxNXdiMEJQUW83YUhSMGNITTZMeTkzZDNjdWVXOTFkSFZpWlM1amIyMHZiR2wyWlY5amFHRjBQM1k5YXpSTGQwbFhUMHREZGpRbWFYTmZjRzl3YjNWMFBURWdBZyUzRCUzRCiPz5-Os-PkAjAAOABAAUorCAAQABgAIAAqDnN0YXRpY2NoZWNrc3VtOgBAAEoCCAFQgJqXjrPj5AJYA1CRwciOs-PkAli3pNq1k-PkAmgBggEECAEQAIgBAKABjbfnjrPj5AI%253D",
+  "timing": {
+    "info": {
+      "st": 81
+    }
+  }
 }

tests/testdata/default/newSponsor_current.json

@@ -0,0 +1,98 @@
+{
+  "responseContext": {
+    "webResponseContextExtensionData": ""
+  },
+  "continuationContents": {
+    "liveChatContinuation": {
+      "continuations": [
+        {
+          "invalidationContinuationData": {
+            "invalidationId": {
+              "objectSource": 1000,
+              "objectId": "___objectId___",
+              "topic": "chat~00000000000~0000000",
+              "subscribeToGcmTopics": true,
+              "protoCreationTimestampMs": "1577804400000"
+            },
+            "timeoutMs": 10000,
+            "continuation": "___continuation___"
+          }
+        }
+      ],
+      "actions": [
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatMembershipItemRenderer": {
+                "id": "dummy_id",
+                "timestampUsec": 1570678496000000,
+                "authorExternalChannelId": "author_channel_id",
+                "headerSubtext": {
+                  "runs": [
+                    {
+                      "text": "新規メンバー"
+                    }
+                  ]
+                },
+                "authorName": {
+                  "simpleText": "author_name"
+                },
+                "authorPhoto": {
+                  "thumbnails": [
+                    {
+                      "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                      "width": 32,
+                      "height": 32
+                    },
+                    {
+                      "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                      "width": 64,
+                      "height": 64
+                    }
+                  ]
+                },
+                "authorBadges": [
+                  {
+                    "liveChatAuthorBadgeRenderer": {
+                      "customThumbnail": {
+                        "thumbnails": [
+                          {
+                            "url": "https://yt3.ggpht.com/X=s32-c-k"
+                          },
+                          {
+                            "url": "https://yt3.ggpht.com/X=s64-c-k"
+                          }
+                        ]
+                      },
+                      "tooltip": "新規メンバー",
+                      "accessibility": {
+                        "accessibilityData": {
+                          "label": "新規メンバー"
+                        }
+                      }
+                    }
+                  }
+                ],
+                "contextMenuEndpoint": {
+                  "commandMetadata": {
+                    "webCommandMetadata": {
+                      "ignoreNavigation": true
+                    }
+                  },
+                  "liveChatItemContextMenuEndpoint": {
+                    "params": "___params___"
+                  }
+                },
+                "contextMenuAccessibility": {
+                  "accessibilityData": {
+                    "label": "コメントの操作"
+                  }
+                }
+              }
+            }
+          }
+        }
+      ]
+    }
+  }
+}

tests/testdata/default/newSponsor_lagacy.json

@@ -0,0 +1,80 @@
+{
+  "responseContext": {
+    "webResponseContextExtensionData": ""
+  },
+  "continuationContents": {
+    "liveChatContinuation": {
+      "continuations": [
+        {
+          "invalidationContinuationData": {
+            "invalidationId": {
+              "objectSource": 1000,
+              "objectId": "___objectId___",
+              "topic": "chat~00000000000~0000000",
+              "subscribeToGcmTopics": true,
+              "protoCreationTimestampMs": "1577804400000"
+            },
+            "timeoutMs": 10000,
+            "continuation": "___continuation___"
+          }
+        }
+      ],
+      "actions": [
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatLegacyPaidMessageRenderer": {
+                "id": "dummy_id",
+                "timestampUsec": 1570678496000000,
+                "eventText": {
+                  "runs": [
+                    {
+                      "text": "新規メンバー"
+                    }
+                  ]
+                },
+                "detailText": {
+                  "simpleText": "ようこそ、author_name！"
+                },
+                "authorName": {
+                  "simpleText": "author_name"
+                },
+                "authorPhoto": {
+                  "thumbnails": [
+                    {
+                      "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                      "width": 32,
+                      "height": 32
+                    },
+                    {
+                      "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                      "width": 64,
+                      "height": 64
+                    }
+                  ]
+                },
+                "authorExternalChannelId": "author_channel_id",
+                "contextMenuEndpoint": {
+                  "clickTrackingParams": "___clickTrackingParams___",
+                  "commandMetadata": {
+                    "webCommandMetadata": {
+                      "ignoreNavigation": true
+                    }
+                  },
+                  "liveChatItemContextMenuEndpoint": {
+                    "params": "___params___"
+                  }
+                },
+                "contextMenuAccessibility": {
+                  "accessibilityData": {
+                    "label": "コメントの操作"
+                  }
+                }
+              }
+            }
+          }
+        }
+      ]
+    }
+  }
+}

tests/testdata/default/replay_member_text.json

@@ -0,0 +1,110 @@
+{
+  "responseContext": {
+    "webResponseContextExtensionData": "data"
+  },
+  "continuationContents": {
+    "liveChatContinuation": {
+      "continuations": [
+        {
+          "liveChatReplayContinuationData": {
+            "invalidationId": {
+              "objectSource": 1000,
+              "objectId": "___objectId___",
+              "topic": "chat~00000000000~0000000",
+              "subscribeToGcmTopics": true,
+              "protoCreationTimestampMs": "1577804400000"
+            },
+            "timeoutMs": 10000,
+            "continuation": "___continuation___"
+          }
+        }
+      ],
+      "actions": [
+        {
+          "replayChatItemAction": {
+            "actions": [
+              {
+                "addChatItemAction": {
+                  "item": {
+                    "liveChatTextMessageRenderer": {
+                      "message": {
+                        "runs": [
+                          {
+                            "text": "dummy_message"
+                          }
+                        ]
+                      },
+                      "authorName": {
+                        "simpleText": "author_name"
+                      },
+                      "authorPhoto": {
+                        "thumbnails": [
+                          {
+                            "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                            "width": 32,
+                            "height": 32
+                          },
+                          {
+                            "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                            "width": 64,
+                            "height": 64
+                          }
+                        ]
+                      },
+                      "contextMenuEndpoint": {
+                        "clickTrackingParams": "___clickTrackingParams___",
+                        "commandMetadata": {
+                          "webCommandMetadata": {
+                            "ignoreNavigation": true
+                          }
+                        },
+                        "liveChatItemContextMenuEndpoint": {
+                          "params": "___params___"
+                        }
+                      },
+                      "id": "dummy_id",
+                      "timestampUsec": 1570678496000000,
+                      "authorBadges": [
+                        {
+                          "liveChatAuthorBadgeRenderer": {
+                            "customThumbnail": {
+                              "thumbnails": [
+                                {
+                                  "url": "https://yt3.ggpht.com/X=s16-c-k"
+                                },
+                                {
+                                  "url": "https://yt3.ggpht.com/X=s32-c-k"
+                                }
+                              ]
+                            },
+                            "tooltip": "メンバー（1 か月）",
+                            "accessibility": {
+                              "accessibilityData": {
+                                "label": "メンバー（1 か月）"
+                              }
+                            }
+                          }
+                        }
+                      ],
+                      "authorExternalChannelId": "author_channel_id",
+                      "contextMenuAccessibility": {
+                        "accessibilityData": {
+                          "label": "コメントの操作"
+                        }
+                      },
+                      "timestampText": {
+                        "simpleText": "1:23:45"
+                      }
+                    }
+                  },
+                  "clientId": "dummy_client_id"
+                }
+              }
+            ],
+            "videoOffsetTimeMsec": "5025120"
+          }
+        }
+      ]
+    }
+  }
+}

tests/testdata/default/superchat.json

@@ -0,0 +1,182 @@
+{
+  "responseContext": {
+    "webResponseContextExtensionData": ""
+  },
+  "continuationContents": {
+    "liveChatContinuation": {
+      "continuations": [
+        {
+          "invalidationContinuationData": {
+            "invalidationId": {
+              "objectSource": 1000,
+              "objectId": "___objectId___",
+              "topic": "chat~00000000000~0000000",
+              "subscribeToGcmTopics": true,
+              "protoCreationTimestampMs": "1577804400000"
+            },
+            "timeoutMs": 10000,
+            "continuation": "___continuation___"
+          }
+        }
+      ],
+      "actions": [
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatPaidMessageRenderer": {
+                "id": "dummy_id",
+                "timestampUsec": 1570678496000000,
+                "authorName": {
+                  "simpleText": "author_name"
+                },
+                "authorPhoto": {
+                  "thumbnails": [
+                    {
+                      "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                      "width": 32,
+                      "height": 32
+                    },
+                    {
+                      "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                      "width": 64,
+                      "height": 64
+                    }
+                  ]
+                },
+                "purchaseAmountText": {
+                  "simpleText": "￥800"
+                },
+                "message": {
+                  "runs": [
+                    {
+                      "text": "dummy_message"
+                    }
+                  ]
+                },
+                "headerBackgroundColor": 4278239141,
+                "headerTextColor": 4278190080,
+                "bodyBackgroundColor": 4280150454,
+                "bodyTextColor": 4278190080,
+                "authorExternalChannelId": "author_channel_id",
+                "authorNameTextColor": 2315255808,
+                "contextMenuEndpoint": {
+                  "commandMetadata": {
+                    "webCommandMetadata": {
+                      "ignoreNavigation": true
+                    }
+                  },
+                  "liveChatItemContextMenuEndpoint": {
+                    "params": "___params___"
+                  }
+                },
+                "timestampColor": 2147483648,
+                "contextMenuAccessibility": {
+                  "accessibilityData": {
+                    "label": "コメントの操作"
+                  }
+                }
+              }
+            }
+          }
+        },
+        {
+          "addLiveChatTickerItemAction": {
+            "item": {
+              "liveChatTickerPaidMessageItemRenderer": {
+                "id": "dummy_id",
+                "amount": {
+                  "simpleText": "￥846"
+                },
+                "amountTextColor": 4278190080,
+                "startBackgroundColor": 4280150454,
+                "endBackgroundColor": 4278239141,
+                "authorPhoto": {
+                  "thumbnails": [
+                    {
+                      "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                      "width": 32,
+                      "height": 32
+                    },
+                    {
+                      "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                      "width": 64,
+                      "height": 64
+                    }
+                  ]
+                },
+                "durationSec": 120,
+                "showItemEndpoint": {
+                  "commandMetadata": {
+                    "webCommandMetadata": {
+                      "ignoreNavigation": true
+                    }
+                  },
+                  "showLiveChatItemEndpoint": {
+                    "renderer": {
+                      "liveChatPaidMessageRenderer": {
+                        "id": "dummy_id",
+                        "timestampUsec": 1570678496000000,
+                        "authorName": {
+                          "simpleText": "author_name"
+                        },
+                        "authorPhoto": {
+                          "thumbnails": [
+                            {
+                              "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                              "width": 32,
+                              "height": 32
+                            },
+                            {
+                              "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                              "width": 64,
+                              "height": 64
+                            }
+                          ]
+                        },
+                        "purchaseAmountText": {
+                          "simpleText": "￥846"
+                        },
+                        "message": {
+                          "runs": [
+                            {
+                              "text": "dummy_message"
+                            }
+                          ]
+                        },
+                        "headerBackgroundColor": 4278239141,
+                        "headerTextColor": 4278190080,
+                        "bodyBackgroundColor": 4280150454,
+                        "bodyTextColor": 4278190080,
+                        "authorExternalChannelId": "author_channel_id",
+                        "authorNameTextColor": 2315255808,
+                        "contextMenuEndpoint": {
+                          "commandMetadata": {
+                            "webCommandMetadata": {
+                              "ignoreNavigation": true
+                            }
+                          },
+                          "liveChatItemContextMenuEndpoint": {
+                            "params": "___params___"
+                          }
+                        },
+                        "timestampColor": 2147483648,
+                        "contextMenuAccessibility": {
+                          "accessibilityData": {
+                            "label": "コメントの操作"
+                          }
+                        }
+                      }
+                    }
+                  }
+                },
+                "authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url",
+                "fullDurationSec": 120
+              }
+            },
+            "durationSec": "120"
+          }
+        }
+      ]
+    }
+  }
+}

tests/testdata/default/supersticker.json

@@ -0,0 +1,97 @@
+{
+  "responseContext": {
+    "webResponseContextExtensionData": ""
+  },
+  "continuationContents": {
+    "liveChatContinuation": {
+      "continuations": [
+        {
+          "invalidationContinuationData": {
+            "invalidationId": {
+              "objectSource": 1000,
+              "objectId": "___objectId___",
+              "topic": "chat~00000000000~0000000",
+              "subscribeToGcmTopics": true,
+              "protoCreationTimestampMs": "1577804400000"
+            },
+            "timeoutMs": 10000,
+            "continuation": "___continuation___"
+          }
+        }
+      ],
+      "actions": [
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatPaidStickerRenderer": {
+                "id": "dummy_id",
+                "contextMenuEndpoint": {
+                  "commandMetadata": {
+                    "webCommandMetadata": {
+                      "ignoreNavigation": true
+                    }
+                  },
+                  "liveChatItemContextMenuEndpoint": {
+                    "params": "___params___"
+                  }
+                },
+                "contextMenuAccessibility": {
+                  "accessibilityData": {
+                    "label": "コメントの操作"
+                  }
+                },
+                "timestampUsec": 1570678496000000,
+                "authorPhoto": {
+                  "thumbnails": [
+                    {
+                      "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                      "width": 32,
+                      "height": 32
+                    },
+                    {
+                      "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                      "width": 64,
+                      "height": 64
+                    }
+                  ]
+                },
+                "authorName": {
+                  "simpleText": "author_name"
+                },
+                "authorExternalChannelId": "author_channel_id",
+                "sticker": {
+                  "thumbnails": [
+                    {
+                      "url": "//lh3.googleusercontent.com/param_s=s72-rp",
+                      "width": 72,
+                      "height": 72
+                    },
+                    {
+                      "url": "//lh3.googleusercontent.com/param_s=s144-rp",
+                      "width": 144,
+                      "height": 144
+                    }
+                  ],
+                  "accessibility": {
+                    "accessibilityData": {
+                      "label": "___sticker_label___"
+                    }
+                  }
+                },
+                "moneyChipBackgroundColor": 4278248959,
+                "moneyChipTextColor": 4278190080,
+                "purchaseAmountText": {
+                  "simpleText": "￥200"
+                },
+                "stickerDisplayWidth": 72,
+                "stickerDisplayHeight": 72,
+                "backgroundColor": 4278237396,
+                "authorNameTextColor": 3003121664
+              }
+            }
+          }
+        }
+      ]
+    }
+  }
+}

tests/testdata/default/textmessage.json

@@ -0,0 +1,77 @@
+{
+  "responseContext": {
+    "webResponseContextExtensionData": ""
+  },
+  "continuationContents": {
+    "liveChatContinuation": {
+      "continuations": [
+        {
+          "invalidationContinuationData": {
+            "invalidationId": {
+              "objectSource": 1000,
+              "objectId": "___objectId___",
+              "topic": "chat~00000000000~0000000",
+              "subscribeToGcmTopics": true,
+              "protoCreationTimestampMs": "1577804400000"
+            },
+            "timeoutMs": 10000,
+            "continuation": "___continuation___"
+          }
+        }
+      ],
+      "actions": [
+        {
+          "addChatItemAction": {
+            "item": {
+              "liveChatTextMessageRenderer": {
+                "message": {
+                  "runs": [
+                    {
+                      "text": "dummy_message"
+                    }
+                  ]
+                },
+                "authorName": {
+                  "simpleText": "author_name"
+                },
+                "authorPhoto": {
+                  "thumbnails": [
+                    {
+                      "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                      "width": 32,
+                      "height": 32
+                    },
+                    {
+                      "url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s64-x-x-xx-xx-xx-c0xffffff/photo.jpg",
+                      "width": 64,
+                      "height": 64
+                    }
+                  ]
+                },
+                "contextMenuEndpoint": {
+                  "commandMetadata": {
+                    "webCommandMetadata": {
+                      "ignoreNavigation": true
+                    }
+                  },
+                  "liveChatItemContextMenuEndpoint": {
+                    "params": "___params___"
+                  }
+                },
+                "id": "dummy_id",
+                "timestampUsec": 1570678496000000,
+                "authorExternalChannelId": "author_channel_id",
+                "contextMenuAccessibility": {
+                  "accessibilityData": {
+                    "label": "コメントの操作"
+                  }
+                }
+              }
+            },
+            "clientId": "dummy_client_id"
+          }
+        }
+      ]
+    }
+  }
+}