ramforth/pytchat-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/fix_json_archive_processor' into develop

Browse Source
This commit is contained in:
taizan-hokuto
2020-02-26 22:18:32 +09:00
parent 3c95242ddf f7f9c1cda3
commit 4416e1a79c
42 changed files with 771 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pytchat/__init__.py
View File

@@ -20,7 +20,7 @@ from .api import (
CompatibleProcessor,
DefaultProcessor,
SimpleDisplayProcessor,
JsonfileArchiveProcessor,
JsonfileArchiver,
SpeedCalculator,
DummyProcessor
)

2
pytchat/api.py
View File

@@ -6,7 +6,7 @@ from .processors.chat_processor import ChatProcessor
from .processors.default.processor import DefaultProcessor
from .processors.compatible.processor import CompatibleProcessor
from .processors.simple_display_processor import SimpleDisplayProcessor
from .processors.jsonfile_archive_processor import JsonfileArchiveProcessor
from .processors.jsonfile_archiver import JsonfileArchiver
from .processors.speed_calculator import SpeedCalculator
from .processors.dummy_processor import DummyProcessor
from . import config

46
pytchat/processors/jsonfile_archive_processor.py
View File

@@ -1,46 +0,0 @@
import json
import os
import datetime
from .chat_processor import ChatProcessor
class JsonfileArchiveProcessor(ChatProcessor):
def __init__(self,filepath):
super().__init__()
if os.path.exists(filepath):
print('filepath is already exists!: ')
print(' '+filepath)
newpath=os.path.dirname(filepath) + \
'/'+datetime.datetime.now() \
.strftime('%Y-%m-%d %H-%M-%S')+'.data'
print('created alternate filename:')
print(' '+newpath)
self.filepath = newpath
else:
print('filepath: '+filepath)
self.filepath = filepath
def process(self,chat_components: list):
if chat_components:
with open(self.filepath, mode='a', encoding = 'utf-8') as f:
for component in chat_components:
if component:
chatdata = component.get('chatdata')
for action in chatdata:
if action:
if action.get("addChatItemAction"):
if action["addChatItemAction"]["item"].get(
"liveChatViewerEngagementMessageRenderer"):
continue
s = json.dumps(action,ensure_ascii = False)
#print(s[:200])
f.writelines(s+'\n')
def _parsedir(self,_dir):
if _dir[-1]=='\\' or _dir[-1]=='/':
separator =''
else:
separator ='/'
os.makedirs(_dir + separator, exist_ok=True)
return _dir + separator

66
pytchat/processors/jsonfile_archiver.py Normal file
View File

@@ -0,0 +1,66 @@
import datetime
import json
import os
import re
from .chat_processor import ChatProcessor
PATTERN = re.compile(r"(.*)\(([0-9]+)\)$")
class JsonfileArchiver(ChatProcessor):
"""
JsonfileArchiver saves chat data as text of JSON lines.
Parameter:
----------
save_path : str :
save path of file.If a file with the same name exists,
it is automatically saved under a different name
with suffix '(number)'
"""
def __init__(self,save_path):
super().__init__()
self.save_path = self._checkpath(save_path)
self.line_counter = 0
def process(self,chat_components: list):
"""
Returns
----------
dict :
save_path : str :
Actual save path of file.
total_lines : int :
count of total lines written to the file.
"""
if chat_components is None: return
with open(self.save_path, mode='a', encoding = 'utf-8') as f:
for component in chat_components:
if component is None: continue
chatdata = component.get('chatdata')
if chatdata is None: continue
for action in chatdata:
if action is None: continue
json_line = json.dumps(action, ensure_ascii = False)
f.writelines(json_line+'\n')
self.line_counter+=1
return { "save_path" : self.save_path,
"total_lines": self.line_counter }
def _checkpath(self, filepath):
splitter = os.path.splitext(os.path.basename(filepath))
body = splitter[0]
extention = splitter[1]
newpath = filepath
counter = 0
while os.path.exists(newpath):
match = re.search(PATTERN,body)
if match:
counter=int(match[2])+1
num_with_bracket = f'({str(counter)})'
body = f'{match[1]}{num_with_bracket}'
else:
body = f'{body}({str(counter)})'
newpath = os.path.join(os.path.dirname(filepath),body+extention)
return newpath

0
pytchat/tool/download/__init__.py → pytchat/tool/extract/__init__.py
View File

6
pytchat/tool/download/asyncdl.py → pytchat/tool/extract/asyncdl.py
View File

@@ -3,7 +3,7 @@ import asyncio
import json
from . import parser
from . block import Block
from . dlworker import DownloadWorker
from . worker import ExtractWorker
from . patch import Patch
from ... import config
from ... paramgen import arcparam
@@ -79,11 +79,11 @@ def ready_blocks(video_id, duration, div, callback):
_get_blocks(video_id, duration, div, callback))
return blocks
def download_patch(callback, blocks, video_id):
def fetch_patch(callback, blocks, video_id):
async def _allocate_workers():
workers = [
DownloadWorker(
ExtractWorker(
fetch = _fetch, block = block,
blocks = blocks, video_id = video_id
)

8
pytchat/tool/download/block.py → pytchat/tool/extract/block.py
View File

@@ -16,9 +16,9 @@ class Block:
this value increases as fetching chatdata progresses.
end : int :
target videoOffsetTimeMs of last chat data for download,
target videoOffsetTimeMs of last chat data for extract,
equals to first videoOffsetTimeMs of next block.
when download worker reaches this offset, stop downloading.
when extract worker reaches this offset, stop fetching.
continuation : str :
continuation param of last chat data.
@@ -26,10 +26,10 @@ class Block:
chat_data : list
done : bool :
whether this block has been downloaded.
whether this block has been fetched.
remaining : int :
remaining data to download.
remaining data to extract.
equals end - last.
is_last : bool :

0
pytchat/tool/download/duplcheck.py → pytchat/tool/extract/duplcheck.py
View File

10
pytchat/tool/download/downloader.py → pytchat/tool/extract/extractor.py
View File

@@ -8,7 +8,7 @@ from ... exceptions import InvalidVideoIdException
logger = config.logger(__name__)
headers=config.headers
class Downloader:
class Extractor:
def __init__(self, video_id, duration, div, callback):
if not isinstance(div ,int) or div < 1:
raise ValueError('div must be positive integer.')
@@ -44,7 +44,7 @@ class Downloader:
return self
def _download_blocks(self):
asyncdl.download_patch(self.callback, self.blocks, self.video_id)
asyncdl.fetch_patch(self.callback, self.blocks, self.video_id)
return self
def _remove_duplicate_tail(self):
@@ -57,7 +57,7 @@ class Downloader:
ret.extend(block.chat_data)
return ret
def download(self):
def extract(self):
return (
self._ready_blocks()
._remove_duplicate_head()
@@ -68,7 +68,7 @@ class Downloader:
._combine()
)
def download(video_id, div = 1, callback = None, processor = None):
def extract(video_id, div = 1, callback = None, processor = None):
duration = 0
try:
duration = VideoInfo(video_id).get("duration")
@@ -77,7 +77,7 @@ def download(video_id, div = 1, callback = None, processor = None):
if duration == 0:
print("video is live.")
return []
data = Downloader(video_id, duration, div, callback).download()
data = Extractor(video_id, duration, div, callback).extract()
if processor is None:
return data
return processor.process(

0
pytchat/tool/download/parser.py → pytchat/tool/extract/parser.py
View File

2
pytchat/tool/download/patch.py → pytchat/tool/extract/patch.py
View File

@@ -5,7 +5,7 @@ from typing import NamedTuple
class Patch(NamedTuple):
"""
Patch represents chunk of chat data
which is fetched by asyncdl.download_patch._fetch().
which is fetched by asyncdl.fetch_patch._fetch().
"""
chats : list = []
continuation : str = None

14
pytchat/tool/download/dlworker.py → pytchat/tool/extract/worker.py
View File

@@ -3,17 +3,17 @@ from . block import Block
from . patch import Patch, fill, split
from ... paramgen import arcparam
class DownloadWorker:
class ExtractWorker:
"""
DownloadWorker associates a download session with a block.
ExtractWorker associates a download session with a block.
When the dlworker finishes downloading, the block
being downloaded is splitted and assigned the free dlworker.
When the worker finishes fetching, the block
being fetched is splitted and assigned the free worker.
Parameter
----------
fetch : func :
download function of asyncdl
extract function of asyncdl
block : Block :
Block object that includes chat_data
@@ -40,7 +40,7 @@ class DownloadWorker:
patch = await self.fetch(
self.block.continuation, session)
if patch.continuation is None:
"""TODO : make the dlworker assigned to the last block
"""TODO : make the worker assigned to the last block
to work more than twice as possible.
"""
break
@@ -50,7 +50,7 @@ class DownloadWorker:
else:
fill(self.block, patch)
if self.block.continuation is None:
"""finished downloading this block """
"""finished fetching this block """
self.block.done = True
self.block = _search_new_block(self)

6
pytchat/tool/mining/asyncdl.py
View File

@@ -4,7 +4,7 @@ import asyncio
import json
from . import parser
from . block import Block
from . dlworker import DownloadWorker
from . worker import ExtractWorker
from . patch import Patch
from ... import config
from ... paramgen import arcparam_mining as arcparam
@@ -84,11 +84,11 @@ def ready_blocks(video_id, duration, div, callback):
_get_blocks(video_id, duration, div, callback))
return blocks
def download_patch(callback, blocks, video_id):
def fetch_patch(callback, blocks, video_id):
async def _allocate_workers():
workers = [
DownloadWorker(
ExtractWorker(
fetch = _fetch, block = block,
blocks = blocks, video_id = video_id
)

8
pytchat/tool/mining/block.py
View File

@@ -16,9 +16,9 @@ class Block:
this value increases as fetching chatdata progresses.
end : int :
target videoOffsetTimeMs of last chat data for download,
target videoOffsetTimeMs of last chat data for extract,
equals to first videoOffsetTimeMs of next block.
when download worker reaches this offset, stop downloading.
when extract worker reaches this offset, stop fetching.
continuation : str :
continuation param of last chat data.
@@ -26,10 +26,10 @@ class Block:
chat_data : list
done : bool :
whether this block has been downloaded.
whether this block has been fetched.
remaining : int :
remaining data to download.
remaining data to extract.
equals end - last.
is_last : bool :

2
pytchat/tool/mining/patch.py
View File

@@ -5,7 +5,7 @@ from typing import NamedTuple
class Patch(NamedTuple):
"""
Patch represents chunk of chat data
which is fetched by asyncdl.download_patch._fetch().
which is fetched by asyncdl.fetch_patch._fetch().
"""
chats : list = []
continuation : str = None

10
pytchat/tool/mining/downloader.py → pytchat/tool/mining/superchat_miner.py
View File

@@ -6,7 +6,7 @@ from ... exceptions import InvalidVideoIdException
logger = config.logger(__name__)
headers=config.headers
class Downloader:
class SuperChatMiner:
def __init__(self, video_id, duration, div, callback):
if not isinstance(div ,int) or div < 1:
raise ValueError('div must be positive integer.')
@@ -34,7 +34,7 @@ class Downloader:
return self
def _download_blocks(self):
asyncdl.download_patch(self.callback, self.blocks, self.video_id)
asyncdl.fetch_patch(self.callback, self.blocks, self.video_id)
return self
def _combine(self):
@@ -43,7 +43,7 @@ class Downloader:
ret.extend(block.chat_data)
return ret
def download(self):
def extract(self):
return (
self._ready_blocks()
._set_block_end()
@@ -51,7 +51,7 @@ class Downloader:
._combine()
)
def download(video_id, div = 1, callback = None, processor = None):
def extract(video_id, div = 1, callback = None, processor = None):
duration = 0
try:
duration = VideoInfo(video_id).get("duration")
@@ -60,7 +60,7 @@ def download(video_id, div = 1, callback = None, processor = None):
if duration == 0:
print("video is live.")
return []
data = Downloader(video_id, duration, div, callback).download()
data = SuperChatMiner(video_id, duration, div, callback).extract()
if processor is None:
return data
return processor.process(

10
pytchat/tool/mining/dlworker.py → pytchat/tool/mining/worker.py
View File

@@ -3,17 +3,17 @@ from . block import Block
from . patch import Patch, fill
from ... paramgen import arcparam
INTERVAL = 1
class DownloadWorker:
class ExtractWorker:
"""
DownloadWorker associates a download session with a block.
ExtractWorker associates a download session with a block.
When the dlworker finishes downloading, the block
being downloaded is splitted and assigned the free dlworker.
When the worker finishes fetching, the block
being fetched is splitted and assigned the free worker.
Parameter
----------
fetch : func :
download function of asyncdl
extract function of asyncdl
block : Block :
Block object that includes chat_data

2
pytchat/util/__init__.py
View File

@@ -1,7 +1,7 @@
import requests,json,datetime
from .. import config
def download(url):
def extract(url):
_session = requests.Session()
html = _session.get(url, headers=config.headers)
with open(str(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')

4
tests/test_dl_asyncdl.py → tests/test_extract_asyncdl.py
View File

@@ -1,12 +1,12 @@
import aiohttp
import asyncio
import json
from pytchat.tool.download import parser
from pytchat.tool.extract import parser
import sys
import time
from aioresponses import aioresponses
from concurrent.futures import CancelledError
from pytchat.tool.download import asyncdl
from pytchat.tool.extract import asyncdl
def _open_file(path):
with open(path,mode ='r',encoding = 'utf-8') as f:

14
tests/test_dl_duplcheck.py → tests/test_extract_duplcheck.py
View File

@@ -3,10 +3,10 @@ import asyncio
import json
import os, sys
import time
from pytchat.tool.download import duplcheck
from pytchat.tool.download import parser
from pytchat.tool.download.block import Block
from pytchat.tool.download.duplcheck import _dump
from pytchat.tool.extract import duplcheck
from pytchat.tool.extract import parser
from pytchat.tool.extract.block import Block
from pytchat.tool.extract.duplcheck import _dump
def _open_file(path):
with open(path,mode ='r',encoding = 'utf-8') as f:
return f.read()
@@ -23,7 +23,7 @@ def test_overlap():
def load_chatdata(filename):
return parser.parse(
json.loads(_open_file("tests/testdata/dl_duplcheck/overlap/"+filename))
json.loads(_open_file("tests/testdata/extract_duplcheck/overlap/"+filename))
)[1]
blocks = (
@@ -54,7 +54,7 @@ def test_duplicate_head():
def load_chatdata(filename):
return parser.parse(
json.loads(_open_file("tests/testdata/dl_duplcheck/head/"+filename))
json.loads(_open_file("tests/testdata/extract_duplcheck/head/"+filename))
)[1]
"""
@@ -103,7 +103,7 @@ def test_duplicate_tail():
"""
def load_chatdata(filename):
return parser.parse(
json.loads(_open_file("tests/testdata/dl_duplcheck/head/"+filename))
json.loads(_open_file("tests/testdata/extract_duplcheck/head/"+filename))
)[1]
#chat data offsets are ignored.
blocks = (

26
tests/test_dl_patch.py → tests/test_extract_patch.py
View File

@@ -4,18 +4,18 @@ import json
import os, sys
import time
from aioresponses import aioresponses
from pytchat.tool.download import duplcheck
from pytchat.tool.download import parser
from pytchat.tool.download.block import Block
from pytchat.tool.download.patch import Patch, fill, split, set_patch
from pytchat.tool.download.duplcheck import _dump
from pytchat.tool.extract import duplcheck
from pytchat.tool.extract import parser
from pytchat.tool.extract.block import Block
from pytchat.tool.extract.patch import Patch, fill, split, set_patch
from pytchat.tool.extract.duplcheck import _dump
def _open_file(path):
with open(path,mode ='r',encoding = 'utf-8') as f:
return f.read()
def load_chatdata(filename):
return parser.parse(
json.loads(_open_file("tests/testdata/dl_patch/"+filename))
json.loads(_open_file("tests/testdata/fetch_patch/"+filename))
)[1]
@@ -25,7 +25,7 @@ def test_split_0():
~~~~~~ before ~~~~~~
@parent_block (# = already downloaded)
@parent_block (# = already fetched)
first last end
|########----------------------------------------|
@@ -79,11 +79,11 @@ def test_split_1():
"""patch.first <= parent_block.last
While awaiting at run()->asyncdl._fetch()
downloading parent_block proceeds,
fetching parent_block proceeds,
and parent.block.last exceeds patch.first.
In this case, fetched patch is all discarded,
and dlworker searches other processing block again.
and worker searches other processing block again.
~~~~~~ before ~~~~~~
@@ -135,7 +135,7 @@ def test_split_2():
~~~~~~ before ~~~~~~
@parent_block (# = already downloaded)
@parent_block (# = already fetched)
first last end (before split)
|########------------------------------|
@@ -163,7 +163,7 @@ def test_split_2():
first last=end |
|#################|...... cut extra data.
^
continuation : None (download complete)
continuation : None (extract complete)
@fetched patch
|-------- patch --------|
@@ -188,11 +188,11 @@ def test_split_none():
"""patch.last <= parent_block.last
While awaiting at run()->asyncdl._fetch()
downloading parent_block proceeds,
fetching parent_block proceeds,
and parent.block.last exceeds patch.first.
In this case, fetched patch is all discarded,
and dlworker searches other processing block again.
and worker searches other processing block again.
~~~~~~ before ~~~~~~

48
tests/test_jsonfile_archiver.py Normal file
View File

@@ -0,0 +1,48 @@
import json
from pytchat.processors.jsonfile_archiver import JsonfileArchiver
from unittest.mock import patch, mock_open
from tests.testdata.jsonfile_archiver.chat_component import chat_component
def _open_file(path):
with open(path,mode ='r',encoding = 'utf-8') as f:
return f.read()
def test_checkpath(mocker):
processor = JsonfileArchiver("path")
mocker.patch('os.path.exists').side_effect = exists_file
'''Test no duplicate file.'''
assert processor._checkpath("z:/other.txt") == "z:/other.txt"
'''Test duplicate filename.
The case the name first renamed ('test.txt -> test(0).txt')
is also duplicated.
'''
assert processor._checkpath("z:/test.txt") == "z:/test(1).txt"
'''Test no extention file (duplicate).'''
assert processor._checkpath("z:/test") == "z:/test(0)"
def test_read_write():
'''Test read and write chatdata'''
mock = mock_open(read_data = "")
with patch('builtins.open',mock):
processor = JsonfileArchiver("path")
save_path = processor.process([chat_component])
fh = mock()
actuals = [args[0] for (args, kwargs) in fh.writelines.call_args_list]
'''write format is json dump string with 0x0A'''
to_be_written = [json.dumps(action, ensure_ascii=False)+'\n'
for action in chat_component["chatdata"]]
for i in range(len(actuals)):
assert actuals[i] == to_be_written[i]
assert save_path == {'save_path': 'path', 'total_lines': 7}
def exists_file(path):
if path == "z:/test.txt":
return True
if path == "z:/test(0).txt":
return True
if path == "z:/test":
return True

0
tests/testdata/dl_duplcheck/head/dp0-0.json → tests/testdata/extract_duplcheck/head/dp0-0.json vendored
View File

0
tests/testdata/dl_duplcheck/head/dp0-1.json → tests/testdata/extract_duplcheck/head/dp0-1.json vendored
View File

0
tests/testdata/dl_duplcheck/head/dp0-2.json → tests/testdata/extract_duplcheck/head/dp0-2.json vendored
View File

0
tests/testdata/dl_duplcheck/head/dp0-3.json → tests/testdata/extract_duplcheck/head/dp0-3.json vendored
View File

0
tests/testdata/dl_duplcheck/head/dp0-4.json → tests/testdata/extract_duplcheck/head/dp0-4.json vendored
View File

0
tests/testdata/dl_duplcheck/head/dp0-5.json → tests/testdata/extract_duplcheck/head/dp0-5.json vendored
View File

0
tests/testdata/dl_duplcheck/overlap/dp0-0.json → tests/testdata/extract_duplcheck/overlap/dp0-0.json vendored
View File

0
tests/testdata/dl_duplcheck/overlap/dp0-1.json → tests/testdata/extract_duplcheck/overlap/dp0-1.json vendored
View File

0
tests/testdata/dl_duplcheck/overlap/dp0-2.json → tests/testdata/extract_duplcheck/overlap/dp0-2.json vendored
View File

0
tests/testdata/dl_duplcheck/overlap/dp0-3.json → tests/testdata/extract_duplcheck/overlap/dp0-3.json vendored
View File

0
tests/testdata/dl_duplcheck/overlap/dp0-4.json → tests/testdata/extract_duplcheck/overlap/dp0-4.json vendored
View File

0
tests/testdata/dl_duplcheck/overlap/dp0-5.json → tests/testdata/extract_duplcheck/overlap/dp0-5.json vendored
View File

0
tests/testdata/dl_patch/pt0-0.json → tests/testdata/fetch_patch/pt0-0.json vendored
View File

0
tests/testdata/dl_patch/pt0-1.json → tests/testdata/fetch_patch/pt0-1.json vendored
View File

0
tests/testdata/dl_patch/pt0-3.json → tests/testdata/fetch_patch/pt0-3.json vendored
View File

0
tests/testdata/dl_patch/pt0-4.json → tests/testdata/fetch_patch/pt0-4.json vendored
View File

0
tests/testdata/dl_patch/pt0-5.json → tests/testdata/fetch_patch/pt0-5.json vendored
View File

487
tests/testdata/jsonfile_archiver/chat_component.py vendored Normal file
View File

@@ -0,0 +1,487 @@
chat_component = {
"video_id" : "video_id",
"timeout" : 10,
"chatdata": [
{
"addChatItemAction": {
"item": {
"liveChatTextMessageRenderer": {
"message": {
"runs": [
{
"text": "This is normal message."
}
]
},
"authorName": {
"simpleText": "author_name"
},
"authorPhoto": {
"thumbnails": [
{
"url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
"width": 32,
"height": 32
},
{
"url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
"width": 64,
"height": 64
}
]
},
"contextMenuEndpoint": {
"commandMetadata": {
"webCommandMetadata": {
"ignoreNavigation": True
}
},
"liveChatItemContextMenuEndpoint": {
"params": "___params___"
}
},
"id": "dummy_id",
"timestampUsec": 0,
"authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url",
"contextMenuAccessibility": {
"accessibilityData": {
"label": "コメントの操作"
}
}
}
},
"clientId": "dummy_client_id"
}
},
{
"addChatItemAction": {
"item": {
"liveChatTextMessageRenderer": {
"message": {
"runs": [
{
"text": "This is members's message"
}
]
},
"authorName": {
"simpleText": "author_name"
},
"authorPhoto": {
"thumbnails": [
{
"url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
"width": 32,
"height": 32
},
{
"url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
"width": 64,
"height": 64
}
]
},
"contextMenuEndpoint": {
"commandMetadata": {
"webCommandMetadata": {
"ignoreNavigation": True
}
},
"liveChatItemContextMenuEndpoint": {
"params": "___params___"
}
},
"id": "dummy_id",
"timestampUsec": 0,
"authorBadges": [
{
"liveChatAuthorBadgeRenderer": {
"customThumbnail": {
"thumbnails": [
{
"url": "https://yt3.ggpht.com/X=s32-c-k"
},
{
"url": "https://yt3.ggpht.com/X=s32-c-k"
}
]
},
"tooltip": "メンバー（2 か月）",
"accessibility": {
"accessibilityData": {
"label": "メンバー（2 か月）"
}
}
}
}
],
"authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url",
"contextMenuAccessibility": {
"accessibilityData": {
"label": "コメントの操作"
}
}
}
},
"clientId": "dummy_client_id"
}
},
{
"addChatItemAction": {
"item": {
"liveChatPlaceholderItemRenderer": {
"id": "dummy_id",
"timestampUsec": 0
}
},
"clientId": "dummy_client_id"
}
},
{
"addLiveChatTickerItemAction": {
"item": {
"liveChatTickerPaidMessageItemRenderer": {
"id": "dummy_id",
"amount": {
"simpleText": "￥10,000"
},
"amountTextColor": 4294967295,
"startBackgroundColor": 4293271831,
"endBackgroundColor": 4291821568,
"authorPhoto": {
"thumbnails": [
{
"url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
"width": 32,
"height": 32
},
{
"url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
"width": 64,
"height": 64
}
]
},
"durationSec": 3600,
"showItemEndpoint": {
"commandMetadata": {
"webCommandMetadata": {
"ignoreNavigation": True
}
},
"showLiveChatItemEndpoint": {
"renderer": {
"liveChatPaidMessageRenderer": {
"id": "dummy_id",
"timestampUsec": 0,
"authorName": {
"simpleText": "author_name"
},
"authorPhoto": {
"thumbnails": [
{
"url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
"width": 32,
"height": 32
},
{
"url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
"width": 64,
"height": 64
}
]
},
"purchaseAmountText": {
"simpleText": "￥10,000"
},
"message": {
"runs": [
{
"text": "This is superchat message."
}
]
},
"headerBackgroundColor": 4291821568,
"headerTextColor": 4294967295,
"bodyBackgroundColor": 4293271831,
"bodyTextColor": 4294967295,
"authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url",
"authorNameTextColor": 3019898879,
"contextMenuEndpoint": {
"commandMetadata": {
"webCommandMetadata": {
"ignoreNavigation": True
}
},
"liveChatItemContextMenuEndpoint": {
"params": "___params___"
}
},
"timestampColor": 2164260863,
"contextMenuAccessibility": {
"accessibilityData": {
"label": "コメントの操作"
}
}
}
}
}
},
"authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url",
"fullDurationSec": 3600
}
},
"durationSec": "3600"
}
},
{
"addChatItemAction": {
"item": {
"liveChatPaidMessageRenderer": {
"id": "dummy_id",
"timestampUsec": 0,
"authorName": {
"simpleText": "author_name"
},
"authorPhoto": {
"thumbnails": [
{
"url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
"width": 32,
"height": 32
},
{
"url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
"width": 64,
"height": 64
}
]
},
"purchaseAmountText": {
"simpleText": "￥10,800"
},
"message": {
"runs": [
{
"text": "This is superchat message."
}
]
},
"headerBackgroundColor": 4291821568,
"headerTextColor": 4294967295,
"bodyBackgroundColor": 4293271831,
"bodyTextColor": 4294967295,
"authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url",
"authorNameTextColor": 3019898879,
"contextMenuEndpoint": {
"commandMetadata": {
"webCommandMetadata": {
"ignoreNavigation": True
}
},
"liveChatItemContextMenuEndpoint": {
"params": "___params___"
}
},
"timestampColor": 2164260863,
"contextMenuAccessibility": {
"accessibilityData": {
"label": "コメントの操作"
}
}
}
}
}
},
{
"addChatItemAction": {
"item": {
"liveChatPaidStickerRenderer": {
"id": "dummy_id",
"contextMenuEndpoint": {
"clickTrackingParams": "___clickTrackingParams___",
"commandMetadata": {
"webCommandMetadata": {
"ignoreNavigation": True
}
},
"liveChatItemContextMenuEndpoint": {
"params": "___params___"
}
},
"contextMenuAccessibility": {
"accessibilityData": {
"label": "コメントの操作"
}
},
"timestampUsec": 0,
"authorPhoto": {
"thumbnails": [
{
"url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
"width": 32,
"height": 32
},
{
"url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
"width": 64,
"height": 64
}
]
},
"authorName": {
"simpleText": "author_name"
},
"authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url",
"sticker": {
"thumbnails": [
{
"url": "//lh3.googleusercontent.com/param_s=s40-rp",
"width": 40,
"height": 40
},
{
"url": "//lh3.googleusercontent.com/param_s=s80-rp",
"width": 80,
"height": 80
}
],
"accessibility": {
"accessibilityData": {
"label": "___sticker_label___"
}
}
},
"moneyChipBackgroundColor": 4280191205,
"moneyChipTextColor": 4294967295,
"purchaseAmountText": {
"simpleText": "￥150"
},
"stickerDisplayWidth": 40,
"stickerDisplayHeight": 40,
"backgroundColor": 4279592384,
"authorNameTextColor": 3019898879,
"trackingParams": "___trackingParams___"
}
}
}
},
{
"addLiveChatTickerItemAction": {
"item": {
"liveChatTickerSponsorItemRenderer": {
"id": "dummy_id",
"detailText": {
"runs": [
{
"text": "メンバー"
}
]
},
"detailTextColor": 4294967295,
"startBackgroundColor": 4279213400,
"endBackgroundColor": 4278943811,
"sponsorPhoto": {
"thumbnails": [
{
"url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
"width": 32,
"height": 32
},
{
"url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
"width": 64,
"height": 64
}
]
},
"durationSec": 300,
"showItemEndpoint": {
"commandMetadata": {
"webCommandMetadata": {
"ignoreNavigation": True
}
},
"showLiveChatItemEndpoint": {
"renderer": {
"liveChatMembershipItemRenderer": {
"id": "dummy_id",
"timestampUsec": 0,
"authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url",
"headerSubtext": {
"runs": [
{
"text": "メンバーシップ"
},
{
"text": " へようこそ！"
}
]
},
"authorName": {
"simpleText": "author_name"
},
"authorPhoto": {
"thumbnails": [
{
"url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
"width": 32,
"height": 32
},
{
"url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
"width": 64,
"height": 64
}
]
},
"authorBadges": [
{
"liveChatAuthorBadgeRenderer": {
"customThumbnail": {
"thumbnails": [
{
"url": "https://yt3.ggpht.com/X=s32-c-k"
},
{
"url": "https://yt3.ggpht.com/X=s32-c-k"
}
]
},
"tooltip": "新規メンバー",
"accessibility": {
"accessibilityData": {
"label": "新規メンバー"
}
}
}
}
],
"contextMenuEndpoint": {
"commandMetadata": {
"webCommandMetadata": {
"ignoreNavigation": True
}
},
"liveChatItemContextMenuEndpoint": {
"params": "___params___"
}
},
"contextMenuAccessibility": {
"accessibilityData": {
"label": "コメントの操作"
}
}
}
}
}
},
"authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url",
"fullDurationSec": 300
}
},
"durationSec": "300"
}
}
]
}

18
tests/testdata/jsonfile_archiver/replay_end.json vendored Normal file
View File

@@ -0,0 +1,18 @@
{
"response": {
"responseContext": {
"webResponseContextExtensionData": ""
},
"continuationContents": {
"liveChatContinuation": {
"continuations": [
{
"playerSeekContinuationData": {
"continuation": "___reload_continuation___"
}
}
]
}
}
}
}

89
tests/testdata/jsonfile_archiver/text_only.json vendored Normal file
View File

@@ -0,0 +1,89 @@
{
"response": {
"responseContext": {
"webResponseContextExtensionData": ""
},
"continuationContents": {
"liveChatContinuation": {
"continuations": [
{
"invalidationContinuationData": {
"invalidationId": {
"objectSource": 1000,
"objectId": "___objectId___",
"topic": "chat~00000000000~0000000",
"subscribeToGcmTopics": true,
"protoCreationTimestampMs": "1577804400000"
},
"timeoutMs": 10000,
"continuation": "___continuation___"
}
}
],
"actions": [
{
"replayChatItemAction": {
"actions": [
{
"addChatItemAction": {
"item": {
"liveChatTextMessageRenderer": {
"message": {
"runs": [
{
"text": "dummy_message"
}
]
},
"authorName": {
"simpleText": "author_name"
},
"authorPhoto": {
"thumbnails": [
{
"url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
"width": 32,
"height": 32
},
{
"url": "https://yt3.ggpht.com/------------/AAAAAAAAAAA/AAAAAAAAAAA/xxxxxxxxxxxx/s32-x-x-xx-xx-xx-c0xffffff/photo.jpg",
"width": 64,
"height": 64
}
]
},
"contextMenuEndpoint": {
"commandMetadata": {
"webCommandMetadata": {
"ignoreNavigation": true
}
},
"liveChatItemContextMenuEndpoint": {
"params": "___params___"
}
},
"id": "dummy_id",
"timestampUsec": 0,
"authorExternalChannelId": "http://www.youtube.com/channel/author_channel_url",
"contextMenuAccessibility": {
"accessibilityData": {
"label": "コメントの操作"
}
},
"timestampText": {
"simpleText": "0:00"
}
}
},
"clientId": "dummy_client_id"
}
}
],
"videoOffsetTimeMsec": "10000"
}
}
]
}
}
}
}