ramforth/pytchat-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Format code

Browse Source
This commit is contained in:
taizan-hokuto
2020-06-04 23:10:26 +09:00
parent e6dbc8772e
commit 2474207691
50 changed files with 635 additions and 622 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pytchat/__init__.py
View File

@@ -28,3 +28,5 @@ from .api import (
SuperchatCalculator, SuperchatCalculator,
VideoInfo VideoInfo
) )
# flake8: noqa

2
pytchat/api.py
View File

@@ -14,3 +14,5 @@ from .processors.speed.calculator import SpeedCalculator
from .processors.superchat.calculator import SuperchatCalculator from .processors.superchat.calculator import SuperchatCalculator
from .tool.extract.extractor import Extractor from .tool.extract.extractor import Extractor
from .tool.videoinfo import VideoInfo from .tool.videoinfo import VideoInfo
# flake8: noqa

30
pytchat/cli/__init__.py
View File

@@ -1,11 +1,7 @@
import argparse import argparse
import os
from pathlib import Path from pathlib import Path
from typing import List, Callable
from .arguments import Arguments from .arguments import Arguments
from .. exceptions import InvalidVideoIdException, NoContentsException from .. exceptions import InvalidVideoIdException, NoContentsException
from .. processors.tsv_archiver import TSVArchiver
from .. processors.html_archiver import HTMLArchiver from .. processors.html_archiver import HTMLArchiver
from .. tool.extract.extractor import Extractor from .. tool.extract.extractor import Extractor
from .. tool.videoinfo import VideoInfo from .. tool.videoinfo import VideoInfo
@@ -18,16 +14,18 @@ https://github.com/PetterKraabol/Twitch-Chat-Downloader
(MIT License) (MIT License)
''' '''
def main(): def main():
# Arguments # Arguments
parser = argparse.ArgumentParser(description=f'pytchat v{__version__}') parser = argparse.ArgumentParser(description=f'pytchat v{__version__}')
parser.add_argument('-v', f'--{Arguments.Name.VIDEO}', type=str, parser.add_argument('-v', f'--{Arguments.Name.VIDEO}', type=str,
help='Video IDs separated by commas without space.\n' help='Video IDs separated by commas without space.\n'
'If ID starts with a hyphen (-), enclose the ID in square brackets.') 'If ID starts with a hyphen (-), enclose the ID in square brackets.')
parser.add_argument('-o', f'--{Arguments.Name.OUTPUT}', type=str, parser.add_argument('-o', f'--{Arguments.Name.OUTPUT}', type=str,
help='Output directory (end with "/"). default="./"', default='./') help='Output directory (end with "/"). default="./"', default='./')
parser.add_argument(f'--{Arguments.Name.VERSION}', action='store_true', parser.add_argument(f'--{Arguments.Name.VERSION}', action='store_true',
help='Settings version') help='Settings version')
Arguments(parser.parse_args().__dict__) Arguments(parser.parse_args().__dict__)
if Arguments().print_version: if Arguments().print_version:
print(f'pytchat v{__version__}') print(f'pytchat v{__version__}')
@@ -37,24 +35,26 @@ def main():
if Arguments().video_ids: if Arguments().video_ids:
for video_id in Arguments().video_ids: for video_id in Arguments().video_ids:
if '[' in video_id: if '[' in video_id:
video_id = video_id.replace('[','').replace(']','') video_id = video_id.replace('[', '').replace(']', '')
try: try:
info = VideoInfo(video_id) info = VideoInfo(video_id)
print(f"Extracting...\n" print(f"Extracting...\n"
f" video_id: {video_id}\n" f" video_id: {video_id}\n"
f" channel: {info.get_channel_name()}\n" f" channel: {info.get_channel_name()}\n"
f" title: {info.get_title()}") f" title: {info.get_title()}")
path = Path(Arguments().output+video_id+'.html') path = Path(Arguments().output + video_id + '.html')
print(f"output path: {path.resolve()}") print(f"output path: {path.resolve()}")
Extractor(video_id, Extractor(video_id,
processor = HTMLArchiver(Arguments().output+video_id+'.html'), processor=HTMLArchiver(
callback = _disp_progress Arguments().output + video_id + '.html'),
).extract() callback=_disp_progress
).extract()
print("\nExtraction end.\n") print("\nExtraction end.\n")
except (InvalidVideoIdException, NoContentsException) as e: except (InvalidVideoIdException, NoContentsException) as e:
print(e) print(e)
return return
parser.print_help() parser.print_help()
def _disp_progress(a,b):
print('.',end="",flush=True) def _disp_progress(a, b):
print('.', end="", flush=True)

5
pytchat/cli/arguments.py
View File

@@ -8,6 +8,7 @@ https://github.com/PetterKraabol/Twitch-Chat-Downloader
(MIT License) (MIT License)
''' '''
class Arguments(metaclass=Singleton): class Arguments(metaclass=Singleton):
""" """
Arguments singleton Arguments singleton
@@ -19,7 +20,7 @@ class Arguments(metaclass=Singleton):
VIDEO: str = 'video' VIDEO: str = 'video'
def __init__(self, def __init__(self,
arguments: Optional[Dict[str, Union[str, bool, int]]] = None): arguments: Optional[Dict[str, Union[str, bool, int]]] = None):
""" """
Initialize arguments Initialize arguments
:param arguments: Arguments from cli :param arguments: Arguments from cli
@@ -36,4 +37,4 @@ class Arguments(metaclass=Singleton):
# Videos # Videos
if arguments[Arguments.Name.VIDEO]: if arguments[Arguments.Name.VIDEO]:
self.video_ids = [video_id self.video_ids = [video_id
for video_id in arguments[Arguments.Name.VIDEO].split(',')] for video_id in arguments[Arguments.Name.VIDEO].split(',')]

2
pytchat/cli/singleton.py
View File

@@ -4,6 +4,8 @@ Petter Kraabøl's Twitch-Chat-Downloader
https://github.com/PetterKraabol/Twitch-Chat-Downloader https://github.com/PetterKraabol/Twitch-Chat-Downloader
(MIT License) (MIT License)
''' '''
class Singleton(type): class Singleton(type):
""" """
Abstract class for singletons Abstract class for singletons

10
pytchat/config/__init__.py
View File

@@ -1,11 +1,9 @@
import logging
from . import mylogger from . import mylogger
headers = { headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'} 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'}
def logger(module_name: str, loglevel = None):
module_logger = mylogger.get_logger(module_name, loglevel = loglevel) def logger(module_name: str, loglevel=None):
module_logger = mylogger.get_logger(module_name, loglevel=loglevel)
return module_logger return module_logger

16
pytchat/config/mylogger.py
View File

@@ -1,31 +1,31 @@
from logging import NullHandler, getLogger, StreamHandler, FileHandler, Formatter from logging import NullHandler, getLogger, StreamHandler, FileHandler
import logging import logging
from datetime import datetime from datetime import datetime
def get_logger(modname,loglevel=logging.DEBUG): def get_logger(modname, loglevel=logging.DEBUG):
logger = getLogger(modname) logger = getLogger(modname)
if loglevel == None: if loglevel is None:
logger.addHandler(NullHandler()) logger.addHandler(NullHandler())
return logger return logger
logger.setLevel(loglevel) logger.setLevel(loglevel)
#create handler1 for showing info # create handler1 for showing info
handler1 = StreamHandler() handler1 = StreamHandler()
my_formatter = MyFormatter() my_formatter = MyFormatter()
handler1.setFormatter(my_formatter) handler1.setFormatter(my_formatter)
handler1.setLevel(loglevel) handler1.setLevel(loglevel)
logger.addHandler(handler1) logger.addHandler(handler1)
#create handler2 for recording log file # create handler2 for recording log file
if loglevel <= logging.DEBUG: if loglevel <= logging.DEBUG:
handler2 = FileHandler(filename="log.txt", encoding='utf-8') handler2 = FileHandler(filename="log.txt", encoding='utf-8')
handler2.setLevel(logging.ERROR) handler2.setLevel(logging.ERROR)
handler2.setFormatter(my_formatter) handler2.setFormatter(my_formatter)
logger.addHandler(handler2) logger.addHandler(handler2)
return logger return logger
class MyFormatter(logging.Formatter): class MyFormatter(logging.Formatter):
def format(self, record): def format(self, record):
timestamp = ( timestamp = (
@@ -35,4 +35,4 @@ class MyFormatter(logging.Formatter):
lineno = str(record.lineno).rjust(4) lineno = str(record.lineno).rjust(4)
message = record.getMessage() message = record.getMessage()
return timestamp+'| '+module+' { '+funcname+':'+lineno+'} - '+message return timestamp + '| ' + module + ' { ' + funcname + ':' + lineno + '} - ' + message

9
pytchat/core_async/buffer.py
View File

@@ -1,5 +1,7 @@
import asyncio import asyncio
class Buffer(asyncio.Queue): class Buffer(asyncio.Queue):
''' '''
チャットデータを格納するバッファの役割を持つFIFOキュー チャットデータを格納するバッファの役割を持つFIFOキュー
@@ -10,17 +12,18 @@ class Buffer(asyncio.Queue):
格納するチャットブロックの最大個数。0の場合は無限。 格納するチャットブロックの最大個数。0の場合は無限。
最大値を超える場合は古いチャットブロックから破棄される。 最大値を超える場合は古いチャットブロックから破棄される。
''' '''
def __init__(self,maxsize = 0):
def __init__(self, maxsize=0):
super().__init__(maxsize) super().__init__(maxsize)
async def put(self,item): async def put(self, item):
if item is None: if item is None:
return return
if super().full(): if super().full():
super().get_nowait() super().get_nowait()
await super().put(item) await super().put(item)
def put_nowait(self,item): def put_nowait(self, item):
if item is None: if item is None:
return return
if super().full(): if super().full():

7
pytchat/core_async/livechat.py
View File

@@ -169,7 +169,7 @@ class LiveChatAsync:
continuation, session, headers) continuation, session, headers)
metadata, chatdata = self._parser.parse(contents) metadata, chatdata = self._parser.parse(contents)
timeout = metadata['timeoutMs']/1000 timeout = metadata['timeoutMs'] / 1000
chat_component = { chat_component = {
"video_id": self.video_id, "video_id": self.video_id,
"timeout": timeout, "timeout": timeout,
@@ -177,14 +177,15 @@ class LiveChatAsync:
} }
time_mark = time.time() time_mark = time.time()
if self._direct_mode: if self._direct_mode:
processed_chat = self.processor.process([chat_component]) processed_chat = self.processor.process(
[chat_component])
if isinstance(processed_chat, tuple): if isinstance(processed_chat, tuple):
await self._callback(*processed_chat) await self._callback(*processed_chat)
else: else:
await self._callback(processed_chat) await self._callback(processed_chat)
else: else:
await self._buffer.put(chat_component) await self._buffer.put(chat_component)
diff_time = timeout - (time.time()-time_mark) diff_time = timeout - (time.time() - time_mark)
await asyncio.sleep(diff_time) await asyncio.sleep(diff_time)
continuation = metadata.get('continuation') continuation = metadata.get('continuation')
except ChatParseException as e: except ChatParseException as e:

8
pytchat/core_multithread/buffer.py
View File

@@ -1,6 +1,7 @@
import queue import queue
class Buffer(queue.Queue): class Buffer(queue.Queue):
''' '''
チャットデータを格納するバッファの役割を持つFIFOキュー チャットデータを格納するバッファの役割を持つFIFOキュー
@@ -11,10 +12,11 @@ class Buffer(queue.Queue):
格納するチャットブロックの最大個数。0の場合は無限。 格納するチャットブロックの最大個数。0の場合は無限。
最大値を超える場合は古いチャットブロックから破棄される。 最大値を超える場合は古いチャットブロックから破棄される。
''' '''
def __init__(self,maxsize = 0):
def __init__(self, maxsize=0):
super().__init__(maxsize=maxsize) super().__init__(maxsize=maxsize)
def put(self,item): def put(self, item):
if item is None: if item is None:
return return
if super().full(): if super().full():
@@ -22,7 +24,7 @@ class Buffer(queue.Queue):
else: else:
super().put(item) super().put(item)
def put_nowait(self,item): def put_nowait(self, item):
if item is None: if item is None:
return return
if super().full(): if super().full():

4
pytchat/core_multithread/livechat.py
View File

@@ -156,7 +156,7 @@ class LiveChat:
continuation, session, headers) continuation, session, headers)
metadata, chatdata = self._parser.parse(contents) metadata, chatdata = self._parser.parse(contents)
timeout = metadata['timeoutMs']/1000 timeout = metadata['timeoutMs'] / 1000
chat_component = { chat_component = {
"video_id": self.video_id, "video_id": self.video_id,
"timeout": timeout, "timeout": timeout,
@@ -172,7 +172,7 @@ class LiveChat:
self._callback(processed_chat) self._callback(processed_chat)
else: else:
self._buffer.put(chat_component) self._buffer.put(chat_component)
diff_time = timeout - (time.time()-time_mark) diff_time = timeout - (time.time() - time_mark)
time.sleep(diff_time if diff_time > 0 else 0) time.sleep(diff_time if diff_time > 0 else 0)
continuation = metadata.get('continuation') continuation = metadata.get('continuation')
except ChatParseException as e: except ChatParseException as e:

7
pytchat/exceptions.py
View File

@@ -4,18 +4,21 @@ class ChatParseException(Exception):
''' '''
pass pass
class NoYtinitialdataException(ChatParseException): class NoYtinitialdataException(ChatParseException):
''' '''
Thrown when the video is not found. Thrown when the video is not found.
''' '''
pass pass
class ResponseContextError(ChatParseException): class ResponseContextError(ChatParseException):
''' '''
Thrown when chat data is invalid. Thrown when chat data is invalid.
''' '''
pass pass
class NoLivechatRendererException(ChatParseException): class NoLivechatRendererException(ChatParseException):
''' '''
Thrown when livechatRenderer is missing in JSON. Thrown when livechatRenderer is missing in JSON.
@@ -29,12 +32,14 @@ class NoContentsException(ChatParseException):
''' '''
pass pass
class NoContinuationsException(ChatParseException): class NoContinuationsException(ChatParseException):
''' '''
Thrown when continuation is missing in ContinuationContents. Thrown when continuation is missing in ContinuationContents.
''' '''
pass pass
class IllegalFunctionCall(Exception): class IllegalFunctionCall(Exception):
''' '''
Thrown when get () is called even though Thrown when get () is called even though
@@ -42,11 +47,13 @@ class IllegalFunctionCall(Exception):
''' '''
pass pass
class InvalidVideoIdException(Exception): class InvalidVideoIdException(Exception):
''' '''
Thrown when the video_id is not exist (VideoInfo). Thrown when the video_id is not exist (VideoInfo).
''' '''
pass pass
class UnknownConnectionError(Exception): class UnknownConnectionError(Exception):
pass pass

2
pytchat/paramgen/arcparam.py
View File

@@ -32,7 +32,7 @@ def _build(video_id, seektime, topchat_only) -> str:
elif seektime == 0: elif seektime == 0:
timestamp = 1 timestamp = 1
else: else:
timestamp = int(seektime*1000000) timestamp = int(seektime * 1000000)
continuation = Continuation() continuation = Continuation()
entity = continuation.entity entity = continuation.entity
entity.header = _gen_vid(video_id) entity.header = _gen_vid(video_id)

34
pytchat/paramgen/arcparam_mining.py
View File

@@ -36,9 +36,10 @@ def _gen_vid_long(video_id):
] ]
return urllib.parse.quote( return urllib.parse.quote(
b64enc(reduce(lambda x, y: x+y, item)).decode() b64enc(reduce(lambda x, y: x + y, item)).decode()
).encode() ).encode()
def _gen_vid(video_id): def _gen_vid(video_id):
"""generate video_id parameter. """generate video_id parameter.
Parameter Parameter
@@ -50,7 +51,7 @@ def _gen_vid(video_id):
bytes : base64 encoded video_id parameter. bytes : base64 encoded video_id parameter.
""" """
header_magic = b'\x0A\x0F\x1A\x0D\x0A' header_magic = b'\x0A\x0F\x1A\x0D\x0A'
header_id = video_id.encode() header_id = video_id.encode()
header_terminator = b'\x20\x01' header_terminator = b'\x20\x01'
item = [ item = [
@@ -61,9 +62,10 @@ def _gen_vid(video_id):
] ]
return urllib.parse.quote( return urllib.parse.quote(
b64enc(reduce(lambda x, y: x+y, item)).decode() b64enc(reduce(lambda x, y: x + y, item)).decode()
).encode() ).encode()
def _nval(val): def _nval(val):
"""convert value to byte array""" """convert value to byte array"""
if val < 0: if val < 0:
@@ -84,19 +86,19 @@ def _build(video_id, seektime, topchat_only):
if seektime == 0: if seektime == 0:
times = b'' times = b''
else: else:
times = _nval(int(seektime*1000)) times = _nval(int(seektime * 1000))
if seektime > 0: if seektime > 0:
_len_time = b'\x5A' + (len(times)+1).to_bytes(1, 'big') + b'\x10' _len_time = b'\x5A' + (len(times) + 1).to_bytes(1, 'big') + b'\x10'
else: else:
_len_time = b'' _len_time = b''
header_magic = b'\xA2\x9D\xB0\xD3\x04' header_magic = b'\xA2\x9D\xB0\xD3\x04'
sep_0 = b'\x1A' sep_0 = b'\x1A'
vid = _gen_vid(video_id) vid = _gen_vid(video_id)
_tag = b'\x40\x01' _tag = b'\x40\x01'
timestamp1 = times timestamp1 = times
sep_1 = b'\x60\x04\x72\x02\x08' sep_1 = b'\x60\x04\x72\x02\x08'
terminator = b'\x78\x01' terminator = b'\x78\x01'
body = [ body = [
sep_0, sep_0,
@@ -110,14 +112,12 @@ def _build(video_id, seektime, topchat_only):
terminator terminator
] ]
body = reduce(lambda x, y: x+y, body) body = reduce(lambda x, y: x + y, body)
return urllib.parse.quote( return urllib.parse.quote(
b64enc(header_magic + b64enc(header_magic + _nval(len(body)) + body
_nval(len(body)) + ).decode()
body )
).decode()
)
def getparam(video_id, seektime=0.0, topchat_only=False): def getparam(video_id, seektime=0.0, topchat_only=False):

6
pytchat/paramgen/liveparam.py
View File

@@ -68,12 +68,12 @@ def _build(video_id, ts1, ts2, ts3, ts4, ts5, topchat_only) -> str:
def _times(past_sec): def _times(past_sec):
n = int(time.time()) n = int(time.time())
_ts1 = n - random.uniform(0, 1*3) _ts1 = n - random.uniform(0, 1 * 3)
_ts2 = n - random.uniform(0.01, 0.99) _ts2 = n - random.uniform(0.01, 0.99)
_ts3 = n - past_sec + random.uniform(0, 1) _ts3 = n - past_sec + random.uniform(0, 1)
_ts4 = n - random.uniform(10*60, 60*60) _ts4 = n - random.uniform(10 * 60, 60 * 60)
_ts5 = n - random.uniform(0.01, 0.99) _ts5 = n - random.uniform(0.01, 0.99)
return list(map(lambda x: int(x*1000000), [_ts1, _ts2, _ts3, _ts4, _ts5])) return list(map(lambda x: int(x * 1000000), [_ts1, _ts2, _ts3, _ts4, _ts5]))
def getparam(video_id, past_sec=0, topchat_only=False) -> str: def getparam(video_id, past_sec=0, topchat_only=False) -> str:

14
pytchat/parser/live.py
View File

@@ -22,7 +22,8 @@ class Parser:
if jsn is None: if jsn is None:
raise ChatParseException('Called with none JSON object.') raise ChatParseException('Called with none JSON object.')
if jsn['response']['responseContext'].get('errors'): if jsn['response']['responseContext'].get('errors'):
raise ResponseContextError('The video_id would be wrong, or video is deleted or private.') raise ResponseContextError(
'The video_id would be wrong, or video is deleted or private.')
contents = jsn['response'].get('continuationContents') contents = jsn['response'].get('continuationContents')
return contents return contents
@@ -50,17 +51,18 @@ class Parser:
cont = contents['liveChatContinuation']['continuations'][0] cont = contents['liveChatContinuation']['continuations'][0]
if cont is None: if cont is None:
raise NoContinuationsException('No Continuation') raise NoContinuationsException('No Continuation')
metadata = (cont.get('invalidationContinuationData') or metadata = (cont.get('invalidationContinuationData')
cont.get('timedContinuationData') or or cont.get('timedContinuationData')
cont.get('reloadContinuationData') or or cont.get('reloadContinuationData')
cont.get('liveChatReplayContinuationData') or cont.get('liveChatReplayContinuationData')
) )
if metadata is None: if metadata is None:
if cont.get("playerSeekContinuationData"): if cont.get("playerSeekContinuationData"):
raise ChatParseException('Finished chat data') raise ChatParseException('Finished chat data')
unknown = list(cont.keys())[0] unknown = list(cont.keys())[0]
if unknown: if unknown:
raise ChatParseException(f"Received unknown continuation type:{unknown}") raise ChatParseException(
f"Received unknown continuation type:{unknown}")
else: else:
raise ChatParseException('Cannot extract continuation data') raise ChatParseException('Cannot extract continuation data')
return self._create_data(metadata, contents) return self._create_data(metadata, contents)

6
pytchat/processors/chat_processor.py
View File

@@ -3,6 +3,7 @@ class ChatProcessor:
Abstract class that processes chat data. Abstract class that processes chat data.
Receive chat data (actions) from Listener. Receive chat data (actions) from Listener.
''' '''
def process(self, chat_components: list): def process(self, chat_components: list):
''' '''
Interface that represents processing of chat data. Interface that represents processing of chat data.
@@ -20,8 +21,3 @@ class ChatProcessor:
} }
''' '''
pass pass

5
pytchat/processors/combinator.py
View File

@@ -1,5 +1,6 @@
from .chat_processor import ChatProcessor from .chat_processor import ChatProcessor
class Combinator(ChatProcessor): class Combinator(ChatProcessor):
''' '''
Combinator combines multiple chat processors. Combinator combines multiple chat processors.
@@ -34,6 +35,4 @@ class Combinator(ChatProcessor):
Tuple of chat data processed by each chat processor. Tuple of chat data processed by each chat processor.
''' '''
return tuple(processor.process(chat_components) return tuple(processor.process(chat_components)
for processor in self.processors) for processor in self.processors)

6
pytchat/processors/compatible/processor.py
View File

@@ -1,5 +1,3 @@
import datetime
import time
from .renderer.textmessage import LiveChatTextMessageRenderer from .renderer.textmessage import LiveChatTextMessageRenderer
from .renderer.paidmessage import LiveChatPaidMessageRenderer from .renderer.paidmessage import LiveChatPaidMessageRenderer
from .renderer.paidsticker import LiveChatPaidStickerRenderer from .renderer.paidsticker import LiveChatPaidStickerRenderer
@@ -39,7 +37,7 @@ class CompatibleProcessor(ChatProcessor):
chat = self.parse(action) chat = self.parse(action)
if chat: if chat:
chatlist.append(chat) chatlist.append(chat)
ret["pollingIntervalMillis"] = int(timeout*1000) ret["pollingIntervalMillis"] = int(timeout * 1000)
ret["pageInfo"] = { ret["pageInfo"] = {
"totalResults": len(chatlist), "totalResults": len(chatlist),
"resultsPerPage": len(chatlist), "resultsPerPage": len(chatlist),
@@ -58,7 +56,7 @@ class CompatibleProcessor(ChatProcessor):
rd = {} rd = {}
try: try:
renderer = self.get_renderer(item) renderer = self.get_renderer(item)
if renderer == None: if renderer is None:
return None return None
rd["kind"] = "youtube#liveChatMessage" rd["kind"] = "youtube#liveChatMessage"

62
pytchat/processors/compatible/renderer/base.py
View File

@@ -1,68 +1,67 @@
import datetime, pytz import datetime
import pytz
class BaseRenderer: class BaseRenderer:
def __init__(self, item, chattype): def __init__(self, item, chattype):
self.renderer = list(item.values())[0] self.renderer = list(item.values())[0]
self.chattype = chattype self.chattype = chattype
def get_snippet(self): def get_snippet(self):
message = self.get_message(self.renderer) message = self.get_message(self.renderer)
return { return {
"type" : self.chattype, "type": self.chattype,
"liveChatId" : "", "liveChatId": "",
"authorChannelId" : self.renderer.get("authorExternalChannelId"), "authorChannelId": self.renderer.get("authorExternalChannelId"),
"publishedAt" : self.get_publishedat(self.renderer.get("timestampUsec",0)), "publishedAt": self.get_publishedat(self.renderer.get("timestampUsec", 0)),
"hasDisplayContent" : True, "hasDisplayContent": True,
"displayMessage" : message, "displayMessage": message,
"textMessageDetails": { "textMessageDetails": {
"messageText" : message "messageText": message
} }
} }
def get_authordetails(self): def get_authordetails(self):
authorExternalChannelId = self.renderer.get("authorExternalChannelId") authorExternalChannelId = self.renderer.get("authorExternalChannelId")
#parse subscriber type # parse subscriber type
isVerified, isChatOwner, isChatSponsor, isChatModerator = ( isVerified, isChatOwner, isChatSponsor, isChatModerator = (
self.get_badges(self.renderer) self.get_badges(self.renderer)
) )
return { return {
"channelId" : authorExternalChannelId, "channelId": authorExternalChannelId,
"channelUrl" : "http://www.youtube.com/channel/"+authorExternalChannelId, "channelUrl": "http://www.youtube.com/channel/" + authorExternalChannelId,
"displayName" : self.renderer["authorName"]["simpleText"], "displayName": self.renderer["authorName"]["simpleText"],
"profileImageUrl" : self.renderer["authorPhoto"]["thumbnails"][1]["url"] , "profileImageUrl": self.renderer["authorPhoto"]["thumbnails"][1]["url"],
"isVerified" : isVerified, "isVerified": isVerified,
"isChatOwner" : isChatOwner, "isChatOwner": isChatOwner,
"isChatSponsor" : isChatSponsor, "isChatSponsor": isChatSponsor,
"isChatModerator" : isChatModerator "isChatModerator": isChatModerator
} }
def get_message(self, renderer):
def get_message(self,renderer):
message = '' message = ''
if renderer.get("message"): if renderer.get("message"):
runs=renderer["message"].get("runs") runs = renderer["message"].get("runs")
if runs: if runs:
for r in runs: for r in runs:
if r: if r:
if r.get('emoji'): if r.get('emoji'):
message += r['emoji'].get('shortcuts',[''])[0] message += r['emoji'].get('shortcuts', [''])[0]
else: else:
message += r.get('text','') message += r.get('text', '')
return message return message
def get_badges(self,renderer): def get_badges(self, renderer):
isVerified = False isVerified = False
isChatOwner = False isChatOwner = False
isChatSponsor = False isChatSponsor = False
isChatModerator = False isChatModerator = False
badges=renderer.get("authorBadges") badges = renderer.get("authorBadges")
if badges: if badges:
for badge in badges: for badge in badges:
author_type = badge["liveChatAuthorBadgeRenderer"]["accessibility"]["accessibilityData"]["label"] author_type = badge["liveChatAuthorBadgeRenderer"]["accessibility"]["accessibilityData"]["label"]
if author_type == '確認済み': if author_type == '確認済み':
isVerified = True isVerified = True
if author_type == '所有者': if author_type == '所有者':
@@ -76,8 +75,7 @@ class BaseRenderer:
def get_id(self): def get_id(self):
return self.renderer.get('id') return self.renderer.get('id')
def get_publishedat(self,timestamp): def get_publishedat(self, timestamp):
dt = datetime.datetime.fromtimestamp(int(timestamp)/1000000) dt = datetime.datetime.fromtimestamp(int(timestamp) / 1000000)
return dt.astimezone(pytz.utc).isoformat( return dt.astimezone(pytz.utc).isoformat(
timespec='milliseconds').replace('+00:00','Z') timespec='milliseconds').replace('+00:00', 'Z')

41
pytchat/processors/compatible/renderer/legacypaid.py
View File

@@ -1,4 +1,6 @@
from .base import BaseRenderer from .base import BaseRenderer
class LiveChatLegacyPaidMessageRenderer(BaseRenderer): class LiveChatLegacyPaidMessageRenderer(BaseRenderer):
def __init__(self, item): def __init__(self, item):
super().__init__(item, "newSponsorEvent") super().__init__(item, "newSponsorEvent")
@@ -8,36 +10,33 @@ class LiveChatLegacyPaidMessageRenderer(BaseRenderer):
message = self.get_message(self.renderer) message = self.get_message(self.renderer)
return { return {
"type" : self.chattype, "type": self.chattype,
"liveChatId" : "", "liveChatId": "",
"authorChannelId" : self.renderer.get("authorExternalChannelId"), "authorChannelId": self.renderer.get("authorExternalChannelId"),
"publishedAt" : self.get_publishedat(self.renderer.get("timestampUsec",0)), "publishedAt": self.get_publishedat(self.renderer.get("timestampUsec", 0)),
"hasDisplayContent" : True, "hasDisplayContent": True,
"displayMessage" : message, "displayMessage": message,
} }
def get_authordetails(self): def get_authordetails(self):
authorExternalChannelId = self.renderer.get("authorExternalChannelId") authorExternalChannelId = self.renderer.get("authorExternalChannelId")
#parse subscriber type # parse subscriber type
isVerified, isChatOwner, _, isChatModerator = ( isVerified, isChatOwner, _, isChatModerator = (
self.get_badges(self.renderer) self.get_badges(self.renderer)
) )
return { return {
"channelId" : authorExternalChannelId, "channelId": authorExternalChannelId,
"channelUrl" : "http://www.youtube.com/channel/"+authorExternalChannelId, "channelUrl": "http://www.youtube.com/channel/" + authorExternalChannelId,
"displayName" : self.renderer["authorName"]["simpleText"], "displayName": self.renderer["authorName"]["simpleText"],
"profileImageUrl" : self.renderer["authorPhoto"]["thumbnails"][1]["url"] , "profileImageUrl": self.renderer["authorPhoto"]["thumbnails"][1]["url"],
"isVerified" : isVerified, "isVerified": isVerified,
"isChatOwner" : isChatOwner, "isChatOwner": isChatOwner,
"isChatSponsor" : True, "isChatSponsor": True,
"isChatModerator" : isChatModerator "isChatModerator": isChatModerator
} }
def get_message(self, renderer):
def get_message(self,renderer):
message = (renderer["eventText"]["runs"][0]["text"] message = (renderer["eventText"]["runs"][0]["text"]
)+' / '+(renderer["detailText"]["simpleText"]) ) + ' / ' + (renderer["detailText"]["simpleText"])
return message return message

6
pytchat/processors/compatible/renderer/membership.py
View File

@@ -25,7 +25,7 @@ class LiveChatMembershipItemRenderer(BaseRenderer):
) )
return { return {
"channelId": authorExternalChannelId, "channelId": authorExternalChannelId,
"channelUrl": "http://www.youtube.com/channel/"+authorExternalChannelId, "channelUrl": "http://www.youtube.com/channel/" + authorExternalChannelId,
"displayName": self.renderer["authorName"]["simpleText"], "displayName": self.renderer["authorName"]["simpleText"],
"profileImageUrl": self.renderer["authorPhoto"]["thumbnails"][1]["url"], "profileImageUrl": self.renderer["authorPhoto"]["thumbnails"][1]["url"],
"isVerified": isVerified, "isVerified": isVerified,
@@ -35,6 +35,6 @@ class LiveChatMembershipItemRenderer(BaseRenderer):
} }
def get_message(self, renderer): def get_message(self, renderer):
message = ''.join([mes.get("text", "") for mes in renderer["headerSubtext"]["runs"]]) message = ''.join([mes.get("text", "")
for mes in renderer["headerSubtext"]["runs"]])
return message, [message] return message, [message]

31
pytchat/processors/compatible/renderer/paidmessage.py
View File

@@ -3,6 +3,7 @@ from . import currency
from .base import BaseRenderer from .base import BaseRenderer
superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$") superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
class LiveChatPaidMessageRenderer(BaseRenderer): class LiveChatPaidMessageRenderer(BaseRenderer):
def __init__(self, item): def __init__(self, item):
super().__init__(item, "superChatEvent") super().__init__(item, "superChatEvent")
@@ -10,31 +11,31 @@ class LiveChatPaidMessageRenderer(BaseRenderer):
def get_snippet(self): def get_snippet(self):
authorName = self.renderer["authorName"]["simpleText"] authorName = self.renderer["authorName"]["simpleText"]
message = self.get_message(self.renderer) message = self.get_message(self.renderer)
amountDisplayString, symbol, amountMicros =( amountDisplayString, symbol, amountMicros = (
self.get_amountdata(self.renderer) self.get_amountdata(self.renderer)
) )
return { return {
"type" : self.chattype, "type": self.chattype,
"liveChatId" : "", "liveChatId": "",
"authorChannelId" : self.renderer.get("authorExternalChannelId"), "authorChannelId": self.renderer.get("authorExternalChannelId"),
"publishedAt" : self.get_publishedat(self.renderer.get("timestampUsec",0)), "publishedAt": self.get_publishedat(self.renderer.get("timestampUsec", 0)),
"hasDisplayContent" : True, "hasDisplayContent": True,
"displayMessage" : amountDisplayString+" from "+authorName+': \"'+ message+'\"', "displayMessage": amountDisplayString + " from " + authorName + ': \"' + message + '\"',
"superChatDetails" : { "superChatDetails": {
"amountMicros" : amountMicros, "amountMicros": amountMicros,
"currency" : currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol, "currency": currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol,
"amountDisplayString" : amountDisplayString, "amountDisplayString": amountDisplayString,
"tier" : 0, "tier": 0,
"backgroundColor" : self.renderer.get("bodyBackgroundColor", 0) "backgroundColor": self.renderer.get("bodyBackgroundColor", 0)
} }
} }
def get_amountdata(self,renderer): def get_amountdata(self, renderer):
amountDisplayString = renderer["purchaseAmountText"]["simpleText"] amountDisplayString = renderer["purchaseAmountText"]["simpleText"]
m = superchat_regex.search(amountDisplayString) m = superchat_regex.search(amountDisplayString)
if m: if m:
symbol = m.group(1) symbol = m.group(1)
amountMicros = int(float(m.group(2).replace(',',''))*1000000) amountMicros = int(float(m.group(2).replace(',', '')) * 1000000)
else: else:
symbol = "" symbol = ""
amountMicros = 0 amountMicros = 0

35
pytchat/processors/compatible/renderer/paidsticker.py
View File

@@ -3,46 +3,45 @@ from . import currency
from .base import BaseRenderer from .base import BaseRenderer
superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$") superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
class LiveChatPaidStickerRenderer(BaseRenderer): class LiveChatPaidStickerRenderer(BaseRenderer):
def __init__(self, item): def __init__(self, item):
super().__init__(item, "superStickerEvent") super().__init__(item, "superStickerEvent")
def get_snippet(self): def get_snippet(self):
authorName = self.renderer["authorName"]["simpleText"] authorName = self.renderer["authorName"]["simpleText"]
amountDisplayString, symbol, amountMicros =( amountDisplayString, symbol, amountMicros = (
self.get_amountdata(self.renderer) self.get_amountdata(self.renderer)
) )
return { return {
"type" : self.chattype, "type": self.chattype,
"liveChatId" : "", "liveChatId": "",
"authorChannelId" : self.renderer.get("authorExternalChannelId"), "authorChannelId": self.renderer.get("authorExternalChannelId"),
"publishedAt" : self.get_publishedat(self.renderer.get("timestampUsec",0)), "publishedAt": self.get_publishedat(self.renderer.get("timestampUsec", 0)),
"hasDisplayContent" : True, "hasDisplayContent": True,
"displayMessage" : "Super Sticker " + amountDisplayString + " from "+authorName, "displayMessage": "Super Sticker " + amountDisplayString + " from " + authorName,
"superStickerDetails" : { "superStickerDetails": {
"superStickerMetaData" : { "superStickerMetaData": {
"stickerId": "", "stickerId": "",
"altText": "", "altText": "",
"language": "" "language": ""
}, },
"amountMicros" : amountMicros, "amountMicros": amountMicros,
"currency" : currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol, "currency": currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol,
"amountDisplayString" : amountDisplayString, "amountDisplayString": amountDisplayString,
"tier" : 0, "tier": 0,
"backgroundColor" : self.renderer.get("bodyBackgroundColor", 0) "backgroundColor": self.renderer.get("bodyBackgroundColor", 0)
} }
} }
def get_amountdata(self,renderer): def get_amountdata(self, renderer):
amountDisplayString = renderer["purchaseAmountText"]["simpleText"] amountDisplayString = renderer["purchaseAmountText"]["simpleText"]
m = superchat_regex.search(amountDisplayString) m = superchat_regex.search(amountDisplayString)
if m: if m:
symbol = m.group(1) symbol = m.group(1)
amountMicros = int(float(m.group(2).replace(',',''))*1000000) amountMicros = int(float(m.group(2).replace(',', '')) * 1000000)
else: else:
symbol = "" symbol = ""
amountMicros = 0 amountMicros = 0
return amountDisplayString, symbol, amountMicros return amountDisplayString, symbol, amountMicros

2
pytchat/processors/compatible/renderer/textmessage.py
View File

@@ -1,4 +1,6 @@
from .base import BaseRenderer from .base import BaseRenderer
class LiveChatTextMessageRenderer(BaseRenderer): class LiveChatTextMessageRenderer(BaseRenderer):
def __init__(self, item): def __init__(self, item):
super().__init__(item, "textMessageEvent") super().__init__(item, "textMessageEvent")

6
pytchat/processors/default/processor.py
View File

@@ -20,13 +20,13 @@ class Chatdata:
if self.interval == 0: if self.interval == 0:
time.sleep(1) time.sleep(1)
return return
time.sleep(self.interval/len(self.items)) time.sleep(self.interval / len(self.items))
async def tick_async(self): async def tick_async(self):
if self.interval == 0: if self.interval == 0:
await asyncio.sleep(1) await asyncio.sleep(1)
return return
await asyncio.sleep(self.interval/len(self.items)) await asyncio.sleep(self.interval / len(self.items))
class DefaultProcessor(ChatProcessor): class DefaultProcessor(ChatProcessor):
@@ -62,7 +62,7 @@ class DefaultProcessor(ChatProcessor):
return None return None
try: try:
renderer = self._get_renderer(item) renderer = self._get_renderer(item)
if renderer == None: if renderer is None:
return None return None
renderer.get_snippet() renderer.get_snippet()

60
pytchat/processors/default/renderer/base.py
View File

@@ -1,6 +1,10 @@
from datetime import datetime from datetime import datetime
class Author: class Author:
pass pass
class BaseRenderer: class BaseRenderer:
def __init__(self, item, chattype): def __init__(self, item, chattype):
self.renderer = list(item.values())[0] self.renderer = list(item.values())[0]
@@ -10,65 +14,62 @@ class BaseRenderer:
def get_snippet(self): def get_snippet(self):
self.type = self.chattype self.type = self.chattype
self.id = self.renderer.get('id') self.id = self.renderer.get('id')
timestampUsec = int(self.renderer.get("timestampUsec",0)) timestampUsec = int(self.renderer.get("timestampUsec", 0))
self.timestamp = int(timestampUsec/1000) self.timestamp = int(timestampUsec / 1000)
tst = self.renderer.get("timestampText") tst = self.renderer.get("timestampText")
if tst: if tst:
self.elapsedTime = tst.get("simpleText") self.elapsedTime = tst.get("simpleText")
else: else:
self.elapsedTime = "" self.elapsedTime = ""
self.datetime = self.get_datetime(timestampUsec) self.datetime = self.get_datetime(timestampUsec)
self.message ,self.messageEx = self.get_message(self.renderer) self.message, self.messageEx = self.get_message(self.renderer)
self.id = self.renderer.get('id') self.id = self.renderer.get('id')
self.amountValue= 0.0 self.amountValue = 0.0
self.amountString = "" self.amountString = ""
self.currency= "" self.currency = ""
self.bgColor = 0 self.bgColor = 0
def get_authordetails(self): def get_authordetails(self):
self.author.badgeUrl = "" self.author.badgeUrl = ""
(self.author.isVerified, (self.author.isVerified,
self.author.isChatOwner, self.author.isChatOwner,
self.author.isChatSponsor, self.author.isChatSponsor,
self.author.isChatModerator) = ( self.author.isChatModerator) = (
self.get_badges(self.renderer) self.get_badges(self.renderer)
) )
self.author.channelId = self.renderer.get("authorExternalChannelId") self.author.channelId = self.renderer.get("authorExternalChannelId")
self.author.channelUrl = "http://www.youtube.com/channel/"+self.author.channelId self.author.channelUrl = "http://www.youtube.com/channel/" + self.author.channelId
self.author.name = self.renderer["authorName"]["simpleText"] self.author.name = self.renderer["authorName"]["simpleText"]
self.author.imageUrl= self.renderer["authorPhoto"]["thumbnails"][1]["url"] self.author.imageUrl = self.renderer["authorPhoto"]["thumbnails"][1]["url"]
def get_message(self, renderer):
def get_message(self,renderer):
message = '' message = ''
message_ex = [] message_ex = []
if renderer.get("message"): if renderer.get("message"):
runs=renderer["message"].get("runs") runs = renderer["message"].get("runs")
if runs: if runs:
for r in runs: for r in runs:
if r: if r:
if r.get('emoji'): if r.get('emoji'):
message += r['emoji'].get('shortcuts',[''])[0] message += r['emoji'].get('shortcuts', [''])[0]
message_ex.append(r['emoji']['image']['thumbnails'][1].get('url')) message_ex.append(
r['emoji']['image']['thumbnails'][1].get('url'))
else: else:
message += r.get('text','') message += r.get('text', '')
message_ex.append(r.get('text','')) message_ex.append(r.get('text', ''))
return message, message_ex return message, message_ex
def get_badges(self, renderer):
def get_badges(self,renderer):
self.author.type = '' self.author.type = ''
isVerified = False isVerified = False
isChatOwner = False isChatOwner = False
isChatSponsor = False isChatSponsor = False
isChatModerator = False isChatModerator = False
badges=renderer.get("authorBadges") badges = renderer.get("authorBadges")
if badges: if badges:
for badge in badges: for badge in badges:
if badge["liveChatAuthorBadgeRenderer"].get("icon"): if badge["liveChatAuthorBadgeRenderer"].get("icon"):
author_type = badge["liveChatAuthorBadgeRenderer"]["icon"]["iconType"] author_type = badge["liveChatAuthorBadgeRenderer"]["icon"]["iconType"]
self.author.type = author_type self.author.type = author_type
if author_type == 'VERIFIED': if author_type == 'VERIFIED':
isVerified = True isVerified = True
@@ -82,12 +83,9 @@ class BaseRenderer:
self.get_badgeurl(badge) self.get_badgeurl(badge)
return isVerified, isChatOwner, isChatSponsor, isChatModerator return isVerified, isChatOwner, isChatSponsor, isChatModerator
def get_badgeurl(self, badge):
def get_badgeurl(self,badge):
self.author.badgeUrl = badge["liveChatAuthorBadgeRenderer"]["customThumbnail"]["thumbnails"][0]["url"] self.author.badgeUrl = badge["liveChatAuthorBadgeRenderer"]["customThumbnail"]["thumbnails"][0]["url"]
def get_datetime(self, timestamp):
dt = datetime.fromtimestamp(timestamp / 1000000)
def get_datetime(self,timestamp):
dt = datetime.fromtimestamp(timestamp/1000000)
return dt.strftime('%Y-%m-%d %H:%M:%S') return dt.strftime('%Y-%m-%d %H:%M:%S')

13
pytchat/processors/default/renderer/legacypaid.py
View File

@@ -1,18 +1,15 @@
from .base import BaseRenderer from .base import BaseRenderer
class LiveChatLegacyPaidMessageRenderer(BaseRenderer): class LiveChatLegacyPaidMessageRenderer(BaseRenderer):
def __init__(self, item): def __init__(self, item):
super().__init__(item, "newSponsor") super().__init__(item, "newSponsor")
def get_authordetails(self): def get_authordetails(self):
super().get_authordetails() super().get_authordetails()
self.author.isChatSponsor = True self.author.isChatSponsor = True
def get_message(self, renderer):
def get_message(self,renderer):
message = (renderer["eventText"]["runs"][0]["text"] message = (renderer["eventText"]["runs"][0]["text"]
)+' / '+(renderer["detailText"]["simpleText"]) ) + ' / ' + (renderer["detailText"]["simpleText"])
return message return message

4
pytchat/processors/default/renderer/membership.py
View File

@@ -10,6 +10,6 @@ class LiveChatMembershipItemRenderer(BaseRenderer):
self.author.isChatSponsor = True self.author.isChatSponsor = True
def get_message(self, renderer): def get_message(self, renderer):
message = ''.join([mes.get("text", "") for mes in renderer["headerSubtext"]["runs"]]) message = ''.join([mes.get("text", "")
for mes in renderer["headerSubtext"]["runs"]])
return message, [message] return message, [message]

17
pytchat/processors/default/renderer/paidmessage.py
View File

@@ -3,29 +3,28 @@ from . import currency
from .base import BaseRenderer from .base import BaseRenderer
superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$") superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
class LiveChatPaidMessageRenderer(BaseRenderer): class LiveChatPaidMessageRenderer(BaseRenderer):
def __init__(self, item): def __init__(self, item):
super().__init__(item, "superChat") super().__init__(item, "superChat")
def get_snippet(self): def get_snippet(self):
super().get_snippet() super().get_snippet()
amountDisplayString, symbol, amount =( amountDisplayString, symbol, amount = (
self.get_amountdata(self.renderer) self.get_amountdata(self.renderer)
) )
self.amountValue= amount self.amountValue = amount
self.amountString = amountDisplayString self.amountString = amountDisplayString
self.currency= currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol self.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get(
self.bgColor= self.renderer.get("bodyBackgroundColor", 0) symbol) else symbol
self.bgColor = self.renderer.get("bodyBackgroundColor", 0)
def get_amountdata(self, renderer):
def get_amountdata(self,renderer):
amountDisplayString = renderer["purchaseAmountText"]["simpleText"] amountDisplayString = renderer["purchaseAmountText"]["simpleText"]
m = superchat_regex.search(amountDisplayString) m = superchat_regex.search(amountDisplayString)
if m: if m:
symbol = m.group(1) symbol = m.group(1)
amount = float(m.group(2).replace(',','')) amount = float(m.group(2).replace(',', ''))
else: else:
symbol = "" symbol = ""
amount = 0.0 amount = 0.0

22
pytchat/processors/default/renderer/paidsticker.py
View File

@@ -3,37 +3,31 @@ from . import currency
from .base import BaseRenderer from .base import BaseRenderer
superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$") superchat_regex = re.compile(r"^(\D*)(\d{1,3}(,\d{3})*(\.\d*)*\b)$")
class LiveChatPaidStickerRenderer(BaseRenderer): class LiveChatPaidStickerRenderer(BaseRenderer):
def __init__(self, item): def __init__(self, item):
super().__init__(item, "superSticker") super().__init__(item, "superSticker")
def get_snippet(self): def get_snippet(self):
super().get_snippet() super().get_snippet()
amountDisplayString, symbol, amount =( amountDisplayString, symbol, amount = (
self.get_amountdata(self.renderer) self.get_amountdata(self.renderer)
) )
self.amountValue = amount self.amountValue = amount
self.amountString = amountDisplayString self.amountString = amountDisplayString
self.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get(symbol) else symbol self.currency = currency.symbols[symbol]["fxtext"] if currency.symbols.get(
symbol) else symbol
self.bgColor = self.renderer.get("moneyChipBackgroundColor", 0) self.bgColor = self.renderer.get("moneyChipBackgroundColor", 0)
self.sticker = "https:"+self.renderer["sticker"]["thumbnails"][0]["url"] self.sticker = "https:" + \
self.renderer["sticker"]["thumbnails"][0]["url"]
def get_amountdata(self, renderer):
def get_amountdata(self,renderer):
amountDisplayString = renderer["purchaseAmountText"]["simpleText"] amountDisplayString = renderer["purchaseAmountText"]["simpleText"]
m = superchat_regex.search(amountDisplayString) m = superchat_regex.search(amountDisplayString)
if m: if m:
symbol = m.group(1) symbol = m.group(1)
amount = float(m.group(2).replace(',','')) amount = float(m.group(2).replace(',', ''))
else: else:
symbol = "" symbol = ""
amount = 0.0 amount = 0.0
return amountDisplayString, symbol, amount return amountDisplayString, symbol, amount

2
pytchat/processors/default/renderer/textmessage.py
View File

@@ -1,4 +1,6 @@
from .base import BaseRenderer from .base import BaseRenderer
class LiveChatTextMessageRenderer(BaseRenderer): class LiveChatTextMessageRenderer(BaseRenderer):
def __init__(self, item): def __init__(self, item):
super().__init__(item, "textMessage") super().__init__(item, "textMessage")

2
pytchat/processors/dummy_processor.py
View File

@@ -1,8 +1,10 @@
from .chat_processor import ChatProcessor from .chat_processor import ChatProcessor
class DummyProcessor(ChatProcessor): class DummyProcessor(ChatProcessor):
''' '''
Dummy processor just returns received chat_components directly. Dummy processor just returns received chat_components directly.
''' '''
def process(self, chat_components: list): def process(self, chat_components: list):
return chat_components return chat_components

41
pytchat/processors/html_archiver.py
View File

@@ -1,18 +1,18 @@
import csv
import os import os
import re import re
from .chat_processor import ChatProcessor from .chat_processor import ChatProcessor
from .default.processor import DefaultProcessor from .default.processor import DefaultProcessor
PATTERN = re.compile(r"(.*)\(([0-9]+)\)$") PATTERN = re.compile(r"(.*)\(([0-9]+)\)$")
fmt_headers = ['datetime','elapsed','authorName','message','superchat' fmt_headers = ['datetime', 'elapsed', 'authorName',
,'type','authorChannel'] 'message', 'superchat', 'type', 'authorChannel']
HEADER_HTML = ''' HEADER_HTML = '''
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
<meta http-equiv="Content-Type" content="text/html;charset=UTF-8"> <meta http-equiv="Content-Type" content="text/html;charset=UTF-8">
''' '''
class HTMLArchiver(ChatProcessor): class HTMLArchiver(ChatProcessor):
''' '''
HtmlArchiver saves chat data as HTML table format. HtmlArchiver saves chat data as HTML table format.
@@ -21,7 +21,7 @@ class HTMLArchiver(ChatProcessor):
def __init__(self, save_path): def __init__(self, save_path):
super().__init__() super().__init__()
self.save_path = self._checkpath(save_path) self.save_path = self._checkpath(save_path)
with open(self.save_path, mode='a', encoding = 'utf-8') as f: with open(self.save_path, mode='a', encoding='utf-8') as f:
f.write(HEADER_HTML) f.write(HEADER_HTML)
f.write('<table border="1" style="border-collapse: collapse">') f.write('<table border="1" style="border-collapse: collapse">')
f.writelines(self._parse_html_header(fmt_headers)) f.writelines(self._parse_html_header(fmt_headers))
@@ -34,14 +34,14 @@ class HTMLArchiver(ChatProcessor):
newpath = filepath newpath = filepath
counter = 0 counter = 0
while os.path.exists(newpath): while os.path.exists(newpath):
match = re.search(PATTERN,body) match = re.search(PATTERN, body)
if match: if match:
counter=int(match[2])+1 counter = int(match[2]) + 1
num_with_bracket = f'({str(counter)})' num_with_bracket = f'({str(counter)})'
body = f'{match[1]}{num_with_bracket}' body = f'{match[1]}{num_with_bracket}'
else: else:
body = f'{body}({str(counter)})' body = f'{body}({str(counter)})'
newpath = os.path.join(os.path.dirname(filepath),body+extention) newpath = os.path.join(os.path.dirname(filepath), body + extention)
return newpath return newpath
def process(self, chat_components: list): def process(self, chat_components: list):
@@ -54,10 +54,10 @@ class HTMLArchiver(ChatProcessor):
total_lines : int : total_lines : int :
count of total lines written to the file. count of total lines written to the file.
""" """
if chat_components is None or len (chat_components) == 0: if chat_components is None or len(chat_components) == 0:
return return
with open(self.save_path, mode='a', encoding = 'utf-8') as f: with open(self.save_path, mode='a', encoding='utf-8') as f:
chats = self.processor.process(chat_components).items chats = self.processor.process(chat_components).items
for c in chats: for c in chats:
f.writelines( f.writelines(
@@ -76,23 +76,22 @@ class HTMLArchiver(ChatProcessor):
Comment out below line to prevent the table Comment out below line to prevent the table
display from collapsing. display from collapsing.
''' '''
#f.write('</table>') # f.write('</table>')
def _parse_html_line(self, raw_line): def _parse_html_line(self, raw_line):
html = '' html = ''
html+=' <tr>' html += ' <tr>'
for cell in raw_line: for cell in raw_line:
html+='<td>'+cell+'</td>' html += '<td>' + cell + '</td>'
html+='</tr>\n' html += '</tr>\n'
return html return html
def _parse_html_header(self,raw_line): def _parse_html_header(self, raw_line):
html = '' html = ''
html+='<thead>\n' html += '<thead>\n'
html+=' <tr>' html += ' <tr>'
for cell in raw_line: for cell in raw_line:
html+='<th>'+cell+'</th>' html += '<th>' + cell + '</th>'
html+='</tr>\n' html += '</tr>\n'
html+='</thead>\n' html += '</thead>\n'
return html return html

41
pytchat/processors/jsonfile_archiver.py
View File

@@ -1,10 +1,10 @@
import datetime
import json import json
import os import os
import re import re
from .chat_processor import ChatProcessor from .chat_processor import ChatProcessor
PATTERN = re.compile(r"(.*)\(([0-9]+)\)$") PATTERN = re.compile(r"(.*)\(([0-9]+)\)$")
class JsonfileArchiver(ChatProcessor): class JsonfileArchiver(ChatProcessor):
""" """
@@ -17,12 +17,13 @@ class JsonfileArchiver(ChatProcessor):
it is automatically saved under a different name it is automatically saved under a different name
with suffix '(number)' with suffix '(number)'
""" """
def __init__(self,save_path):
def __init__(self, save_path):
super().__init__() super().__init__()
self.save_path = self._checkpath(save_path) self.save_path = self._checkpath(save_path)
self.line_counter = 0 self.line_counter = 0
def process(self,chat_components: list): def process(self, chat_components: list):
""" """
Returns Returns
---------- ----------
@@ -32,19 +33,23 @@ class JsonfileArchiver(ChatProcessor):
total_lines : int : total_lines : int :
count of total lines written to the file. count of total lines written to the file.
""" """
if chat_components is None: return if chat_components is None:
with open(self.save_path, mode='a', encoding = 'utf-8') as f: return
with open(self.save_path, mode='a', encoding='utf-8') as f:
for component in chat_components: for component in chat_components:
if component is None: continue if component is None:
continue
chatdata = component.get('chatdata') chatdata = component.get('chatdata')
if chatdata is None: continue if chatdata is None:
continue
for action in chatdata: for action in chatdata:
if action is None: continue if action is None:
json_line = json.dumps(action, ensure_ascii = False) continue
f.writelines(json_line+'\n') json_line = json.dumps(action, ensure_ascii=False)
self.line_counter+=1 f.writelines(json_line + '\n')
return { "save_path" : self.save_path, self.line_counter += 1
"total_lines": self.line_counter } return {"save_path": self.save_path,
"total_lines": self.line_counter}
def _checkpath(self, filepath): def _checkpath(self, filepath):
splitter = os.path.splitext(os.path.basename(filepath)) splitter = os.path.splitext(os.path.basename(filepath))
@@ -53,14 +58,12 @@ class JsonfileArchiver(ChatProcessor):
newpath = filepath newpath = filepath
counter = 0 counter = 0
while os.path.exists(newpath): while os.path.exists(newpath):
match = re.search(PATTERN,body) match = re.search(PATTERN, body)
if match: if match:
counter=int(match[2])+1 counter = int(match[2]) + 1
num_with_bracket = f'({str(counter)})' num_with_bracket = f'({str(counter)})'
body = f'{match[1]}{num_with_bracket}' body = f'{match[1]}{num_with_bracket}'
else: else:
body = f'{body}({str(counter)})' body = f'{body}({str(counter)})'
newpath = os.path.join(os.path.dirname(filepath),body+extention) newpath = os.path.join(os.path.dirname(filepath), body + extention)
return newpath return newpath

42
pytchat/processors/simple_display_processor.py
View File

@@ -1,10 +1,6 @@
import json
import os
import traceback
import datetime
import time
from .chat_processor import ChatProcessor from .chat_processor import ChatProcessor
##version 2
class SimpleDisplayProcessor(ChatProcessor): class SimpleDisplayProcessor(ChatProcessor):
def process(self, chat_components: list): def process(self, chat_components: list):
@@ -12,36 +8,42 @@ class SimpleDisplayProcessor(ChatProcessor):
timeout = 0 timeout = 0
if chat_components is None: if chat_components is None:
return {"timeout":timeout, "chatlist":chatlist} return {"timeout": timeout, "chatlist": chatlist}
for component in chat_components: for component in chat_components:
timeout += component.get('timeout', 0) timeout += component.get('timeout', 0)
chatdata = component.get('chatdata') chatdata = component.get('chatdata')
if chatdata is None:break if chatdata is None:
break
for action in chatdata: for action in chatdata:
if action is None:continue if action is None:
if action.get('addChatItemAction') is None:continue continue
if action['addChatItemAction'].get('item') is None:continue if action.get('addChatItemAction') is None:
continue
if action['addChatItemAction'].get('item') is None:
continue
root = action['addChatItemAction']['item'].get('liveChatTextMessageRenderer') root = action['addChatItemAction']['item'].get(
'liveChatTextMessageRenderer')
if root: if root:
author_name = root['authorName']['simpleText'] author_name = root['authorName']['simpleText']
message = self._parse_message(root.get('message')) message = self._parse_message(root.get('message'))
purchase_amount_text = '' purchase_amount_text = ''
else: else:
root = ( action['addChatItemAction']['item'].get('liveChatPaidMessageRenderer') or root = (action['addChatItemAction']['item'].get('liveChatPaidMessageRenderer')
action['addChatItemAction']['item'].get('liveChatPaidStickerRenderer') ) or action['addChatItemAction']['item'].get('liveChatPaidStickerRenderer'))
if root: if root:
author_name = root['authorName']['simpleText'] author_name = root['authorName']['simpleText']
message = self._parse_message(root.get('message')) message = self._parse_message(root.get('message'))
purchase_amount_text = root['purchaseAmountText']['simpleText'] purchase_amount_text = root['purchaseAmountText']['simpleText']
else: else:
continue continue
chatlist.append(f'[{author_name}]: {message} {purchase_amount_text}') chatlist.append(
return {"timeout":timeout, "chatlist":chatlist} f'[{author_name}]: {message} {purchase_amount_text}')
return {"timeout": timeout, "chatlist": chatlist}
def _parse_message(self,message): def _parse_message(self, message):
if message is None: if message is None:
return '' return ''
if message.get('simpleText'): if message.get('simpleText'):
@@ -51,11 +53,9 @@ class SimpleDisplayProcessor(ChatProcessor):
tmp = '' tmp = ''
for run in runs: for run in runs:
if run.get('emoji'): if run.get('emoji'):
tmp+=(run['emoji']['shortcuts'][0]) tmp += (run['emoji']['shortcuts'][0])
elif run.get('text'): elif run.get('text'):
tmp+=(run['text']) tmp += (run['text'])
return tmp return tmp
else: else:
return '' return ''

68
pytchat/processors/speed/calculator.py
View File

@@ -5,6 +5,8 @@ Calculate speed of chat.
""" """
import time import time
from .. chat_processor import ChatProcessor from .. chat_processor import ChatProcessor
class RingQueue: class RingQueue:
""" """
リング型キュー リング型キュー
@@ -50,17 +52,17 @@ class RingQueue:
""" """
if self.mergin: if self.mergin:
self.items.append(item) self.items.append(item)
self.last_pos = len(self.items)-1 self.last_pos = len(self.items) - 1
if self.last_pos == self.capacity-1: if self.last_pos == self.capacity - 1:
self.mergin = False self.mergin = False
return return
self.last_pos += 1 self.last_pos += 1
if self.last_pos > self.capacity-1: if self.last_pos > self.capacity - 1:
self.last_pos = 0 self.last_pos = 0
self.items[self.last_pos] = item self.items[self.last_pos] = item
self.first_pos += 1 self.first_pos += 1
if self.first_pos > self.capacity-1: if self.first_pos > self.capacity - 1:
self.first_pos = 0 self.first_pos = 0
def get(self): def get(self):
@@ -77,6 +79,7 @@ class RingQueue:
def item_count(self): def item_count(self):
return len(self.items) return len(self.items)
class SpeedCalculator(ChatProcessor, RingQueue): class SpeedCalculator(ChatProcessor, RingQueue):
""" """
チャットの勢いを計算する。 チャットの勢いを計算する。
@@ -91,7 +94,7 @@ class SpeedCalculator(ChatProcessor, RingQueue):
RingQueueに格納するチャット勢い算出用データの最大数 RingQueueに格納するチャット勢い算出用データの最大数
""" """
def __init__(self, capacity = 10): def __init__(self, capacity=10):
super().__init__(capacity) super().__init__(capacity)
self.speed = 0 self.speed = 0
@@ -106,7 +109,6 @@ class SpeedCalculator(ChatProcessor, RingQueue):
self.speed = self._calc_speed() self.speed = self._calc_speed()
return self.speed return self.speed
def _calc_speed(self): def _calc_speed(self):
""" """
RingQueue内のチャット勢い算出用データリストを元に、 RingQueue内のチャット勢い算出用データリストを元に、
@@ -117,13 +119,12 @@ class SpeedCalculator(ChatProcessor, RingQueue):
チャット速度（１分間で換算したチャット数） チャット速度（１分間で換算したチャット数）
""" """
try: try:
#キュー内の総チャット数 # キュー内の総チャット数
total = sum(item['chat_count'] for item in self.items) total = sum(item['chat_count'] for item in self.items)
#キュー内の最初と最後のチャットの時間差 # キュー内の最初と最後のチャットの時間差
duration = (self.items[self.last_pos]['endtime'] duration = (self.items[self.last_pos]['endtime'] - self.items[self.first_pos]['starttime'])
- self.items[self.first_pos]['starttime'])
if duration != 0: if duration != 0:
return int(total*60/duration) return int(total * 60 / duration)
return 0 return 0
except IndexError: except IndexError:
return 0 return 0
@@ -143,61 +144,60 @@ class SpeedCalculator(ChatProcessor, RingQueue):
''' '''
チャットデータがない場合に空のデータをキューに投入する。 チャットデータがない場合に空のデータをキューに投入する。
''' '''
timestamp_now = int(time.time()) timestamp_now = int(time.time())
self.put({ self.put({
'chat_count':0, 'chat_count': 0,
'starttime':int(timestamp_now), 'starttime': int(timestamp_now),
'endtime':int(timestamp_now) 'endtime': int(timestamp_now)
}) })
def _get_timestamp(action :dict): def _get_timestamp(action: dict):
""" """
チャットデータから時刻データを取り出す。 チャットデータから時刻データを取り出す。
""" """
try: try:
item = action['addChatItemAction']['item'] item = action['addChatItemAction']['item']
timestamp = int(item[list(item.keys())[0]]['timestampUsec']) timestamp = int(item[list(item.keys())[0]]['timestampUsec'])
except (KeyError,TypeError): except (KeyError, TypeError):
return None return None
return timestamp return timestamp
if actions is None or len(actions)==0: if actions is None or len(actions) == 0:
_put_emptydata() _put_emptydata()
return return
#actions内の時刻データを持つチャットデータの数 # actions内の時刻データを持つチャットデータの数
counter=0 counter = 0
#actions内の最初のチャットデータの時刻 # actions内の最初のチャットデータの時刻
starttime= None starttime = None
#actions内の最後のチャットデータの時刻 # actions内の最後のチャットデータの時刻
endtime=None endtime = None
for action in actions: for action in actions:
#チャットデータからtimestampUsecを読み取る # チャットデータからtimestampUsecを読み取る
gettime = _get_timestamp(action) gettime = _get_timestamp(action)
#時刻のないデータだった場合は次の行のデータで読み取り試行 # 時刻のないデータだった場合は次の行のデータで読み取り試行
if gettime is None: if gettime is None:
continue continue
#最初に有効な時刻を持つデータのtimestampをstarttimeに設定 # 最初に有効な時刻を持つデータのtimestampをstarttimeに設定
if starttime is None: if starttime is None:
starttime = gettime starttime = gettime
#最後のtimestampを設定(途中で時刻のないデータの場合もあるので上書きしていく) # 最後のtimestampを設定(途中で時刻のないデータの場合もあるので上書きしていく)
endtime = gettime endtime = gettime
#チャットの数をインクリメント # チャットの数をインクリメント
counter += 1 counter += 1
#チャット速度用のデータをRingQueueに送る # チャット速度用のデータをRingQueueに送る
if starttime is None or endtime is None: if starttime is None or endtime is None:
_put_emptydata() _put_emptydata()
return return
self.put({ self.put({
'chat_count':counter, 'chat_count': counter,
'starttime':int(starttime/1000000), 'starttime': int(starttime / 1000000),
'endtime':int(endtime/1000000) 'endtime': int(endtime / 1000000)
}) })

19
pytchat/processors/superchat/calculator.py
View File

@@ -15,10 +15,12 @@ items_sticker = [
'liveChatPaidStickerRenderer' 'liveChatPaidStickerRenderer'
] ]
class SuperchatCalculator(ChatProcessor): class SuperchatCalculator(ChatProcessor):
""" """
Calculate the amount of SuperChat by currency. Calculate the amount of SuperChat by currency.
""" """
def __init__(self): def __init__(self):
self.results = {} self.results = {}
@@ -34,14 +36,16 @@ class SuperchatCalculator(ChatProcessor):
return self.results return self.results
for component in chat_components: for component in chat_components:
chatdata = component.get('chatdata') chatdata = component.get('chatdata')
if chatdata is None: continue if chatdata is None:
continue
for action in chatdata: for action in chatdata:
renderer = self._get_item(action, items_paid) or \ renderer = self._get_item(action, items_paid) or \
self._get_item(action, items_sticker) self._get_item(action, items_sticker)
if renderer is None: continue if renderer is None:
continue
symbol, amount = self._parse(renderer) symbol, amount = self._parse(renderer)
self.results.setdefault(symbol,0) self.results.setdefault(symbol, 0)
self.results[symbol]+=amount self.results[symbol] += amount
return self.results return self.results
def _parse(self, renderer): def _parse(self, renderer):
@@ -49,7 +53,7 @@ class SuperchatCalculator(ChatProcessor):
m = superchat_regex.search(purchase_amount_text) m = superchat_regex.search(purchase_amount_text)
if m: if m:
symbol = m.group(1) symbol = m.group(1)
amount = float(m.group(2).replace(',','')) amount = float(m.group(2).replace(',', ''))
else: else:
symbol = "" symbol = ""
amount = 0.0 amount = 0.0
@@ -69,6 +73,3 @@ class SuperchatCalculator(ChatProcessor):
continue continue
return None return None
return dict_body return dict_body

23
pytchat/processors/tsv_archiver.py
View File

@@ -4,9 +4,10 @@ import re
from .chat_processor import ChatProcessor from .chat_processor import ChatProcessor
from .default.processor import DefaultProcessor from .default.processor import DefaultProcessor
PATTERN = re.compile(r"(.*)\(([0-9]+)\)$") PATTERN = re.compile(r"(.*)\(([0-9]+)\)$")
fmt_headers = ['datetime','elapsed','authorName','message','superchatAmount' fmt_headers = ['datetime', 'elapsed', 'authorName', 'message',
,'authorType','authorChannel'] 'superchatAmount', 'authorType', 'authorChannel']
class TSVArchiver(ChatProcessor): class TSVArchiver(ChatProcessor):
''' '''
@@ -16,7 +17,7 @@ class TSVArchiver(ChatProcessor):
def __init__(self, save_path): def __init__(self, save_path):
super().__init__() super().__init__()
self.save_path = self._checkpath(save_path) self.save_path = self._checkpath(save_path)
with open(self.save_path, mode='a', encoding = 'utf-8') as f: with open(self.save_path, mode='a', encoding='utf-8') as f:
writer = csv.writer(f, delimiter='\t') writer = csv.writer(f, delimiter='\t')
writer.writerow(fmt_headers) writer.writerow(fmt_headers)
self.processor = DefaultProcessor() self.processor = DefaultProcessor()
@@ -28,14 +29,14 @@ class TSVArchiver(ChatProcessor):
newpath = filepath newpath = filepath
counter = 0 counter = 0
while os.path.exists(newpath): while os.path.exists(newpath):
match = re.search(PATTERN,body) match = re.search(PATTERN, body)
if match: if match:
counter=int(match[2])+1 counter = int(match[2]) + 1
num_with_bracket = f'({str(counter)})' num_with_bracket = f'({str(counter)})'
body = f'{match[1]}{num_with_bracket}' body = f'{match[1]}{num_with_bracket}'
else: else:
body = f'{body}({str(counter)})' body = f'{body}({str(counter)})'
newpath = os.path.join(os.path.dirname(filepath),body+extention) newpath = os.path.join(os.path.dirname(filepath), body + extention)
return newpath return newpath
def process(self, chat_components: list): def process(self, chat_components: list):
@@ -48,10 +49,10 @@ class TSVArchiver(ChatProcessor):
total_lines : int : total_lines : int :
count of total lines written to the file. count of total lines written to the file.
""" """
if chat_components is None or len (chat_components) == 0: if chat_components is None or len(chat_components) == 0:
return return
with open(self.save_path, mode='a', encoding = 'utf-8') as f: with open(self.save_path, mode='a', encoding='utf-8') as f:
writer = csv.writer(f, delimiter='\t') writer = csv.writer(f, delimiter='\t')
chats = self.processor.process(chat_components).items chats = self.processor.process(chat_components).items
for c in chats: for c in chats:
@@ -64,7 +65,3 @@ class TSVArchiver(ChatProcessor):
c.author.type, c.author.type,
c.author.channelId c.author.channelId
]) ])

69
pytchat/tool/extract/asyncdl.py
View File

@@ -17,7 +17,8 @@ REPLAY_URL = "https://www.youtube.com/live_chat_replay/" \
"get_live_chat_replay?continuation=" "get_live_chat_replay?continuation="
MAX_RETRY_COUNT = 3 MAX_RETRY_COUNT = 3
def _split(start, end, count, min_interval_sec = 120):
def _split(start, end, count, min_interval_sec=120):
""" """
Split section from `start` to `end` into `count` pieces, Split section from `start` to `end` into `count` pieces,
and returns the beginning of each piece. and returns the beginning of each piece.
@@ -28,41 +29,43 @@ def _split(start, end, count, min_interval_sec = 120):
-------- --------
List of the offset of each block's first chat data. List of the offset of each block's first chat data.
""" """
if not (isinstance(start, int) or isinstance(start, float)) or \
if not (isinstance(start,int) or isinstance(start,float)) or \ not (isinstance(end, int) or isinstance(end, float)):
not (isinstance(end,int) or isinstance(end,float)):
raise ValueError("start/end must be int or float") raise ValueError("start/end must be int or float")
if not isinstance(count,int): if not isinstance(count, int):
raise ValueError("count must be int") raise ValueError("count must be int")
if start>end: if start > end:
raise ValueError("end must be equal to or greater than start.") raise ValueError("end must be equal to or greater than start.")
if count<1: if count < 1:
raise ValueError("count must be equal to or greater than 1.") raise ValueError("count must be equal to or greater than 1.")
if (end-start)/count < min_interval_sec: if (end - start) / count < min_interval_sec:
count = int((end-start)/min_interval_sec) count = int((end - start) / min_interval_sec)
if count == 0 : count = 1 if count == 0:
interval= (end-start)/count count = 1
interval = (end - start) / count
if count == 1: if count == 1:
return [start] return [start]
return sorted( list(set( [int(start + interval*j) return sorted(list(set([int(start + interval * j)
for j in range(count) ]))) for j in range(count)])))
def ready_blocks(video_id, duration, div, callback): def ready_blocks(video_id, duration, div, callback):
if div <= 0: raise ValueError if div <= 0:
raise ValueError
async def _get_blocks( video_id, duration, div, callback): async def _get_blocks(video_id, duration, div, callback):
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
tasks = [_create_block(session, video_id, seektime, callback) tasks = [_create_block(session, video_id, seektime, callback)
for seektime in _split(-1, duration, div)] for seektime in _split(-1, duration, div)]
return await asyncio.gather(*tasks) return await asyncio.gather(*tasks)
async def _create_block(session, video_id, seektime, callback): async def _create_block(session, video_id, seektime, callback):
continuation = arcparam.getparam(video_id, seektime = seektime) continuation = arcparam.getparam(video_id, seektime=seektime)
url = f"{REPLAY_URL}{quote(continuation)}&pbj=1" url = f"{REPLAY_URL}{quote(continuation)}&pbj=1"
for _ in range(MAX_RETRY_COUNT): for _ in range(MAX_RETRY_COUNT):
try : try:
async with session.get(url, headers = headers) as resp: async with session.get(url, headers=headers) as resp:
text = await resp.text() text = await resp.text()
next_continuation, actions = parser.parse(json.loads(text)) next_continuation, actions = parser.parse(json.loads(text))
break break
@@ -76,12 +79,12 @@ def ready_blocks(video_id, duration, div, callback):
first = parser.get_offset(actions[0]) first = parser.get_offset(actions[0])
last = parser.get_offset(actions[-1]) last = parser.get_offset(actions[-1])
if callback: if callback:
callback(actions,last-first) callback(actions, last - first)
return Block( return Block(
continuation = next_continuation, continuation=next_continuation,
chat_data = actions, chat_data=actions,
first = first, first=first,
last = last last=last
) )
""" """
@@ -92,13 +95,14 @@ def ready_blocks(video_id, duration, div, callback):
_get_blocks(video_id, duration, div, callback)) _get_blocks(video_id, duration, div, callback))
return blocks return blocks
def fetch_patch(callback, blocks, video_id): def fetch_patch(callback, blocks, video_id):
async def _allocate_workers(): async def _allocate_workers():
workers = [ workers = [
ExtractWorker( ExtractWorker(
fetch = _fetch, block = block, fetch=_fetch, block=block,
blocks = blocks, video_id = video_id blocks=blocks, video_id=video_id
) )
for block in blocks for block in blocks
] ]
@@ -106,11 +110,11 @@ def fetch_patch(callback, blocks, video_id):
tasks = [worker.run(session) for worker in workers] tasks = [worker.run(session) for worker in workers]
return await asyncio.gather(*tasks) return await asyncio.gather(*tasks)
async def _fetch(continuation,session) -> Patch: async def _fetch(continuation, session) -> Patch:
url = f"{REPLAY_URL}{quote(continuation)}&pbj=1" url = f"{REPLAY_URL}{quote(continuation)}&pbj=1"
for _ in range(MAX_RETRY_COUNT): for _ in range(MAX_RETRY_COUNT):
try: try:
async with session.get(url,headers = config.headers) as resp: async with session.get(url, headers=config.headers) as resp:
chat_json = await resp.text() chat_json = await resp.text()
continuation, actions = parser.parse(json.loads(chat_json)) continuation, actions = parser.parse(json.loads(chat_json))
break break
@@ -126,7 +130,7 @@ def fetch_patch(callback, blocks, video_id):
if callback: if callback:
callback(actions, last - first) callback(actions, last - first)
return Patch(actions, continuation, first, last) return Patch(actions, continuation, first, last)
return Patch(continuation = continuation) return Patch(continuation=continuation)
""" """
allocate workers and assign blocks. allocate workers and assign blocks.
@@ -137,10 +141,11 @@ def fetch_patch(callback, blocks, video_id):
except CancelledError: except CancelledError:
pass pass
async def _shutdown(): async def _shutdown():
print("\nshutdown...") print("\nshutdown...")
tasks = [t for t in asyncio.all_tasks() tasks = [t for t in asyncio.all_tasks()
if t is not asyncio.current_task()] if t is not asyncio.current_task()]
for task in tasks: for task in tasks:
task.cancel() task.cancel()
try: try:
@@ -148,7 +153,7 @@ async def _shutdown():
except asyncio.CancelledError: except asyncio.CancelledError:
pass pass
def cancel(): def cancel():
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
loop.create_task(_shutdown()) loop.create_task(_shutdown())

11
pytchat/tool/extract/block.py
View File

@@ -1,4 +1,3 @@
from . import parser
class Block: class Block:
"""Block object represents something like a box """Block object represents something like a box
to join chunk of chatdata. to join chunk of chatdata.
@@ -40,12 +39,12 @@ class Block:
while True, this block is excluded from duplicate split procedure. while True, this block is excluded from duplicate split procedure.
""" """
__slots__ = ['first','last','end','continuation','chat_data','remaining', __slots__ = ['first', 'last', 'end', 'continuation', 'chat_data', 'remaining',
'done','is_last','during_split'] 'done', 'is_last', 'during_split']
def __init__(self, first = 0, last = 0, end = 0, def __init__(self, first=0, last=0, end=0,
continuation = '', chat_data = [], is_last = False, continuation='', chat_data=[], is_last=False,
during_split = False): during_split=False):
self.first = first self.first = first
self.last = last self.last = last
self.end = end self.end = end

109
pytchat/tool/extract/duplcheck.py
View File

@@ -1,7 +1,8 @@
from . import parser from . import parser
def check_duplicate(chatdata): def check_duplicate(chatdata):
max_range = len(chatdata)-1 max_range = len(chatdata) - 1
tbl_offset = [None] * max_range tbl_offset = [None] * max_range
tbl_id = [None] * max_range tbl_id = [None] * max_range
tbl_type = [None] * max_range tbl_type = [None] * max_range
@@ -15,27 +16,25 @@ def check_duplicate(chatdata):
def is_duplicate(i, j): def is_duplicate(i, j):
return ( return (
tbl_offset[i] == tbl_offset[j] tbl_offset[i] == tbl_offset[j]
and and tbl_id[i] == tbl_id[j]
tbl_id[i] == tbl_id[j] and tbl_type[i] == tbl_type[j]
and
tbl_type[i] == tbl_type[j]
) )
print("creating table...") print("creating table...")
create_table(chatdata,max_range) create_table(chatdata, max_range)
print("searching duplicate data...") print("searching duplicate data...")
return [{ "i":{ return [{"i": {
"index" : i, "id" : parser.get_id(chatdata[i]), "index": i, "id": parser.get_id(chatdata[i]),
"offsetTime" : parser.get_offset(chatdata[i]), "offsetTime": parser.get_offset(chatdata[i]),
"type" : parser.get_type(chatdata[i]) "type": parser.get_type(chatdata[i])
}, },
"j":{ "j":{
"index" : j, "id" : parser.get_id(chatdata[j]), "index": j, "id": parser.get_id(chatdata[j]),
"offsetTime" : parser.get_offset(chatdata[j]), "offsetTime": parser.get_offset(chatdata[j]),
"type" : parser.get_type(chatdata[j]) "type": parser.get_type(chatdata[j])
} }
} }
for i in range(max_range) for j in range(i+1,max_range) for i in range(max_range) for j in range(i + 1, max_range)
if is_duplicate(i,j)] if is_duplicate(i, j)]
def check_duplicate_offset(chatdata): def check_duplicate_offset(chatdata):
@@ -53,21 +52,21 @@ def check_duplicate_offset(chatdata):
def is_duplicate(i, j): def is_duplicate(i, j):
return ( return (
tbl_offset[i] == tbl_offset[j] tbl_offset[i] == tbl_offset[j]
and and tbl_id[i] == tbl_id[j]
tbl_id[i] == tbl_id[j]
) )
print("creating table...") print("creating table...")
create_table(chatdata,max_range) create_table(chatdata, max_range)
print("searching duplicate data...") print("searching duplicate data...")
return [{ return [{
"index" : i, "id" : tbl_id[i], "index": i, "id": tbl_id[i],
"offsetTime" : tbl_offset[i], "offsetTime": tbl_offset[i],
"type:" : tbl_type[i] "type:": tbl_type[i]
} }
for i in range(max_range-1) for i in range(max_range - 1)
if is_duplicate(i,i+1)] if is_duplicate(i, i + 1)]
def remove_duplicate_head(blocks): def remove_duplicate_head(blocks):
if len(blocks) == 0 or len(blocks) == 1: if len(blocks) == 0 or len(blocks) == 1:
@@ -77,26 +76,25 @@ def remove_duplicate_head(blocks):
if len(blocks[index].chat_data) == 0: if len(blocks[index].chat_data) == 0:
return True return True
elif len(blocks[index+1].chat_data) == 0: elif len(blocks[index + 1].chat_data) == 0:
return False return False
id_0 = parser.get_id(blocks[index].chat_data[0]) id_0 = parser.get_id(blocks[index].chat_data[0])
id_1 = parser.get_id(blocks[index+1].chat_data[0]) id_1 = parser.get_id(blocks[index + 1].chat_data[0])
type_0 = parser.get_type(blocks[index].chat_data[0]) type_0 = parser.get_type(blocks[index].chat_data[0])
type_1 = parser.get_type(blocks[index+1].chat_data[0]) type_1 = parser.get_type(blocks[index + 1].chat_data[0])
return ( return (
blocks[index].first == blocks[index+1].first blocks[index].first == blocks[index + 1].first
and and id_0 == id_1
id_0 == id_1 and type_0 == type_1
and
type_0 == type_1
) )
ret = [blocks[i] for i in range(len(blocks)-1) ret = [blocks[i] for i in range(len(blocks) - 1)
if (len(blocks[i].chat_data)>0 and if (len(blocks[i].chat_data) > 0
not is_duplicate_head(i) )] and not is_duplicate_head(i))]
ret.append(blocks[-1]) ret.append(blocks[-1])
return ret return ret
def remove_duplicate_tail(blocks): def remove_duplicate_tail(blocks):
if len(blocks) == 0 or len(blocks) == 1: if len(blocks) == 0 or len(blocks) == 1:
return blocks return blocks
@@ -104,24 +102,23 @@ def remove_duplicate_tail(blocks):
def is_duplicate_tail(index): def is_duplicate_tail(index):
if len(blocks[index].chat_data) == 0: if len(blocks[index].chat_data) == 0:
return True return True
elif len(blocks[index-1].chat_data) == 0: elif len(blocks[index - 1].chat_data) == 0:
return False return False
id_0 = parser.get_id(blocks[index-1].chat_data[-1]) id_0 = parser.get_id(blocks[index - 1].chat_data[-1])
id_1 = parser.get_id(blocks[index].chat_data[-1]) id_1 = parser.get_id(blocks[index].chat_data[-1])
type_0 = parser.get_type(blocks[index-1].chat_data[-1]) type_0 = parser.get_type(blocks[index - 1].chat_data[-1])
type_1 = parser.get_type(blocks[index].chat_data[-1]) type_1 = parser.get_type(blocks[index].chat_data[-1])
return ( return (
blocks[index-1].last == blocks[index].last blocks[index - 1].last == blocks[index].last
and and id_0 == id_1
id_0 == id_1 and type_0 == type_1
and
type_0 == type_1
) )
ret = [blocks[i] for i in range(0,len(blocks)) ret = [blocks[i] for i in range(0, len(blocks))
if i == 0 or not is_duplicate_tail(i) ] if i == 0 or not is_duplicate_tail(i)]
return ret return ret
def remove_overlap(blocks): def remove_overlap(blocks):
""" """
Fix overlapped blocks after ready_blocks(). Fix overlapped blocks after ready_blocks().
@@ -134,7 +131,7 @@ def remove_overlap(blocks):
for block in blocks: for block in blocks:
if block.is_last: if block.is_last:
break break
if len(block.chat_data)==0: if len(block.chat_data) == 0:
continue continue
block_end = block.end block_end = block.end
if block.last >= block_end: if block.last >= block_end:
@@ -143,14 +140,14 @@ def remove_overlap(blocks):
break break
block.chat_data.pop() block.chat_data.pop()
block.last = parser.get_offset(line) block.last = parser.get_offset(line)
block.remaining=0 block.remaining = 0
block.done=True block.done = True
block.continuation = None block.continuation = None
return blocks return blocks
def _dump(blocks): def _dump(blocks):
print(f"---------- first last end---") print("---------- first last end---")
for i,block in enumerate(blocks): for i, block in enumerate(blocks):
print(f"block[{i:3}] {block.first:>10} {block.last:>10} {block.end:>10}") print(
f"block[{i:3}] {block.first:>10} {block.last:>10} {block.end:>10}")

25
pytchat/tool/extract/extractor.py
View File

@@ -1,16 +1,16 @@
from . import asyncdl from . import asyncdl
from . import duplcheck from . import duplcheck
from . import parser
from .. videoinfo import VideoInfo from .. videoinfo import VideoInfo
from ... import config from ... import config
from ... exceptions import InvalidVideoIdException from ... exceptions import InvalidVideoIdException
logger = config.logger(__name__) logger = config.logger(__name__)
headers=config.headers headers = config.headers
class Extractor: class Extractor:
def __init__(self, video_id, div = 1, callback = None, processor = None): def __init__(self, video_id, div=1, callback=None, processor=None):
if not isinstance(div ,int) or div < 1: if not isinstance(div, int) or div < 1:
raise ValueError('div must be positive integer.') raise ValueError('div must be positive integer.')
elif div > 10: elif div > 10:
div = 10 div = 10
@@ -41,10 +41,10 @@ class Extractor:
def _set_block_end(self): def _set_block_end(self):
if len(self.blocks) > 0: if len(self.blocks) > 0:
for i in range(len(self.blocks)-1): for i in range(len(self.blocks) - 1):
self.blocks[i].end = self.blocks[i+1].first self.blocks[i].end = self.blocks[i + 1].first
self.blocks[-1].end = self.duration*1000 self.blocks[-1].end = self.duration * 1000
self.blocks[-1].is_last =True self.blocks[-1].is_last = True
return self return self
def _remove_overlap(self): def _remove_overlap(self):
@@ -82,11 +82,12 @@ class Extractor:
return [] return []
data = self._execute_extract_operations() data = self._execute_extract_operations()
if self.processor is None: if self.processor is None:
return data return data
return self.processor.process( return self.processor.process(
[{'video_id':None,'timeout':1,'chatdata' : (action [{'video_id': None,
["replayChatItemAction"]["actions"][0] for action in data)}] 'timeout': 1,
) 'chatdata': (action["replayChatItemAction"]["actions"][0] for action in data)}]
)
def cancel(self): def cancel(self):
asyncdl.cancel() asyncdl.cancel()

16
pytchat/tool/extract/parser.py
View File

@@ -1,12 +1,12 @@
import json
from ... import config from ... import config
from ... exceptions import ( from ... exceptions import (
ResponseContextError, ResponseContextError,
NoContentsException, NoContentsException,
NoContinuationsException ) NoContinuationsException)
logger = config.logger(__name__) logger = config.logger(__name__)
def parse(jsn): def parse(jsn):
""" """
Parse replay chat data. Parse replay chat data.
@@ -24,8 +24,8 @@ def parse(jsn):
raise ValueError("parameter JSON is None") raise ValueError("parameter JSON is None")
if jsn['response']['responseContext'].get('errors'): if jsn['response']['responseContext'].get('errors'):
raise ResponseContextError( raise ResponseContextError(
'video_id is invalid or private/deleted.') 'video_id is invalid or private/deleted.')
contents=jsn['response'].get('continuationContents') contents = jsn['response'].get('continuationContents')
if contents is None: if contents is None:
raise NoContentsException('No chat data.') raise NoContentsException('No chat data.')
@@ -43,12 +43,12 @@ def parse(jsn):
def get_offset(item): def get_offset(item):
return int(item['replayChatItemAction']["videoOffsetTimeMsec"]) return int(item['replayChatItemAction']["videoOffsetTimeMsec"])
def get_id(item): def get_id(item):
return list((list(item['replayChatItemAction']["actions"][0].values() return list((list(item['replayChatItemAction']["actions"][0].values()
)[0])['item'].values())[0].get('id') )[0])['item'].values())[0].get('id')
def get_type(item): def get_type(item):
return list((list(item['replayChatItemAction']["actions"][0].values() return list((list(item['replayChatItemAction']["actions"][0].values()
)[0])['item'].keys())[0] )[0])['item'].keys())[0]

29
pytchat/tool/extract/patch.py
View File

@@ -2,17 +2,19 @@ from . import parser
from . block import Block from . block import Block
from typing import NamedTuple from typing import NamedTuple
class Patch(NamedTuple): class Patch(NamedTuple):
""" """
Patch represents chunk of chat data Patch represents chunk of chat data
which is fetched by asyncdl.fetch_patch._fetch(). which is fetched by asyncdl.fetch_patch._fetch().
""" """
chats : list = [] chats: list = []
continuation : str = None continuation: str = None
first : int = None first: int = None
last : int = None last: int = None
def fill(block:Block, patch:Patch):
def fill(block: Block, patch: Patch):
block_end = block.end block_end = block.end
if patch.last < block_end or block.is_last: if patch.last < block_end or block.is_last:
set_patch(block, patch) set_patch(block, patch)
@@ -23,15 +25,15 @@ def fill(block:Block, patch:Patch):
break break
patch.chats.pop() patch.chats.pop()
set_patch(block, patch._replace( set_patch(block, patch._replace(
continuation = None, continuation=None,
last = line_offset last=line_offset
)
) )
block.remaining=0 )
block.done=True block.remaining = 0
block.done = True
def split(parent_block:Block, child_block:Block, patch:Patch): def split(parent_block: Block, child_block: Block, patch: Patch):
parent_block.during_split = False parent_block.during_split = False
if patch.first <= parent_block.last: if patch.first <= parent_block.last:
''' When patch overlaps with parent_block, ''' When patch overlaps with parent_block,
@@ -46,9 +48,8 @@ def split(parent_block:Block, child_block:Block, patch:Patch):
fill(child_block, patch) fill(child_block, patch)
def set_patch(block:Block, patch:Patch): def set_patch(block: Block, patch: Patch):
block.continuation = patch.continuation block.continuation = patch.continuation
block.chat_data.extend(patch.chats) block.chat_data.extend(patch.chats)
block.last = patch.last block.last = patch.last
block.remaining = block.end-block.last block.remaining = block.end - block.last

26
pytchat/tool/extract/worker.py
View File

@@ -1,8 +1,8 @@
from . import parser
from . block import Block from . block import Block
from . patch import Patch, fill, split from . patch import fill, split
from ... paramgen import arcparam from ... paramgen import arcparam
class ExtractWorker: class ExtractWorker:
""" """
ExtractWorker associates a download session with a block. ExtractWorker associates a download session with a block.
@@ -28,7 +28,7 @@ class ExtractWorker:
""" """
__slots__ = ['block', 'fetch', 'blocks', 'video_id', 'parent_block'] __slots__ = ['block', 'fetch', 'blocks', 'video_id', 'parent_block']
def __init__(self, fetch, block, blocks, video_id ): def __init__(self, fetch, block, blocks, video_id):
self.block = block self.block = block
self.fetch = fetch self.fetch = fetch
self.blocks = blocks self.blocks = blocks
@@ -54,26 +54,28 @@ class ExtractWorker:
self.block.done = True self.block.done = True
self.block = _search_new_block(self) self.block = _search_new_block(self)
def _search_new_block(worker) -> Block: def _search_new_block(worker) -> Block:
index, undone_block = _get_undone_block(worker.blocks) index, undone_block = _get_undone_block(worker.blocks)
if undone_block is None: if undone_block is None:
return Block(continuation = None) return Block(continuation=None)
mean = (undone_block.last + undone_block.end)/2 mean = (undone_block.last + undone_block.end) / 2
continuation = arcparam.getparam(worker.video_id, seektime = mean/1000) continuation = arcparam.getparam(worker.video_id, seektime=mean / 1000)
worker.parent_block = undone_block worker.parent_block = undone_block
worker.parent_block.during_split = True worker.parent_block.during_split = True
new_block = Block( new_block = Block(
end = undone_block.end, end=undone_block.end,
chat_data = [], chat_data=[],
continuation = continuation, continuation=continuation,
during_split = True, during_split=True,
is_last = worker.parent_block.is_last) is_last=worker.parent_block.is_last)
'''swap last block''' '''swap last block'''
if worker.parent_block.is_last: if worker.parent_block.is_last:
worker.parent_block.is_last = False worker.parent_block.is_last = False
worker.blocks.insert(index+1, new_block) worker.blocks.insert(index + 1, new_block)
return new_block return new_block
def _get_undone_block(blocks) -> (int, Block): def _get_undone_block(blocks) -> (int, Block):
min_interval_ms = 120000 min_interval_ms = 120000
max_remaining = 0 max_remaining = 0

15
pytchat/tool/videoinfo.py
View File

@@ -2,14 +2,13 @@ import json
import re import re
import requests import requests
from .. import config from .. import config
from .. import util
from ..exceptions import InvalidVideoIdException from ..exceptions import InvalidVideoIdException
headers = config.headers headers = config.headers
pattern = re.compile(r"yt\.setConfig\({'PLAYER_CONFIG': ({.*})}\);") pattern = re.compile(r"yt\.setConfig\({'PLAYER_CONFIG': ({.*})}\);")
item_channel_id =[ item_channel_id = [
"videoDetails", "videoDetails",
"embeddedPlayerOverlayVideoDetailsRenderer", "embeddedPlayerOverlayVideoDetailsRenderer",
"channelThumbnailEndpoint", "channelThumbnailEndpoint",
@@ -29,7 +28,7 @@ item_response = [
"embedded_player_response" "embedded_player_response"
] ]
item_author_image =[ item_author_image = [
"videoDetails", "videoDetails",
"embeddedPlayerOverlayVideoDetailsRenderer", "embeddedPlayerOverlayVideoDetailsRenderer",
"channelThumbnail", "channelThumbnail",
@@ -63,6 +62,7 @@ item_moving_thumbnail = [
"url" "url"
] ]
class VideoInfo: class VideoInfo:
''' '''
VideoInfo object retrieves YouTube video information. VideoInfo object retrieves YouTube video information.
@@ -76,6 +76,7 @@ class VideoInfo:
InvalidVideoIdException : InvalidVideoIdException :
Occurs when video_id does not exist on YouTube. Occurs when video_id does not exist on YouTube.
''' '''
def __init__(self, video_id): def __init__(self, video_id):
self.video_id = video_id self.video_id = video_id
text = self._get_page_text(video_id) text = self._get_page_text(video_id)
@@ -83,13 +84,13 @@ class VideoInfo:
def _get_page_text(self, video_id): def _get_page_text(self, video_id):
url = f"https://www.youtube.com/embed/{video_id}" url = f"https://www.youtube.com/embed/{video_id}"
resp = requests.get(url, headers = headers) resp = requests.get(url, headers=headers)
resp.raise_for_status() resp.raise_for_status()
return resp.text return resp.text
def _parse(self, text): def _parse(self, text):
result = re.search(pattern, text) result = re.search(pattern, text)
res= json.loads(result.group(1)) res = json.loads(result.group(1))
response = self._get_item(res, item_response) response = self._get_item(res, item_response)
if response is None: if response is None:
self._check_video_is_private(res.get("args")) self._check_video_is_private(res.get("args"))
@@ -98,7 +99,7 @@ class VideoInfo:
raise InvalidVideoIdException( raise InvalidVideoIdException(
f"No renderer found in video_id: [{self.video_id}].") f"No renderer found in video_id: [{self.video_id}].")
def _check_video_is_private(self,args): def _check_video_is_private(self, args):
if args and args.get("video_id"): if args and args.get("video_id"):
raise InvalidVideoIdException( raise InvalidVideoIdException(
f"video_id [{self.video_id}] is private or deleted.") f"video_id [{self.video_id}] is private or deleted.")
@@ -131,7 +132,7 @@ class VideoInfo:
def get_title(self): def get_title(self):
if self._renderer.get("title"): if self._renderer.get("title"):
return [''.join(run["text"]) return [''.join(run["text"])
for run in self._renderer["title"]["runs"]][0] for run in self._renderer["title"]["runs"]][0]
return None return None
def get_channel_id(self): def get_channel_id(self):

15
pytchat/util/__init__.py
View File

@@ -1,15 +1,18 @@
import requests,json,datetime import requests
import json
import datetime
from .. import config from .. import config
def extract(url): def extract(url):
_session = requests.Session() _session = requests.Session()
html = _session.get(url, headers=config.headers) html = _session.get(url, headers=config.headers)
with open(str(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S') with open(str(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')
)+'test.json',mode ='w',encoding='utf-8') as f: ) + 'test.json', mode='w', encoding='utf-8') as f:
json.dump(html.json(),f,ensure_ascii=False) json.dump(html.json(), f, ensure_ascii=False)
def save(data,filename,extention): def save(data, filename, extention):
with open(filename+"_"+(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S') with open(filename + "_" + (datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')) + extention,
)+extention,mode ='w',encoding='utf-8') as f: mode='w', encoding='utf-8') as f:
f.writelines(data) f.writelines(data)