ramforth/pytchat-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Return continuation even if no chat data

Browse Source
This commit is contained in:
taizan-hokuto
2020-02-16 23:31:25 +09:00
parent d8c1c4491d
commit eae485b914
4 changed files with 34 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
pytchat/tool/download/asyncdl.py
View File

@@ -1,4 +1,3 @@
import aiohttp import aiohttp
import asyncio import asyncio
import json import json
@@ -105,7 +104,7 @@ def download_patch(callback, blocks, video_id):
if callback: if callback:
callback(actions, last - first) callback(actions, last - first)
return Patch(actions, continuation, first, last) return Patch(actions, continuation, first, last)
return Patch() return Patch(continuation = continuation)
""" """
allocate workers and assign blocks. allocate workers and assign blocks.
""" """

10
pytchat/tool/download/patch.py
View File

@@ -22,7 +22,6 @@ def fill(block:Block, patch:Patch):
if line_offset < block_end: if line_offset < block_end:
break break
patch.chats.pop() patch.chats.pop()
set_patch(block, patch._replace( set_patch(block, patch._replace(
continuation = None, continuation = None,
last = line_offset last = line_offset
@@ -34,15 +33,16 @@ def fill(block:Block, patch:Patch):
def split(parent_block:Block, child_block:Block, patch:Patch): def split(parent_block:Block, child_block:Block, patch:Patch):
parent_block.during_split = False parent_block.during_split = False
"""patch overlaps with parent_block"""
if patch.first <= parent_block.last: if patch.first <= parent_block.last:
''' When patch overlaps with parent_block,
discard this block. '''
child_block.continuation = None child_block.continuation = None
''' Leave child_block.during_split == True ''' Leave child_block.during_split == True
to exclude from during_split sequence.''' to exclude from during_split sequence. '''
return return
child_block.during_split = False child_block.during_split = False
child_block.first=patch.first child_block.first = patch.first
parent_block.end =patch.first parent_block.end = patch.first
fill(child_block, patch) fill(child_block, patch)

2
tests/test_dl_duplcheck.py
View File

@@ -66,7 +66,7 @@ def test_duplicate_head():
[4] , [5] -> append [4] [4] , [5] -> append [4]
append [5] append [5]
result : [0] , [3] , [5] result : [2] , [4] , [5]
""" """
#chat data offsets are ignored. #chat data offsets are ignored.

49
tests/test_patch.py
View File

@@ -23,6 +23,8 @@ def test_split_0():
""" """
Normal case Normal case
~~~~~~ before ~~~~~~
@parent_block (# = already downloaded) @parent_block (# = already downloaded)
first last end first last end
@@ -31,18 +33,21 @@ def test_split_0():
@child_block @child_block
first = last = 0 end=parent_end first = last = 0 end (=parent_end)
---------------------------------------------------| | |
@fetched patch @fetched patch
|-- patch --| |-- patch --|
| |
| |
V V
~~~~~~ after ~~~~~~
@parent_block @parent_block
first last end (after split) first last end (after split)
@@ -83,20 +88,20 @@ def test_split_1():
~~~~~~ before ~~~~~~ ~~~~~~ before ~~~~~~
patch.first patch.first
first | last end first | last end
|####################|#####|---------------------| |####################|#####|---------------------|
^ ^
@child_block @child_block
first = last = 0 end=parent_end first = last = 0 end (=parent_end)
---------------------------------------------------| | |
@fetched patch @fetched patch
|-- patch --| |-- patch --|
| |
| |
V V
~~~~~~ after ~~~~~~ ~~~~~~ after ~~~~~~
@@ -135,8 +140,8 @@ def test_split_2():
|########------------------------------| |########------------------------------|
@child_block @child_block
first = last = 0 end=parent_end first = last = 0 end (=parent_end)
-----------------------------------------| | |
continuation:succeed from patch continuation:succeed from patch
@@ -144,9 +149,9 @@ def test_split_2():
|-------- patch --------| |-------- patch --------|
| |
| |
V V
~~~~~~ after ~~~~~~ ~~~~~~ after ~~~~~~
@@ -196,23 +201,23 @@ def test_split_none():
|####################|###################|-------| |####################|###################|-------|
^ ^
@child_block @child_block
first = last = 0 end=parent_end first = last = 0 end (=parent_end)
---------------------------------------------------| | |
@fetched patch @fetched patch
|-- patch --| |-- patch --|
patch.last < parent_block.last . patch.last < parent_block.last .
| |
| |
V V
~~~~~~ after ~~~~~~ ~~~~~~ after ~~~~~~
@parent_block @parent_block
first last end (before split) first last end (before split)
|########################################|-------| |########################################|-------|
.
@child_block @child_block
............ -> discard all data. ............ -> discard all data.