diff options
Diffstat (limited to 'src/portal/py/tests')
| -rw-r--r-- | src/portal/py/tests/archive_query.py | 108 | ||||
| -rw-r--r-- | src/portal/py/tests/test.py | 21 |
2 files changed, 98 insertions, 31 deletions
diff --git a/src/portal/py/tests/archive_query.py b/src/portal/py/tests/archive_query.py index 1ed9db4..9c4cf41 100644 --- a/src/portal/py/tests/archive_query.py +++ b/src/portal/py/tests/archive_query.py @@ -11,10 +11,12 @@ from post import PostEncoder, PostType, DateType, User from modules import ALL_MODULES from tests.logger import Logger -#mode = 'pixiv_web' -mode = 'fanbox' +mode = 'pixiv_web' +#mode = 'fanbox' +#mode = 'patreon' cmd = 'user' #cmd = 'bookmarks' +#cmd = 'search' module = ALL_MODULES[mode][0] if not module.init(): sys.exit(1) @@ -41,13 +43,14 @@ def default_media_download(obj, post, output_base): return False class QueryDownloadThread(threading.Thread): - def __init__(self, log, complete, query, output_dir): + def __init__(self, log, complete, query, output_dir, start_page): super().__init__() self.log = log self.complete = complete self.query = query self.output_dir = output_dir self.media_download = default_media_download + self.start_page = start_page def make_path(self, post): prefix: str @@ -142,22 +145,23 @@ class QueryDownloadThread(threading.Thread): return completed = True arg = self.query.split(':')[-1] - num = 0 + num = self.start_page count = 0 while completed: self.log.write(f'Starting page {num} of {arg}.') page = search.get_page(num) + for hash, response in module.raw_responses.items(): + raw_response_path = f'{self.output_dir}/raw_responses/{hash}.json.gz' + if not os.path.isfile(raw_response_path): + if not self.write_to_file(raw_response_path, 'wb+', response, True): + completed = False + break if not page: break num += 1 for unique_id in page: if unique_id == 'twitter:t:1597637140833529856': self.log.write('---------Hit target---------') - with quit_mutex: - if not is_running: - self.complete.write(f'{mode}:{cmd}:{arg}:i:{str(count)},{str(num)}') - completed = False - break post = module.get_item(unique_id) if not post: self.log.write(f'Missing post with id {unique_id}.') @@ -165,6 +169,11 @@ class QueryDownloadThread(threading.Thread): if not self.write_post_to_disk(post): continue count += 1 + with quit_mutex: + if not is_running: + self.complete.write(f'{mode}:{cmd}:{arg}:i:{str(count)},{str(num)}') + completed = False + break if completed: self.complete.write(f'{mode}:{cmd}:{arg}:c:{str(count)},{str(num)}') with queue_cond: @@ -204,25 +213,72 @@ def download_args(): output_dir = sys.argv[1] if not os.path.isdir(output_dir): os.mkdir(output_dir) + os.mkdir(f'{output_dir}/raw_responses') #args = module.search(f'following:{22781328}') - #args = ['22781328'] - #args = ['anoh223'] - #args = ['532891', '1324074',] - #args = ['6600030', '163436'] - #args = ['85117546', '29506', '3298353', '21674944', '2034628', '3085731'] - #args = ['6827013', '11742', '4671', '36323560', '5128785', '30970895'] - #args = ['mugi49'] - #args = ['71598064', '9204502', '689283', '19956185', '42987864', '103682732', '4675776', '7780', '160755', '250142', '3444594', '1995476', '357647'] - #args = ['10473280'] - #args = ['178217', '106414', '882896', '3040749'] - #args = ['21245269'] - #args = ['1969907', '292644', '746841'] - #args = ['1319954', '43718238', '2721319', '53459337'] + #args = [ + # '5838770', + # '44194940' + #] # TODO: - args = ['yzr'] - #args = ['11187954'] - # twitter test: butcha_u, sep__rina + #args = [ + # '68839843', + # '16105069', + # '67090304', + # '188984', + # '15041022', + # '3948', + # '2232374', + # '194668', + # '7835', + # '3067312', + # '59375709', + # '9077832', + # '199750', + # '5806160', + # '33726', + # '15261563', + # '7246', + # '8146', + # '40259810', + # '28865', + # '18362', + # '2750098', + # '2157729', + # '65523978', + # '154858', + # '37625681', + # '34642665', + # '24023', + # '3564336', + # '2658856', + # '990017', + # '55392960', + # '8967182', + # '8965979', + # '14305327', + # '469378', + # '8885157', + # '1107939', + # '18159122', + # '57304923', + # '6534119', + # '91521', + # '1074517', + # '14394', + # '46563991', + # '33694842', + # '84663', + # '4028970', + # '8876470', + # '1829995', + # '14113571', + # '56349311', + # '143555', + # '36388895', + # '14188490' + #] + # twitter test: butcha_u, sep__rina, nagayori000, sattinittas, threads = [] start = True @@ -256,7 +312,7 @@ def download_args(): if not is_running: break log.write(f'***********************\nStarting download of {arg}\n***********************\n') - arg_thread = QueryDownloadThread(log, complete, arg, output_dir) + arg_thread = QueryDownloadThread(log, complete, arg, output_dir, 0) threads.append(arg_thread) arg_thread.start() diff --git a/src/portal/py/tests/test.py b/src/portal/py/tests/test.py index 15bde0a..2fe2eda 100644 --- a/src/portal/py/tests/test.py +++ b/src/portal/py/tests/test.py @@ -1,3 +1,4 @@ +import sys import json import httpx import os.path @@ -5,11 +6,11 @@ from base import Method from typing import Optional, Any from post import DateType, DateMeta, Date, PostEncoder, Post, MediaType from modules import ALL_MODULES -#from twitter.scraper import Scraper +from twitter.scraper import Scraper #scraper = Scraper(cookies = { -# 'ct0': '5a3c98b7b50a74dc556fa497f497932e8958b5b2ef7ba74440ce3f341bb4753f663d3fe7633222d62a52d4ce121673234b927d97eeef806129d42bbfba7ab4d4cfbd2749c20fe6f8da480d0a1e6629e1', -# 'auth_token': 'ea66b4c2ebd04e22de855dce6f1366221635bf2e' +# 'ct0': '', +# 'auth_token': '' #}) # #media = scraper.media([1557919548904419328], limit=10) @@ -55,14 +56,24 @@ from modules import ALL_MODULES # else: # print('Skipping file') -module = ALL_MODULES['instagram'][0] + +module = ALL_MODULES['pixiv_web'][0] module.init() -search = module.search('plottttwistttttt') +search = module.search('illust:91352621') page = search.get_page(0) for unique_id in page: post = module.get_item(unique_id) print(json.dumps(post, indent=4, cls=PostEncoder)) +#module = ALL_MODULES['instagram'][0] +#module.init() +#search = module.search('plottttwistttttt') +#page = search.get_page(0) +#for unique_id in page: +# post = module.get_item(unique_id) +# print(json.dumps(post, indent=4, cls=PostEncoder)) + + #d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.EDITED_AT_UNKNOWN_TIME) #d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.NONE) #print(json.dumps(d, indent=4, cls=PostEncoder)) |