diff options
Diffstat (limited to 'src/portal/py/tests')
| -rw-r--r-- | src/portal/py/tests/archive_query.py | 21 |
1 files changed, 17 insertions, 4 deletions
diff --git a/src/portal/py/tests/archive_query.py b/src/portal/py/tests/archive_query.py index 8ecbb52..f944ca0 100644 --- a/src/portal/py/tests/archive_query.py +++ b/src/portal/py/tests/archive_query.py @@ -24,8 +24,8 @@ class Logger(): def close(self): self.file.close() -mode = 'pixiv_web' -#mode = 'fanbox' +#mode = 'pixiv_web' +mode = 'fanbox' #mode = 'patreon' #mode = 'instagram' #mode = 'twitter' @@ -42,6 +42,18 @@ queue_mutex = threading.Lock() queue_cond = threading.Condition(queue_mutex) is_running = True +def media_check_disk(obj, post, output_base): + for key in post.media.keys(): + download_params = module.get_download(post.unique_id, key) + if not download_params: + obj.log.write(f'Ignoring media{key} for post {post.unique_id}.') + continue + for url in download_params['urls']: + media_path = f'{output_base}_media{key}.{url.ext}' + if not os.path.isfile(media_path): + return False + return True + def default_media_download(obj, post, output_base): for key in post.media.keys(): download_params = module.get_download(post.unique_id, key) @@ -133,7 +145,7 @@ class QueryDownloadThread(threading.Thread): self.log.write('Rewriting post with quote.') has_mp4 = True ''' - if os.path.isfile(output): + if os.path.isfile(output) and media_check_disk(self, post, output_base): skip_message = f'Skipping already downloaded post {post.unique_id}' if post.type != PostType.TOMBSTONE: skip_message += f' from {post.author.unique_id}.' @@ -233,6 +245,7 @@ def download_args(): os.mkdir(f'{output_dir}/raw_responses') args = [] + start_page = 0 threads = [] start = True @@ -265,7 +278,7 @@ def download_args(): if not is_running: break log.write(f'***********************\nStarting download of {arg}\n***********************\n') - arg_thread = QueryDownloadThread(log, complete, arg, output_dir, 0) + arg_thread = QueryDownloadThread(log, complete, arg, output_dir, start_page) threads.append(arg_thread) arg_thread.start() |