Update portal to avoid TLS fingerprinting

Signed-off-by: Andrew Opalach <andrew@akon.city>

1 files changed, 17 insertions, 4 deletions

diff --git a/src/portal/py/tests/archive_query.py b/src/portal/py/tests/archive_query.py
index 8ecbb52..f944ca0 100644
--- a/src/portal/py/tests/archive_query.py
+++ b/src/portal/py/tests/archive_query.py
@@ -24,8 +24,8 @@ class Logger():
     def close(self):
         self.file.close()
 
-mode = 'pixiv_web'
-#mode = 'fanbox'
+#mode = 'pixiv_web'
+mode = 'fanbox'
 #mode = 'patreon'
 #mode = 'instagram'
 #mode = 'twitter'
@@ -42,6 +42,18 @@ queue_mutex = threading.Lock()
 queue_cond = threading.Condition(queue_mutex)
 is_running = True
 
+def media_check_disk(obj, post, output_base):
+    for key in post.media.keys():
+        download_params = module.get_download(post.unique_id, key)
+        if not download_params:
+            obj.log.write(f'Ignoring media{key} for post {post.unique_id}.')
+            continue
+        for url in download_params['urls']:
+            media_path = f'{output_base}_media{key}.{url.ext}'
+            if not os.path.isfile(media_path):
+                return False
+    return True
+
 def default_media_download(obj, post, output_base):
     for key in post.media.keys():
         download_params = module.get_download(post.unique_id, key)
@@ -133,7 +145,7 @@ class QueryDownloadThread(threading.Thread):
             self.log.write('Rewriting post with quote.')
             has_mp4 = True
         '''
-        if os.path.isfile(output):
+        if os.path.isfile(output) and media_check_disk(self, post, output_base):
             skip_message = f'Skipping already downloaded post {post.unique_id}'
             if post.type != PostType.TOMBSTONE:
                 skip_message += f' from {post.author.unique_id}.'
@@ -233,6 +245,7 @@ def download_args():
         os.mkdir(f'{output_dir}/raw_responses')
 
     args = []
+    start_page = 0
     threads = []
     start = True
 
@@ -265,7 +278,7 @@ def download_args():
             if not is_running:
                 break
         log.write(f'***********************\nStarting download of {arg}\n***********************\n')
-        arg_thread = QueryDownloadThread(log, complete, arg, output_dir, 0)
+        arg_thread = QueryDownloadThread(log, complete, arg, output_dir, start_page)
         threads.append(arg_thread)
         arg_thread.start()