Update portal to avoid TLS fingerprinting

Signed-off-by: Andrew Opalach <andrew@akon.city>

13 files changed, 104 insertions, 76 deletions

diff --git a/src/portal/cpy/portal.pyx b/src/portal/cpy/portal.pyx
index 6307a2d..ae209b5 100644
--- a/src/portal/cpy/portal.pyx
+++ b/src/portal/cpy/portal.pyx
@@ -7,16 +7,16 @@ import sys
 sys.path.append('./py')
 
 cdef int log_info(char *message) except -1:
-    return lib.log_info("%s", message)
+    return lib.log_info('%s', message)
 
 cdef int log_warn(char *message) except -1:
-    return lib.log_warn("%s", message)
+    return lib.log_warn('%s', message)
 
 cdef int log_error(char *message) except -1:
-    return lib.log_error("%s", message)
+    return lib.log_error('%s', message)
 
 cdef int log_debug(char *message) except -1:
-    return lib.log_debug("%s", message)
+    return lib.log_debug('%s', message)
 
 def encode_str(str):
     return str.encode('UTF-8')
diff --git a/src/portal/cpy/setup.py b/src/portal/cpy/setup.py
index d14068f..9d6e462 100644
--- a/src/portal/cpy/setup.py
+++ b/src/portal/cpy/setup.py
@@ -2,5 +2,5 @@ from setuptools import Extension
 from Cython.Build import cythonize
 from Cython.Compiler import Options
 Options.embed = True
-alabaster_inc = "../../../subprojects/libalabaster/include"
-cythonize([Extension("portal", ["./portal.pyx"], include_dirs=[alabaster_inc], extra_link_args=[], define_macros=[("AL_LOG_SECTION", "portal")])], language_level=3)
+alabaster_inc = '../../../subprojects/libalabaster/include'
+cythonize([Extension('portal', ['./portal.pyx'], include_dirs=[alabaster_inc], extra_link_args=[], define_macros=[('AL_LOG_SECTION', 'portal')])], language_level=3)
diff --git a/src/portal/py/base.py b/src/portal/py/base.py
index bd68925..a310cf5 100644
--- a/src/portal/py/base.py
+++ b/src/portal/py/base.py
@@ -1,7 +1,7 @@
 import log
 import time
 import json
-import httpx
+import tls_client
 import binascii
 import threading
 import config
@@ -12,13 +12,14 @@ from typing import Optional, Self, Any
 from post import Post, User
 
 type ParsedJson = Any
+type HttpResponse = tls_client.Response
 
 USER_AGENT = config.USER_AGENT
 
 class Method(Enum):
-    HEAD = "HEAD"
-    GET = "GET"
-    POST = "POST"
+    HEAD = 'HEAD'
+    GET = 'GET'
+    POST = 'POST'
 
 class Search():
     def __init__(self) -> None:
@@ -66,27 +67,29 @@ class Module():
         self.mutex: threading.Lock = threading.Lock()
         self.raw_responses: dict[str, str] = {}
         self.unique_id_map: OrderedDict = OrderedDict()
-        self.session: httpx.Client = httpx.Client(follow_redirects=True, timeout=20.0, http2=False)
+        self.session = tls_client.Session(client_identifier=config.TLS_CLIENT_IDENTIFIER, random_tls_extension_order=True)
 
     def init(self) -> bool:
         return True
 
-    def do_request(self, method: Method, url: str, params: Optional[dict[str, str]] = None, retries: int = 2) -> Optional[httpx.Response]:
+    def do_request(self, method: Method, url: str, params: Optional[dict[str, str]] = None, retries: int = 2) -> Optional[HttpResponse]:
         log.debug(f'HTTP {method.value} request {url} {params}.')
-        response: Optional[httpx.Response] = None
+        response: Optional[HttpResponse] = None
         for _ in range(0, retries + 1):
             try:
-                response = self.session.request(method.value, url, headers=self.headers, cookies=self.cookies, params=params)
+                response = self.session.execute_request(method.value, url, headers=self.headers, cookies=self.cookies, params=params, timeout=600)
             except Exception as e:
-                log.info(f'Connection exception ({e}), retrying in 20 seconds...')
+                log.info(f'Connection exception ({repr(e)}), retrying in 20 seconds...')
                 time.sleep(20)
                 continue
-            if not response:
-                continue
+            if response is None:
+                log.error('Invalid None response from requests library.')
+                break
             if response.status_code == 429:
-                log.info('Got too many requests, waiting for 3 minutes...')
-                time.sleep(60 * 3)
+                log.info('Got too many requests, waiting for 4 minutes...')
+                time.sleep(60 * 4)
                 response = None
+                retries += 1
                 continue
             if response.status_code == 404 or response.status_code == 403: # Shortcut 404, 403
                 log.warn(f'Returning None, {response.status_code}')
@@ -96,6 +99,7 @@ class Module():
                 time.sleep(5)
                 response = None
                 continue
+            '''
             try:
                 response.read()
             except Exception as e:
@@ -105,6 +109,8 @@ class Module():
                 continue
             else:
                 break
+            '''
+            break
         return response
 
     def add_raw_response(self, data: ParsedJson) -> str:
diff --git a/src/portal/py/modules/__init__.py b/src/portal/py/modules/__init__.py
index a9b50eb..480ff3c 100644
--- a/src/portal/py/modules/__init__.py
+++ b/src/portal/py/modules/__init__.py
@@ -5,9 +5,9 @@ from modules.youtube import YoutubeModule
 #from modules.twitter import TwitterModule
 #from modules.pixiv_app import PixivAppModule
 #from modules.pixiv_web import PixivWebModule
-#from modules.fanbox import FanboxModule
+from modules.fanbox import FanboxModule
 #from modules.instagram import InstagramModule
-#from modules.patreon import PatreonModule
+from modules.patreon import PatreonModule
 
 ALL_MODULES = {
     'youtube': (YoutubeModule(), []),
@@ -15,9 +15,9 @@ ALL_MODULES = {
 #    'twitter': (TwitterModule(config.TWITTER_COOKIES_PATH), []),
 #    'pixiv_web': (PixivWebModule(config.PIXIV_SESSID, config.PIXIV_USERID), []),
 #    'pixiv_app': (PixivAppModule(config.PIXIV_REFRESH_TOKEN), []),
-#    'fanbox': (FanboxModule(config.FANBOX_SESSID, config.FANBOX_CF_CLEARANCE), []),
+    'fanbox': (FanboxModule(config.FANBOX_SESSID, config.FANBOX_CF_CLEARANCE), []),
 #    'instagram': (InstagramModule(
 #        config.INSTAGRAM_USER_AGENT, config.INSTAGRAM_SETTINGS_PATH,
 #        config.INSTAGRAM_SESSION_ID), []),
-#    'patreon': (PatreonModule(config.PATREON_SESSION_ID, config.PATREON_UUID, config.PATREON_USER_ID), [])
+    'patreon': (PatreonModule(config.PATREON_SESSION_ID, config.PATREON_UUID, config.PATREON_USER_ID), [])
 }
diff --git a/src/portal/py/modules/common.py b/src/portal/py/modules/common.py
index 3b87884..f4c8b8c 100644
--- a/src/portal/py/modules/common.py
+++ b/src/portal/py/modules/common.py
@@ -1,9 +1,10 @@
 from datetime import datetime, timezone
 
-def reformat_pixiv_date(date_str: str) -> datetime:
+def get_current_utc_time() -> datetime:
+    return datetime.now(timezone.utc)
+
+def parse_pixiv_date(date_str: str) -> datetime:
     rindex = date_str.rfind(':')
     date_str = date_str[:rindex] + date_str[rindex + 1:]
-    return datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S%z")
+    return datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S%z')
 
-def get_current_utc_time() -> datetime:
-    return datetime.now(timezone.utc)
diff --git a/src/portal/py/modules/fanbox.py b/src/portal/py/modules/fanbox.py
index 6e0f188..d50c173 100644
--- a/src/portal/py/modules/fanbox.py
+++ b/src/portal/py/modules/fanbox.py
@@ -1,21 +1,20 @@
 import log
 import json
-import httpx
 from typing import Optional, Any
 from json.decoder import JSONDecodeError
-from base import USER_AGENT, Search, Module, Method, ParsedJson
+from base import USER_AGENT, HttpResponse, Search, Module, Method, ParsedJson
 from post import Url, PostType, Date, DateType, DateMeta, Post, User, Image, File, Tag
 from query_parser import QueryParser
-from modules.common import reformat_pixiv_date, get_current_utc_time
+from modules.common import parse_pixiv_date, get_current_utc_time
 
-BASE_URL = "https://api.fanbox.cc"
+BASE_URL = 'https://api.fanbox.cc'
 
 class FanboxBase(Search):
     def __init__(self, userdata: Any) -> None:
         super().__init__()
         self.module: FanboxModule = userdata
 
-    def check_api_response(self, response: Optional[httpx.Response]) -> Optional[ParsedJson]:
+    def check_api_response(self, response: Optional[HttpResponse]) -> Optional[ParsedJson]:
         if not response:
             return None
         try:
@@ -32,13 +31,13 @@ class FanboxBase(Search):
         kwargs['unique_id'] = unique_id
         kwargs['raw_responses'] = {}
         kwargs['raw_responses']['api'] = json.dumps(data, separators=(',', ':'), indent=None)
-        publish_date = reformat_pixiv_date(data['publishedDatetime']).timestamp()
+        publish_date = parse_pixiv_date(data['publishedDatetime']).timestamp()
         current_date = get_current_utc_time().timestamp()
         kwargs['dates'] = [
             Date(DateType.CREATED, (publish_date, publish_date), DateMeta.NONE),
             Date(DateType.RETRIEVED, (current_date, current_date), DateMeta.NONE)
         ]
-        update_date = reformat_pixiv_date(data['updatedDatetime']).timestamp()
+        update_date = parse_pixiv_date(data['updatedDatetime']).timestamp()
         if update_date != publish_date:
             kwargs['dates'].append(Date(DateType.EDITED, (update_date, update_date), DateMeta.NONE))
         user = User(unique_id=f'fanbox:u:{data['user']['userId']}', username=data['user']['name'])
@@ -81,6 +80,12 @@ class FanboxBase(Search):
         return post
 
     def request_post(self, post_id: int) -> Optional[Post]:
+        '''
+        url = f'{BASE_URL}/post.getComments?postId={post_id}&limit=10'
+        obj = self.check_api_response(self.module.do_request(Method.GET, url))
+        if not obj:
+            return None
+        '''
         url = f'{BASE_URL}/post.info?postId={post_id}'
         obj = self.check_api_response(self.module.do_request(Method.GET, url))
         if not obj:
@@ -124,11 +129,14 @@ class FanboxUser(FanboxBase):
             return False
         for link in obj:
             self.pagination.append(link)
+        if len(self.pagination) == 0:
+            log.warn(f'Found 0 posts for user.')
         return True
 
     def request_page(self, num: int) -> bool:
         if len(self.pagination) == 0:
             if not self.request_pagination():
+                log.error('Failed to load user\'s posts.')
                 self.errored = True
                 return False
         if num >= len(self.pagination):
@@ -169,6 +177,7 @@ class FanboxModule(Module):
         self.parser.add_command('post', FanboxPost)
         self.parser.add_command('user', FanboxUser)
         self.parser.add_command('supporting', FanboxSupporting)
+        # @TODO: At some point verify if tls-client supports zstd.
         self.headers['Accept-Encoding'] = 'gzip, deflate, br, zstd'
         self.headers['Accept-Language'] = 'en-US,en;q=0.5'
         self.headers['Alt-Used'] = 'api.fanbox.cc'
@@ -176,9 +185,9 @@ class FanboxModule(Module):
         self.headers['Referer'] = 'https://www.fanbox.cc/'
         self.headers['User-Agent'] = USER_AGENT
         self.cookies['FANBOXSESSID'] = sessid
+        self.cookies['cf_clearance'] = cf_clearance
         self.cookies['privacy_policy_agreement'] = '7'
         self.cookies['privacy_policy_notification'] = '0'
-        self.cookies['cf_clearance'] = cf_clearance
 
     def search(self, query: str, *extra_args: Any) -> Optional[Search]:
         return self.parser.parse_query(query)
diff --git a/src/portal/py/modules/patreon.py b/src/portal/py/modules/patreon.py
index 9f672c1..6b1d2cb 100644
--- a/src/portal/py/modules/patreon.py
+++ b/src/portal/py/modules/patreon.py
@@ -1,15 +1,17 @@
 import log
-import httpx
 from json import JSONDecodeError
 from typing import Optional, Any
 from datetime import datetime
-from base import USER_AGENT, Search, Module, Method, ParsedJson
+from base import USER_AGENT, HttpResponse, Search, Module, Method, ParsedJson
 from modules.common import get_current_utc_time
 from post import Date, DateType, DateMeta, Post, User, Url, Media, File
 from query_parser import QueryParser
 
 BASE_URL = 'https://www.patreon.com/api'
 
+def parse_patreon_date(date_str: str) -> datetime:
+    return datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S.%f%z')
+
 class PatreonBase(Search):
     REACH_LIMIT = 3
 
@@ -19,7 +21,7 @@ class PatreonBase(Search):
         self.page: int = 0
         self.cursor: Optional[str] = None
 
-    def check_api_response(self, response: Optional[httpx.Response]) -> Optional[ParsedJson]:
+    def check_api_response(self, response: Optional[HttpResponse]) -> Optional[ParsedJson]:
         if not response:
             return None
         try:
@@ -86,14 +88,14 @@ class PatreonUser(PatreonBase):
         kwargs['raw_responses'] = {}
         kwargs['raw_responses']['api'] = hash
         kwargs['url'] = data['attributes']['url']
-        published_at = datetime.strptime(data['attributes']['published_at'], "%Y-%m-%dT%H:%M:%S.%f%z").timestamp()
+        published_at = parse_patreon_date(data['attributes']['published_at']).timestamp()
         current_time = get_current_utc_time().timestamp()
         kwargs['dates'] = [
             Date(DateType.CREATED, (published_at, published_at), DateMeta.NONE),
             Date(DateType.RETRIEVED, (current_time, current_time), DateMeta.NONE)
         ]
         if 'edited_at' in data['attributes']:
-            edited_at = datetime.strptime(data['attributes']['edited_at'], "%Y-%m-%dT%H:%M:%S.%f%z").timestamp()
+            edited_at = parse_patreon_date(data['attributes']['edited_at']).timestamp()
             kwargs['dates'].append(Date(DateType.EDITED, (edited_at, edited_at), DateMeta.NONE))
         kwargs['author'] = self.user
         kwargs['title'] = data['attributes']['title']
diff --git a/src/portal/py/modules/pixiv_app.py b/src/portal/py/modules/pixiv_app.py
index 7bd341b..0b4cd6f 100644
--- a/src/portal/py/modules/pixiv_app.py
+++ b/src/portal/py/modules/pixiv_app.py
@@ -4,7 +4,7 @@ from base import USER_AGENT, Search, Module, ParsedJson
 from post import Url, PostType, Date, DateType, DateMeta, User, Post, Image, Tag, TagType
 from query_parser import QueryParser
 from pixivpy3 import AppPixivAPI, models
-from modules.common import get_current_utc_time, reformat_pixiv_date
+from modules.common import get_current_utc_time, parse_pixiv_date
 
 class PixivAppBase(Search):
     def __init__(self, userdata: Any) -> None:
@@ -63,7 +63,7 @@ class PixivAppBase(Search):
             Date(DateType.RETRIEVED, (retrieved_date, retrieved_date), DateMeta.NONE)
         ]
         if not is_deleted:
-            create_date = reformat_pixiv_date(data['create_date']).timestamp()
+            create_date = parse_pixiv_date(data['create_date']).timestamp()
             kwargs['dates'].append(Date(DateType.CREATED, (create_date, create_date), DateMeta.NONE))
         user = User(unique_id=f'pixiv:u:{data['user']['id']}', username=data['user']['account'], display_name=data['user']['name'])
         if not is_deleted:
@@ -192,10 +192,10 @@ class PixivAppModule(Module):
         self.parser.add_command('following', PixivAppFollowing)
         self.parser.add_command('illust', PixivAppIllust)
         headers = {
-            "app-os": "ios",
-            "app-os-version": "16.7.2",
-            "app-version": "7.19.6",
-            "user-agent": "PixivIOSApp/7.19.6 (iOS 16.7.2; iPhone13,2)"
+            'app-os': 'ios',
+            'app-os-version': '16.7.2',
+            'app-version': '7.19.6',
+            'user-agent': 'PixivIOSApp/7.19.6 (iOS 16.7.2; iPhone13,2)'
         }
         self.api: AppPixivAPI = AppPixivAPI(headers=headers)
         self.api.auth(refresh_token=refresh_token)
diff --git a/src/portal/py/modules/pixiv_web.py b/src/portal/py/modules/pixiv_web.py
index 43d017f..9150c71 100644
--- a/src/portal/py/modules/pixiv_web.py
+++ b/src/portal/py/modules/pixiv_web.py
@@ -1,13 +1,12 @@
 import log
 import json
-import httpx
 import urllib.parse
 from typing import Optional, Any
 from json.decoder import JSONDecodeError
-from base import USER_AGENT, Search, Module, Method, ParsedJson
+from base import USER_AGENT, HttpResponse, Search, Module, Method, ParsedJson
 from post import Url, PostType, Date, DateType, DateMeta, Post, User, Media, Image, Animation, Tag, TagType
 from query_parser import QueryParser
-from modules.common import reformat_pixiv_date, get_current_utc_time
+from modules.common import parse_pixiv_date, get_current_utc_time
 
 BASE_URL = 'https://www.pixiv.net/ajax'
 LANG = 'en'
@@ -18,7 +17,7 @@ class PixivWebBase(Search):
         super().__init__()
         self.module: PixivWebModule = userdata
 
-    def check_api_response(self, response: Optional[httpx.Response]) -> Optional[ParsedJson]:
+    def check_api_response(self, response: Optional[HttpResponse]) -> Optional[ParsedJson]:
         if not response:
             return None
         try:
@@ -72,14 +71,14 @@ class PixivWebBase(Search):
         kwargs['raw_responses'] = {}
         kwargs['raw_responses']['api'] = json.dumps(data, separators=(',', ':'), indent=None)
         kwargs['url'] = f'https://www.pixiv.net/{LANG}/artworks/{data['illustId']}'
-        create_date = reformat_pixiv_date(data['createDate']).timestamp()
+        create_date = parse_pixiv_date(data['createDate']).timestamp()
         if not retrieved_date:
             retrieved_date = get_current_utc_time().timestamp()
         kwargs['dates'] = [
             Date(DateType.CREATED, (create_date, create_date), DateMeta.NONE),
             Date(DateType.RETRIEVED, (retrieved_date, retrieved_date), DateMeta.NONE)
         ]
-        upload_date = reformat_pixiv_date(data['uploadDate']).timestamp()
+        upload_date = parse_pixiv_date(data['uploadDate']).timestamp()
         if upload_date != create_date:
             kwargs['dates'].append(Date(DateType.EDITED, (upload_date, upload_date), DateMeta.NONE))
         user = User(unique_id=f'pixiv:u:{data['userId']}', username=data['userAccount'], display_name=data['userName'])
diff --git a/src/portal/py/modules/twitter.py b/src/portal/py/modules/twitter.py
index 79e7f31..be81c0f 100644
--- a/src/portal/py/modules/twitter.py
+++ b/src/portal/py/modules/twitter.py
@@ -1,8 +1,7 @@
 import json
-import httpx
 import http.cookiejar
 from typing import Optional, Any, Iterator
-from base import USER_AGENT, Search, Module, ParsedJson, Method
+from base import USER_AGENT, HttpResponse, Search, Module, ParsedJson, Method
 from post import Post
 from query_parser import QueryParser
 from modules.twitter_common import parse_graphql_raw_response
@@ -21,7 +20,7 @@ class TwitterUser(TwitterBase):
         super().__init__(userdata)
         self.user_id = ''
 
-    def check_api_response(self, response: Optional[httpx.Response]) -> Optional[ParsedJson]:
+    def check_api_response(self, response: Optional[HttpResponse]) -> Optional[ParsedJson]:
         if not response:
             return None
         try:
@@ -85,7 +84,7 @@ class TwitterUser(TwitterBase):
             'features': build_params(features),
             'fieldToggles': build_params(fieldToggles)
         }
-        url = "https://x.com/i/api/graphql/DdW3L2nfGW6FZDcSSSG4yg/UserMedia"
+        url = 'https://x.com/i/api/graphql/DdW3L2nfGW6FZDcSSSG4yg/UserMedia'
         return self.check_api_response(self.module.do_request(Method.GET, url, params=params))
 
     #print(json.dumps(media, indent=4))
diff --git a/src/portal/py/tests/archive_query.py b/src/portal/py/tests/archive_query.py
index 8ecbb52..f944ca0 100644
--- a/src/portal/py/tests/archive_query.py
+++ b/src/portal/py/tests/archive_query.py
@@ -24,8 +24,8 @@ class Logger():
     def close(self):
         self.file.close()
 
-mode = 'pixiv_web'
-#mode = 'fanbox'
+#mode = 'pixiv_web'
+mode = 'fanbox'
 #mode = 'patreon'
 #mode = 'instagram'
 #mode = 'twitter'
@@ -42,6 +42,18 @@ queue_mutex = threading.Lock()
 queue_cond = threading.Condition(queue_mutex)
 is_running = True
 
+def media_check_disk(obj, post, output_base):
+    for key in post.media.keys():
+        download_params = module.get_download(post.unique_id, key)
+        if not download_params:
+            obj.log.write(f'Ignoring media{key} for post {post.unique_id}.')
+            continue
+        for url in download_params['urls']:
+            media_path = f'{output_base}_media{key}.{url.ext}'
+            if not os.path.isfile(media_path):
+                return False
+    return True
+
 def default_media_download(obj, post, output_base):
     for key in post.media.keys():
         download_params = module.get_download(post.unique_id, key)
@@ -133,7 +145,7 @@ class QueryDownloadThread(threading.Thread):
             self.log.write('Rewriting post with quote.')
             has_mp4 = True
         '''
-        if os.path.isfile(output):
+        if os.path.isfile(output) and media_check_disk(self, post, output_base):
             skip_message = f'Skipping already downloaded post {post.unique_id}'
             if post.type != PostType.TOMBSTONE:
                 skip_message += f' from {post.author.unique_id}.'
@@ -233,6 +245,7 @@ def download_args():
         os.mkdir(f'{output_dir}/raw_responses')
 
     args = []
+    start_page = 0
     threads = []
     start = True
 
@@ -265,7 +278,7 @@ def download_args():
             if not is_running:
                 break
         log.write(f'***********************\nStarting download of {arg}\n***********************\n')
-        arg_thread = QueryDownloadThread(log, complete, arg, output_dir, 0)
+        arg_thread = QueryDownloadThread(log, complete, arg, output_dir, start_page)
         threads.append(arg_thread)
         arg_thread.start()
 
diff --git a/src/portal/requirements.txt b/src/portal/requirements.txt
index c2e2915..b084ba0 100644
--- a/src/portal/requirements.txt
+++ b/src/portal/requirements.txt
@@ -1,3 +1,2 @@
-httpx[http2,brotli,zstd]
-blake3
-pillow
+tls-client2==1.1
+blake3==1.0.8
diff --git a/src/portal/scripts/create_vendor.sh b/src/portal/scripts/create_vendor.sh
index 6e5aaef..5daa3a3 100755
--- a/src/portal/scripts/create_vendor.sh
+++ b/src/portal/scripts/create_vendor.sh
@@ -16,25 +16,25 @@ cd yt-dlp/
 pip3 install . --upgrade --target=../vendor
 cd ../
 
-cd snscrape/
+cd gallery-dl/
 pip3 install . --upgrade --target=../vendor
 cd ../
 
-cd twitter-api-client/
-pip3 install . --upgrade --target=../vendor
-cd ../
+#cd snscrape/
+#pip3 install . --upgrade --target=../vendor
+#cd ../
 
-cd pixivpy/
-pip3 install . --upgrade --target=../vendor
-cd ../
+#cd twitter-api-client/
+#pip3 install . --upgrade --target=../vendor
+#cd ../
 
-cd instagrapi/
-pip3 install . --upgrade --target=../vendor
-cd ../
+#cd pixivpy/
+#pip3 install . --upgrade --target=../vendor
+#cd ../
 
-cd gallery-dl/
-pip3 install . --upgrade --target=../vendor
-cd ../
+#cd instagrapi/
+#pip3 install . --upgrade --target=../vendor
+#cd ../
 
 cd ../
 rm -r venv