Update portal to avoid TLS fingerprinting

Signed-off-by: Andrew Opalach <andrew@akon.city>

7 files changed, 47 insertions, 37 deletions

diff --git a/src/portal/py/modules/__init__.py b/src/portal/py/modules/__init__.py
index a9b50eb..480ff3c 100644
--- a/src/portal/py/modules/__init__.py
+++ b/src/portal/py/modules/__init__.py
@@ -5,9 +5,9 @@ from modules.youtube import YoutubeModule
 #from modules.twitter import TwitterModule
 #from modules.pixiv_app import PixivAppModule
 #from modules.pixiv_web import PixivWebModule
-#from modules.fanbox import FanboxModule
+from modules.fanbox import FanboxModule
 #from modules.instagram import InstagramModule
-#from modules.patreon import PatreonModule
+from modules.patreon import PatreonModule
 
 ALL_MODULES = {
     'youtube': (YoutubeModule(), []),
@@ -15,9 +15,9 @@ ALL_MODULES = {
 #    'twitter': (TwitterModule(config.TWITTER_COOKIES_PATH), []),
 #    'pixiv_web': (PixivWebModule(config.PIXIV_SESSID, config.PIXIV_USERID), []),
 #    'pixiv_app': (PixivAppModule(config.PIXIV_REFRESH_TOKEN), []),
-#    'fanbox': (FanboxModule(config.FANBOX_SESSID, config.FANBOX_CF_CLEARANCE), []),
+    'fanbox': (FanboxModule(config.FANBOX_SESSID, config.FANBOX_CF_CLEARANCE), []),
 #    'instagram': (InstagramModule(
 #        config.INSTAGRAM_USER_AGENT, config.INSTAGRAM_SETTINGS_PATH,
 #        config.INSTAGRAM_SESSION_ID), []),
-#    'patreon': (PatreonModule(config.PATREON_SESSION_ID, config.PATREON_UUID, config.PATREON_USER_ID), [])
+    'patreon': (PatreonModule(config.PATREON_SESSION_ID, config.PATREON_UUID, config.PATREON_USER_ID), [])
 }
diff --git a/src/portal/py/modules/common.py b/src/portal/py/modules/common.py
index 3b87884..f4c8b8c 100644
--- a/src/portal/py/modules/common.py
+++ b/src/portal/py/modules/common.py
@@ -1,9 +1,10 @@
 from datetime import datetime, timezone
 
-def reformat_pixiv_date(date_str: str) -> datetime:
+def get_current_utc_time() -> datetime:
+    return datetime.now(timezone.utc)
+
+def parse_pixiv_date(date_str: str) -> datetime:
     rindex = date_str.rfind(':')
     date_str = date_str[:rindex] + date_str[rindex + 1:]
-    return datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S%z")
+    return datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S%z')
 
-def get_current_utc_time() -> datetime:
-    return datetime.now(timezone.utc)
diff --git a/src/portal/py/modules/fanbox.py b/src/portal/py/modules/fanbox.py
index 6e0f188..d50c173 100644
--- a/src/portal/py/modules/fanbox.py
+++ b/src/portal/py/modules/fanbox.py
@@ -1,21 +1,20 @@
 import log
 import json
-import httpx
 from typing import Optional, Any
 from json.decoder import JSONDecodeError
-from base import USER_AGENT, Search, Module, Method, ParsedJson
+from base import USER_AGENT, HttpResponse, Search, Module, Method, ParsedJson
 from post import Url, PostType, Date, DateType, DateMeta, Post, User, Image, File, Tag
 from query_parser import QueryParser
-from modules.common import reformat_pixiv_date, get_current_utc_time
+from modules.common import parse_pixiv_date, get_current_utc_time
 
-BASE_URL = "https://api.fanbox.cc"
+BASE_URL = 'https://api.fanbox.cc'
 
 class FanboxBase(Search):
     def __init__(self, userdata: Any) -> None:
         super().__init__()
         self.module: FanboxModule = userdata
 
-    def check_api_response(self, response: Optional[httpx.Response]) -> Optional[ParsedJson]:
+    def check_api_response(self, response: Optional[HttpResponse]) -> Optional[ParsedJson]:
         if not response:
             return None
         try:
@@ -32,13 +31,13 @@ class FanboxBase(Search):
         kwargs['unique_id'] = unique_id
         kwargs['raw_responses'] = {}
         kwargs['raw_responses']['api'] = json.dumps(data, separators=(',', ':'), indent=None)
-        publish_date = reformat_pixiv_date(data['publishedDatetime']).timestamp()
+        publish_date = parse_pixiv_date(data['publishedDatetime']).timestamp()
         current_date = get_current_utc_time().timestamp()
         kwargs['dates'] = [
             Date(DateType.CREATED, (publish_date, publish_date), DateMeta.NONE),
             Date(DateType.RETRIEVED, (current_date, current_date), DateMeta.NONE)
         ]
-        update_date = reformat_pixiv_date(data['updatedDatetime']).timestamp()
+        update_date = parse_pixiv_date(data['updatedDatetime']).timestamp()
         if update_date != publish_date:
             kwargs['dates'].append(Date(DateType.EDITED, (update_date, update_date), DateMeta.NONE))
         user = User(unique_id=f'fanbox:u:{data['user']['userId']}', username=data['user']['name'])
@@ -81,6 +80,12 @@ class FanboxBase(Search):
         return post
 
     def request_post(self, post_id: int) -> Optional[Post]:
+        '''
+        url = f'{BASE_URL}/post.getComments?postId={post_id}&limit=10'
+        obj = self.check_api_response(self.module.do_request(Method.GET, url))
+        if not obj:
+            return None
+        '''
         url = f'{BASE_URL}/post.info?postId={post_id}'
         obj = self.check_api_response(self.module.do_request(Method.GET, url))
         if not obj:
@@ -124,11 +129,14 @@ class FanboxUser(FanboxBase):
             return False
         for link in obj:
             self.pagination.append(link)
+        if len(self.pagination) == 0:
+            log.warn(f'Found 0 posts for user.')
         return True
 
     def request_page(self, num: int) -> bool:
         if len(self.pagination) == 0:
             if not self.request_pagination():
+                log.error('Failed to load user\'s posts.')
                 self.errored = True
                 return False
         if num >= len(self.pagination):
@@ -169,6 +177,7 @@ class FanboxModule(Module):
         self.parser.add_command('post', FanboxPost)
         self.parser.add_command('user', FanboxUser)
         self.parser.add_command('supporting', FanboxSupporting)
+        # @TODO: At some point verify if tls-client supports zstd.
         self.headers['Accept-Encoding'] = 'gzip, deflate, br, zstd'
         self.headers['Accept-Language'] = 'en-US,en;q=0.5'
         self.headers['Alt-Used'] = 'api.fanbox.cc'
@@ -176,9 +185,9 @@ class FanboxModule(Module):
         self.headers['Referer'] = 'https://www.fanbox.cc/'
         self.headers['User-Agent'] = USER_AGENT
         self.cookies['FANBOXSESSID'] = sessid
+        self.cookies['cf_clearance'] = cf_clearance
         self.cookies['privacy_policy_agreement'] = '7'
         self.cookies['privacy_policy_notification'] = '0'
-        self.cookies['cf_clearance'] = cf_clearance
 
     def search(self, query: str, *extra_args: Any) -> Optional[Search]:
         return self.parser.parse_query(query)
diff --git a/src/portal/py/modules/patreon.py b/src/portal/py/modules/patreon.py
index 9f672c1..6b1d2cb 100644
--- a/src/portal/py/modules/patreon.py
+++ b/src/portal/py/modules/patreon.py
@@ -1,15 +1,17 @@
 import log
-import httpx
 from json import JSONDecodeError
 from typing import Optional, Any
 from datetime import datetime
-from base import USER_AGENT, Search, Module, Method, ParsedJson
+from base import USER_AGENT, HttpResponse, Search, Module, Method, ParsedJson
 from modules.common import get_current_utc_time
 from post import Date, DateType, DateMeta, Post, User, Url, Media, File
 from query_parser import QueryParser
 
 BASE_URL = 'https://www.patreon.com/api'
 
+def parse_patreon_date(date_str: str) -> datetime:
+    return datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S.%f%z')
+
 class PatreonBase(Search):
     REACH_LIMIT = 3
 
@@ -19,7 +21,7 @@ class PatreonBase(Search):
         self.page: int = 0
         self.cursor: Optional[str] = None
 
-    def check_api_response(self, response: Optional[httpx.Response]) -> Optional[ParsedJson]:
+    def check_api_response(self, response: Optional[HttpResponse]) -> Optional[ParsedJson]:
         if not response:
             return None
         try:
@@ -86,14 +88,14 @@ class PatreonUser(PatreonBase):
         kwargs['raw_responses'] = {}
         kwargs['raw_responses']['api'] = hash
         kwargs['url'] = data['attributes']['url']
-        published_at = datetime.strptime(data['attributes']['published_at'], "%Y-%m-%dT%H:%M:%S.%f%z").timestamp()
+        published_at = parse_patreon_date(data['attributes']['published_at']).timestamp()
         current_time = get_current_utc_time().timestamp()
         kwargs['dates'] = [
             Date(DateType.CREATED, (published_at, published_at), DateMeta.NONE),
             Date(DateType.RETRIEVED, (current_time, current_time), DateMeta.NONE)
         ]
         if 'edited_at' in data['attributes']:
-            edited_at = datetime.strptime(data['attributes']['edited_at'], "%Y-%m-%dT%H:%M:%S.%f%z").timestamp()
+            edited_at = parse_patreon_date(data['attributes']['edited_at']).timestamp()
             kwargs['dates'].append(Date(DateType.EDITED, (edited_at, edited_at), DateMeta.NONE))
         kwargs['author'] = self.user
         kwargs['title'] = data['attributes']['title']
diff --git a/src/portal/py/modules/pixiv_app.py b/src/portal/py/modules/pixiv_app.py
index 7bd341b..0b4cd6f 100644
--- a/src/portal/py/modules/pixiv_app.py
+++ b/src/portal/py/modules/pixiv_app.py
@@ -4,7 +4,7 @@ from base import USER_AGENT, Search, Module, ParsedJson
 from post import Url, PostType, Date, DateType, DateMeta, User, Post, Image, Tag, TagType
 from query_parser import QueryParser
 from pixivpy3 import AppPixivAPI, models
-from modules.common import get_current_utc_time, reformat_pixiv_date
+from modules.common import get_current_utc_time, parse_pixiv_date
 
 class PixivAppBase(Search):
     def __init__(self, userdata: Any) -> None:
@@ -63,7 +63,7 @@ class PixivAppBase(Search):
             Date(DateType.RETRIEVED, (retrieved_date, retrieved_date), DateMeta.NONE)
         ]
         if not is_deleted:
-            create_date = reformat_pixiv_date(data['create_date']).timestamp()
+            create_date = parse_pixiv_date(data['create_date']).timestamp()
             kwargs['dates'].append(Date(DateType.CREATED, (create_date, create_date), DateMeta.NONE))
         user = User(unique_id=f'pixiv:u:{data['user']['id']}', username=data['user']['account'], display_name=data['user']['name'])
         if not is_deleted:
@@ -192,10 +192,10 @@ class PixivAppModule(Module):
         self.parser.add_command('following', PixivAppFollowing)
         self.parser.add_command('illust', PixivAppIllust)
         headers = {
-            "app-os": "ios",
-            "app-os-version": "16.7.2",
-            "app-version": "7.19.6",
-            "user-agent": "PixivIOSApp/7.19.6 (iOS 16.7.2; iPhone13,2)"
+            'app-os': 'ios',
+            'app-os-version': '16.7.2',
+            'app-version': '7.19.6',
+            'user-agent': 'PixivIOSApp/7.19.6 (iOS 16.7.2; iPhone13,2)'
         }
         self.api: AppPixivAPI = AppPixivAPI(headers=headers)
         self.api.auth(refresh_token=refresh_token)
diff --git a/src/portal/py/modules/pixiv_web.py b/src/portal/py/modules/pixiv_web.py
index 43d017f..9150c71 100644
--- a/src/portal/py/modules/pixiv_web.py
+++ b/src/portal/py/modules/pixiv_web.py
@@ -1,13 +1,12 @@
 import log
 import json
-import httpx
 import urllib.parse
 from typing import Optional, Any
 from json.decoder import JSONDecodeError
-from base import USER_AGENT, Search, Module, Method, ParsedJson
+from base import USER_AGENT, HttpResponse, Search, Module, Method, ParsedJson
 from post import Url, PostType, Date, DateType, DateMeta, Post, User, Media, Image, Animation, Tag, TagType
 from query_parser import QueryParser
-from modules.common import reformat_pixiv_date, get_current_utc_time
+from modules.common import parse_pixiv_date, get_current_utc_time
 
 BASE_URL = 'https://www.pixiv.net/ajax'
 LANG = 'en'
@@ -18,7 +17,7 @@ class PixivWebBase(Search):
         super().__init__()
         self.module: PixivWebModule = userdata
 
-    def check_api_response(self, response: Optional[httpx.Response]) -> Optional[ParsedJson]:
+    def check_api_response(self, response: Optional[HttpResponse]) -> Optional[ParsedJson]:
         if not response:
             return None
         try:
@@ -72,14 +71,14 @@ class PixivWebBase(Search):
         kwargs['raw_responses'] = {}
         kwargs['raw_responses']['api'] = json.dumps(data, separators=(',', ':'), indent=None)
         kwargs['url'] = f'https://www.pixiv.net/{LANG}/artworks/{data['illustId']}'
-        create_date = reformat_pixiv_date(data['createDate']).timestamp()
+        create_date = parse_pixiv_date(data['createDate']).timestamp()
         if not retrieved_date:
             retrieved_date = get_current_utc_time().timestamp()
         kwargs['dates'] = [
             Date(DateType.CREATED, (create_date, create_date), DateMeta.NONE),
             Date(DateType.RETRIEVED, (retrieved_date, retrieved_date), DateMeta.NONE)
         ]
-        upload_date = reformat_pixiv_date(data['uploadDate']).timestamp()
+        upload_date = parse_pixiv_date(data['uploadDate']).timestamp()
         if upload_date != create_date:
             kwargs['dates'].append(Date(DateType.EDITED, (upload_date, upload_date), DateMeta.NONE))
         user = User(unique_id=f'pixiv:u:{data['userId']}', username=data['userAccount'], display_name=data['userName'])
diff --git a/src/portal/py/modules/twitter.py b/src/portal/py/modules/twitter.py
index 79e7f31..be81c0f 100644
--- a/src/portal/py/modules/twitter.py
+++ b/src/portal/py/modules/twitter.py
@@ -1,8 +1,7 @@
 import json
-import httpx
 import http.cookiejar
 from typing import Optional, Any, Iterator
-from base import USER_AGENT, Search, Module, ParsedJson, Method
+from base import USER_AGENT, HttpResponse, Search, Module, ParsedJson, Method
 from post import Post
 from query_parser import QueryParser
 from modules.twitter_common import parse_graphql_raw_response
@@ -21,7 +20,7 @@ class TwitterUser(TwitterBase):
         super().__init__(userdata)
         self.user_id = ''
 
-    def check_api_response(self, response: Optional[httpx.Response]) -> Optional[ParsedJson]:
+    def check_api_response(self, response: Optional[HttpResponse]) -> Optional[ParsedJson]:
         if not response:
             return None
         try:
@@ -85,7 +84,7 @@ class TwitterUser(TwitterBase):
             'features': build_params(features),
             'fieldToggles': build_params(fieldToggles)
         }
-        url = "https://x.com/i/api/graphql/DdW3L2nfGW6FZDcSSSG4yg/UserMedia"
+        url = 'https://x.com/i/api/graphql/DdW3L2nfGW6FZDcSSSG4yg/UserMedia'
         return self.check_api_response(self.module.do_request(Method.GET, url, params=params))
 
     #print(json.dumps(media, indent=4))