summaryrefslogtreecommitdiff
path: root/src/portal/py
diff options
context:
space:
mode:
Diffstat (limited to 'src/portal/py')
-rw-r--r--src/portal/py/base.py32
-rw-r--r--src/portal/py/modules/__init__.py8
-rw-r--r--src/portal/py/modules/common.py9
-rw-r--r--src/portal/py/modules/fanbox.py25
-rw-r--r--src/portal/py/modules/patreon.py12
-rw-r--r--src/portal/py/modules/pixiv_app.py12
-rw-r--r--src/portal/py/modules/pixiv_web.py11
-rw-r--r--src/portal/py/modules/twitter.py7
-rw-r--r--src/portal/py/tests/archive_query.py21
9 files changed, 83 insertions, 54 deletions
diff --git a/src/portal/py/base.py b/src/portal/py/base.py
index bd68925..a310cf5 100644
--- a/src/portal/py/base.py
+++ b/src/portal/py/base.py
@@ -1,7 +1,7 @@
import log
import time
import json
-import httpx
+import tls_client
import binascii
import threading
import config
@@ -12,13 +12,14 @@ from typing import Optional, Self, Any
from post import Post, User
type ParsedJson = Any
+type HttpResponse = tls_client.Response
USER_AGENT = config.USER_AGENT
class Method(Enum):
- HEAD = "HEAD"
- GET = "GET"
- POST = "POST"
+ HEAD = 'HEAD'
+ GET = 'GET'
+ POST = 'POST'
class Search():
def __init__(self) -> None:
@@ -66,27 +67,29 @@ class Module():
self.mutex: threading.Lock = threading.Lock()
self.raw_responses: dict[str, str] = {}
self.unique_id_map: OrderedDict = OrderedDict()
- self.session: httpx.Client = httpx.Client(follow_redirects=True, timeout=20.0, http2=False)
+ self.session = tls_client.Session(client_identifier=config.TLS_CLIENT_IDENTIFIER, random_tls_extension_order=True)
def init(self) -> bool:
return True
- def do_request(self, method: Method, url: str, params: Optional[dict[str, str]] = None, retries: int = 2) -> Optional[httpx.Response]:
+ def do_request(self, method: Method, url: str, params: Optional[dict[str, str]] = None, retries: int = 2) -> Optional[HttpResponse]:
log.debug(f'HTTP {method.value} request {url} {params}.')
- response: Optional[httpx.Response] = None
+ response: Optional[HttpResponse] = None
for _ in range(0, retries + 1):
try:
- response = self.session.request(method.value, url, headers=self.headers, cookies=self.cookies, params=params)
+ response = self.session.execute_request(method.value, url, headers=self.headers, cookies=self.cookies, params=params, timeout=600)
except Exception as e:
- log.info(f'Connection exception ({e}), retrying in 20 seconds...')
+ log.info(f'Connection exception ({repr(e)}), retrying in 20 seconds...')
time.sleep(20)
continue
- if not response:
- continue
+ if response is None:
+ log.error('Invalid None response from requests library.')
+ break
if response.status_code == 429:
- log.info('Got too many requests, waiting for 3 minutes...')
- time.sleep(60 * 3)
+ log.info('Got too many requests, waiting for 4 minutes...')
+ time.sleep(60 * 4)
response = None
+ retries += 1
continue
if response.status_code == 404 or response.status_code == 403: # Shortcut 404, 403
log.warn(f'Returning None, {response.status_code}')
@@ -96,6 +99,7 @@ class Module():
time.sleep(5)
response = None
continue
+ '''
try:
response.read()
except Exception as e:
@@ -105,6 +109,8 @@ class Module():
continue
else:
break
+ '''
+ break
return response
def add_raw_response(self, data: ParsedJson) -> str:
diff --git a/src/portal/py/modules/__init__.py b/src/portal/py/modules/__init__.py
index a9b50eb..480ff3c 100644
--- a/src/portal/py/modules/__init__.py
+++ b/src/portal/py/modules/__init__.py
@@ -5,9 +5,9 @@ from modules.youtube import YoutubeModule
#from modules.twitter import TwitterModule
#from modules.pixiv_app import PixivAppModule
#from modules.pixiv_web import PixivWebModule
-#from modules.fanbox import FanboxModule
+from modules.fanbox import FanboxModule
#from modules.instagram import InstagramModule
-#from modules.patreon import PatreonModule
+from modules.patreon import PatreonModule
ALL_MODULES = {
'youtube': (YoutubeModule(), []),
@@ -15,9 +15,9 @@ ALL_MODULES = {
# 'twitter': (TwitterModule(config.TWITTER_COOKIES_PATH), []),
# 'pixiv_web': (PixivWebModule(config.PIXIV_SESSID, config.PIXIV_USERID), []),
# 'pixiv_app': (PixivAppModule(config.PIXIV_REFRESH_TOKEN), []),
-# 'fanbox': (FanboxModule(config.FANBOX_SESSID, config.FANBOX_CF_CLEARANCE), []),
+ 'fanbox': (FanboxModule(config.FANBOX_SESSID, config.FANBOX_CF_CLEARANCE), []),
# 'instagram': (InstagramModule(
# config.INSTAGRAM_USER_AGENT, config.INSTAGRAM_SETTINGS_PATH,
# config.INSTAGRAM_SESSION_ID), []),
-# 'patreon': (PatreonModule(config.PATREON_SESSION_ID, config.PATREON_UUID, config.PATREON_USER_ID), [])
+ 'patreon': (PatreonModule(config.PATREON_SESSION_ID, config.PATREON_UUID, config.PATREON_USER_ID), [])
}
diff --git a/src/portal/py/modules/common.py b/src/portal/py/modules/common.py
index 3b87884..f4c8b8c 100644
--- a/src/portal/py/modules/common.py
+++ b/src/portal/py/modules/common.py
@@ -1,9 +1,10 @@
from datetime import datetime, timezone
-def reformat_pixiv_date(date_str: str) -> datetime:
+def get_current_utc_time() -> datetime:
+ return datetime.now(timezone.utc)
+
+def parse_pixiv_date(date_str: str) -> datetime:
rindex = date_str.rfind(':')
date_str = date_str[:rindex] + date_str[rindex + 1:]
- return datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S%z")
+ return datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S%z')
-def get_current_utc_time() -> datetime:
- return datetime.now(timezone.utc)
diff --git a/src/portal/py/modules/fanbox.py b/src/portal/py/modules/fanbox.py
index 6e0f188..d50c173 100644
--- a/src/portal/py/modules/fanbox.py
+++ b/src/portal/py/modules/fanbox.py
@@ -1,21 +1,20 @@
import log
import json
-import httpx
from typing import Optional, Any
from json.decoder import JSONDecodeError
-from base import USER_AGENT, Search, Module, Method, ParsedJson
+from base import USER_AGENT, HttpResponse, Search, Module, Method, ParsedJson
from post import Url, PostType, Date, DateType, DateMeta, Post, User, Image, File, Tag
from query_parser import QueryParser
-from modules.common import reformat_pixiv_date, get_current_utc_time
+from modules.common import parse_pixiv_date, get_current_utc_time
-BASE_URL = "https://api.fanbox.cc"
+BASE_URL = 'https://api.fanbox.cc'
class FanboxBase(Search):
def __init__(self, userdata: Any) -> None:
super().__init__()
self.module: FanboxModule = userdata
- def check_api_response(self, response: Optional[httpx.Response]) -> Optional[ParsedJson]:
+ def check_api_response(self, response: Optional[HttpResponse]) -> Optional[ParsedJson]:
if not response:
return None
try:
@@ -32,13 +31,13 @@ class FanboxBase(Search):
kwargs['unique_id'] = unique_id
kwargs['raw_responses'] = {}
kwargs['raw_responses']['api'] = json.dumps(data, separators=(',', ':'), indent=None)
- publish_date = reformat_pixiv_date(data['publishedDatetime']).timestamp()
+ publish_date = parse_pixiv_date(data['publishedDatetime']).timestamp()
current_date = get_current_utc_time().timestamp()
kwargs['dates'] = [
Date(DateType.CREATED, (publish_date, publish_date), DateMeta.NONE),
Date(DateType.RETRIEVED, (current_date, current_date), DateMeta.NONE)
]
- update_date = reformat_pixiv_date(data['updatedDatetime']).timestamp()
+ update_date = parse_pixiv_date(data['updatedDatetime']).timestamp()
if update_date != publish_date:
kwargs['dates'].append(Date(DateType.EDITED, (update_date, update_date), DateMeta.NONE))
user = User(unique_id=f'fanbox:u:{data['user']['userId']}', username=data['user']['name'])
@@ -81,6 +80,12 @@ class FanboxBase(Search):
return post
def request_post(self, post_id: int) -> Optional[Post]:
+ '''
+ url = f'{BASE_URL}/post.getComments?postId={post_id}&limit=10'
+ obj = self.check_api_response(self.module.do_request(Method.GET, url))
+ if not obj:
+ return None
+ '''
url = f'{BASE_URL}/post.info?postId={post_id}'
obj = self.check_api_response(self.module.do_request(Method.GET, url))
if not obj:
@@ -124,11 +129,14 @@ class FanboxUser(FanboxBase):
return False
for link in obj:
self.pagination.append(link)
+ if len(self.pagination) == 0:
+ log.warn(f'Found 0 posts for user.')
return True
def request_page(self, num: int) -> bool:
if len(self.pagination) == 0:
if not self.request_pagination():
+ log.error('Failed to load user\'s posts.')
self.errored = True
return False
if num >= len(self.pagination):
@@ -169,6 +177,7 @@ class FanboxModule(Module):
self.parser.add_command('post', FanboxPost)
self.parser.add_command('user', FanboxUser)
self.parser.add_command('supporting', FanboxSupporting)
+ # @TODO: At some point verify if tls-client supports zstd.
self.headers['Accept-Encoding'] = 'gzip, deflate, br, zstd'
self.headers['Accept-Language'] = 'en-US,en;q=0.5'
self.headers['Alt-Used'] = 'api.fanbox.cc'
@@ -176,9 +185,9 @@ class FanboxModule(Module):
self.headers['Referer'] = 'https://www.fanbox.cc/'
self.headers['User-Agent'] = USER_AGENT
self.cookies['FANBOXSESSID'] = sessid
+ self.cookies['cf_clearance'] = cf_clearance
self.cookies['privacy_policy_agreement'] = '7'
self.cookies['privacy_policy_notification'] = '0'
- self.cookies['cf_clearance'] = cf_clearance
def search(self, query: str, *extra_args: Any) -> Optional[Search]:
return self.parser.parse_query(query)
diff --git a/src/portal/py/modules/patreon.py b/src/portal/py/modules/patreon.py
index 9f672c1..6b1d2cb 100644
--- a/src/portal/py/modules/patreon.py
+++ b/src/portal/py/modules/patreon.py
@@ -1,15 +1,17 @@
import log
-import httpx
from json import JSONDecodeError
from typing import Optional, Any
from datetime import datetime
-from base import USER_AGENT, Search, Module, Method, ParsedJson
+from base import USER_AGENT, HttpResponse, Search, Module, Method, ParsedJson
from modules.common import get_current_utc_time
from post import Date, DateType, DateMeta, Post, User, Url, Media, File
from query_parser import QueryParser
BASE_URL = 'https://www.patreon.com/api'
+def parse_patreon_date(date_str: str) -> datetime:
+ return datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S.%f%z')
+
class PatreonBase(Search):
REACH_LIMIT = 3
@@ -19,7 +21,7 @@ class PatreonBase(Search):
self.page: int = 0
self.cursor: Optional[str] = None
- def check_api_response(self, response: Optional[httpx.Response]) -> Optional[ParsedJson]:
+ def check_api_response(self, response: Optional[HttpResponse]) -> Optional[ParsedJson]:
if not response:
return None
try:
@@ -86,14 +88,14 @@ class PatreonUser(PatreonBase):
kwargs['raw_responses'] = {}
kwargs['raw_responses']['api'] = hash
kwargs['url'] = data['attributes']['url']
- published_at = datetime.strptime(data['attributes']['published_at'], "%Y-%m-%dT%H:%M:%S.%f%z").timestamp()
+ published_at = parse_patreon_date(data['attributes']['published_at']).timestamp()
current_time = get_current_utc_time().timestamp()
kwargs['dates'] = [
Date(DateType.CREATED, (published_at, published_at), DateMeta.NONE),
Date(DateType.RETRIEVED, (current_time, current_time), DateMeta.NONE)
]
if 'edited_at' in data['attributes']:
- edited_at = datetime.strptime(data['attributes']['edited_at'], "%Y-%m-%dT%H:%M:%S.%f%z").timestamp()
+ edited_at = parse_patreon_date(data['attributes']['edited_at']).timestamp()
kwargs['dates'].append(Date(DateType.EDITED, (edited_at, edited_at), DateMeta.NONE))
kwargs['author'] = self.user
kwargs['title'] = data['attributes']['title']
diff --git a/src/portal/py/modules/pixiv_app.py b/src/portal/py/modules/pixiv_app.py
index 7bd341b..0b4cd6f 100644
--- a/src/portal/py/modules/pixiv_app.py
+++ b/src/portal/py/modules/pixiv_app.py
@@ -4,7 +4,7 @@ from base import USER_AGENT, Search, Module, ParsedJson
from post import Url, PostType, Date, DateType, DateMeta, User, Post, Image, Tag, TagType
from query_parser import QueryParser
from pixivpy3 import AppPixivAPI, models
-from modules.common import get_current_utc_time, reformat_pixiv_date
+from modules.common import get_current_utc_time, parse_pixiv_date
class PixivAppBase(Search):
def __init__(self, userdata: Any) -> None:
@@ -63,7 +63,7 @@ class PixivAppBase(Search):
Date(DateType.RETRIEVED, (retrieved_date, retrieved_date), DateMeta.NONE)
]
if not is_deleted:
- create_date = reformat_pixiv_date(data['create_date']).timestamp()
+ create_date = parse_pixiv_date(data['create_date']).timestamp()
kwargs['dates'].append(Date(DateType.CREATED, (create_date, create_date), DateMeta.NONE))
user = User(unique_id=f'pixiv:u:{data['user']['id']}', username=data['user']['account'], display_name=data['user']['name'])
if not is_deleted:
@@ -192,10 +192,10 @@ class PixivAppModule(Module):
self.parser.add_command('following', PixivAppFollowing)
self.parser.add_command('illust', PixivAppIllust)
headers = {
- "app-os": "ios",
- "app-os-version": "16.7.2",
- "app-version": "7.19.6",
- "user-agent": "PixivIOSApp/7.19.6 (iOS 16.7.2; iPhone13,2)"
+ 'app-os': 'ios',
+ 'app-os-version': '16.7.2',
+ 'app-version': '7.19.6',
+ 'user-agent': 'PixivIOSApp/7.19.6 (iOS 16.7.2; iPhone13,2)'
}
self.api: AppPixivAPI = AppPixivAPI(headers=headers)
self.api.auth(refresh_token=refresh_token)
diff --git a/src/portal/py/modules/pixiv_web.py b/src/portal/py/modules/pixiv_web.py
index 43d017f..9150c71 100644
--- a/src/portal/py/modules/pixiv_web.py
+++ b/src/portal/py/modules/pixiv_web.py
@@ -1,13 +1,12 @@
import log
import json
-import httpx
import urllib.parse
from typing import Optional, Any
from json.decoder import JSONDecodeError
-from base import USER_AGENT, Search, Module, Method, ParsedJson
+from base import USER_AGENT, HttpResponse, Search, Module, Method, ParsedJson
from post import Url, PostType, Date, DateType, DateMeta, Post, User, Media, Image, Animation, Tag, TagType
from query_parser import QueryParser
-from modules.common import reformat_pixiv_date, get_current_utc_time
+from modules.common import parse_pixiv_date, get_current_utc_time
BASE_URL = 'https://www.pixiv.net/ajax'
LANG = 'en'
@@ -18,7 +17,7 @@ class PixivWebBase(Search):
super().__init__()
self.module: PixivWebModule = userdata
- def check_api_response(self, response: Optional[httpx.Response]) -> Optional[ParsedJson]:
+ def check_api_response(self, response: Optional[HttpResponse]) -> Optional[ParsedJson]:
if not response:
return None
try:
@@ -72,14 +71,14 @@ class PixivWebBase(Search):
kwargs['raw_responses'] = {}
kwargs['raw_responses']['api'] = json.dumps(data, separators=(',', ':'), indent=None)
kwargs['url'] = f'https://www.pixiv.net/{LANG}/artworks/{data['illustId']}'
- create_date = reformat_pixiv_date(data['createDate']).timestamp()
+ create_date = parse_pixiv_date(data['createDate']).timestamp()
if not retrieved_date:
retrieved_date = get_current_utc_time().timestamp()
kwargs['dates'] = [
Date(DateType.CREATED, (create_date, create_date), DateMeta.NONE),
Date(DateType.RETRIEVED, (retrieved_date, retrieved_date), DateMeta.NONE)
]
- upload_date = reformat_pixiv_date(data['uploadDate']).timestamp()
+ upload_date = parse_pixiv_date(data['uploadDate']).timestamp()
if upload_date != create_date:
kwargs['dates'].append(Date(DateType.EDITED, (upload_date, upload_date), DateMeta.NONE))
user = User(unique_id=f'pixiv:u:{data['userId']}', username=data['userAccount'], display_name=data['userName'])
diff --git a/src/portal/py/modules/twitter.py b/src/portal/py/modules/twitter.py
index 79e7f31..be81c0f 100644
--- a/src/portal/py/modules/twitter.py
+++ b/src/portal/py/modules/twitter.py
@@ -1,8 +1,7 @@
import json
-import httpx
import http.cookiejar
from typing import Optional, Any, Iterator
-from base import USER_AGENT, Search, Module, ParsedJson, Method
+from base import USER_AGENT, HttpResponse, Search, Module, ParsedJson, Method
from post import Post
from query_parser import QueryParser
from modules.twitter_common import parse_graphql_raw_response
@@ -21,7 +20,7 @@ class TwitterUser(TwitterBase):
super().__init__(userdata)
self.user_id = ''
- def check_api_response(self, response: Optional[httpx.Response]) -> Optional[ParsedJson]:
+ def check_api_response(self, response: Optional[HttpResponse]) -> Optional[ParsedJson]:
if not response:
return None
try:
@@ -85,7 +84,7 @@ class TwitterUser(TwitterBase):
'features': build_params(features),
'fieldToggles': build_params(fieldToggles)
}
- url = "https://x.com/i/api/graphql/DdW3L2nfGW6FZDcSSSG4yg/UserMedia"
+ url = 'https://x.com/i/api/graphql/DdW3L2nfGW6FZDcSSSG4yg/UserMedia'
return self.check_api_response(self.module.do_request(Method.GET, url, params=params))
#print(json.dumps(media, indent=4))
diff --git a/src/portal/py/tests/archive_query.py b/src/portal/py/tests/archive_query.py
index 8ecbb52..f944ca0 100644
--- a/src/portal/py/tests/archive_query.py
+++ b/src/portal/py/tests/archive_query.py
@@ -24,8 +24,8 @@ class Logger():
def close(self):
self.file.close()
-mode = 'pixiv_web'
-#mode = 'fanbox'
+#mode = 'pixiv_web'
+mode = 'fanbox'
#mode = 'patreon'
#mode = 'instagram'
#mode = 'twitter'
@@ -42,6 +42,18 @@ queue_mutex = threading.Lock()
queue_cond = threading.Condition(queue_mutex)
is_running = True
+def media_check_disk(obj, post, output_base):
+ for key in post.media.keys():
+ download_params = module.get_download(post.unique_id, key)
+ if not download_params:
+ obj.log.write(f'Ignoring media{key} for post {post.unique_id}.')
+ continue
+ for url in download_params['urls']:
+ media_path = f'{output_base}_media{key}.{url.ext}'
+ if not os.path.isfile(media_path):
+ return False
+ return True
+
def default_media_download(obj, post, output_base):
for key in post.media.keys():
download_params = module.get_download(post.unique_id, key)
@@ -133,7 +145,7 @@ class QueryDownloadThread(threading.Thread):
self.log.write('Rewriting post with quote.')
has_mp4 = True
'''
- if os.path.isfile(output):
+ if os.path.isfile(output) and media_check_disk(self, post, output_base):
skip_message = f'Skipping already downloaded post {post.unique_id}'
if post.type != PostType.TOMBSTONE:
skip_message += f' from {post.author.unique_id}.'
@@ -233,6 +245,7 @@ def download_args():
os.mkdir(f'{output_dir}/raw_responses')
args = []
+ start_page = 0
threads = []
start = True
@@ -265,7 +278,7 @@ def download_args():
if not is_running:
break
log.write(f'***********************\nStarting download of {arg}\n***********************\n')
- arg_thread = QueryDownloadThread(log, complete, arg, output_dir, 0)
+ arg_thread = QueryDownloadThread(log, complete, arg, output_dir, start_page)
threads.append(arg_thread)
arg_thread.start()
generated by cgit v1.2.3-101-g0448 (git 2.54.0) at 2026-07-25 15:02:56 +0000