From 2f9a0945bfeee3296cec3d38d094e4c49f9cb65f Mon Sep 17 00:00:00 2001 From: Andrew Opalach Date: Sat, 25 May 2024 20:22:59 -0400 Subject: wip Signed-off-by: Andrew Opalach --- src/portal/py/modules/__init__.py | 6 +- src/portal/py/modules/fanbox.py | 4 +- src/portal/py/modules/instagram.py | 4 +- src/portal/py/modules/patreon.py | 124 +++++++++++++++++++++++++++++++++++++ src/portal/py/modules/pixiv_app.py | 4 +- src/portal/py/modules/pixiv_web.py | 28 +++++---- src/portal/py/modules/twitter.py | 4 +- 7 files changed, 152 insertions(+), 22 deletions(-) create mode 100644 src/portal/py/modules/patreon.py (limited to 'src/portal/py/modules') diff --git a/src/portal/py/modules/__init__.py b/src/portal/py/modules/__init__.py index 480660c..994b25a 100644 --- a/src/portal/py/modules/__init__.py +++ b/src/portal/py/modules/__init__.py @@ -6,17 +6,19 @@ from modules.pixiv_app import PixivAppModule from modules.pixiv_web import PixivWebModule from modules.fanbox import FanboxModule from modules.instagram import InstagramModule +from modules.patreon import PatreonModule ALL_MODULES = { 'youtube': (YoutubeModule(), []), # 'twitter': (TwitterModule( # config.TWITTER_ACCESS_TOKEN, config.TWITTER_ACCESS_TOKEN_SECRET, # config.TWITTER_CONSUMER_TOKEN, config.TWITTER_CONSUMER_TOKEN_SECRET), []), -# 'twitter': (TwitterScrapeModule(config.TWITTER_COOKIES_PATH), []), -# 'pixiv_web': (PixivWebModule(config.PIXIV_SESSID), []), + 'twitter': (TwitterScrapeModule(config.TWITTER_COOKIES_PATH), []), + 'pixiv_web': (PixivWebModule(config.PIXIV_SESSID, config.PIXIV_USERID), []), # 'pixiv_app': (PixivAppModule(config.PIXIV_REFRESH_TOKEN), []), # 'fanbox': (FanboxModule(config.FANBOX_SESSID), []), # 'instagram': (InstagramModule( # config.INSTAGRAM_USER_AGENT, config.INSTAGRAM_SETTINGS_PATH, # config.INSTAGRAM_SESSION_ID), []) + 'patreon': (PatreonModule(config.PATREON_SESSION_ID, config.PATREON_UUID, config.PATREON_USER_ID), []) } diff --git a/src/portal/py/modules/fanbox.py b/src/portal/py/modules/fanbox.py index 01e66c3..4323d6b 100644 --- a/src/portal/py/modules/fanbox.py +++ b/src/portal/py/modules/fanbox.py @@ -2,7 +2,7 @@ import json import httpx from typing import Optional, Any from json.decoder import JSONDecodeError -from base import Search, Module, Method, ParsedJson +from base import USER_AGENT, Search, Module, Method, ParsedJson from post import MediaUrl, PostType, DateType, Post, User, Image, File, Tag from query_parser import QueryParser from modules.common import parse_pixiv_date, get_current_utc_time @@ -173,7 +173,7 @@ class FanboxModule(Module): self.headers['Accept-Language'] = 'en-US,en;q=0.5' self.headers['Origin'] = 'https://www.fanbox.cc' self.headers['Referer'] = 'https://www.fanbox.cc/' - self.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0' + self.headers['User-Agent'] = USER_AGENT self.cookies['FANBOXSESSID'] = sessid def search(self, query: str, *extra_args: Any) -> Optional[Search]: diff --git a/src/portal/py/modules/instagram.py b/src/portal/py/modules/instagram.py index 325caf6..d107566 100644 --- a/src/portal/py/modules/instagram.py +++ b/src/portal/py/modules/instagram.py @@ -3,7 +3,7 @@ import log import email.utils from typing import Optional, Any from pathlib import Path -from base import Module, Search +from base import USER_AGENT, Module, Search from post import MediaUrl, PostType, DateType, Post, User, Image, Video from modules.common import get_current_utc_time from instagrapi import Client @@ -149,6 +149,6 @@ class InstagramModule(Module): return { 'urls': [post.media[key].url], 'headers': { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0' + 'User-Agent': USER_AGENT } } diff --git a/src/portal/py/modules/patreon.py b/src/portal/py/modules/patreon.py new file mode 100644 index 0000000..40c3255 --- /dev/null +++ b/src/portal/py/modules/patreon.py @@ -0,0 +1,124 @@ +import json +import httpx +from json import JSONDecodeError +from typing import Optional, Any +from base import USER_AGENT, Search, Module, Method, ParsedJson +from post import MediaUrl, PostType, DateType, Post, User, Media, Image, Animation, Tag, TagType +from query_parser import QueryParser + +BASE_URL = 'https://www.patreon.com/api' + +class PatreonBase(Search): + REACH_LIMIT = 3 + + def __init__(self, userdata: Any): + super().__init__() + self.module: PatreonModule = userdata + self.page: int = 0 + self.cursor: Optional[str] = None + + def check_api_response(self, response: Optional[httpx.Response]) -> Optional[ParsedJson]: + if not response: + return None + try: + obj = response.json() + except JSONDecodeError: + return None + return obj + + def search_user(self, query: str) -> Optional[User]: + url = f'{BASE_URL}/search' + params = { + 'q': query, + 'page[number]': '1', + 'json-api-version': '1.0', + 'json-api-use-default-includes': 'false', + 'include': '[]' + } + obj = self.check_api_response(self.module.do_request(Method.GET, url, params=params)) + if not obj: + return None + for result in obj['data']: + prefix = result['id'].find('campaign_') + if prefix >= 0: + campaign = result['id'][prefix + len('campaign_'):] + unique_id = 'patreon:c:' + campaign + user = User(unique_id=unique_id, username=result['attributes']['creator_name'], display_name=result['attributes']['name']) + return user + return None + +class PatreonUser(PatreonBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.username: str = arg + self.user: Optional[User] = None + self.campaign: str + + def request_page(self, num: int) -> bool: + if num >= self.page + self.REACH_LIMIT: + return False + if not self.user: + user = self.search_user(self.username) + if not user: + self.errored = True + return False + self.campaign = user.unique_id[len('patreon:c:'):] + request_satisfied = False + for i in range(self.page, num + 1): + url = f'{BASE_URL}/posts' + params = { + 'include': 'campaign,access_rules,attachments,audio,audio_preview.null,drop,images,media,native_video_insights,poll.choices,poll.current_user_responses.user,poll.current_user_responses.choice,poll.current_user_responses.poll,user,user_defined_tags,ti_checks,content_unlock_options.product_variant.null', + 'fields[campaign]': 'currency,show_audio_post_download_links,avatar_photo_url,avatar_photo_image_urls,earnings_visibility,is_nsfw,is_monthly,name,url', + 'fields[post]': 'change_visibility_at,comment_count,commenter_count,content,current_user_can_comment,current_user_can_delete,current_user_can_report,current_user_can_view,current_user_comment_disallowed_reason,current_user_has_liked,embed,image,insights_last_updated_at,is_paid,like_count,meta_image_url,min_cents_pledged_to_view,post_file,post_metadata,published_at,patreon_url,post_type,pledge_url,preview_asset_type,thumbnail,share_images,thumbnail_url,teaser_text,title,upgrade_url,url,was_posted_by_campaign_owner,has_ti_violation,moderation_status,post_level_suspension_removal_date,pls_one_liners_by_category,video_preview,view_count', + 'fields[post_tag]': 'tag_type,value', + 'fields[user]': 'image_url,full_name,url', + 'fields[access_rule]': 'access_rule_type,amount_cents', + 'fields[media]': 'id,image_urls,display,download_url,metadata,file_name', + 'fields[native_video_insights]': 'average_view_duration,average_view_pct,has_preview,id,last_updated_at,num_views,preview_views,video_duration', + 'fields[content-unlock-option]': 'content_unlock_type', + 'fields[product-variant]': 'price_cents,currency_code,checkout_url,is_hidden,published_at_datetime,orders_count,access_metadata', + 'filter[campaign_id]': self.campaign, + 'filter[contains_exclusive_posts]': 'true', + 'filter[is_draft]': 'false', + 'filter[accessible_by_user_id]': self.module.user_id, + 'sort': '-published_at', + 'json-api-version': '1.0', + 'json-api-use-default-includes': 'false' + } + if self.cursor: + params['page[cursor]'] = self.cursor + obj = self.check_api_response(self.module.do_request(Method.GET, url, params=params)) + if not obj: + self.errored = True + return False + print(json.dumps(obj, indent=4)) + return True + +class PatreonModule(Module): + def __init__(self, session_id: str, uuid: str, user_id: str): + super().__init__() + self.user_id: str = user_id + self.parser: QueryParser = QueryParser(self, 'user') + self.parser.add_command('user', PatreonUser) + self.headers['Accept-Encoding'] = 'gzip, deflate, br' + self.headers['Accept-Language'] = 'en-US,en;q=0.5' + self.headers['x-patreon-uuid'] = uuid + self.cookies['patreon_locale_code'] = 'en-US' + self.cookies['patreon_location_country_code'] = 'US' + self.cookies['session_id'] = session_id + + def search(self, query: str, *extra_args: Any) -> Optional[Search]: + return self.parser.parse_query(query) + + def get_download(self, unique_id: str, key: str) -> Optional[dict[str, Any]]: + if unique_id not in self.unique_id_map: + return None + post = self.unique_id_map[unique_id] + if key not in post.media: + return None + return { + 'urls': [post.media[key].url], + 'headers': { + 'User-Agent': USER_AGENT + } + } diff --git a/src/portal/py/modules/pixiv_app.py b/src/portal/py/modules/pixiv_app.py index 8be3f9c..c84030d 100644 --- a/src/portal/py/modules/pixiv_app.py +++ b/src/portal/py/modules/pixiv_app.py @@ -1,6 +1,6 @@ import json from typing import Optional, Any -from base import Search, Module, ParsedJson +from base import USER_AGENT, Search, Module, ParsedJson from post import MediaUrl, PostType, DateType, User, Post, Image, Tag, TagType from query_parser import QueryParser from pixivpy3 import * @@ -195,7 +195,7 @@ class PixivAppModule(Module): return { 'urls': [post.media[key].url], 'headers': { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0', + 'User-Agent': USER_AGENT, 'Referer': 'https://www.pixiv.net/' } } diff --git a/src/portal/py/modules/pixiv_web.py b/src/portal/py/modules/pixiv_web.py index 7bc043b..be68b7e 100644 --- a/src/portal/py/modules/pixiv_web.py +++ b/src/portal/py/modules/pixiv_web.py @@ -4,7 +4,7 @@ import httpx import urllib.parse from typing import Optional, Any from json.decoder import JSONDecodeError -from base import Search, Module, Method, ParsedJson +from base import USER_AGENT, Search, Module, Method, ParsedJson from post import MediaUrl, PostType, DateType, Post, User, Media, Image, Animation, Tag, TagType from query_parser import QueryParser from modules.common import parse_pixiv_date, get_current_utc_time @@ -199,13 +199,17 @@ class PixivWebUser(PixivWebBase): self.errored = True return False self.pages[num] = [] + keys = [] + if obj['illusts']: + keys.extend([int(x) for x in obj['illusts'].keys()]) if obj['manga']: - for illust in obj['manga'].keys(): - post = self.request_illust(int(illust)) - if not post: - self.errored = True - return False - self.pages[num].append(post.unique_id) + keys.extend([int(x) for x in obj['manga'].keys()]) + for illust in keys: + post = self.request_illust(illust) + if not post: + self.errored = True + return False + self.pages[num].append(post.unique_id) self.completed = True return True @@ -231,18 +235,18 @@ class PixivWebFollowing(PixivWebBase): return True class PixivWebModule(Module): - def __init__(self, sessid: str): + def __init__(self, sessid: str, user_id: str): super().__init__() self.parser: QueryParser = QueryParser(self, 'search') self.parser.add_command('search', PixivWebSearch) self.parser.add_command('illust', PixivWebIllust) self.parser.add_command('user', PixivWebUser) self.parser.add_command('following', PixivWebFollowing) - self.headers['Accept-Encoding'] = 'gzip, deflate, br' + self.headers['Accept-Encoding'] = 'gzip, deflate, br' # add 'zstd' when httpx update self.headers['Accept-Language'] = 'en-US,en;q=0.5' self.headers['Referer'] = 'https://www.pixiv.net/' - self.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0' - self.headers['x-user-id'] = '22781328' + self.headers['User-Agent'] = USER_AGENT + self.headers['x-user-id'] = user_id self.cookies['PHPSESSID'] = sessid def search(self, query: str, *extra_args: Any) -> Optional[Search]: @@ -257,7 +261,7 @@ class PixivWebModule(Module): return { 'urls': [post.media[key].url], 'headers': { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0', + 'User-Agent': USER_AGENT, 'Referer': 'https://www.pixiv.net/' } } diff --git a/src/portal/py/modules/twitter.py b/src/portal/py/modules/twitter.py index 7e77253..2996d0e 100644 --- a/src/portal/py/modules/twitter.py +++ b/src/portal/py/modules/twitter.py @@ -2,7 +2,7 @@ import log import http.cookiejar from typing import Optional, Any, Iterator from datetime import timezone -from base import Search, Module +from base import USER_AGENT, Search, Module from post import MediaUrl, PostType, DateType, PostRef, Post, User, Media, Image, Video from query_parser import QueryParser from modules.common import get_current_utc_time @@ -185,6 +185,6 @@ class TwitterScrapeModule(Module): return { 'urls': [post.media[key].url], 'headers': { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0' + 'User-Agent': USER_AGENT } } -- cgit v1.2.3-101-g0448