wip

Signed-off-by: Andrew Opalach <andrew@akon.city>

7 files changed, 152 insertions, 22 deletions

diff --git a/src/portal/py/modules/__init__.py b/src/portal/py/modules/__init__.py
index 480660c..994b25a 100644
--- a/src/portal/py/modules/__init__.py
+++ b/src/portal/py/modules/__init__.py
@@ -6,17 +6,19 @@ from modules.pixiv_app import PixivAppModule
 from modules.pixiv_web import PixivWebModule
 from modules.fanbox import FanboxModule
 from modules.instagram import InstagramModule
+from modules.patreon import PatreonModule
 
 ALL_MODULES = {
     'youtube': (YoutubeModule(), []),
 #    'twitter': (TwitterModule(
 #        config.TWITTER_ACCESS_TOKEN, config.TWITTER_ACCESS_TOKEN_SECRET,
 #        config.TWITTER_CONSUMER_TOKEN, config.TWITTER_CONSUMER_TOKEN_SECRET), []),
-#    'twitter': (TwitterScrapeModule(config.TWITTER_COOKIES_PATH), []),
-#    'pixiv_web': (PixivWebModule(config.PIXIV_SESSID), []),
+    'twitter': (TwitterScrapeModule(config.TWITTER_COOKIES_PATH), []),
+    'pixiv_web': (PixivWebModule(config.PIXIV_SESSID, config.PIXIV_USERID), []),
 #    'pixiv_app': (PixivAppModule(config.PIXIV_REFRESH_TOKEN), []),
 #    'fanbox': (FanboxModule(config.FANBOX_SESSID), []),
 #    'instagram': (InstagramModule(
 #        config.INSTAGRAM_USER_AGENT, config.INSTAGRAM_SETTINGS_PATH,
 #        config.INSTAGRAM_SESSION_ID), [])
+    'patreon': (PatreonModule(config.PATREON_SESSION_ID, config.PATREON_UUID, config.PATREON_USER_ID), [])
 }
diff --git a/src/portal/py/modules/fanbox.py b/src/portal/py/modules/fanbox.py
index 01e66c3..4323d6b 100644
--- a/src/portal/py/modules/fanbox.py
+++ b/src/portal/py/modules/fanbox.py
@@ -2,7 +2,7 @@ import json
 import httpx
 from typing import Optional, Any
 from json.decoder import JSONDecodeError
-from base import Search, Module, Method, ParsedJson
+from base import USER_AGENT, Search, Module, Method, ParsedJson
 from post import MediaUrl, PostType, DateType, Post, User, Image, File, Tag
 from query_parser import QueryParser
 from modules.common import parse_pixiv_date, get_current_utc_time
@@ -173,7 +173,7 @@ class FanboxModule(Module):
         self.headers['Accept-Language'] = 'en-US,en;q=0.5'
         self.headers['Origin'] =  'https://www.fanbox.cc'
         self.headers['Referer'] =  'https://www.fanbox.cc/'
-        self.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0'
+        self.headers['User-Agent'] = USER_AGENT
         self.cookies['FANBOXSESSID'] = sessid
 
     def search(self, query: str, *extra_args: Any) -> Optional[Search]:
diff --git a/src/portal/py/modules/instagram.py b/src/portal/py/modules/instagram.py
index 325caf6..d107566 100644
--- a/src/portal/py/modules/instagram.py
+++ b/src/portal/py/modules/instagram.py
@@ -3,7 +3,7 @@ import log
 import email.utils
 from typing import Optional, Any
 from pathlib import Path
-from base import Module, Search
+from base import USER_AGENT, Module, Search
 from post import MediaUrl, PostType, DateType, Post, User, Image, Video
 from modules.common import get_current_utc_time
 from instagrapi import Client
@@ -149,6 +149,6 @@ class InstagramModule(Module):
         return {
             'urls': [post.media[key].url],
             'headers': {
-                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0'
+                'User-Agent': USER_AGENT
             }
         }
diff --git a/src/portal/py/modules/patreon.py b/src/portal/py/modules/patreon.py
new file mode 100644
index 0000000..40c3255
--- /dev/null
+++ b/src/portal/py/modules/patreon.py
@@ -0,0 +1,124 @@
+import json
+import httpx
+from json import JSONDecodeError
+from typing import Optional, Any
+from base import USER_AGENT, Search, Module, Method, ParsedJson
+from post import MediaUrl, PostType, DateType, Post, User, Media, Image, Animation, Tag, TagType
+from query_parser import QueryParser
+
+BASE_URL = 'https://www.patreon.com/api'
+
+class PatreonBase(Search):
+    REACH_LIMIT = 3
+
+    def __init__(self, userdata: Any):
+        super().__init__()
+        self.module: PatreonModule = userdata
+        self.page: int = 0
+        self.cursor: Optional[str] = None
+
+    def check_api_response(self, response: Optional[httpx.Response]) -> Optional[ParsedJson]:
+        if not response:
+            return None
+        try:
+            obj = response.json()
+        except JSONDecodeError:
+            return None
+        return obj
+
+    def search_user(self, query: str) -> Optional[User]:
+        url = f'{BASE_URL}/search'
+        params = {
+            'q': query,
+            'page[number]': '1',
+            'json-api-version': '1.0',
+            'json-api-use-default-includes': 'false',
+            'include': '[]'
+        }
+        obj = self.check_api_response(self.module.do_request(Method.GET, url, params=params))
+        if not obj:
+            return None
+        for result in obj['data']:
+            prefix = result['id'].find('campaign_')
+            if prefix >= 0:
+                campaign = result['id'][prefix + len('campaign_'):]
+                unique_id = 'patreon:c:' + campaign
+                user = User(unique_id=unique_id, username=result['attributes']['creator_name'], display_name=result['attributes']['name'])
+                return user
+        return None
+
+class PatreonUser(PatreonBase):
+    def __init__(self, userdata: Any, arg: str):
+        super().__init__(userdata)
+        self.username: str = arg
+        self.user: Optional[User] = None
+        self.campaign: str
+
+    def request_page(self, num: int) -> bool:
+        if num >= self.page + self.REACH_LIMIT:
+            return False
+        if not self.user:
+            user = self.search_user(self.username)
+            if not user:
+                self.errored = True
+                return False
+            self.campaign = user.unique_id[len('patreon:c:'):]
+        request_satisfied = False
+        for i in range(self.page, num + 1):
+            url = f'{BASE_URL}/posts'
+            params = {
+                'include': 'campaign,access_rules,attachments,audio,audio_preview.null,drop,images,media,native_video_insights,poll.choices,poll.current_user_responses.user,poll.current_user_responses.choice,poll.current_user_responses.poll,user,user_defined_tags,ti_checks,content_unlock_options.product_variant.null',
+                'fields[campaign]': 'currency,show_audio_post_download_links,avatar_photo_url,avatar_photo_image_urls,earnings_visibility,is_nsfw,is_monthly,name,url',
+                'fields[post]': 'change_visibility_at,comment_count,commenter_count,content,current_user_can_comment,current_user_can_delete,current_user_can_report,current_user_can_view,current_user_comment_disallowed_reason,current_user_has_liked,embed,image,insights_last_updated_at,is_paid,like_count,meta_image_url,min_cents_pledged_to_view,post_file,post_metadata,published_at,patreon_url,post_type,pledge_url,preview_asset_type,thumbnail,share_images,thumbnail_url,teaser_text,title,upgrade_url,url,was_posted_by_campaign_owner,has_ti_violation,moderation_status,post_level_suspension_removal_date,pls_one_liners_by_category,video_preview,view_count',
+                'fields[post_tag]': 'tag_type,value',
+                'fields[user]': 'image_url,full_name,url',
+                'fields[access_rule]': 'access_rule_type,amount_cents',
+                'fields[media]': 'id,image_urls,display,download_url,metadata,file_name',
+                'fields[native_video_insights]': 'average_view_duration,average_view_pct,has_preview,id,last_updated_at,num_views,preview_views,video_duration',
+                'fields[content-unlock-option]': 'content_unlock_type',
+                'fields[product-variant]': 'price_cents,currency_code,checkout_url,is_hidden,published_at_datetime,orders_count,access_metadata',
+                'filter[campaign_id]': self.campaign,
+                'filter[contains_exclusive_posts]': 'true',
+                'filter[is_draft]': 'false',
+                'filter[accessible_by_user_id]': self.module.user_id,
+                'sort': '-published_at',
+                'json-api-version': '1.0',
+                'json-api-use-default-includes': 'false'
+            }
+            if self.cursor:
+                params['page[cursor]'] = self.cursor
+            obj = self.check_api_response(self.module.do_request(Method.GET, url, params=params))
+            if not obj:
+                self.errored = True
+                return False
+            print(json.dumps(obj, indent=4))
+        return True
+
+class PatreonModule(Module):
+    def __init__(self, session_id: str, uuid: str, user_id: str):
+        super().__init__()
+        self.user_id: str = user_id
+        self.parser: QueryParser = QueryParser(self, 'user')
+        self.parser.add_command('user', PatreonUser)
+        self.headers['Accept-Encoding'] = 'gzip, deflate, br'
+        self.headers['Accept-Language'] = 'en-US,en;q=0.5'
+        self.headers['x-patreon-uuid'] = uuid
+        self.cookies['patreon_locale_code'] = 'en-US'
+        self.cookies['patreon_location_country_code'] = 'US'
+        self.cookies['session_id'] = session_id
+
+    def search(self, query: str, *extra_args: Any) -> Optional[Search]:
+        return self.parser.parse_query(query)
+
+    def get_download(self, unique_id: str, key: str) -> Optional[dict[str, Any]]:
+        if unique_id not in self.unique_id_map:
+            return None
+        post = self.unique_id_map[unique_id]
+        if key not in post.media:
+            return None
+        return {
+            'urls': [post.media[key].url],
+            'headers': {
+                'User-Agent': USER_AGENT
+            }
+        }
diff --git a/src/portal/py/modules/pixiv_app.py b/src/portal/py/modules/pixiv_app.py
index 8be3f9c..c84030d 100644
--- a/src/portal/py/modules/pixiv_app.py
+++ b/src/portal/py/modules/pixiv_app.py
@@ -1,6 +1,6 @@
 import json
 from typing import Optional, Any
-from base import Search, Module, ParsedJson
+from base import USER_AGENT, Search, Module, ParsedJson
 from post import MediaUrl, PostType, DateType, User, Post, Image, Tag, TagType
 from query_parser import QueryParser
 from pixivpy3 import *
@@ -195,7 +195,7 @@ class PixivAppModule(Module):
         return {
             'urls': [post.media[key].url],
             'headers': {
-                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0',
+                'User-Agent': USER_AGENT,
                 'Referer': 'https://www.pixiv.net/'
             }
         }
diff --git a/src/portal/py/modules/pixiv_web.py b/src/portal/py/modules/pixiv_web.py
index 7bc043b..be68b7e 100644
--- a/src/portal/py/modules/pixiv_web.py
+++ b/src/portal/py/modules/pixiv_web.py
@@ -4,7 +4,7 @@ import httpx
 import urllib.parse
 from typing import Optional, Any
 from json.decoder import JSONDecodeError
-from base import Search, Module, Method, ParsedJson
+from base import USER_AGENT, Search, Module, Method, ParsedJson
 from post import MediaUrl, PostType, DateType, Post, User, Media, Image, Animation, Tag, TagType
 from query_parser import QueryParser
 from modules.common import parse_pixiv_date, get_current_utc_time
@@ -199,13 +199,17 @@ class PixivWebUser(PixivWebBase):
             self.errored = True
             return False
         self.pages[num] = []
+        keys = []
+        if obj['illusts']:
+            keys.extend([int(x) for x in obj['illusts'].keys()])
         if obj['manga']:
-            for illust in obj['manga'].keys():
-                post = self.request_illust(int(illust))
-                if not post:
-                    self.errored = True
-                    return False
-                self.pages[num].append(post.unique_id)
+            keys.extend([int(x) for x in obj['manga'].keys()])
+        for illust in keys:
+            post = self.request_illust(illust)
+            if not post:
+                self.errored = True
+                return False
+            self.pages[num].append(post.unique_id)
         self.completed = True
         return True
 
@@ -231,18 +235,18 @@ class PixivWebFollowing(PixivWebBase):
         return True
 
 class PixivWebModule(Module):
-    def __init__(self, sessid: str):
+    def __init__(self, sessid: str, user_id: str):
         super().__init__()
         self.parser: QueryParser = QueryParser(self, 'search')
         self.parser.add_command('search', PixivWebSearch)
         self.parser.add_command('illust', PixivWebIllust)
         self.parser.add_command('user', PixivWebUser)
         self.parser.add_command('following', PixivWebFollowing)
-        self.headers['Accept-Encoding'] = 'gzip, deflate, br'
+        self.headers['Accept-Encoding'] = 'gzip, deflate, br' # add 'zstd' when httpx update
         self.headers['Accept-Language'] = 'en-US,en;q=0.5'
         self.headers['Referer'] =  'https://www.pixiv.net/'
-        self.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0'
-        self.headers['x-user-id'] = '22781328'
+        self.headers['User-Agent'] = USER_AGENT
+        self.headers['x-user-id'] = user_id
         self.cookies['PHPSESSID'] = sessid
 
     def search(self, query: str, *extra_args: Any) -> Optional[Search]:
@@ -257,7 +261,7 @@ class PixivWebModule(Module):
         return {
             'urls': [post.media[key].url],
             'headers': {
-                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0',
+                'User-Agent': USER_AGENT,
                 'Referer': 'https://www.pixiv.net/'
             }
         }
diff --git a/src/portal/py/modules/twitter.py b/src/portal/py/modules/twitter.py
index 7e77253..2996d0e 100644
--- a/src/portal/py/modules/twitter.py
+++ b/src/portal/py/modules/twitter.py
@@ -2,7 +2,7 @@ import log
 import http.cookiejar
 from typing import Optional, Any, Iterator
 from datetime import timezone
-from base import Search, Module
+from base import USER_AGENT, Search, Module
 from post import MediaUrl, PostType, DateType, PostRef, Post, User, Media, Image, Video
 from query_parser import QueryParser
 from modules.common import get_current_utc_time
@@ -185,6 +185,6 @@ class TwitterScrapeModule(Module):
         return {
             'urls': [post.media[key].url],
             'headers': {
-                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0'
+                'User-Agent': USER_AGENT
             }
         }