summaryrefslogtreecommitdiff
path: root/src/portal
diff options
context:
space:
mode:
Diffstat (limited to 'src/portal')
-rw-r--r--src/portal/py/base.py4
-rw-r--r--src/portal/py/config.def.py6
-rw-r--r--src/portal/py/modules/__init__.py6
-rw-r--r--src/portal/py/modules/fanbox.py4
-rw-r--r--src/portal/py/modules/instagram.py4
-rw-r--r--src/portal/py/modules/patreon.py124
-rw-r--r--src/portal/py/modules/pixiv_app.py4
-rw-r--r--src/portal/py/modules/pixiv_web.py28
-rw-r--r--src/portal/py/modules/twitter.py4
-rw-r--r--src/portal/py/post.py43
-rw-r--r--src/portal/src/search.c2
-rw-r--r--src/portal/tests/test.py6
12 files changed, 190 insertions, 45 deletions
diff --git a/src/portal/py/base.py b/src/portal/py/base.py
index c0a0b4f..cc883af 100644
--- a/src/portal/py/base.py
+++ b/src/portal/py/base.py
@@ -7,12 +7,14 @@ from collections import OrderedDict
from typing import Optional, Self, Any
from post import Post, User
+USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0'
+
class Method(Enum):
HEAD = "HEAD"
GET = "GET"
POST = "POST"
-ParsedJson = Any
+type ParsedJson = Any
class Search():
def __init__(self):
diff --git a/src/portal/py/config.def.py b/src/portal/py/config.def.py
index c4e11c6..8c0dab9 100644
--- a/src/portal/py/config.def.py
+++ b/src/portal/py/config.def.py
@@ -11,6 +11,8 @@ TWITTER_TIMEOUT = 5 # seconds
FANBOX_SESSID = ''
+PIXIV_SESSID = ''
+PIXIV_USERID = ''
PIXIV_REFRESH_TOKEN = ''
INSTAGRAM_USERNAME = ''
@@ -18,3 +20,7 @@ INSTAGRAM_PASSWORD = ''
INSTAGRAM_SESSION_ID = ''
INSTAGRAM_SETTINGS_PATH = Path('{}/ig_settings.json'.format(BASE_DIR))
INSTAGRAM_USER_AGENT = ''
+
+PATREON_USER_ID = ''
+PATREON_UUID = ''
+PATREON_SESSION_ID = ''
diff --git a/src/portal/py/modules/__init__.py b/src/portal/py/modules/__init__.py
index 480660c..994b25a 100644
--- a/src/portal/py/modules/__init__.py
+++ b/src/portal/py/modules/__init__.py
@@ -6,17 +6,19 @@ from modules.pixiv_app import PixivAppModule
from modules.pixiv_web import PixivWebModule
from modules.fanbox import FanboxModule
from modules.instagram import InstagramModule
+from modules.patreon import PatreonModule
ALL_MODULES = {
'youtube': (YoutubeModule(), []),
# 'twitter': (TwitterModule(
# config.TWITTER_ACCESS_TOKEN, config.TWITTER_ACCESS_TOKEN_SECRET,
# config.TWITTER_CONSUMER_TOKEN, config.TWITTER_CONSUMER_TOKEN_SECRET), []),
-# 'twitter': (TwitterScrapeModule(config.TWITTER_COOKIES_PATH), []),
-# 'pixiv_web': (PixivWebModule(config.PIXIV_SESSID), []),
+ 'twitter': (TwitterScrapeModule(config.TWITTER_COOKIES_PATH), []),
+ 'pixiv_web': (PixivWebModule(config.PIXIV_SESSID, config.PIXIV_USERID), []),
# 'pixiv_app': (PixivAppModule(config.PIXIV_REFRESH_TOKEN), []),
# 'fanbox': (FanboxModule(config.FANBOX_SESSID), []),
# 'instagram': (InstagramModule(
# config.INSTAGRAM_USER_AGENT, config.INSTAGRAM_SETTINGS_PATH,
# config.INSTAGRAM_SESSION_ID), [])
+ 'patreon': (PatreonModule(config.PATREON_SESSION_ID, config.PATREON_UUID, config.PATREON_USER_ID), [])
}
diff --git a/src/portal/py/modules/fanbox.py b/src/portal/py/modules/fanbox.py
index 01e66c3..4323d6b 100644
--- a/src/portal/py/modules/fanbox.py
+++ b/src/portal/py/modules/fanbox.py
@@ -2,7 +2,7 @@ import json
import httpx
from typing import Optional, Any
from json.decoder import JSONDecodeError
-from base import Search, Module, Method, ParsedJson
+from base import USER_AGENT, Search, Module, Method, ParsedJson
from post import MediaUrl, PostType, DateType, Post, User, Image, File, Tag
from query_parser import QueryParser
from modules.common import parse_pixiv_date, get_current_utc_time
@@ -173,7 +173,7 @@ class FanboxModule(Module):
self.headers['Accept-Language'] = 'en-US,en;q=0.5'
self.headers['Origin'] = 'https://www.fanbox.cc'
self.headers['Referer'] = 'https://www.fanbox.cc/'
- self.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0'
+ self.headers['User-Agent'] = USER_AGENT
self.cookies['FANBOXSESSID'] = sessid
def search(self, query: str, *extra_args: Any) -> Optional[Search]:
diff --git a/src/portal/py/modules/instagram.py b/src/portal/py/modules/instagram.py
index 325caf6..d107566 100644
--- a/src/portal/py/modules/instagram.py
+++ b/src/portal/py/modules/instagram.py
@@ -3,7 +3,7 @@ import log
import email.utils
from typing import Optional, Any
from pathlib import Path
-from base import Module, Search
+from base import USER_AGENT, Module, Search
from post import MediaUrl, PostType, DateType, Post, User, Image, Video
from modules.common import get_current_utc_time
from instagrapi import Client
@@ -149,6 +149,6 @@ class InstagramModule(Module):
return {
'urls': [post.media[key].url],
'headers': {
- 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0'
+ 'User-Agent': USER_AGENT
}
}
diff --git a/src/portal/py/modules/patreon.py b/src/portal/py/modules/patreon.py
new file mode 100644
index 0000000..40c3255
--- /dev/null
+++ b/src/portal/py/modules/patreon.py
@@ -0,0 +1,124 @@
+import json
+import httpx
+from json import JSONDecodeError
+from typing import Optional, Any
+from base import USER_AGENT, Search, Module, Method, ParsedJson
+from post import MediaUrl, PostType, DateType, Post, User, Media, Image, Animation, Tag, TagType
+from query_parser import QueryParser
+
+BASE_URL = 'https://www.patreon.com/api'
+
+class PatreonBase(Search):
+ REACH_LIMIT = 3
+
+ def __init__(self, userdata: Any):
+ super().__init__()
+ self.module: PatreonModule = userdata
+ self.page: int = 0
+ self.cursor: Optional[str] = None
+
+ def check_api_response(self, response: Optional[httpx.Response]) -> Optional[ParsedJson]:
+ if not response:
+ return None
+ try:
+ obj = response.json()
+ except JSONDecodeError:
+ return None
+ return obj
+
+ def search_user(self, query: str) -> Optional[User]:
+ url = f'{BASE_URL}/search'
+ params = {
+ 'q': query,
+ 'page[number]': '1',
+ 'json-api-version': '1.0',
+ 'json-api-use-default-includes': 'false',
+ 'include': '[]'
+ }
+ obj = self.check_api_response(self.module.do_request(Method.GET, url, params=params))
+ if not obj:
+ return None
+ for result in obj['data']:
+ prefix = result['id'].find('campaign_')
+ if prefix >= 0:
+ campaign = result['id'][prefix + len('campaign_'):]
+ unique_id = 'patreon:c:' + campaign
+ user = User(unique_id=unique_id, username=result['attributes']['creator_name'], display_name=result['attributes']['name'])
+ return user
+ return None
+
+class PatreonUser(PatreonBase):
+ def __init__(self, userdata: Any, arg: str):
+ super().__init__(userdata)
+ self.username: str = arg
+ self.user: Optional[User] = None
+ self.campaign: str
+
+ def request_page(self, num: int) -> bool:
+ if num >= self.page + self.REACH_LIMIT:
+ return False
+ if not self.user:
+ user = self.search_user(self.username)
+ if not user:
+ self.errored = True
+ return False
+ self.campaign = user.unique_id[len('patreon:c:'):]
+ request_satisfied = False
+ for i in range(self.page, num + 1):
+ url = f'{BASE_URL}/posts'
+ params = {
+ 'include': 'campaign,access_rules,attachments,audio,audio_preview.null,drop,images,media,native_video_insights,poll.choices,poll.current_user_responses.user,poll.current_user_responses.choice,poll.current_user_responses.poll,user,user_defined_tags,ti_checks,content_unlock_options.product_variant.null',
+ 'fields[campaign]': 'currency,show_audio_post_download_links,avatar_photo_url,avatar_photo_image_urls,earnings_visibility,is_nsfw,is_monthly,name,url',
+ 'fields[post]': 'change_visibility_at,comment_count,commenter_count,content,current_user_can_comment,current_user_can_delete,current_user_can_report,current_user_can_view,current_user_comment_disallowed_reason,current_user_has_liked,embed,image,insights_last_updated_at,is_paid,like_count,meta_image_url,min_cents_pledged_to_view,post_file,post_metadata,published_at,patreon_url,post_type,pledge_url,preview_asset_type,thumbnail,share_images,thumbnail_url,teaser_text,title,upgrade_url,url,was_posted_by_campaign_owner,has_ti_violation,moderation_status,post_level_suspension_removal_date,pls_one_liners_by_category,video_preview,view_count',
+ 'fields[post_tag]': 'tag_type,value',
+ 'fields[user]': 'image_url,full_name,url',
+ 'fields[access_rule]': 'access_rule_type,amount_cents',
+ 'fields[media]': 'id,image_urls,display,download_url,metadata,file_name',
+ 'fields[native_video_insights]': 'average_view_duration,average_view_pct,has_preview,id,last_updated_at,num_views,preview_views,video_duration',
+ 'fields[content-unlock-option]': 'content_unlock_type',
+ 'fields[product-variant]': 'price_cents,currency_code,checkout_url,is_hidden,published_at_datetime,orders_count,access_metadata',
+ 'filter[campaign_id]': self.campaign,
+ 'filter[contains_exclusive_posts]': 'true',
+ 'filter[is_draft]': 'false',
+ 'filter[accessible_by_user_id]': self.module.user_id,
+ 'sort': '-published_at',
+ 'json-api-version': '1.0',
+ 'json-api-use-default-includes': 'false'
+ }
+ if self.cursor:
+ params['page[cursor]'] = self.cursor
+ obj = self.check_api_response(self.module.do_request(Method.GET, url, params=params))
+ if not obj:
+ self.errored = True
+ return False
+ print(json.dumps(obj, indent=4))
+ return True
+
+class PatreonModule(Module):
+ def __init__(self, session_id: str, uuid: str, user_id: str):
+ super().__init__()
+ self.user_id: str = user_id
+ self.parser: QueryParser = QueryParser(self, 'user')
+ self.parser.add_command('user', PatreonUser)
+ self.headers['Accept-Encoding'] = 'gzip, deflate, br'
+ self.headers['Accept-Language'] = 'en-US,en;q=0.5'
+ self.headers['x-patreon-uuid'] = uuid
+ self.cookies['patreon_locale_code'] = 'en-US'
+ self.cookies['patreon_location_country_code'] = 'US'
+ self.cookies['session_id'] = session_id
+
+ def search(self, query: str, *extra_args: Any) -> Optional[Search]:
+ return self.parser.parse_query(query)
+
+ def get_download(self, unique_id: str, key: str) -> Optional[dict[str, Any]]:
+ if unique_id not in self.unique_id_map:
+ return None
+ post = self.unique_id_map[unique_id]
+ if key not in post.media:
+ return None
+ return {
+ 'urls': [post.media[key].url],
+ 'headers': {
+ 'User-Agent': USER_AGENT
+ }
+ }
diff --git a/src/portal/py/modules/pixiv_app.py b/src/portal/py/modules/pixiv_app.py
index 8be3f9c..c84030d 100644
--- a/src/portal/py/modules/pixiv_app.py
+++ b/src/portal/py/modules/pixiv_app.py
@@ -1,6 +1,6 @@
import json
from typing import Optional, Any
-from base import Search, Module, ParsedJson
+from base import USER_AGENT, Search, Module, ParsedJson
from post import MediaUrl, PostType, DateType, User, Post, Image, Tag, TagType
from query_parser import QueryParser
from pixivpy3 import *
@@ -195,7 +195,7 @@ class PixivAppModule(Module):
return {
'urls': [post.media[key].url],
'headers': {
- 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0',
+ 'User-Agent': USER_AGENT,
'Referer': 'https://www.pixiv.net/'
}
}
diff --git a/src/portal/py/modules/pixiv_web.py b/src/portal/py/modules/pixiv_web.py
index 7bc043b..be68b7e 100644
--- a/src/portal/py/modules/pixiv_web.py
+++ b/src/portal/py/modules/pixiv_web.py
@@ -4,7 +4,7 @@ import httpx
import urllib.parse
from typing import Optional, Any
from json.decoder import JSONDecodeError
-from base import Search, Module, Method, ParsedJson
+from base import USER_AGENT, Search, Module, Method, ParsedJson
from post import MediaUrl, PostType, DateType, Post, User, Media, Image, Animation, Tag, TagType
from query_parser import QueryParser
from modules.common import parse_pixiv_date, get_current_utc_time
@@ -199,13 +199,17 @@ class PixivWebUser(PixivWebBase):
self.errored = True
return False
self.pages[num] = []
+ keys = []
+ if obj['illusts']:
+ keys.extend([int(x) for x in obj['illusts'].keys()])
if obj['manga']:
- for illust in obj['manga'].keys():
- post = self.request_illust(int(illust))
- if not post:
- self.errored = True
- return False
- self.pages[num].append(post.unique_id)
+ keys.extend([int(x) for x in obj['manga'].keys()])
+ for illust in keys:
+ post = self.request_illust(illust)
+ if not post:
+ self.errored = True
+ return False
+ self.pages[num].append(post.unique_id)
self.completed = True
return True
@@ -231,18 +235,18 @@ class PixivWebFollowing(PixivWebBase):
return True
class PixivWebModule(Module):
- def __init__(self, sessid: str):
+ def __init__(self, sessid: str, user_id: str):
super().__init__()
self.parser: QueryParser = QueryParser(self, 'search')
self.parser.add_command('search', PixivWebSearch)
self.parser.add_command('illust', PixivWebIllust)
self.parser.add_command('user', PixivWebUser)
self.parser.add_command('following', PixivWebFollowing)
- self.headers['Accept-Encoding'] = 'gzip, deflate, br'
+ self.headers['Accept-Encoding'] = 'gzip, deflate, br' # add 'zstd' when httpx update
self.headers['Accept-Language'] = 'en-US,en;q=0.5'
self.headers['Referer'] = 'https://www.pixiv.net/'
- self.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0'
- self.headers['x-user-id'] = '22781328'
+ self.headers['User-Agent'] = USER_AGENT
+ self.headers['x-user-id'] = user_id
self.cookies['PHPSESSID'] = sessid
def search(self, query: str, *extra_args: Any) -> Optional[Search]:
@@ -257,7 +261,7 @@ class PixivWebModule(Module):
return {
'urls': [post.media[key].url],
'headers': {
- 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0',
+ 'User-Agent': USER_AGENT,
'Referer': 'https://www.pixiv.net/'
}
}
diff --git a/src/portal/py/modules/twitter.py b/src/portal/py/modules/twitter.py
index 7e77253..2996d0e 100644
--- a/src/portal/py/modules/twitter.py
+++ b/src/portal/py/modules/twitter.py
@@ -2,7 +2,7 @@ import log
import http.cookiejar
from typing import Optional, Any, Iterator
from datetime import timezone
-from base import Search, Module
+from base import USER_AGENT, Search, Module
from post import MediaUrl, PostType, DateType, PostRef, Post, User, Media, Image, Video
from query_parser import QueryParser
from modules.common import get_current_utc_time
@@ -185,6 +185,6 @@ class TwitterScrapeModule(Module):
return {
'urls': [post.media[key].url],
'headers': {
- 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0'
+ 'User-Agent': USER_AGENT
}
}
diff --git a/src/portal/py/post.py b/src/portal/py/post.py
index b064f97..8149e4c 100644
--- a/src/portal/py/post.py
+++ b/src/portal/py/post.py
@@ -1,6 +1,6 @@
import dataclasses
from dataclasses import field
-from typing import Optional
+from typing import Optional, Any
from enum import Enum
from json import JSONEncoder
@@ -34,6 +34,9 @@ class TagType(int, Enum):
META = 5
DEPRECATED = 6
+def df(c: Any) -> Any:
+ return field(default_factory=lambda: c)
+
@dataclasses.dataclass
class MediaUrl():
url: str = ''
@@ -42,7 +45,7 @@ class MediaUrl():
@dataclasses.dataclass
class Media():
type: MediaType = MediaType.UNKNOWN
- url: MediaUrl = field(default_factory=lambda: MediaUrl())
+ url: MediaUrl = df(MediaUrl())
@dataclasses.dataclass
class File(Media):
@@ -56,38 +59,38 @@ class Audio(Media):
@dataclasses.dataclass
class Image(Media):
type: MediaType = MediaType.IMAGE
- thumbnail_url: MediaUrl = field(default_factory=lambda: MediaUrl())
+ thumbnail_url: MediaUrl = df(MediaUrl())
@dataclasses.dataclass
class Video(Media):
type: MediaType = MediaType.VIDEO
- thumbnail_url: MediaUrl = field(default_factory=lambda: MediaUrl())
+ thumbnail_url: MediaUrl = df(MediaUrl())
@dataclasses.dataclass
class VideoSplit(Media):
type: MediaType = MediaType.VIDEO_SPLIT
- audio_url: MediaUrl = field(default_factory=lambda: MediaUrl())
- subtitle_url: MediaUrl = field(default_factory=lambda: MediaUrl())
- thumbnail_url: MediaUrl = field(default_factory=lambda: MediaUrl())
+ audio_url: MediaUrl = df(MediaUrl())
+ subtitle_url: MediaUrl = df(MediaUrl())
+ thumbnail_url: MediaUrl = df(MediaUrl())
@dataclasses.dataclass
class Animation(Media):
type: MediaType = MediaType.ANIMATION
- thumbnail_url: MediaUrl = field(default_factory=lambda: MediaUrl())
- frames: list[tuple[str, int]] = field(default_factory=lambda: [])
+ thumbnail_url: MediaUrl = df(MediaUrl())
+ frames: list[tuple[str, int]] = df([])
@dataclasses.dataclass
class Tag():
type: TagType = TagType.UNKNOWN
name: str = ''
- alts: dict[str, str] = field(default_factory=lambda: {})
+ alts: dict[str, str] = df({})
@dataclasses.dataclass
class User():
unique_id: str = ''
username: str = ''
display_name: str = ''
- profile_picture_url: MediaUrl = field(default_factory=lambda: MediaUrl())
+ profile_picture_url: MediaUrl = df(MediaUrl())
@dataclasses.dataclass
class PostRef():
@@ -104,10 +107,10 @@ class Post():
version: int = 3
type: PostType = PostType.UNKNOWN
unique_id: str = ''
- raw_responses: dict[str, str] = field(default_factory=lambda: {})
+ raw_responses: dict[str, str] = df({})
url: str = ''
- dates: dict[DateType, float] = field(default_factory=lambda: {})
- author: User = field(default_factory=lambda: User())
+ dates: dict[DateType, float] = df({})
+ author: User = df(User())
title: str = ''
text: str = ''
likes: Optional[int] = None
@@ -116,12 +119,12 @@ class Post():
quotes: Optional[int] = None
comments: Optional[int] = None
views: Optional[int] = None
- tags: list[Tag] = field(default_factory=lambda: [])
- links: list[str] = field(default_factory=lambda: [])
- media: dict[str, Media] = field(default_factory=lambda: {})
- post: PostRef = field(default_factory=lambda: PostRef())
- quoted: PostRef = field(default_factory=lambda: PostRef())
- in_reply_to: PostRef = field(default_factory=lambda: PostRef())
+ tags: list[Tag] = df([])
+ links: list[str] = df([])
+ media: dict[str, Media] = df({})
+ post: PostRef = df(PostRef())
+ quoted: PostRef = df(PostRef())
+ in_reply_to: PostRef = df(PostRef())
class PostEncoder(JSONEncoder):
def default(self, o):
diff --git a/src/portal/src/search.c b/src/portal/src/search.c
index ec40d0e..7334c41 100644
--- a/src/portal/src/search.c
+++ b/src/portal/src/search.c
@@ -71,7 +71,7 @@ s32 camu_portal_create_search(struct camu_portal *portal, str *module, str *quer
search->portal = portal;
al_array_push(portal->searches, search);
}
- al_log_info("portal", "New search %i (%.*s).", id, AL_STR_PRINTF(query));
+ al_log_info("portal", "New search %x (%.*s).", id, AL_STR_PRINTF(query));
return id;
}
diff --git a/src/portal/tests/test.py b/src/portal/tests/test.py
index cdf7178..a5139f6 100644
--- a/src/portal/tests/test.py
+++ b/src/portal/tests/test.py
@@ -3,7 +3,7 @@ import httpx
import sys
sys.path.append('../py')
from base import Method
-from post import DateType, PostEncoder
+from post import DateType, PostEncoder, Media
from modules import ALL_MODULES
#ALL_MODULES['instagram'][0].init()
@@ -12,6 +12,10 @@ from modules import ALL_MODULES
#print(page)
#print(json.dumps(page, indent=4, cls=PostEncoder))
+d = Media()
+print(d)
+sys.exit(1)
+
#module = ALL_MODULES['fanbox'][0]
#module = ALL_MODULES['pixiv_web'][0]
module = ALL_MODULES['instagram'][0]
generated by cgit v1.2.3-101-g0448 (git 2.54.0) at 2026-07-25 19:32:31 +0000