summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/portal/py/base.py3
-rw-r--r--src/portal/py/modules/__init__.py18
-rw-r--r--src/portal/py/modules/fanbox.py32
-rw-r--r--src/portal/py/modules/instagram.py30
-rw-r--r--src/portal/py/modules/pixiv_web.py74
-rw-r--r--src/portal/py/modules/youtube.py2
-rw-r--r--src/portal/py/post.py14
-rw-r--r--src/portal/py/test.py3
-rw-r--r--src/portal/py/tests/archive_query.py7
-rw-r--r--src/portal/py/tests/test.py10
10 files changed, 116 insertions, 77 deletions
diff --git a/src/portal/py/base.py b/src/portal/py/base.py
index 660843c..18dd961 100644
--- a/src/portal/py/base.py
+++ b/src/portal/py/base.py
@@ -4,6 +4,7 @@ import json
import httpx
import binascii
import threading
+import config
from enum import Enum
from blake3 import blake3
from collections import OrderedDict
@@ -12,7 +13,7 @@ from post import Post, User
type ParsedJson = Any
-USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; rv:131.0) Gecko/20100101 Firefox/131.0'
+USER_AGENT = config.USER_AGENT
class Method(Enum):
HEAD = "HEAD"
diff --git a/src/portal/py/modules/__init__.py b/src/portal/py/modules/__init__.py
index 7fe56b8..c44e571 100644
--- a/src/portal/py/modules/__init__.py
+++ b/src/portal/py/modules/__init__.py
@@ -1,11 +1,11 @@
-#import config
+import config
from modules.youtube import YoutubeModule
#from modules.twitter import TwitterScrapeModule
#from modules.pixiv_app import PixivAppModule
-#from modules.pixiv_web import PixivWebModule
-#from modules.fanbox import FanboxModule
-#from modules.instagram import InstagramModule
+from modules.pixiv_web import PixivWebModule
+from modules.fanbox import FanboxModule
+from modules.instagram import InstagramModule
#from modules.patreon import PatreonModule
ALL_MODULES = {
@@ -14,11 +14,11 @@ ALL_MODULES = {
# config.TWITTER_ACCESS_TOKEN, config.TWITTER_ACCESS_TOKEN_SECRET,
# config.TWITTER_CONSUMER_TOKEN, config.TWITTER_CONSUMER_TOKEN_SECRET), []),
# 'twitter': (TwitterScrapeModule(config.TWITTER_COOKIES_PATH), []),
-# 'pixiv_web': (PixivWebModule(config.PIXIV_SESSID, config.PIXIV_USERID), []),
+ 'pixiv_web': (PixivWebModule(config.PIXIV_SESSID, config.PIXIV_USERID), []),
# 'pixiv_app': (PixivAppModule(config.PIXIV_REFRESH_TOKEN), []),
-# 'fanbox': (FanboxModule(config.FANBOX_SESSID), []),
-# 'instagram': (InstagramModule(
-# config.INSTAGRAM_USER_AGENT, config.INSTAGRAM_SETTINGS_PATH,
-# config.INSTAGRAM_SESSION_ID), []),
+ 'fanbox': (FanboxModule(config.FANBOX_SESSID, config.FANBOX_CF_CLEARANCE), []),
+ 'instagram': (InstagramModule(
+ config.INSTAGRAM_USER_AGENT, config.INSTAGRAM_SETTINGS_PATH,
+ config.INSTAGRAM_SESSION_ID), []),
# 'patreon': (PatreonModule(config.PATREON_SESSION_ID, config.PATREON_UUID, config.PATREON_USER_ID), [])
}
diff --git a/src/portal/py/modules/fanbox.py b/src/portal/py/modules/fanbox.py
index fefe1c2..ba9a9f7 100644
--- a/src/portal/py/modules/fanbox.py
+++ b/src/portal/py/modules/fanbox.py
@@ -3,7 +3,7 @@ import httpx
from typing import Optional, Any
from json.decoder import JSONDecodeError
from base import USER_AGENT, Search, Module, Method, ParsedJson
-from post import Url, PostType, DateType, Post, User, Image, File, Tag
+from post import Url, PostType, Date, DateType, DateMeta, Post, User, Image, File, Tag
from query_parser import QueryParser
from modules.common import reformat_pixiv_date, get_current_utc_time
@@ -23,9 +23,6 @@ class FanboxBase(Search):
return None
return obj['body']
- def create_media_url(self, url: str) -> Url:
- return Url(url, url.split('.')[-1])
-
def create_post(self, data: ParsedJson) -> Optional[Post]:
kwargs = {}
post_id = data['id']
@@ -34,15 +31,16 @@ class FanboxBase(Search):
kwargs['raw_responses'] = {}
kwargs['raw_responses']['api'] = json.dumps(data)
publish_date = reformat_pixiv_date(data['publishedDatetime']).timestamp()
- kwargs['dates'] = {
- DateType.CREATED: publish_date,
- DateType.RETRIEVED: get_current_utc_time().timestamp()
- }
+ current_date = get_current_utc_time().timestamp()
+ kwargs['dates'] = [
+ Date(DateType.CREATED, (publish_date, publish_date), DateMeta.NONE),
+ Date(DateType.RETRIEVED, (current_date, current_date), DateMeta.NONE)
+ ]
update_date = reformat_pixiv_date(data['updatedDatetime']).timestamp()
if update_date != publish_date:
- kwargs['dates'][DateType.EDITED] = update_date
+ kwargs['dates'].append(Date(DateType.EDITED, (update_date, update_date), DateMeta.NONE))
user = User(unique_id=f'fanbox:u:{data['user']['userId']}', username=data['user']['name'])
- user.profile_picture_url = self.create_media_url(data['user']['iconUrl'])
+ user.profile_picture_url = Url(data['user']['iconUrl'])
kwargs['author'] = user
kwargs['title'] = data['title']
kwargs['likes'] = data['likeCount']
@@ -57,7 +55,7 @@ class FanboxBase(Search):
if data['type'] == 'image':
text = data['body']['text']
for i, value in enumerate(data['body']['images']):
- media[str(i)] = Image(url=Url(value['originalUrl'], value['extension']), thumbnail_url=self.create_media_url(value['thumbnailUrl']))
+ media[str(i)] = Image(url=Url(value['originalUrl'], value['extension']), thumbnail_url=Url(value['thumbnailUrl']))
elif data['type'] == 'file':
text = data['body']['text']
for i, value in enumerate(data['body']['files']):
@@ -71,7 +69,7 @@ class FanboxBase(Search):
elif block['type'] == 'url_embed':
text += f'embed::{block['urlEmbedId']}[]' + '\n'
for key, value in data['body']['imageMap'].items():
- media[key] = Image(url=Url(value['originalUrl'], value['extension']), thumbnail_url=self.create_media_url(value['thumbnailUrl']))
+ media[key] = Image(url=Url(value['originalUrl'], value['extension']), thumbnail_url=Url(value['thumbnailUrl']))
for key, value in data['body']['fileMap'].items():
media[key] = File(url=Url(value['url'], value['extension']), name=value['name'])
kwargs['text'] = text
@@ -90,7 +88,7 @@ class FanboxBase(Search):
def create_user(self, data: ParsedJson) -> User:
unique_id = f'fanbox:u:{data['id']}'
user = User(unique_id=unique_id, username=data['creatorId'], display_name=data['user']['name'])
- user.profile_picture_url = self.create_media_url(data['user']['iconUrl'])
+ user.profile_picture_url = Url(data['user']['iconUrl'])
self.module.add_to_map(unique_id, user)
return user
@@ -163,7 +161,7 @@ class FanboxSupporting(FanboxBase):
return True
class FanboxModule(Module):
- def __init__(self, sessid: str) -> None:
+ def __init__(self, sessid: str, cf_clearance: str) -> None:
super().__init__()
self.parser: QueryParser = QueryParser(self, 'post')
self.parser.add_command('post', FanboxPost)
@@ -178,11 +176,7 @@ class FanboxModule(Module):
self.cookies['FANBOXSESSID'] = sessid
self.cookies['privacy_policy_agreement'] = '7'
self.cookies['privacy_policy_notification'] = '0'
- self.cookies['p_ab_id'] = '0'
- self.cookies['p_ab_id_2'] = '6'
- self.cookies['p_ab_d_id'] = '251960935'
- self.cookies['cf_clearance'] = ''
- self.cookies['__cf_bm'] = ''
+ self.cookies['cf_clearance'] = cf_clearance
def search(self, query: str, *extra_args: Any) -> Optional[Search]:
return self.parser.parse_query(query)
diff --git a/src/portal/py/modules/instagram.py b/src/portal/py/modules/instagram.py
index 553ce56..014ffc1 100644
--- a/src/portal/py/modules/instagram.py
+++ b/src/portal/py/modules/instagram.py
@@ -3,7 +3,7 @@ import log
from typing import Optional, Any
from pathlib import Path
from base import USER_AGENT, Module, Search
-from post import Url, PostType, DateType, Post, User, Image, Video
+from post import Url, PostType, Date, DateType, DateMeta, Post, User, Image, Video
from modules.common import get_current_utc_time
from instagrapi import Client
from instagrapi.exceptions import UserNotFound, LoginRequired, ChallengeRequired
@@ -18,14 +18,6 @@ class InstagramSearch(Search):
self.page: int = 0
self.cursor: Any = None
- def create_media_url(self, url: str) -> Url:
- question = url.find('?')
- if question >= 0:
- ext = url[:question].split('.')[-1]
- else:
- ext = url.split('.')[-1]
- return Url(url, ext)
-
def request_page(self, num: int) -> bool:
if num >= self.page + self.REACH_LIMIT:
return False
@@ -53,17 +45,19 @@ class InstagramSearch(Search):
kwargs['raw_responses']['tile'] = m.json()
kwargs['url'] = 'https://www.instagram.com/p/{}'.format(m.code)
#kwargs['url'] = 'https://www.instagram.com/reel/{}'.format(m.code)
- kwargs['dates'] = {
- DateType.CREATED: m.taken_at.timestamp(),
- DateType.RETRIEVED: get_current_utc_time().timestamp()
- }
+ create_date = m.taken_at.timestamp()
+ current_date = get_current_utc_time().timestamp()
+ kwargs['dates'] = [
+ Date(DateType.CREATED, (create_date, create_date), DateMeta.NONE),
+ Date(DateType.RETRIEVED, (current_date, current_date), DateMeta.NONE)
+ ]
user = User(unique_id='instagram:u:{}'.format(m.user.pk))
if m.user.username:
user.username = m.user.username
if m.user.full_name:
user.display_name = m.user.full_name
if m.user.profile_pic_url:
- user.profile_picture_url = self.create_media_url(str(m.user.profile_pic_url))
+ user.profile_picture_url = Url(str(m.user.profile_pic_url))
kwargs['author'] = user
kwargs['title'] = m.title
kwargs['text'] = m.caption_text
@@ -73,15 +67,15 @@ class InstagramSearch(Search):
if m.media_type == 8: # Album
for k, r in enumerate(m.resources):
if r.media_type == 1: # Photo
- media[str(k)] = Image(url=self.create_media_url(str(r.thumbnail_url)))
+ media[str(k)] = Image(url=Url(str(r.thumbnail_url)))
elif r.media_type == 2: # Video
- media[str(k)] = Video(url=self.create_media_url(str(r.video_url)), thumbnail_url=self.create_media_url(str(r.thumbnail_url)))
+ media[str(k)] = Video(url=Url(str(r.video_url)), thumbnail_url=Url(str(r.thumbnail_url)))
elif m.media_type == 2: # Video
kwargs['views'] = m.play_count
- media[str(0)] = Video(url=self.create_media_url(str(m.video_url)), thumbnail_url=self.create_media_url(str(m.thumbnail_url)))
+ media[str(0)] = Video(url=Url(str(m.video_url)), thumbnail_url=Url(str(m.thumbnail_url)))
elif m.media_type == 1: # Photo
candidates = sorted(m.image_versions2['candidates'], key=lambda x: x['width'], reverse=True)
- media[str(0)] = Image(url=self.create_media_url(candidates[0]['url']), thumbnail_url=self.create_media_url(candidates[1]['url']))
+ media[str(0)] = Image(url=Url(candidates[0]['url']), thumbnail_url=Url(candidates[1]['url']))
kwargs['media'] = media
post = Post(**kwargs)
self.module.add_to_map(unique_id, post)
diff --git a/src/portal/py/modules/pixiv_web.py b/src/portal/py/modules/pixiv_web.py
index 33daeb3..60fa812 100644
--- a/src/portal/py/modules/pixiv_web.py
+++ b/src/portal/py/modules/pixiv_web.py
@@ -5,7 +5,7 @@ import urllib.parse
from typing import Optional, Any
from json.decoder import JSONDecodeError
from base import USER_AGENT, Search, Module, Method, ParsedJson
-from post import Url, PostType, DateType, Post, User, Media, Image, Animation, Tag, TagType
+from post import Url, PostType, Date, DateType, DateMeta, Post, User, Media, Image, Animation, Tag, TagType
from query_parser import QueryParser
from modules.common import reformat_pixiv_date, get_current_utc_time
@@ -30,17 +30,14 @@ class PixivWebBase(Search):
return None
return obj['body']
- def create_media_url(self, url: str) -> Url:
- return Url(url, url.split('.')[-1])
-
def parse_pages(self, data: ParsedJson) -> dict[str, Media]:
media = {}
for i, m in enumerate(data):
- media[str(i)] = Image(url=self.create_media_url(m['urls']['original']), thumbnail_url=self.create_media_url(m['urls']['small']))
+ media[str(i)] = Image(url=Url(m['urls']['original']), thumbnail_url=Url(m['urls']['small']))
return media
def parse_ugoira(self, data: ParsedJson, thumbnail_url: str) -> Animation:
- animation = Animation(url=self.create_media_url(data['originalSrc']), thumbnail_url=self.create_media_url(thumbnail_url))
+ animation = Animation(url=Url(data['originalSrc']), thumbnail_url=Url(thumbnail_url))
for frame in data['frames']:
animation.frames.append((frame['file'], frame['delay']))
return animation
@@ -50,7 +47,7 @@ class PixivWebBase(Search):
if illust is None:
continue
if 'profileImageUrl' in illust:
- return self.create_media_url(illust['profileImageUrl'])
+ return Url(illust['profileImageUrl'])
return None
def create_tag(self, data: ParsedJson) -> Tag:
@@ -73,13 +70,13 @@ class PixivWebBase(Search):
kwargs['url'] = f'https://www.pixiv.net/{LANG}/artworks/{data['illustId']}'
create_date = reformat_pixiv_date(data['createDate']).timestamp()
current_date = get_current_utc_time().timestamp()
- kwargs['dates'] = {
- DateType.CREATED: (create_date, create_date, 0),
- DateType.RETRIEVED: (current_date, current_date, 0)
- }
+ kwargs['dates'] = [
+ Date(DateType.CREATED, (create_date, create_date), DateMeta.NONE),
+ Date(DateType.RETRIEVED, (current_date, current_date), DateMeta.NONE)
+ ]
upload_date = reformat_pixiv_date(data['uploadDate']).timestamp()
if upload_date != create_date:
- kwargs['dates'][DateType.EDITED] = (upload_date, upload_date, 0)
+ kwargs['dates'].append(Date(DateType.EDITED, (upload_date, upload_date), DateMeta.NONE))
user = User(unique_id=f'pixiv:u:{data['userId']}', username=data['userAccount'], display_name=data['userName'])
profile_picture_url = self.seek_profile_picture_url(data['userIllusts'])
if profile_picture_url:
@@ -114,6 +111,16 @@ class PixivWebBase(Search):
self.module.add_to_map(unique_id, post)
return post
+ def create_tombstone(self, data: ParsedJson) -> Post:
+ kwargs = {}
+ kwargs['type'] = PostType.TOMBSTONE
+ illust_id = str(data['id'])
+ unique_id = f'pixiv:i:{illust_id}'
+ kwargs['unique_id'] = unique_id
+ post = Post(**kwargs)
+ self.module.add_to_map(unique_id, post)
+ return post
+
def remake_post(self, raw_responses: ParsedJson, original_date: float) -> Optional[Post]:
api = json.loads(raw_responses['api'])
pages = raw_responses.get('pages', None)
@@ -137,7 +144,7 @@ class PixivWebBase(Search):
def create_user(self, data: ParsedJson) -> User:
unique_id=f'pixiv:u:{data['userId']}'
user = User(unique_id=unique_id, display_name=data['userName'])
- user.profile_picture_url = self.create_media_url(data['profileImageUrl'])
+ user.profile_picture_url = Url(data['profileImageUrl'])
self.module.add_to_map(unique_id, user)
return user
@@ -214,6 +221,42 @@ class PixivWebUser(PixivWebBase):
self.completed = True
return True
+class PixivWebBookmarks(PixivWebBase):
+ LIMIT = 48
+
+ def __init__(self, userdata: Any, arg: str) -> None:
+ super().__init__(userdata)
+ try:
+ self.id: int = int(arg)
+ except ValueError:
+ self.errored = True
+ self.total: int = 0
+ self.count: int = 0
+
+ # rest=show - public
+ # rest=hide - private
+ def request_page(self, num: int) -> bool:
+ url = f'{BASE_URL}/user/{self.id}/illusts/bookmarks?tag=&offset={self.LIMIT * num}&limit={self.LIMIT}&rest=show&lang={LANG}&version={VERSION}'
+ obj = self.check_api_response(self.module.do_request(Method.GET, url))
+ if not obj:
+ self.errored = True
+ return False
+ self.total = obj['total']
+ self.pages[num] = []
+ for illust in obj['works']:
+ if illust['userId'] == 0:
+ post = self.create_tombstone(illust)
+ else:
+ post = self.request_illust(int(illust['id']))
+ if not post:
+ self.errored = True
+ return False
+ self.pages[num].append(post.unique_id)
+ self.count += 1
+ if self.count >= self.total:
+ self.completed = True
+ return True
+
class PixivWebFollowing(PixivWebBase):
LIMIT = 24
@@ -224,8 +267,10 @@ class PixivWebFollowing(PixivWebBase):
except ValueError:
self.errored = True
+ # rest=show - public
+ # rest=hide - private
def request_page(self, num: int) -> bool:
- url = f'{BASE_URL}/user/{self.id}/following?offset={self.LIMIT * num}&limit={self.LIMIT}&rest=show&tag=&acceptingRequests=0&lang={LANG}&version={VERSION}'
+ url = f'{BASE_URL}/user/{self.id}/following?tag=&offset={self.LIMIT * num}&limit={self.LIMIT}&rest=show&acceptingRequests=0&lang={LANG}&version={VERSION}'
obj = self.check_api_response(self.module.do_request(Method.GET, url))
if not obj:
self.errored = True
@@ -242,6 +287,7 @@ class PixivWebModule(Module):
self.parser.add_command('search', PixivWebSearch)
self.parser.add_command('illust', PixivWebIllust)
self.parser.add_command('user', PixivWebUser)
+ self.parser.add_command('bookmarks', PixivWebBookmarks)
self.parser.add_command('following', PixivWebFollowing)
self.headers['User-Agent'] = USER_AGENT
self.headers['Accept-Encoding'] = 'gzip, deflate, br, zstd'
diff --git a/src/portal/py/modules/youtube.py b/src/portal/py/modules/youtube.py
index 16d7f67..5125f85 100644
--- a/src/portal/py/modules/youtube.py
+++ b/src/portal/py/modules/youtube.py
@@ -94,7 +94,7 @@ class YoutubeBase(Search):
if info.get('direct'):
url = info['url']
else:
- url = get_playback_url(info, video=False)
+ url = get_playback_url(info, video=True)
if not url:
return False
unique_id = f'youtube:v:{info['id']}'
diff --git a/src/portal/py/post.py b/src/portal/py/post.py
index ca7cf4e..e926c01 100644
--- a/src/portal/py/post.py
+++ b/src/portal/py/post.py
@@ -67,13 +67,15 @@ class Url():
url: str
ext: str
- def __init__(self, url: str, ext: str = 'unknown') -> None:
+ def __init__(self, url: str, ext: str = None) -> None:
self.url = url
- ext_guess = default_url_ext_guess(url)
- if ext_guess:
- self.ext = ext_guess
- else:
- self.ext = ext
+ if not ext:
+ ext_guess = default_url_ext_guess(url)
+ if ext_guess:
+ ext = ext_guess
+ else:
+ ext = 'unknown'
+ self.ext = ext
@dataclasses.dataclass
class Media():
diff --git a/src/portal/py/test.py b/src/portal/py/test.py
index c5687b1..0ca89a4 100644
--- a/src/portal/py/test.py
+++ b/src/portal/py/test.py
@@ -1 +1,2 @@
-import tests.test
+#import tests.test
+import tests.archive_query
diff --git a/src/portal/py/tests/archive_query.py b/src/portal/py/tests/archive_query.py
index 1097634..cff5e5d 100644
--- a/src/portal/py/tests/archive_query.py
+++ b/src/portal/py/tests/archive_query.py
@@ -90,7 +90,7 @@ class QueryDownloadThread(threading.Thread):
return False
del post.raw_responses['detached_api']
if post.type != PostType.TOMBSTONE:
- message += f':{datetime.fromtimestamp(post.dates[DateType.CREATED], tz=timezone.utc).isoformat()} from {post.author.username}({post.author.display_name}):{post.author.unique_id}.'
+ message += f':{datetime.fromtimestamp(post.dates[0].range[0], tz=timezone.utc).isoformat()} from {post.author.username}({post.author.display_name}):{post.author.unique_id}.'
else:
message += '.'
output_path = self.make_path(post)
@@ -258,6 +258,5 @@ def signal_handler(sig, frame):
else:
is_running = False
-if __name__ == "__main__":
- signal.signal(signal.SIGINT, signal_handler)
- download_args()
+signal.signal(signal.SIGINT, signal_handler)
+download_args()
diff --git a/src/portal/py/tests/test.py b/src/portal/py/tests/test.py
index bee7265..15bde0a 100644
--- a/src/portal/py/tests/test.py
+++ b/src/portal/py/tests/test.py
@@ -55,11 +55,13 @@ from modules import ALL_MODULES
# else:
# print('Skipping file')
-ALL_MODULES['youtube'][0].init()
-search = ALL_MODULES['youtube'][0].search('yoyoyoy')
+module = ALL_MODULES['instagram'][0]
+module.init()
+search = module.search('plottttwistttttt')
page = search.get_page(0)
-print(page)
-#print(json.dumps(page, indent=4, cls=PostEncoder))
+for unique_id in page:
+ post = module.get_item(unique_id)
+ print(json.dumps(post, indent=4, cls=PostEncoder))
#d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.EDITED_AT_UNKNOWN_TIME)
#d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.NONE)
generated by cgit v1.2.3-101-g0448 (git 2.54.0) at 2026-07-25 19:19:32 +0000