Misc portal updates

Signed-off-by: Andrew Opalach <andrew@akon.city>

5 files changed, 95 insertions, 61 deletions

diff --git a/src/portal/py/modules/__init__.py b/src/portal/py/modules/__init__.py
index 7fe56b8..c44e571 100644
--- a/src/portal/py/modules/__init__.py
+++ b/src/portal/py/modules/__init__.py
@@ -1,11 +1,11 @@
-#import config
+import config
 
 from modules.youtube import YoutubeModule
 #from modules.twitter import TwitterScrapeModule
 #from modules.pixiv_app import PixivAppModule
-#from modules.pixiv_web import PixivWebModule
-#from modules.fanbox import FanboxModule
-#from modules.instagram import InstagramModule
+from modules.pixiv_web import PixivWebModule
+from modules.fanbox import FanboxModule
+from modules.instagram import InstagramModule
 #from modules.patreon import PatreonModule
 
 ALL_MODULES = {
@@ -14,11 +14,11 @@ ALL_MODULES = {
 #        config.TWITTER_ACCESS_TOKEN, config.TWITTER_ACCESS_TOKEN_SECRET,
 #        config.TWITTER_CONSUMER_TOKEN, config.TWITTER_CONSUMER_TOKEN_SECRET), []),
 #    'twitter': (TwitterScrapeModule(config.TWITTER_COOKIES_PATH), []),
-#    'pixiv_web': (PixivWebModule(config.PIXIV_SESSID, config.PIXIV_USERID), []),
+    'pixiv_web': (PixivWebModule(config.PIXIV_SESSID, config.PIXIV_USERID), []),
 #    'pixiv_app': (PixivAppModule(config.PIXIV_REFRESH_TOKEN), []),
-#    'fanbox': (FanboxModule(config.FANBOX_SESSID), []),
-#    'instagram': (InstagramModule(
-#        config.INSTAGRAM_USER_AGENT, config.INSTAGRAM_SETTINGS_PATH,
-#        config.INSTAGRAM_SESSION_ID), []),
+    'fanbox': (FanboxModule(config.FANBOX_SESSID, config.FANBOX_CF_CLEARANCE), []),
+    'instagram': (InstagramModule(
+        config.INSTAGRAM_USER_AGENT, config.INSTAGRAM_SETTINGS_PATH,
+        config.INSTAGRAM_SESSION_ID), []),
 #    'patreon': (PatreonModule(config.PATREON_SESSION_ID, config.PATREON_UUID, config.PATREON_USER_ID), [])
 }
diff --git a/src/portal/py/modules/fanbox.py b/src/portal/py/modules/fanbox.py
index fefe1c2..ba9a9f7 100644
--- a/src/portal/py/modules/fanbox.py
+++ b/src/portal/py/modules/fanbox.py
@@ -3,7 +3,7 @@ import httpx
 from typing import Optional, Any
 from json.decoder import JSONDecodeError
 from base import USER_AGENT, Search, Module, Method, ParsedJson
-from post import Url, PostType, DateType, Post, User, Image, File, Tag
+from post import Url, PostType, Date, DateType, DateMeta, Post, User, Image, File, Tag
 from query_parser import QueryParser
 from modules.common import reformat_pixiv_date, get_current_utc_time
 
@@ -23,9 +23,6 @@ class FanboxBase(Search):
             return None
         return obj['body']
 
-    def create_media_url(self, url: str) -> Url:
-        return Url(url, url.split('.')[-1])
-
     def create_post(self, data: ParsedJson) -> Optional[Post]:
         kwargs = {}
         post_id = data['id']
@@ -34,15 +31,16 @@ class FanboxBase(Search):
         kwargs['raw_responses'] = {}
         kwargs['raw_responses']['api'] = json.dumps(data)
         publish_date = reformat_pixiv_date(data['publishedDatetime']).timestamp()
-        kwargs['dates'] = {
-            DateType.CREATED: publish_date,
-            DateType.RETRIEVED: get_current_utc_time().timestamp()
-        }
+        current_date = get_current_utc_time().timestamp()
+        kwargs['dates'] = [
+            Date(DateType.CREATED, (publish_date, publish_date), DateMeta.NONE),
+            Date(DateType.RETRIEVED, (current_date, current_date), DateMeta.NONE)
+        ]
         update_date = reformat_pixiv_date(data['updatedDatetime']).timestamp()
         if update_date != publish_date:
-            kwargs['dates'][DateType.EDITED] = update_date
+            kwargs['dates'].append(Date(DateType.EDITED, (update_date, update_date), DateMeta.NONE))
         user = User(unique_id=f'fanbox:u:{data['user']['userId']}', username=data['user']['name'])
-        user.profile_picture_url = self.create_media_url(data['user']['iconUrl'])
+        user.profile_picture_url = Url(data['user']['iconUrl'])
         kwargs['author'] = user
         kwargs['title'] = data['title']
         kwargs['likes'] = data['likeCount']
@@ -57,7 +55,7 @@ class FanboxBase(Search):
             if data['type'] == 'image':
                 text = data['body']['text']
                 for i, value in enumerate(data['body']['images']):
-                    media[str(i)] = Image(url=Url(value['originalUrl'], value['extension']), thumbnail_url=self.create_media_url(value['thumbnailUrl']))
+                    media[str(i)] = Image(url=Url(value['originalUrl'], value['extension']), thumbnail_url=Url(value['thumbnailUrl']))
             elif data['type'] == 'file':
                 text = data['body']['text']
                 for i, value in enumerate(data['body']['files']):
@@ -71,7 +69,7 @@ class FanboxBase(Search):
                     elif block['type'] == 'url_embed':
                         text += f'embed::{block['urlEmbedId']}[]' + '\n'
                 for key, value in data['body']['imageMap'].items():
-                    media[key] = Image(url=Url(value['originalUrl'], value['extension']), thumbnail_url=self.create_media_url(value['thumbnailUrl']))
+                    media[key] = Image(url=Url(value['originalUrl'], value['extension']), thumbnail_url=Url(value['thumbnailUrl']))
                 for key, value in data['body']['fileMap'].items():
                     media[key] = File(url=Url(value['url'], value['extension']), name=value['name'])
             kwargs['text'] = text
@@ -90,7 +88,7 @@ class FanboxBase(Search):
     def create_user(self, data: ParsedJson) -> User:
         unique_id = f'fanbox:u:{data['id']}'
         user = User(unique_id=unique_id, username=data['creatorId'], display_name=data['user']['name'])
-        user.profile_picture_url = self.create_media_url(data['user']['iconUrl'])
+        user.profile_picture_url = Url(data['user']['iconUrl'])
         self.module.add_to_map(unique_id, user)
         return user
 
@@ -163,7 +161,7 @@ class FanboxSupporting(FanboxBase):
         return True
 
 class FanboxModule(Module):
-    def __init__(self, sessid: str) -> None:
+    def __init__(self, sessid: str, cf_clearance: str) -> None:
         super().__init__()
         self.parser: QueryParser = QueryParser(self, 'post')
         self.parser.add_command('post', FanboxPost)
@@ -178,11 +176,7 @@ class FanboxModule(Module):
         self.cookies['FANBOXSESSID'] = sessid
         self.cookies['privacy_policy_agreement'] = '7'
         self.cookies['privacy_policy_notification'] = '0'
-        self.cookies['p_ab_id'] = '0'
-        self.cookies['p_ab_id_2'] = '6'
-        self.cookies['p_ab_d_id'] = '251960935'
-        self.cookies['cf_clearance'] = ''
-        self.cookies['__cf_bm'] = ''
+        self.cookies['cf_clearance'] = cf_clearance
 
     def search(self, query: str, *extra_args: Any) -> Optional[Search]:
         return self.parser.parse_query(query)
diff --git a/src/portal/py/modules/instagram.py b/src/portal/py/modules/instagram.py
index 553ce56..014ffc1 100644
--- a/src/portal/py/modules/instagram.py
+++ b/src/portal/py/modules/instagram.py
@@ -3,7 +3,7 @@ import log
 from typing import Optional, Any
 from pathlib import Path
 from base import USER_AGENT, Module, Search
-from post import Url, PostType, DateType, Post, User, Image, Video
+from post import Url, PostType, Date, DateType, DateMeta, Post, User, Image, Video
 from modules.common import get_current_utc_time
 from instagrapi import Client
 from instagrapi.exceptions import UserNotFound, LoginRequired, ChallengeRequired
@@ -18,14 +18,6 @@ class InstagramSearch(Search):
         self.page: int = 0
         self.cursor: Any = None
 
-    def create_media_url(self, url: str) -> Url:
-        question = url.find('?')
-        if question >= 0:
-            ext = url[:question].split('.')[-1]
-        else:
-            ext = url.split('.')[-1]
-        return Url(url, ext)
-
     def request_page(self, num: int) -> bool:
         if num >= self.page + self.REACH_LIMIT:
             return False
@@ -53,17 +45,19 @@ class InstagramSearch(Search):
                 kwargs['raw_responses']['tile'] = m.json()
                 kwargs['url'] = 'https://www.instagram.com/p/{}'.format(m.code)
                 #kwargs['url'] = 'https://www.instagram.com/reel/{}'.format(m.code)
-                kwargs['dates'] = {
-                    DateType.CREATED: m.taken_at.timestamp(),
-                    DateType.RETRIEVED: get_current_utc_time().timestamp()
-                }
+                create_date = m.taken_at.timestamp()
+                current_date = get_current_utc_time().timestamp()
+                kwargs['dates'] = [
+                    Date(DateType.CREATED, (create_date, create_date), DateMeta.NONE),
+                    Date(DateType.RETRIEVED, (current_date, current_date), DateMeta.NONE)
+                ]
                 user = User(unique_id='instagram:u:{}'.format(m.user.pk))
                 if m.user.username:
                     user.username = m.user.username
                 if m.user.full_name:
                     user.display_name = m.user.full_name
                 if m.user.profile_pic_url:
-                    user.profile_picture_url = self.create_media_url(str(m.user.profile_pic_url))
+                    user.profile_picture_url = Url(str(m.user.profile_pic_url))
                 kwargs['author'] = user
                 kwargs['title'] = m.title
                 kwargs['text'] = m.caption_text
@@ -73,15 +67,15 @@ class InstagramSearch(Search):
                 if m.media_type == 8: # Album
                     for k, r in enumerate(m.resources):
                         if r.media_type == 1: # Photo
-                            media[str(k)] = Image(url=self.create_media_url(str(r.thumbnail_url)))
+                            media[str(k)] = Image(url=Url(str(r.thumbnail_url)))
                         elif r.media_type == 2: # Video
-                            media[str(k)] = Video(url=self.create_media_url(str(r.video_url)), thumbnail_url=self.create_media_url(str(r.thumbnail_url)))
+                            media[str(k)] = Video(url=Url(str(r.video_url)), thumbnail_url=Url(str(r.thumbnail_url)))
                 elif m.media_type == 2: # Video
                     kwargs['views'] = m.play_count
-                    media[str(0)] = Video(url=self.create_media_url(str(m.video_url)), thumbnail_url=self.create_media_url(str(m.thumbnail_url)))
+                    media[str(0)] = Video(url=Url(str(m.video_url)), thumbnail_url=Url(str(m.thumbnail_url)))
                 elif m.media_type == 1: # Photo
                     candidates = sorted(m.image_versions2['candidates'], key=lambda x: x['width'], reverse=True)
-                    media[str(0)] = Image(url=self.create_media_url(candidates[0]['url']), thumbnail_url=self.create_media_url(candidates[1]['url']))
+                    media[str(0)] = Image(url=Url(candidates[0]['url']), thumbnail_url=Url(candidates[1]['url']))
                 kwargs['media'] = media
                 post = Post(**kwargs)
                 self.module.add_to_map(unique_id, post)
diff --git a/src/portal/py/modules/pixiv_web.py b/src/portal/py/modules/pixiv_web.py
index 33daeb3..60fa812 100644
--- a/src/portal/py/modules/pixiv_web.py
+++ b/src/portal/py/modules/pixiv_web.py
@@ -5,7 +5,7 @@ import urllib.parse
 from typing import Optional, Any
 from json.decoder import JSONDecodeError
 from base import USER_AGENT, Search, Module, Method, ParsedJson
-from post import Url, PostType, DateType, Post, User, Media, Image, Animation, Tag, TagType
+from post import Url, PostType, Date, DateType, DateMeta, Post, User, Media, Image, Animation, Tag, TagType
 from query_parser import QueryParser
 from modules.common import reformat_pixiv_date, get_current_utc_time
 
@@ -30,17 +30,14 @@ class PixivWebBase(Search):
             return None
         return obj['body']
 
-    def create_media_url(self, url: str) -> Url:
-        return Url(url, url.split('.')[-1])
-
     def parse_pages(self, data: ParsedJson) -> dict[str, Media]:
         media = {}
         for i, m in enumerate(data):
-            media[str(i)] = Image(url=self.create_media_url(m['urls']['original']), thumbnail_url=self.create_media_url(m['urls']['small']))
+            media[str(i)] = Image(url=Url(m['urls']['original']), thumbnail_url=Url(m['urls']['small']))
         return media
 
     def parse_ugoira(self, data: ParsedJson, thumbnail_url: str) -> Animation:
-        animation = Animation(url=self.create_media_url(data['originalSrc']), thumbnail_url=self.create_media_url(thumbnail_url))
+        animation = Animation(url=Url(data['originalSrc']), thumbnail_url=Url(thumbnail_url))
         for frame in data['frames']:
             animation.frames.append((frame['file'], frame['delay']))
         return animation
@@ -50,7 +47,7 @@ class PixivWebBase(Search):
             if illust is None:
                 continue
             if 'profileImageUrl' in illust:
-                return self.create_media_url(illust['profileImageUrl'])
+                return Url(illust['profileImageUrl'])
         return None
 
     def create_tag(self, data: ParsedJson) -> Tag:
@@ -73,13 +70,13 @@ class PixivWebBase(Search):
         kwargs['url'] = f'https://www.pixiv.net/{LANG}/artworks/{data['illustId']}'
         create_date = reformat_pixiv_date(data['createDate']).timestamp()
         current_date = get_current_utc_time().timestamp()
-        kwargs['dates'] = {
-            DateType.CREATED: (create_date, create_date, 0),
-            DateType.RETRIEVED: (current_date, current_date, 0)
-        }
+        kwargs['dates'] = [
+            Date(DateType.CREATED, (create_date, create_date), DateMeta.NONE),
+            Date(DateType.RETRIEVED, (current_date, current_date), DateMeta.NONE)
+        ]
         upload_date = reformat_pixiv_date(data['uploadDate']).timestamp()
         if upload_date != create_date:
-            kwargs['dates'][DateType.EDITED] = (upload_date, upload_date, 0)
+            kwargs['dates'].append(Date(DateType.EDITED, (upload_date, upload_date), DateMeta.NONE))
         user = User(unique_id=f'pixiv:u:{data['userId']}', username=data['userAccount'], display_name=data['userName'])
         profile_picture_url = self.seek_profile_picture_url(data['userIllusts'])
         if profile_picture_url:
@@ -114,6 +111,16 @@ class PixivWebBase(Search):
         self.module.add_to_map(unique_id, post)
         return post
 
+    def create_tombstone(self, data: ParsedJson) -> Post:
+        kwargs = {}
+        kwargs['type'] = PostType.TOMBSTONE
+        illust_id = str(data['id'])
+        unique_id = f'pixiv:i:{illust_id}'
+        kwargs['unique_id'] = unique_id
+        post = Post(**kwargs)
+        self.module.add_to_map(unique_id, post)
+        return post
+
     def remake_post(self, raw_responses: ParsedJson, original_date: float) -> Optional[Post]:
         api = json.loads(raw_responses['api'])
         pages = raw_responses.get('pages', None)
@@ -137,7 +144,7 @@ class PixivWebBase(Search):
     def create_user(self, data: ParsedJson) -> User:
         unique_id=f'pixiv:u:{data['userId']}'
         user = User(unique_id=unique_id, display_name=data['userName'])
-        user.profile_picture_url = self.create_media_url(data['profileImageUrl'])
+        user.profile_picture_url = Url(data['profileImageUrl'])
         self.module.add_to_map(unique_id, user)
         return user
 
@@ -214,6 +221,42 @@ class PixivWebUser(PixivWebBase):
         self.completed = True
         return True
 
+class PixivWebBookmarks(PixivWebBase):
+    LIMIT = 48
+
+    def __init__(self, userdata: Any, arg: str) -> None:
+        super().__init__(userdata)
+        try:
+            self.id: int = int(arg)
+        except ValueError:
+            self.errored = True
+        self.total: int = 0
+        self.count: int = 0
+
+    # rest=show - public
+    # rest=hide - private
+    def request_page(self, num: int) -> bool:
+        url = f'{BASE_URL}/user/{self.id}/illusts/bookmarks?tag=&offset={self.LIMIT * num}&limit={self.LIMIT}&rest=show&lang={LANG}&version={VERSION}'
+        obj = self.check_api_response(self.module.do_request(Method.GET, url))
+        if not obj:
+            self.errored = True
+            return False
+        self.total = obj['total']
+        self.pages[num] = []
+        for illust in obj['works']:
+            if illust['userId'] == 0:
+                post = self.create_tombstone(illust)
+            else:
+                post = self.request_illust(int(illust['id']))
+                if not post:
+                    self.errored = True
+                    return False
+            self.pages[num].append(post.unique_id)
+            self.count += 1
+        if self.count >= self.total:
+            self.completed = True
+        return True
+
 class PixivWebFollowing(PixivWebBase):
     LIMIT = 24
 
@@ -224,8 +267,10 @@ class PixivWebFollowing(PixivWebBase):
         except ValueError:
             self.errored = True
 
+    # rest=show - public
+    # rest=hide - private
     def request_page(self, num: int) -> bool:
-        url = f'{BASE_URL}/user/{self.id}/following?offset={self.LIMIT * num}&limit={self.LIMIT}&rest=show&tag=&acceptingRequests=0&lang={LANG}&version={VERSION}'
+        url = f'{BASE_URL}/user/{self.id}/following?tag=&offset={self.LIMIT * num}&limit={self.LIMIT}&rest=show&acceptingRequests=0&lang={LANG}&version={VERSION}'
         obj = self.check_api_response(self.module.do_request(Method.GET, url))
         if not obj:
             self.errored = True
@@ -242,6 +287,7 @@ class PixivWebModule(Module):
         self.parser.add_command('search', PixivWebSearch)
         self.parser.add_command('illust', PixivWebIllust)
         self.parser.add_command('user', PixivWebUser)
+        self.parser.add_command('bookmarks', PixivWebBookmarks)
         self.parser.add_command('following', PixivWebFollowing)
         self.headers['User-Agent'] = USER_AGENT
         self.headers['Accept-Encoding'] = 'gzip, deflate, br, zstd'
diff --git a/src/portal/py/modules/youtube.py b/src/portal/py/modules/youtube.py
index 16d7f67..5125f85 100644
--- a/src/portal/py/modules/youtube.py
+++ b/src/portal/py/modules/youtube.py
@@ -94,7 +94,7 @@ class YoutubeBase(Search):
         if info.get('direct'):
             url = info['url']
         else:
-            url = get_playback_url(info, video=False)
+            url = get_playback_url(info, video=True)
         if not url:
             return False
         unique_id = f'youtube:v:{info['id']}'