Portal update

Signed-off-by: Andrew Opalach <andrew@akon.city>

17 files changed, 1741 insertions, 311 deletions

diff --git a/src/portal/py/config.def.py b/src/portal/py/config.def.py
index 8c0dab9..0c7ea30 100644
--- a/src/portal/py/config.def.py
+++ b/src/portal/py/config.def.py
@@ -6,7 +6,7 @@ TWITTER_ACCESS_TOKEN = ''
 TWITTER_ACCESS_TOKEN_SECRET = ''
 TWITTER_CONSUMER_TOKEN = ''
 TWITTER_CONSUMER_TOKEN_SECRET = ''
-TWITTER_COOKIES_PATH = '{}/cookies_twitter.txt'.format(BASE_DIR)
+TWITTER_COOKIES_PATH = f'{BASE_DIR}/cookies_twitter.txt'
 TWITTER_TIMEOUT = 5 # seconds
 
 FANBOX_SESSID = ''
@@ -18,7 +18,7 @@ PIXIV_REFRESH_TOKEN = ''
 INSTAGRAM_USERNAME = ''
 INSTAGRAM_PASSWORD = ''
 INSTAGRAM_SESSION_ID = ''
-INSTAGRAM_SETTINGS_PATH = Path('{}/ig_settings.json'.format(BASE_DIR))
+INSTAGRAM_SETTINGS_PATH = Path(f'{BASE_DIR}/ig_settings.json')
 INSTAGRAM_USER_AGENT = ''
 
 PATREON_USER_ID = ''
diff --git a/src/portal/py/modules/__init__.py b/src/portal/py/modules/__init__.py
index fee4c22..8a1e3cb 100644
--- a/src/portal/py/modules/__init__.py
+++ b/src/portal/py/modules/__init__.py
@@ -1,9 +1,9 @@
 import config
 
 from modules.youtube import YoutubeModule
-#from modules.twitter import TwitterScrapeModule
-#from modules.pixiv_app import PixivAppModule
-#from modules.pixiv_web import PixivWebModule
+from modules.twitter import TwitterScrapeModule
+from modules.pixiv_app import PixivAppModule
+from modules.pixiv_web import PixivWebModule
 #from modules.fanbox import FanboxModule
 #from modules.instagram import InstagramModule
 #from modules.patreon import PatreonModule
@@ -13,9 +13,9 @@ ALL_MODULES = {
 #    'twitter': (TwitterModule(
 #        config.TWITTER_ACCESS_TOKEN, config.TWITTER_ACCESS_TOKEN_SECRET,
 #        config.TWITTER_CONSUMER_TOKEN, config.TWITTER_CONSUMER_TOKEN_SECRET), []),
-#    'twitter': (TwitterScrapeModule(config.TWITTER_COOKIES_PATH), []),
-#    'pixiv_web': (PixivWebModule(config.PIXIV_SESSID, config.PIXIV_USERID), []),
-#    'pixiv_app': (PixivAppModule(config.PIXIV_REFRESH_TOKEN), []),
+    'twitter': (TwitterScrapeModule(config.TWITTER_COOKIES_PATH), []),
+    'pixiv_web': (PixivWebModule(config.PIXIV_SESSID, config.PIXIV_USERID), []),
+    'pixiv_app': (PixivAppModule(config.PIXIV_REFRESH_TOKEN), []),
 #    'fanbox': (FanboxModule(config.FANBOX_SESSID, config.FANBOX_CF_CLEARANCE), []),
 #    'instagram': (InstagramModule(
 #        config.INSTAGRAM_USER_AGENT, config.INSTAGRAM_SETTINGS_PATH,
diff --git a/src/portal/py/modules/fanbox.py b/src/portal/py/modules/fanbox.py
index 9d233bb..01e2b8c 100644
--- a/src/portal/py/modules/fanbox.py
+++ b/src/portal/py/modules/fanbox.py
@@ -31,7 +31,7 @@ class FanboxBase(Search):
         unique_id = f'fanbox:p:{post_id}'
         kwargs['unique_id'] = unique_id
         kwargs['raw_responses'] = {}
-        kwargs['raw_responses']['api'] = json.dumps(data)
+        kwargs['raw_responses']['api'] = json.dumps(data, separators=(',', ':'), indent=None)
         publish_date = reformat_pixiv_date(data['publishedDatetime']).timestamp()
         current_date = get_current_utc_time().timestamp()
         kwargs['dates'] = [
@@ -56,12 +56,12 @@ class FanboxBase(Search):
             media = {}
             if data['type'] == 'image':
                 text = data['body']['text']
-                for i, value in enumerate(data['body']['images']):
-                    media[str(i)] = Image(url=Url(value['originalUrl'], value['extension']), thumbnail_url=Url(value['thumbnailUrl']))
+                for index, value in enumerate(data['body']['images']):
+                    media[str(index)] = Image(url=Url(value['originalUrl'], value['extension']), thumbnail_url=Url(value['thumbnailUrl']))
             elif data['type'] == 'file':
                 text = data['body']['text']
-                for i, value in enumerate(data['body']['files']):
-                    media[str(i)] = File(url=Url(value['url'], value['extension']), name=value['name'])
+                for index, value in enumerate(data['body']['files']):
+                    media[str(index)] = File(url=Url(value['url'], value['extension']), name=value['name'])
             elif data['type'] == 'article':
                 for block in data['body']['blocks']:
                     if block['type'] == 'p':
diff --git a/src/portal/py/modules/instagram.py b/src/portal/py/modules/instagram.py
index 014ffc1..7d71f9f 100644
--- a/src/portal/py/modules/instagram.py
+++ b/src/portal/py/modules/instagram.py
@@ -22,7 +22,7 @@ class InstagramSearch(Search):
         if num >= self.page + self.REACH_LIMIT:
             return False
         request_satisfied = False
-        for i in range(self.page, num + 1):
+        for index in range(self.page, num + 1):
             try:
                 media, self.cursor = self.module.cl.user_medias_paginated_v1(self.pk, 0, end_cursor=self.cursor)
                 #media, self.cursor = self.module.cl.user_clips_paginated_v1(self.pk, amount=12, end_cursor=self.cursor)
@@ -34,24 +34,24 @@ class InstagramSearch(Search):
                 return False
             if not self.cursor:
                 self.completed = True
-            self.pages[i] = []
+            self.pages[index] = []
             for m in media:
                 kwargs = {}
                 kwargs['type'] = PostType.POST
-                unique_id = 'instagram:p:{}'.format(m.id)
-                #unique_id = 'instagram:r:{}'.format(m.id)
+                unique_id = f'instagram:p:{m.id}'
+                #unique_id = f'instagram:r:{m.id}'
                 kwargs['unique_id'] = unique_id
                 kwargs['raw_responses'] = {}
                 kwargs['raw_responses']['tile'] = m.json()
-                kwargs['url'] = 'https://www.instagram.com/p/{}'.format(m.code)
-                #kwargs['url'] = 'https://www.instagram.com/reel/{}'.format(m.code)
+                kwargs['url'] = f'https://www.instagram.com/p/{m.code}'
+                #kwargs['url'] = f'https://www.instagram.com/reel/{m.code}'
                 create_date = m.taken_at.timestamp()
                 current_date = get_current_utc_time().timestamp()
                 kwargs['dates'] = [
                     Date(DateType.CREATED, (create_date, create_date), DateMeta.NONE),
                     Date(DateType.RETRIEVED, (current_date, current_date), DateMeta.NONE)
                 ]
-                user = User(unique_id='instagram:u:{}'.format(m.user.pk))
+                user = User(unique_id=f'instagram:u:{m.user.pk}')
                 if m.user.username:
                     user.username = m.user.username
                 if m.user.full_name:
@@ -79,9 +79,9 @@ class InstagramSearch(Search):
                 kwargs['media'] = media
                 post = Post(**kwargs)
                 self.module.add_to_map(unique_id, post)
-                self.pages[i].append(unique_id)
+                self.pages[index].append(unique_id)
             self.page += 1
-            if i == num:
+            if index == num:
                 request_satisfied = True
         return request_satisfied
 
diff --git a/src/portal/py/modules/patreon.py b/src/portal/py/modules/patreon.py
index 0e4345a..bf2f0df 100644
--- a/src/portal/py/modules/patreon.py
+++ b/src/portal/py/modules/patreon.py
@@ -126,7 +126,7 @@ class PatreonUser(PatreonBase):
                 return False
 
         request_satisfied = False
-        for i in range(self.page, num + 1):
+        for index in range(self.page, num + 1):
             url = f'{BASE_URL}/posts'
             #'filter[accessible_by_user_id]': self.module.user_id,
             params = {
@@ -187,17 +187,17 @@ class PatreonUser(PatreonBase):
                 else:
                     log.warn(f'Unhandled include type {inc['type']}.')
 
-            self.pages[i] = []
+            self.pages[index] = []
             for entry in obj['data']:
                 if entry['type'] != 'post':
                     log.warn(f'Unhandled entry type {entry['type']}.')
                     continue
                 post = self.create_post(entry, hash)
                 if post:
-                    self.pages[i].append(post.unique_id)
+                    self.pages[index].append(post.unique_id)
 
             self.page += 1
-            if i == num:
+            if index == num:
                 request_satisfied = True
 
         return request_satisfied
diff --git a/src/portal/py/modules/pixiv_app.py b/src/portal/py/modules/pixiv_app.py
index 02d5254..7bd341b 100644
--- a/src/portal/py/modules/pixiv_app.py
+++ b/src/portal/py/modules/pixiv_app.py
@@ -1,9 +1,9 @@
 import json
 from typing import Optional, Any
 from base import USER_AGENT, Search, Module, ParsedJson
-from post import Url, PostType, DateType, User, Post, Image, Tag, TagType
+from post import Url, PostType, Date, DateType, DateMeta, User, Post, Image, Tag, TagType
 from query_parser import QueryParser
-from pixivpy3 import AppPixivAPI
+from pixivpy3 import AppPixivAPI, models
 from modules.common import get_current_utc_time, reformat_pixiv_date
 
 class PixivAppBase(Search):
@@ -18,63 +18,77 @@ class PixivAppBase(Search):
 
     def request_page(self, num: int) -> bool:
         request_satisfied = False
-        for i in range(self.page, num + 1):
+        for index in range(self.page, num + 1):
             if not self.next_qs:
                 break
-            if i in self.pages:
+            if index in self.pages:
                 continue
-            self.pages[i] = []
-            success, self.next_qs = self.api_request_page(i)
+            self.pages[index] = []
+            success, self.next_qs = self.api_request_page(index)
             if not success:
                 self.errored = True
                 break
-            if i == num:
+            if index == num:
                 request_satisfied = True
             self.page += 1
         return request_satisfied
 
-    def create_media_url(self, url: str) -> Url:
-        return Url(url, url.split('.')[-1])
-
     def create_tag(self, data: ParsedJson) -> Tag:
-        tag = Tag(type=TagType.GENERAL, name=data['name'])
+        kwargs = {}
+        kwargs['type'] = TagType.GENERAL
+        kwargs['name'] = data['name']
+        kwargs['alts'] = {}
         if data['translated_name']:
-            tag.alts['en'] = data['translated_name']
-        return tag
+            kwargs['alts']['en'] = data['translated_name']
+        return Tag(**kwargs)
 
-    def create_post(self, data: ParsedJson) -> Post:
-        if len(data['meta_pages']) == 0:
-            data['meta_pages'].append({
-                'image_urls': {
-                    'medium': data['image_urls']['medium'],
-                    'original': data['meta_single_page']['original_image_url']
-                }
-            })
+    # Other objects returned from pixivpy3 are likely also ModelT but it seems
+    # so inconsistent internally for now keep some params as ParsedJson.
+    def create_post(self, data: ParsedJson | models.ModelT, retrieved_date: Optional[float] = None) -> Post:
+        is_deleted = not data['visible']
         kwargs = {}
+        kwargs['module'] = 'pixiv_app'
         kwargs['type'] = PostType.POST
-        unique_id = 'pixiv:i:{}'.format(data['id'])
+        unique_id = f'pixiv:i:{data['id']}'
         kwargs['unique_id'] = unique_id
         kwargs['raw_responses'] = {}
-        kwargs['raw_responses']['api'] = json.dumps(data)
-        kwargs['url'] = 'https://www.pixiv.net/en/artworks/{}'.format(data['id'])
-        kwargs['dates'] = {
-            DateType.CREATED: reformat_pixiv_date(data['create_date']).timestamp(),
-            DateType.RETRIEVED: get_current_utc_time().timestamp()
-        }
+        try:
+            kwargs['raw_responses']['api'] = json.dumps(data, separators=(',', ':'), indent=None)
+        except TypeError:
+            kwargs['raw_responses']['api'] = data.model_dump_json(indent=None)
+        kwargs['url'] = f'https://www.pixiv.net/en/artworks/{data['id']}'
+        if not retrieved_date:
+            retrieved_date = get_current_utc_time().timestamp()
+        kwargs['dates'] = [
+            Date(DateType.RETRIEVED, (retrieved_date, retrieved_date), DateMeta.NONE)
+        ]
+        if not is_deleted:
+            create_date = reformat_pixiv_date(data['create_date']).timestamp()
+            kwargs['dates'].append(Date(DateType.CREATED, (create_date, create_date), DateMeta.NONE))
         user = User(unique_id=f'pixiv:u:{data['user']['id']}', username=data['user']['account'], display_name=data['user']['name'])
-        user.profile_picture_url = data['user']['profile_image_urls']['medium']
+        if not is_deleted:
+            user.profile_picture_url = Url(data['user']['profile_image_urls']['medium'])
         kwargs['author'] = user
         kwargs['title'] = data['title']
         kwargs['text'] = data['caption']
-        kwargs['likes'] = data['total_bookmarks']
+        if not is_deleted: # Default to None instead of 0.
+            kwargs['bookmarks'] = data['total_bookmarks']
+            kwargs['views'] = data['total_view']
         if 'total_comments' in data:
             kwargs['comments'] = data['total_comments']
-        kwargs['views'] = data['total_view']
-        kwargs['tags'] = [self.create_tag(tag) for tag in data['tags']]
-        media = {}
-        for i, page in enumerate(data['meta_pages']):
-            media[str(i)] = Image(url=self.create_media_url(page['image_urls']['original']), thumbnail_url=self.create_media_url(page['image_urls']['medium']))
-        kwargs['media'] = media
+        kwargs['tags'] = [self.create_tag(t) for t in data['tags']]
+        if not is_deleted:
+            if len(data['meta_pages']) == 0:
+                data['meta_pages'].append({
+                    'image_urls': {
+                        'medium': data['image_urls']['medium'],
+                        'original': data['meta_single_page']['original_image_url']
+                    }
+                })
+            media = {}
+            for index, page in enumerate(data['meta_pages']):
+                media[str(index)] = Image(url=Url(page['image_urls']['original']), thumbnail_url=Url(page['image_urls']['medium']))
+            kwargs['media'] = media
         post = Post(**kwargs)
         self.module.add_to_map(unique_id, post)
         return post
@@ -82,16 +96,23 @@ class PixivAppBase(Search):
     def create_user(self, data: ParsedJson) -> User:
         unique_id = f'pixiv:u:{data['id']}'
         user = User(unique_id=unique_id, username=data['account'], display_name=data['name'])
+        user.profile_picture_url = Url(data['profile_image_urls']['medium'])
         self.module.add_to_map(unique_id, user)
         return user
 
+    def remake_post(self, raw_responses: ParsedJson, original_date: float) -> Optional[Post]:
+        api = json.loads(raw_responses['api'])
+        return self.create_post(api, original_date)
+
 class PixivAppSearch(PixivAppBase):
     def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
         self.next_qs: Optional[dict[str, Any]] = {}
         self.next_qs['word'] = arg
-        self.next_qs['sort'] = 'popular_desc'
-        self.next_qs['search_ai_type'] = 1
+        #self.next_qs['sort'] = 'popular_desc'
+        # https://github.com/upbit/pixivpy/issues/352
+        # We want to include AI results for completeness, to be ignored further down the chain.
+        #self.next_qs['search_ai_type'] = 1
         self.next_qs['start_date'] = '2019-01-01'
         self.next_qs['end_date'] = '2019-12-31'
 
@@ -99,27 +120,10 @@ class PixivAppSearch(PixivAppBase):
         if not self.next_qs:
             return False, None
         obj = self.module.api.search_illust(**self.next_qs)
-        if 'illusts' not in obj:
-            return False, None
-        for post in obj['illusts']:
+        for post in obj.illusts:
             self.pages[num].append(self.create_post(post).unique_id)
         return True, self.module.api.parse_qs(obj['next_url'])
 
-class PixivAppIllust(PixivAppBase):
-    def __init__(self, userdata: Any, arg: str) -> None:
-        super().__init__(userdata)
-        self.next_qs: Optional[dict[str, Any]] = {}
-        self.next_qs['illust_id'] = arg
-
-    def api_request_page(self, num: int) -> tuple[bool, Optional[dict[str, Any]]]:
-        if not self.next_qs:
-            return False, None
-        obj = self.module.api.illust_detail(**self.next_qs)
-        if 'illust' not in obj:
-            return False, None
-        self.pages[num].append(self.create_post(obj['illust']).unique_id)
-        return True, None
-
 class PixivAppUser(PixivAppBase):
     def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
@@ -131,9 +135,7 @@ class PixivAppUser(PixivAppBase):
         if not self.next_qs:
             return False, None
         obj = self.module.api.user_illusts(**self.next_qs)
-        if 'illusts' not in obj:
-            return False, None
-        for post in obj['illusts']:
+        for post in obj.illusts:
             self.pages[num].append(self.create_post(post).unique_id)
         return True, self.module.api.parse_qs(obj['next_url'])
 
@@ -148,9 +150,7 @@ class PixivAppBookmarks(PixivAppBase):
         if not self.next_qs:
             return False, None
         obj = self.module.api.user_bookmarks_illust(**self.next_qs)
-        if 'illusts' not in obj:
-            return False, None
-        for post in obj['illusts']:
+        for post in obj.illusts:
             self.pages[num].append(self.create_post(post).unique_id)
         return True, self.module.api.parse_qs(obj['next_url'])
 
@@ -165,22 +165,39 @@ class PixivAppFollowing(PixivAppBase):
         if not self.next_qs:
             return False, None
         obj = self.module.api.user_following(**self.next_qs)
-        if 'user_previews' not in obj:
-            return False, None
-        for user in obj['user_previews']:
+        for user in obj.user_previews:
             self.pages[num].append(self.create_user(user['user']).unique_id)
         return True, self.module.api.parse_qs(obj['next_url'])
 
+class PixivAppIllust(PixivAppBase):
+    def __init__(self, userdata: Any, arg: str) -> None:
+        super().__init__(userdata)
+        self.next_qs: Optional[dict[str, Any]] = {}
+        self.next_qs['illust_id'] = arg
+
+    def api_request_page(self, num: int) -> tuple[bool, Optional[dict[str, Any]]]:
+        if not self.next_qs:
+            return False, None
+        obj = self.module.api.illust_detail(**self.next_qs)
+        self.pages[num].append(self.create_post(obj.illust).unique_id)
+        return True, None
+
 class PixivAppModule(Module):
     def __init__(self, refresh_token: str) -> None:
         super().__init__()
         self.parser: QueryParser = QueryParser(self, 'search')
         self.parser.add_command('search', PixivAppSearch)
-        self.parser.add_command('illust', PixivAppIllust)
         self.parser.add_command('user', PixivAppUser)
         self.parser.add_command('bookmarks', PixivAppBookmarks)
         self.parser.add_command('following', PixivAppFollowing)
-        self.api: AppPixivAPI = AppPixivAPI()
+        self.parser.add_command('illust', PixivAppIllust)
+        headers = {
+            "app-os": "ios",
+            "app-os-version": "16.7.2",
+            "app-version": "7.19.6",
+            "user-agent": "PixivIOSApp/7.19.6 (iOS 16.7.2; iPhone13,2)"
+        }
+        self.api: AppPixivAPI = AppPixivAPI(headers=headers)
         self.api.auth(refresh_token=refresh_token)
 
     def search(self, query: str, *extra_args: Any) -> Optional[Search]:
diff --git a/src/portal/py/modules/pixiv_web.py b/src/portal/py/modules/pixiv_web.py
index 3c04202..e7399dc 100644
--- a/src/portal/py/modules/pixiv_web.py
+++ b/src/portal/py/modules/pixiv_web.py
@@ -33,8 +33,8 @@ class PixivWebBase(Search):
 
     def parse_pages(self, data: ParsedJson) -> dict[str, Media]:
         media = {}
-        for i, m in enumerate(data):
-            media[str(i)] = Image(url=Url(m['urls']['original']), thumbnail_url=Url(m['urls']['small']))
+        for index, m in enumerate(data):
+            media[str(index)] = Image(url=Url(m['urls']['original']), thumbnail_url=Url(m['urls']['small']))
         return media
 
     def parse_ugoira(self, data: ParsedJson, thumbnail_url: str) -> Animation:
@@ -43,7 +43,7 @@ class PixivWebBase(Search):
             animation.frames.append((frame['file'], frame['delay']))
         return animation
 
-    def seek_profile_picture_url(self, user_illusts: ParsedJson) -> Optional[Url]:
+    def search_profile_picture_url(self, user_illusts: ParsedJson) -> Optional[Url]:
         for illust in user_illusts.values():
             if illust is None:
                 continue
@@ -52,7 +52,7 @@ class PixivWebBase(Search):
         return None
 
     def create_tag(self, data: ParsedJson) -> Tag:
-        kwargs= {}
+        kwargs = {}
         kwargs['type'] = TagType.GENERAL
         kwargs['name'] = data['tag']
         kwargs['alts'] = {}
@@ -63,26 +63,27 @@ class PixivWebBase(Search):
                 kwargs['alts']['en'] = data['translation']['en']
         return Tag(**kwargs)
 
-    def create_post(self, data: ParsedJson, pages: Optional[ParsedJson] = None, ugoira: Optional[ParsedJson] = None) -> Optional[Post]:
+    def create_post(self, data: ParsedJson, pages: Optional[ParsedJson] = None, ugoira: Optional[ParsedJson] = None, retrieved_date: Optional[float] = None) -> Optional[Post]:
         kwargs = {}
         kwargs['type'] = PostType.POST
         illust_id = data['illustId']
         unique_id = f'pixiv:i:{illust_id}'
         kwargs['unique_id'] = unique_id
         kwargs['raw_responses'] = {}
-        kwargs['raw_responses']['api'] = json.dumps(data)
+        kwargs['raw_responses']['api'] = json.dumps(data, separators=(',', ':'), indent=None)
         kwargs['url'] = f'https://www.pixiv.net/{LANG}/artworks/{data['illustId']}'
         create_date = reformat_pixiv_date(data['createDate']).timestamp()
-        current_date = get_current_utc_time().timestamp()
+        if not retrieved_date:
+            retrieved_date = get_current_utc_time().timestamp()
         kwargs['dates'] = [
             Date(DateType.CREATED, (create_date, create_date), DateMeta.NONE),
-            Date(DateType.RETRIEVED, (current_date, current_date), DateMeta.NONE)
+            Date(DateType.RETRIEVED, (retrieved_date, retrieved_date), DateMeta.NONE)
         ]
         upload_date = reformat_pixiv_date(data['uploadDate']).timestamp()
         if upload_date != create_date:
             kwargs['dates'].append(Date(DateType.EDITED, (upload_date, upload_date), DateMeta.NONE))
         user = User(unique_id=f'pixiv:u:{data['userId']}', username=data['userAccount'], display_name=data['userName'])
-        profile_picture_url = self.seek_profile_picture_url(data['userIllusts'])
+        profile_picture_url = self.search_profile_picture_url(data['userIllusts'])
         if profile_picture_url:
             user.profile_picture_url = profile_picture_url
         kwargs['author'] = user
@@ -124,19 +125,6 @@ class PixivWebBase(Search):
         self.module.add_to_map(unique_id, post)
         return post
 
-    def remake_post(self, raw_responses: ParsedJson, original_date: float) -> Optional[Post]:
-        api = json.loads(raw_responses['api'])
-        pages = raw_responses.get('pages', None)
-        if pages:
-            pages = json.loads(pages)
-        ugoira = raw_responses.get('ugoira', None)
-        if ugoira:
-            ugoira = json.loads(ugoira)
-        post = self.create_post(api, pages, ugoira)
-        if post:
-            post.dates[DateType.RETRIEVED] = original_date
-        return post
-
     def request_illust(self, illust_id: int) -> Optional[Post]:
         url = f'{BASE_URL}/illust/{illust_id}?lang={LANG}&version={VERSION}'
         obj = self.check_api_response(self.module.do_request(Method.GET, url))
@@ -151,6 +139,16 @@ class PixivWebBase(Search):
         self.module.add_to_map(unique_id, user)
         return user
 
+    def remake_post(self, raw_responses: ParsedJson, original_date: float) -> Optional[Post]:
+        api = json.loads(raw_responses['api'])
+        pages = raw_responses.get('pages', None)
+        if pages:
+            pages = json.loads(pages)
+        ugoira = raw_responses.get('ugoira', None)
+        if ugoira:
+            ugoira = json.loads(ugoira)
+        return self.create_post(api, pages, ugoira, original_date)
+
 class PixivWebSearch(PixivWebBase):
     def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
@@ -178,23 +176,6 @@ class PixivWebSearch(PixivWebBase):
             self.pages[num].append(post.unique_id)
         return True
 
-class PixivWebIllust(PixivWebBase):
-    def __init__(self, userdata: Any, arg: str) -> None:
-        super().__init__(userdata)
-        try:
-            self.id: int = int(arg)
-        except ValueError:
-            self.errored = True
-
-    def request_page(self, num: int) -> bool:
-        post = self.request_illust(self.id)
-        if not post:
-            self.errored = True
-            return False
-        self.pages[num] = [post.unique_id]
-        self.completed = True
-        return True
-
 class PixivWebUser(PixivWebBase):
     def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
@@ -285,15 +266,32 @@ class PixivWebFollowing(PixivWebBase):
             self.pages[num].append(self.create_user(user).unique_id)
         return True
 
+class PixivWebIllust(PixivWebBase):
+    def __init__(self, userdata: Any, arg: str) -> None:
+        super().__init__(userdata)
+        try:
+            self.id: int = int(arg)
+        except ValueError:
+            self.errored = True
+
+    def request_page(self, num: int) -> bool:
+        post = self.request_illust(self.id)
+        if not post:
+            self.errored = True
+            return False
+        self.pages[num] = [post.unique_id]
+        self.completed = True
+        return True
+
 class PixivWebModule(Module):
     def __init__(self, sessid: str, user_id: str) -> None:
         super().__init__()
         self.parser: QueryParser = QueryParser(self, 'search')
         self.parser.add_command('search', PixivWebSearch)
-        self.parser.add_command('illust', PixivWebIllust)
         self.parser.add_command('user', PixivWebUser)
         self.parser.add_command('bookmarks', PixivWebBookmarks)
         self.parser.add_command('following', PixivWebFollowing)
+        self.parser.add_command('illust', PixivWebIllust)
         self.headers['User-Agent'] = USER_AGENT
         self.headers['Accept-Encoding'] = 'gzip, deflate, br, zstd'
         self.headers['Accept-Language'] = 'en-US,en;q=0.5'
diff --git a/src/portal/py/modules/twitter.py b/src/portal/py/modules/twitter.py
index 2f45ec1..93286c3 100644
--- a/src/portal/py/modules/twitter.py
+++ b/src/portal/py/modules/twitter.py
@@ -3,15 +3,15 @@ import http.cookiejar
 from typing import Optional, Any, Iterator
 from datetime import timezone
 from base import USER_AGENT, Search, Module
-from post import Url, PostType, DateType, PostRef, Post, User, Media, Image, Video
+from post import Url, PostType, Date, DateType, DateMeta, PostRef, Post, User, Media, Image, Video
 from query_parser import QueryParser
 from modules.common import get_current_utc_time
 
 # GraphQL API
-from snscrape.modules import twitter
 from snscrape.base import ScraperException
+from snscrape.modules import twitter
 from snscrape.modules.twitter import (Tweet, TweetRef, Tombstone, UserRef,
-    TwitterSearchScraper, TwitterProfileScraper, TwitterUserScraper, TwitterTweetScraper)
+    TwitterSearchScraper, TwitterSearchScraperMode, TwitterProfileScraper, TwitterUserScraper, TwitterTweetScraper)
 
 class TwitterScrapeBase(Search):
     # Posts per emulated page because snscrape returns an iterator.
@@ -20,7 +20,7 @@ class TwitterScrapeBase(Search):
     # Don't allow requests for a page more than this amount past the current page.
     REACH_LIMIT = 4
 
-    def __init__(self, userdata: Any):
+    def __init__(self, userdata: Any) -> None:
         super().__init__()
         self.module: TwitterScrapeModule = userdata
         self.iterator: Iterator[Tweet | TweetRef | Tombstone]
@@ -28,38 +28,39 @@ class TwitterScrapeBase(Search):
 
     def create_media_url(self, url: str) -> Url:
         format = url.find('format=')
+        ext = None
         if format >= 0:
-            ext = url[format + 7:format + 10]
-        else:
-            question = url.rfind('?')
-            if question >= 0:
-                url = url[0:question]
-            ext = url.split('.')[-1]
+            ext = url[format + 7:]
+            amp = ext.find('&')
+            if amp >= 0:
+                ext = ext[:amp]
         return Url(url, ext)
 
     def parse_media(self, data: list[twitter.Medium]) -> dict[str, Media]:
         media = {}
-        for i, m in enumerate(data):
+        for index, m in enumerate(data):
             if isinstance(m, twitter.Photo):
-                media[str(i)] = Image(url=self.create_media_url(m.fullUrl), thumbnail_url=self.create_media_url(m.previewUrl))
+                media[str(index)] = Image(url=self.create_media_url(m.fullUrl), thumbnail_url=self.create_media_url(m.previewUrl))
             elif isinstance(m, twitter.Video) or isinstance(m, twitter.Gif):
                 videos = sorted(m.variants, key=lambda x: x.bitrate if x.bitrate else 0, reverse=True)
-                media[str(i)] = Video(url=self.create_media_url(videos[0].url), thumbnail_url=self.create_media_url(m.thumbnailUrl))
+                media[str(index)] = Video(url=self.create_media_url(videos[0].url), thumbnail_url=self.create_media_url(m.thumbnailUrl))
         return media
 
     def create_post(self, tweet: Tweet | TweetRef | Tombstone) -> Post:
         kwargs = {}
+        kwargs['module'] = 'twitter_scrape'
         kwargs['unique_id'] = f'twitter:t:{tweet.id}'
         if isinstance(tweet, TweetRef) or isinstance(tweet, Tombstone):
             kwargs['type'] = PostType.TOMBSTONE
             return Post(**kwargs)
         kwargs['raw_responses'] = tweet.rawResponses
         kwargs['url'] = tweet.url
-        # @TODO: Replace correct?
-        kwargs['dates'] = {
-            DateType.CREATED: tweet.date.replace(tzinfo=timezone.utc).timestamp(),
-            DateType.RETRIEVED: get_current_utc_time().timestamp()
-        }
+        create_date = tweet.date.replace(tzinfo=timezone.utc).timestamp()
+        current_date = get_current_utc_time().timestamp()
+        kwargs['dates'] = [
+            Date(DateType.CREATED, (create_date, create_date), DateMeta.NONE),
+            Date(DateType.RETRIEVED, (current_date, current_date), DateMeta.NONE)
+        ]
         author = User(unique_id=f'twitter:u:{tweet.user.id}')
         if not isinstance(tweet.user, UserRef):
             author.username = tweet.user.username
@@ -116,40 +117,40 @@ class TwitterScrapeBase(Search):
         if num - self.page >= self.REACH_LIMIT:
             return False
         request_satisfied = False
-        for i in range(self.page, num + 1):
+        for index in range(self.page, num + 1):
             if self.completed:
                 break
-            if i in self.pages:
+            if index in self.pages:
                 continue
-            self.pages[i] = []
-            if not self.step_iterator_for_page(i):
+            self.pages[index] = []
+            if not self.step_iterator_for_page(index):
                 break
-            self.page = i
-            if i == num:
+            self.page = index
+            if index == num:
                 request_satisfied = True
         return request_satisfied
 
 class TwitterScrapeSearch(TwitterScrapeBase):
-    def __init__(self, userdata: Any, arg: str):
+    def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
-        self.iterator = TwitterSearchScraper(arg, top=True, cookies=self.module.cookies).get_items()
+        self.iterator = TwitterSearchScraper(arg, mode=TwitterSearchScraperMode.TOP, cookies=self.module.cookies).get_items()
 
 class TwitterScrapeUser(TwitterScrapeBase):
-    def __init__(self, userdata: Any, arg: str):
+    def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
         if arg.startswith('@'):
             arg = arg[1:]
         self.iterator = TwitterUserScraper(arg, cookies=self.module.cookies).get_items()
 
 class TwitterScrapeProfile(TwitterScrapeBase):
-    def __init__(self, userdata: Any, arg: str):
+    def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
         if arg.startswith('@'):
             arg = arg[1:]
         self.iterator = TwitterProfileScraper(arg, cookies=self.module.cookies).get_items()
 
 class TwitterScrapeTweet(TwitterScrapeBase):
-    def __init__(self, userdata: Any, arg: str):
+    def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
         if arg.startswith('https://'):
             arg = arg[arg.rfind('/') + 1:]
@@ -159,7 +160,7 @@ class TwitterScrapeTweet(TwitterScrapeBase):
         self.iterator = TwitterTweetScraper(arg, cookies=self.module.cookies).get_items()
 
 class TwitterScrapeModule(Module):
-    def __init__(self, cookies_path: str):
+    def __init__(self, cookies_path: str) -> None:
         super().__init__()
         self.parser: QueryParser = QueryParser(self, 'search')
         self.parser.add_command('search', TwitterScrapeSearch)
diff --git a/src/portal/py/post.py b/src/portal/py/post.py
index 1a285c2..f173d0e 100644
--- a/src/portal/py/post.py
+++ b/src/portal/py/post.py
@@ -4,13 +4,6 @@ from typing import Optional, Any
 from enum import IntEnum, IntFlag
 from json import JSONEncoder
 
-'''
-List of Things:
- - Social media posts
- - Albums
- - Loose files
-'''
-
 class PostType(IntEnum):
     UNKNOWN = 0
     POST = 1
@@ -54,27 +47,26 @@ def df(c: Any) -> Any:
     return field(default_factory=lambda: c)
 
 def default_url_ext_guess(url: str) -> str:
-    # @TODO: List of known extensions.
+    slash = url.find('/')
+    if slash < 0:
+        return ''
+    url = url[slash:]
     question = url.rfind('?')
     if question >= 0:
-        guess = url[0:question].rsplit('.')[-1]
-    else:
-        guess = url.rsplit('.')[-1]
-    return guess
+        url = url[0:question]
+    s = url.split('.')
+    if len(s) > 1:
+        return s[-1]
+    return ''
 
 @dataclasses.dataclass
 class Url():
     url: str
     ext: str
-
     def __init__(self, url: str, ext: str = None) -> None:
         self.url = url
         if not ext:
-            ext_guess = default_url_ext_guess(url)
-            if ext_guess:
-                ext = ext_guess
-            else:
-                ext = 'unknown'
+            ext = default_url_ext_guess(url)
         self.ext = ext
 
 @dataclasses.dataclass
@@ -158,6 +150,7 @@ class PostRef():
 '''
 V4:
  - Improved date.
+ - Track origin module.
  - Ends when all current modules are ported and/or completed.
 V5:
  - Formatted body/text.
@@ -165,6 +158,7 @@ V5:
 @dataclasses.dataclass
 class Post():
     version: int = 4
+    module: str = ''
     type: PostType = PostType.UNKNOWN
     unique_id: str = ''
     raw_responses: dict[str, str] = df({})
diff --git a/src/portal/py/query_parser.py b/src/portal/py/query_parser.py
index cd6c1e6..bf276ce 100644
--- a/src/portal/py/query_parser.py
+++ b/src/portal/py/query_parser.py
@@ -20,7 +20,7 @@ class QueryParser():
 
         sp = query.split(' ')
 
-        for i, word in enumerate(sp):
+        for index, word in enumerate(sp):
             col = word.find(':')
             if col == -1:
                 continue
@@ -34,7 +34,7 @@ class QueryParser():
                 else:
                     arg = None
                 search = self.commands[command]
-                if i == 0:
+                if index == 0:
                     if len(sp) > 1:
                         query = query.replace(word + ' ', '', 1)
                     else:
diff --git a/src/portal/py/test.py b/src/portal/py/test.py
index 0ca89a4..e862bf0 100644
--- a/src/portal/py/test.py
+++ b/src/portal/py/test.py
@@ -1,2 +1,4 @@
 #import tests.test
-import tests.archive_query
+import tests.touch_all
+#import tests.rewrite_post
+#import tests.archive_query
diff --git a/src/portal/py/tests/archive_query.py b/src/portal/py/tests/archive_query.py
index 4cc1e1e..3de4329 100644
--- a/src/portal/py/tests/archive_query.py
+++ b/src/portal/py/tests/archive_query.py
@@ -15,8 +15,8 @@ mode = 'pixiv_web'
 #mode = 'fanbox'
 #mode = 'patreon'
 #mode = 'instagram'
-#cmd = 'user'
-cmd = 'bookmarks'
+cmd = 'user'
+#cmd = 'bookmarks'
 #cmd = 'search'
 module = ALL_MODULES[mode][0]
 if not module.init():
@@ -180,8 +180,8 @@ class QueryDownloadThread(threading.Thread):
         with queue_cond:
             queue_cond.notify()
 
-#RUNTIME_PATH = "./run"
-RUNTIME_PATH = "/mnt/store/files/tmp/run"
+#RUNTIME_PATH = './run'
+RUNTIME_PATH = '/mnt/store/files/tmp/run'
 LOG_FILE = f'{RUNTIME_PATH}/archive4.log'
 ARG_FILE = f'{RUNTIME_PATH}/completed_args4.log'
 
@@ -216,10 +216,7 @@ def download_args():
         os.mkdir(output_dir)
         os.mkdir(f'{output_dir}/raw_responses')
 
-    #args = module.search(f'following:{22781328}')
-    # @TODO:
     args = []
-    # twitter test: butcha_u, sep__rina, nagayori000, sattinittas,
     threads = []
     start = True
 
diff --git a/src/portal/py/tests/old_twitter_api.py b/src/portal/py/tests/old_twitter_api.py
index 40d0346..4427dfd 100644
--- a/src/portal/py/tests/old_twitter_api.py
+++ b/src/portal/py/tests/old_twitter_api.py
@@ -27,14 +27,14 @@ log = Logger(LOG_FILE)
 compat_thread = QueryDownloadThread(log, None, None, sys.argv[1])
 
 def compat_media_download(obj, post, output_base):
-    for i, key in enumerate(post.media.keys()):
+    for index, key in enumerate(post.media.keys()):
         media = post.media[key]
         url = media.url
         media_path = f'{output_base}_media{key}.{url.ext}'
         if os.path.isfile(media_path):
             obj.log.write(f'Skipping media {media_path}.')
             continue
-        origin_path = f'{origin_base}/{post.author.unique_id.replace(':', '_')}/{post.unique_id.replace(':', '_')}_media{i}.{url.ext}'
+        origin_path = f'{origin_base}/{post.author.unique_id.replace(':', '_')}/{post.unique_id.replace(':', '_')}_media{index}.{url.ext}'
         if os.path.isfile(origin_path):
             shutil.copyfile(origin_path, media_path)
             #post.media[key].url.url = url.url.replace('name=orig', 'name=large')
diff --git a/src/portal/py/tests/rewrite_post.py b/src/portal/py/tests/rewrite_post.py
index 3bb152b..f00cfc7 100644
--- a/src/portal/py/tests/rewrite_post.py
+++ b/src/portal/py/tests/rewrite_post.py
@@ -2,20 +2,24 @@ import sys
 import os
 import gzip
 import json
-import hashlib
 sys.path.append('../py')
-import config
-#from post import PostEncoder, DateType
-#from modules.pixiv_web import PixivWebBase, PixivWebModule
-#from modules.twitter import TwitterScrapeBase, TwitterScrapeModule
-from logger import Logger
+from modules import ALL_MODULES
+from post import PostEncoder
+from modules.pixiv_app import PixivAppBase
 
-#module = PixivWebModule(config.PIXIV_SESSID)
-#search = PixivWebBase(module)
+module = ALL_MODULES['pixiv_app'][0]
+module.init()
+search = PixivAppBase(module)
 
-log = Logger('deleted_posts.log')
+target = sys.argv[1]
+with gzip.open(target, 'rb') as f:
+    obj = json.loads(f.read())
+    mtime = os.path.getmtime(target)
+    print(json.dumps(search.remake_post(obj['raw_responses'], mtime), cls=PostEncoder))
 
 '''
+log = Logger('deleted_posts.log')
+
 def rewrite_post(path):
     print(f'Rewriting post @ {path}')
     with gzip.open(path, 'rb') as f:
@@ -27,7 +31,6 @@ def rewrite_post(path):
     #print(json.dumps(post, indent=4, cls=PostEncoder))
     with gzip.open(path, 'wb') as f:
         f.write(json.dumps(post, cls=PostEncoder).encode('utf-8'))
-'''
 
 def dump_raw_response(path, target):
     print(f'Extracting raw_response from {path}')
@@ -54,3 +57,4 @@ for user in os.listdir(target):
         if file.split('.')[-1] == 'gz':
             dump_raw_response(f'{target}/{user}/{file}', raw_responses)
             #rewrite_post(f'{target}/{user}/{file}')
+'''
diff --git a/src/portal/py/tests/test.py b/src/portal/py/tests/test.py
index ef166d9..d3e5640 100644
--- a/src/portal/py/tests/test.py
+++ b/src/portal/py/tests/test.py
@@ -1,13 +1,4 @@
-import sys
-import json
-import httpx
-import os.path
-from base import Method
-from typing import Optional, Any
-from post import DateType, DateMeta, Date, PostEncoder, Post, MediaType
-from modules import ALL_MODULES
-from twitter.scraper import Scraper
-
+#from twitter.scraper import Scraper
 #scraper = Scraper(cookies = {
 #  'ct0': '',
 #  'auth_token': ''
@@ -15,101 +6,3 @@ from twitter.scraper import Scraper
 #
 #media = scraper.media([1557919548904419328], limit=10)
 #print(media)
-
-#output = '/mnt/store/files/ext/patreon_tmp/FPSBlyck'
-#
-#module = ALL_MODULES['patreon'][0]
-#module.init()
-#search = module.search('FPSBlyck')
-#page_num = 0
-#mark = False
-#while not mark:
-#    page = search.get_page(page_num)
-#    if not page:
-#        break
-#    page_num += 1
-#    for unique_id in page:
-#        if unique_id == 'patreon:p:29707872':
-#            print('hit mark')
-#            mark = True
-#        post = module.get_item(unique_id)
-#        print(json.dumps(post, indent=4, cls=PostEncoder))
-#        for key, m in post.media.items():
-#            if m.type == MediaType.FILE:
-#                path = f'{output}/{post.unique_id.replace(':', '_')}_{m.name}'
-#                if not os.path.isfile(path):
-#                    dl = module.get_download(post.unique_id, key)
-#                    retries = 3
-#                    r: Optional[httpx.Response] = None
-#                    while retries >= 0:
-#                        try:
-#                            r = httpx.get(dl['urls'][0].url, headers=dl['headers'], cookies=dl['cookies'], follow_redirects=True)
-#                        except:
-#                            retries -= 1
-#                        else:
-#                            break
-#                    if r:
-#                        with open(path, 'wb+') as f:
-#                            f.write(r.content)
-#                    else:
-#                        print('Download failed')
-#                else:
-#                    print('Skipping file')
-
-
-#module = ALL_MODULES['pixiv_web'][0]
-#module.init()
-#search = module.search('illust:91352621')
-#page = search.get_page(0)
-#for unique_id in page:
-#    post = module.get_item(unique_id)
-#    print(json.dumps(post, indent=4, cls=PostEncoder))
-
-module = ALL_MODULES['instagram'][0]
-module.init()
-search = module.search('plottttwistttttt')
-page = search.get_page(0)
-for unique_id in page:
-    post = module.get_item(unique_id)
-    print(json.dumps(post, indent=4, cls=PostEncoder))
-
-#d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.EDITED_AT_UNKNOWN_TIME)
-#d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.NONE)
-#print(json.dumps(d, indent=4, cls=PostEncoder))
-#sys.exit(1)
-
-#module = ALL_MODULES['fanbox'][0]
-#module = ALL_MODULES['pixiv_web'][0]
-#module = ALL_MODULES['youtube'][0]
-#module = ALL_MODULES['twitter'][0]
-#module.init()
-
-#search = module.search('opium_00pium')
-#search = module.search('search:ddd')
-#page = search.get_page(0)
-#for unique_id in page:
-#    post = module.get_item(unique_id)
-#    print(json.dumps(post, indent=4, cls=PostEncoder))
-#    break
-
-#post = module.get_item(page[0])
-#for key in post.media.keys():
-#  download_params = module.get_download(post.unique_id, key)
-#  for url in download_params['urls']:
-#      #response = module.do_request(Method.GET, url.url)
-#      r = httpx.get(url.url, headers=download_params['headers'])
-#      with open(f'test_{key}.{url.ext}', 'wb+') as f:
-#          f.write(r.content)
-
-#print(json.dumps(page, indent=4, cls=PostEncoder))
-#for i in range(0, len(post.media)):
-#    download_params = module.get_download(post.unique_id, i)
-#    r = httpx.get(download_params['url'], headers=download_params['headers'])
-#    if r.status_code != 200:
-#        print('error{}'.format(r.status_code))
-#    else:
-#        with open('test{}.png'.format(i), 'wb+') as f:
-#            f.write(r.content)
-
-#print(json.dumps(page, indent=4, cls=PostEncoder))
-#print(page)
diff --git a/src/portal/py/tests/touch_all.py b/src/portal/py/tests/touch_all.py
new file mode 100644
index 0000000..6d6bfa1
--- /dev/null
+++ b/src/portal/py/tests/touch_all.py
@@ -0,0 +1,79 @@
+import json
+from typing import Optional, Any
+from post import PostEncoder
+from modules import ALL_MODULES
+
+'''
+post = module.get_item(page[0])
+for key in post.media.keys():
+    download_params = module.get_download(post.unique_id, key)
+    for url in download_params['urls']:
+        #response = module.do_request(Method.GET, url.url)
+        r = httpx.get(url.url, headers=download_params['headers'])
+        if r.status_code != 200:
+            print('error{}'.format(r.status_code))
+        else:
+            with open(f'test_{key}.{url.ext}', 'wb+') as f:
+                f.write(r.content)
+'''
+
+def print_page(module: Any, page: Optional[list[str]]) -> None:
+    if not page:
+        print('Page is None')
+        return
+    count = 0
+    for unique_id in page:
+        post = module.get_item(unique_id)
+        print(json.dumps(post, indent=4, cls=PostEncoder))
+        count = count + 1
+        if count == 2:
+            break
+
+''' Blocked on certain endpoints, snscrape is outdated.
+module = ALL_MODULES['twitter'][0]
+module.init()
+
+search = module.search('search:#小関麗奈誕生祭2024')
+print_page(module, search.get_page(0))
+search = module.search('user:xuuikie')
+print_page(module, search.get_page(0))
+search = module.search('profile:@barackobama')
+print_page(module, search.get_page(0))
+search = module.search('tweet:1905966372288434680')
+print_page(module, search.get_page(0))
+'''
+
+'''
+module = ALL_MODULES['pixiv_app'][0]
+module.init()
+search = module.search('search:サムス・アラン')
+print_page(module, search.get_page(0))
+search = module.search('user:757584')
+print_page(module, search.get_page(0))
+search = module.search('bookmarks:2034628')
+print_page(module, search.get_page(0))
+search = module.search('following:22781328')
+print_page(module, search.get_page(0))
+search = module.search('illust:91352621')
+print_page(module, search.get_page(0))
+search = module.search('illust:26497574') # Deleted.
+print_page(module, search.get_page(0))
+'''
+
+''' No on-demand loading (Will load entire user before returning).
+module = ALL_MODULES['pixiv_web'][0]
+module.init()
+
+search = module.search('search:サムス・アラン')
+print_page(module, search.get_page(0))
+search = module.search('user:757584')
+print_page(module, search.get_page(0))
+search = module.search('bookmarks:2034628')
+print_page(module, search.get_page(0))
+search = module.search('following:22781328')
+print_page(module, search.get_page(0))
+search = module.search('illust:91352621')
+print_page(module, search.get_page(0))
+search = module.search('illust:100455669') # Deleted.
+print_page(module, search.get_page(0))
+'''
diff --git a/src/portal/py/tlds.py b/src/portal/py/tlds.py
new file mode 100644
index 0000000..d0e4cdd
--- /dev/null
+++ b/src/portal/py/tlds.py
@@ -0,0 +1,1445 @@
+# https://data.iana.org/TLD/tlds-alpha-by-domain.txt
+# Version 2025051100, Last Updated Sun May 11 07:07:01 2025 UTC
+tlds = [
+    'aaa',
+    'aarp',
+    'abb',
+    'abbott',
+    'abbvie',
+    'abc',
+    'able',
+    'abogado',
+    'abudhabi',
+    'ac',
+    'academy',
+    'accenture',
+    'accountant',
+    'accountants',
+    'aco',
+    'actor',
+    'ad',
+    'ads',
+    'adult',
+    'ae',
+    'aeg',
+    'aero',
+    'aetna',
+    'af',
+    'afl',
+    'africa',
+    'ag',
+    'agakhan',
+    'agency',
+    'ai',
+    'aig',
+    'airbus',
+    'airforce',
+    'airtel',
+    'akdn',
+    'al',
+    'alibaba',
+    'alipay',
+    'allfinanz',
+    'allstate',
+    'ally',
+    'alsace',
+    'alstom',
+    'am',
+    'amazon',
+    'americanexpress',
+    'americanfamily',
+    'amex',
+    'amfam',
+    'amica',
+    'amsterdam',
+    'analytics',
+    'android',
+    'anquan',
+    'anz',
+    'ao',
+    'aol',
+    'apartments',
+    'app',
+    'apple',
+    'aq',
+    'aquarelle',
+    'ar',
+    'arab',
+    'aramco',
+    'archi',
+    'army',
+    'arpa',
+    'art',
+    'arte',
+    'as',
+    'asda',
+    'asia',
+    'associates',
+    'at',
+    'athleta',
+    'attorney',
+    'au',
+    'auction',
+    'audi',
+    'audible',
+    'audio',
+    'auspost',
+    'author',
+    'auto',
+    'autos',
+    'aw',
+    'aws',
+    'ax',
+    'axa',
+    'az',
+    'azure',
+    'ba',
+    'baby',
+    'baidu',
+    'banamex',
+    'band',
+    'bank',
+    'bar',
+    'barcelona',
+    'barclaycard',
+    'barclays',
+    'barefoot',
+    'bargains',
+    'baseball',
+    'basketball',
+    'bauhaus',
+    'bayern',
+    'bb',
+    'bbc',
+    'bbt',
+    'bbva',
+    'bcg',
+    'bcn',
+    'bd',
+    'be',
+    'beats',
+    'beauty',
+    'beer',
+    'berlin',
+    'best',
+    'bestbuy',
+    'bet',
+    'bf',
+    'bg',
+    'bh',
+    'bharti',
+    'bi',
+    'bible',
+    'bid',
+    'bike',
+    'bing',
+    'bingo',
+    'bio',
+    'biz',
+    'bj',
+    'black',
+    'blackfriday',
+    'blockbuster',
+    'blog',
+    'bloomberg',
+    'blue',
+    'bm',
+    'bms',
+    'bmw',
+    'bn',
+    'bnpparibas',
+    'bo',
+    'boats',
+    'boehringer',
+    'bofa',
+    'bom',
+    'bond',
+    'boo',
+    'book',
+    'booking',
+    'bosch',
+    'bostik',
+    'boston',
+    'bot',
+    'boutique',
+    'box',
+    'br',
+    'bradesco',
+    'bridgestone',
+    'broadway',
+    'broker',
+    'brother',
+    'brussels',
+    'bs',
+    'bt',
+    'build',
+    'builders',
+    'business',
+    'buy',
+    'buzz',
+    'bv',
+    'bw',
+    'by',
+    'bz',
+    'bzh',
+    'ca',
+    'cab',
+    'cafe',
+    'cal',
+    'call',
+    'calvinklein',
+    'cam',
+    'camera',
+    'camp',
+    'canon',
+    'capetown',
+    'capital',
+    'capitalone',
+    'car',
+    'caravan',
+    'cards',
+    'care',
+    'career',
+    'careers',
+    'cars',
+    'casa',
+    'case',
+    'cash',
+    'casino',
+    'cat',
+    'catering',
+    'catholic',
+    'cba',
+    'cbn',
+    'cbre',
+    'cc',
+    'cd',
+    'center',
+    'ceo',
+    'cern',
+    'cf',
+    'cfa',
+    'cfd',
+    'cg',
+    'ch',
+    'chanel',
+    'channel',
+    'charity',
+    'chase',
+    'chat',
+    'cheap',
+    'chintai',
+    'christmas',
+    'chrome',
+    'church',
+    'ci',
+    'cipriani',
+    'circle',
+    'cisco',
+    'citadel',
+    'citi',
+    'citic',
+    'city',
+    'ck',
+    'cl',
+    'claims',
+    'cleaning',
+    'click',
+    'clinic',
+    'clinique',
+    'clothing',
+    'cloud',
+    'club',
+    'clubmed',
+    'cm',
+    'cn',
+    'co',
+    'coach',
+    'codes',
+    'coffee',
+    'college',
+    'cologne',
+    'com',
+    'commbank',
+    'community',
+    'company',
+    'compare',
+    'computer',
+    'comsec',
+    'condos',
+    'construction',
+    'consulting',
+    'contact',
+    'contractors',
+    'cooking',
+    'cool',
+    'coop',
+    'corsica',
+    'country',
+    'coupon',
+    'coupons',
+    'courses',
+    'cpa',
+    'cr',
+    'credit',
+    'creditcard',
+    'creditunion',
+    'cricket',
+    'crown',
+    'crs',
+    'cruise',
+    'cruises',
+    'cu',
+    'cuisinella',
+    'cv',
+    'cw',
+    'cx',
+    'cy',
+    'cymru',
+    'cyou',
+    'cz',
+    'dad',
+    'dance',
+    'data',
+    'date',
+    'dating',
+    'datsun',
+    'day',
+    'dclk',
+    'dds',
+    'de',
+    'deal',
+    'dealer',
+    'deals',
+    'degree',
+    'delivery',
+    'dell',
+    'deloitte',
+    'delta',
+    'democrat',
+    'dental',
+    'dentist',
+    'desi',
+    'design',
+    'dev',
+    'dhl',
+    'diamonds',
+    'diet',
+    'digital',
+    'direct',
+    'directory',
+    'discount',
+    'discover',
+    'dish',
+    'diy',
+    'dj',
+    'dk',
+    'dm',
+    'dnp',
+    'do',
+    'docs',
+    'doctor',
+    'dog',
+    'domains',
+    'dot',
+    'download',
+    'drive',
+    'dtv',
+    'dubai',
+    'dunlop',
+    'dupont',
+    'durban',
+    'dvag',
+    'dvr',
+    'dz',
+    'earth',
+    'eat',
+    'ec',
+    'eco',
+    'edeka',
+    'edu',
+    'education',
+    'ee',
+    'eg',
+    'email',
+    'emerck',
+    'energy',
+    'engineer',
+    'engineering',
+    'enterprises',
+    'epson',
+    'equipment',
+    'er',
+    'ericsson',
+    'erni',
+    'es',
+    'esq',
+    'estate',
+    'et',
+    'eu',
+    'eurovision',
+    'eus',
+    'events',
+    'exchange',
+    'expert',
+    'exposed',
+    'express',
+    'extraspace',
+    'fage',
+    'fail',
+    'fairwinds',
+    'faith',
+    'family',
+    'fan',
+    'fans',
+    'farm',
+    'farmers',
+    'fashion',
+    'fast',
+    'fedex',
+    'feedback',
+    'ferrari',
+    'ferrero',
+    'fi',
+    'fidelity',
+    'fido',
+    'film',
+    'final',
+    'finance',
+    'financial',
+    'fire',
+    'firestone',
+    'firmdale',
+    'fish',
+    'fishing',
+    'fit',
+    'fitness',
+    'fj',
+    'fk',
+    'flickr',
+    'flights',
+    'flir',
+    'florist',
+    'flowers',
+    'fly',
+    'fm',
+    'fo',
+    'foo',
+    'food',
+    'football',
+    'ford',
+    'forex',
+    'forsale',
+    'forum',
+    'foundation',
+    'fox',
+    'fr',
+    'free',
+    'fresenius',
+    'frl',
+    'frogans',
+    'frontier',
+    'ftr',
+    'fujitsu',
+    'fun',
+    'fund',
+    'furniture',
+    'futbol',
+    'fyi',
+    'ga',
+    'gal',
+    'gallery',
+    'gallo',
+    'gallup',
+    'game',
+    'games',
+    'gap',
+    'garden',
+    'gay',
+    'gb',
+    'gbiz',
+    'gd',
+    'gdn',
+    'ge',
+    'gea',
+    'gent',
+    'genting',
+    'george',
+    'gf',
+    'gg',
+    'ggee',
+    'gh',
+    'gi',
+    'gift',
+    'gifts',
+    'gives',
+    'giving',
+    'gl',
+    'glass',
+    'gle',
+    'global',
+    'globo',
+    'gm',
+    'gmail',
+    'gmbh',
+    'gmo',
+    'gmx',
+    'gn',
+    'godaddy',
+    'gold',
+    'goldpoint',
+    'golf',
+    'goo',
+    'goodyear',
+    'goog',
+    'google',
+    'gop',
+    'got',
+    'gov',
+    'gp',
+    'gq',
+    'gr',
+    'grainger',
+    'graphics',
+    'gratis',
+    'green',
+    'gripe',
+    'grocery',
+    'group',
+    'gs',
+    'gt',
+    'gu',
+    'gucci',
+    'guge',
+    'guide',
+    'guitars',
+    'guru',
+    'gw',
+    'gy',
+    'hair',
+    'hamburg',
+    'hangout',
+    'haus',
+    'hbo',
+    'hdfc',
+    'hdfcbank',
+    'health',
+    'healthcare',
+    'help',
+    'helsinki',
+    'here',
+    'hermes',
+    'hiphop',
+    'hisamitsu',
+    'hitachi',
+    'hiv',
+    'hk',
+    'hkt',
+    'hm',
+    'hn',
+    'hockey',
+    'holdings',
+    'holiday',
+    'homedepot',
+    'homegoods',
+    'homes',
+    'homesense',
+    'honda',
+    'horse',
+    'hospital',
+    'host',
+    'hosting',
+    'hot',
+    'hotels',
+    'hotmail',
+    'house',
+    'how',
+    'hr',
+    'hsbc',
+    'ht',
+    'hu',
+    'hughes',
+    'hyatt',
+    'hyundai',
+    'ibm',
+    'icbc',
+    'ice',
+    'icu',
+    'id',
+    'ie',
+    'ieee',
+    'ifm',
+    'ikano',
+    'il',
+    'im',
+    'imamat',
+    'imdb',
+    'immo',
+    'immobilien',
+    'in',
+    'inc',
+    'industries',
+    'infiniti',
+    'info',
+    'ing',
+    'ink',
+    'institute',
+    'insurance',
+    'insure',
+    'int',
+    'international',
+    'intuit',
+    'investments',
+    'io',
+    'ipiranga',
+    'iq',
+    'ir',
+    'irish',
+    'is',
+    'ismaili',
+    'ist',
+    'istanbul',
+    'it',
+    'itau',
+    'itv',
+    'jaguar',
+    'java',
+    'jcb',
+    'je',
+    'jeep',
+    'jetzt',
+    'jewelry',
+    'jio',
+    'jll',
+    'jm',
+    'jmp',
+    'jnj',
+    'jo',
+    'jobs',
+    'joburg',
+    'jot',
+    'joy',
+    'jp',
+    'jpmorgan',
+    'jprs',
+    'juegos',
+    'juniper',
+    'kaufen',
+    'kddi',
+    'ke',
+    'kerryhotels',
+    'kerryproperties',
+    'kfh',
+    'kg',
+    'kh',
+    'ki',
+    'kia',
+    'kids',
+    'kim',
+    'kindle',
+    'kitchen',
+    'kiwi',
+    'km',
+    'kn',
+    'koeln',
+    'komatsu',
+    'kosher',
+    'kp',
+    'kpmg',
+    'kpn',
+    'kr',
+    'krd',
+    'kred',
+    'kuokgroup',
+    'kw',
+    'ky',
+    'kyoto',
+    'kz',
+    'la',
+    'lacaixa',
+    'lamborghini',
+    'lamer',
+    'land',
+    'landrover',
+    'lanxess',
+    'lasalle',
+    'lat',
+    'latino',
+    'latrobe',
+    'law',
+    'lawyer',
+    'lb',
+    'lc',
+    'lds',
+    'lease',
+    'leclerc',
+    'lefrak',
+    'legal',
+    'lego',
+    'lexus',
+    'lgbt',
+    'li',
+    'lidl',
+    'life',
+    'lifeinsurance',
+    'lifestyle',
+    'lighting',
+    'like',
+    'lilly',
+    'limited',
+    'limo',
+    'lincoln',
+    'link',
+    'live',
+    'living',
+    'lk',
+    'llc',
+    'llp',
+    'loan',
+    'loans',
+    'locker',
+    'locus',
+    'lol',
+    'london',
+    'lotte',
+    'lotto',
+    'love',
+    'lpl',
+    'lplfinancial',
+    'lr',
+    'ls',
+    'lt',
+    'ltd',
+    'ltda',
+    'lu',
+    'lundbeck',
+    'luxe',
+    'luxury',
+    'lv',
+    'ly',
+    'ma',
+    'madrid',
+    'maif',
+    'maison',
+    'makeup',
+    'man',
+    'management',
+    'mango',
+    'map',
+    'market',
+    'marketing',
+    'markets',
+    'marriott',
+    'marshalls',
+    'mattel',
+    'mba',
+    'mc',
+    'mckinsey',
+    'md',
+    'me',
+    'med',
+    'media',
+    'meet',
+    'melbourne',
+    'meme',
+    'memorial',
+    'men',
+    'menu',
+    'merckmsd',
+    'mg',
+    'mh',
+    'miami',
+    'microsoft',
+    'mil',
+    'mini',
+    'mint',
+    'mit',
+    'mitsubishi',
+    'mk',
+    'ml',
+    'mlb',
+    'mls',
+    'mm',
+    'mma',
+    'mn',
+    'mo',
+    'mobi',
+    'mobile',
+    'moda',
+    'moe',
+    'moi',
+    'mom',
+    'monash',
+    'money',
+    'monster',
+    'mormon',
+    'mortgage',
+    'moscow',
+    'moto',
+    'motorcycles',
+    'mov',
+    'movie',
+    'mp',
+    'mq',
+    'mr',
+    'ms',
+    'msd',
+    'mt',
+    'mtn',
+    'mtr',
+    'mu',
+    'museum',
+    'music',
+    'mv',
+    'mw',
+    'mx',
+    'my',
+    'mz',
+    'na',
+    'nab',
+    'nagoya',
+    'name',
+    'navy',
+    'nba',
+    'nc',
+    'ne',
+    'nec',
+    'net',
+    'netbank',
+    'netflix',
+    'network',
+    'neustar',
+    'new',
+    'news',
+    'next',
+    'nextdirect',
+    'nexus',
+    'nf',
+    'nfl',
+    'ng',
+    'ngo',
+    'nhk',
+    'ni',
+    'nico',
+    'nike',
+    'nikon',
+    'ninja',
+    'nissan',
+    'nissay',
+    'nl',
+    'no',
+    'nokia',
+    'norton',
+    'now',
+    'nowruz',
+    'nowtv',
+    'np',
+    'nr',
+    'nra',
+    'nrw',
+    'ntt',
+    'nu',
+    'nyc',
+    'nz',
+    'obi',
+    'observer',
+    'office',
+    'okinawa',
+    'olayan',
+    'olayangroup',
+    'ollo',
+    'om',
+    'omega',
+    'one',
+    'ong',
+    'onl',
+    'online',
+    'ooo',
+    'open',
+    'oracle',
+    'orange',
+    'org',
+    'organic',
+    'origins',
+    'osaka',
+    'otsuka',
+    'ott',
+    'ovh',
+    'pa',
+    'page',
+    'panasonic',
+    'paris',
+    'pars',
+    'partners',
+    'parts',
+    'party',
+    'pay',
+    'pccw',
+    'pe',
+    'pet',
+    'pf',
+    'pfizer',
+    'pg',
+    'ph',
+    'pharmacy',
+    'phd',
+    'philips',
+    'phone',
+    'photo',
+    'photography',
+    'photos',
+    'physio',
+    'pics',
+    'pictet',
+    'pictures',
+    'pid',
+    'pin',
+    'ping',
+    'pink',
+    'pioneer',
+    'pizza',
+    'pk',
+    'pl',
+    'place',
+    'play',
+    'playstation',
+    'plumbing',
+    'plus',
+    'pm',
+    'pn',
+    'pnc',
+    'pohl',
+    'poker',
+    'politie',
+    'porn',
+    'post',
+    'pr',
+    'pramerica',
+    'praxi',
+    'press',
+    'prime',
+    'pro',
+    'prod',
+    'productions',
+    'prof',
+    'progressive',
+    'promo',
+    'properties',
+    'property',
+    'protection',
+    'pru',
+    'prudential',
+    'ps',
+    'pt',
+    'pub',
+    'pw',
+    'pwc',
+    'py',
+    'qa',
+    'qpon',
+    'quebec',
+    'quest',
+    'racing',
+    'radio',
+    're',
+    'read',
+    'realestate',
+    'realtor',
+    'realty',
+    'recipes',
+    'red',
+    'redstone',
+    'redumbrella',
+    'rehab',
+    'reise',
+    'reisen',
+    'reit',
+    'reliance',
+    'ren',
+    'rent',
+    'rentals',
+    'repair',
+    'report',
+    'republican',
+    'rest',
+    'restaurant',
+    'review',
+    'reviews',
+    'rexroth',
+    'rich',
+    'richardli',
+    'ricoh',
+    'ril',
+    'rio',
+    'rip',
+    'ro',
+    'rocks',
+    'rodeo',
+    'rogers',
+    'room',
+    'rs',
+    'rsvp',
+    'ru',
+    'rugby',
+    'ruhr',
+    'run',
+    'rw',
+    'rwe',
+    'ryukyu',
+    'sa',
+    'saarland',
+    'safe',
+    'safety',
+    'sakura',
+    'sale',
+    'salon',
+    'samsclub',
+    'samsung',
+    'sandvik',
+    'sandvikcoromant',
+    'sanofi',
+    'sap',
+    'sarl',
+    'sas',
+    'save',
+    'saxo',
+    'sb',
+    'sbi',
+    'sbs',
+    'sc',
+    'scb',
+    'schaeffler',
+    'schmidt',
+    'scholarships',
+    'school',
+    'schule',
+    'schwarz',
+    'science',
+    'scot',
+    'sd',
+    'se',
+    'search',
+    'seat',
+    'secure',
+    'security',
+    'seek',
+    'select',
+    'sener',
+    'services',
+    'seven',
+    'sew',
+    'sex',
+    'sexy',
+    'sfr',
+    'sg',
+    'sh',
+    'shangrila',
+    'sharp',
+    'shell',
+    'shia',
+    'shiksha',
+    'shoes',
+    'shop',
+    'shopping',
+    'shouji',
+    'show',
+    'si',
+    'silk',
+    'sina',
+    'singles',
+    'site',
+    'sj',
+    'sk',
+    'ski',
+    'skin',
+    'sky',
+    'skype',
+    'sl',
+    'sling',
+    'sm',
+    'smart',
+    'smile',
+    'sn',
+    'sncf',
+    'so',
+    'soccer',
+    'social',
+    'softbank',
+    'software',
+    'sohu',
+    'solar',
+    'solutions',
+    'song',
+    'sony',
+    'soy',
+    'spa',
+    'space',
+    'sport',
+    'spot',
+    'sr',
+    'srl',
+    'ss',
+    'st',
+    'stada',
+    'staples',
+    'star',
+    'statebank',
+    'statefarm',
+    'stc',
+    'stcgroup',
+    'stockholm',
+    'storage',
+    'store',
+    'stream',
+    'studio',
+    'study',
+    'style',
+    'su',
+    'sucks',
+    'supplies',
+    'supply',
+    'support',
+    'surf',
+    'surgery',
+    'suzuki',
+    'sv',
+    'swatch',
+    'swiss',
+    'sx',
+    'sy',
+    'sydney',
+    'systems',
+    'sz',
+    'tab',
+    'taipei',
+    'talk',
+    'taobao',
+    'target',
+    'tatamotors',
+    'tatar',
+    'tattoo',
+    'tax',
+    'taxi',
+    'tc',
+    'tci',
+    'td',
+    'tdk',
+    'team',
+    'tech',
+    'technology',
+    'tel',
+    'temasek',
+    'tennis',
+    'teva',
+    'tf',
+    'tg',
+    'th',
+    'thd',
+    'theater',
+    'theatre',
+    'tiaa',
+    'tickets',
+    'tienda',
+    'tips',
+    'tires',
+    'tirol',
+    'tj',
+    'tjmaxx',
+    'tjx',
+    'tk',
+    'tkmaxx',
+    'tl',
+    'tm',
+    'tmall',
+    'tn',
+    'to',
+    'today',
+    'tokyo',
+    'tools',
+    'top',
+    'toray',
+    'toshiba',
+    'total',
+    'tours',
+    'town',
+    'toyota',
+    'toys',
+    'tr',
+    'trade',
+    'trading',
+    'training',
+    'travel',
+    'travelers',
+    'travelersinsurance',
+    'trust',
+    'trv',
+    'tt',
+    'tube',
+    'tui',
+    'tunes',
+    'tushu',
+    'tv',
+    'tvs',
+    'tw',
+    'tz',
+    'ua',
+    'ubank',
+    'ubs',
+    'ug',
+    'uk',
+    'unicom',
+    'university',
+    'uno',
+    'uol',
+    'ups',
+    'us',
+    'uy',
+    'uz',
+    'va',
+    'vacations',
+    'vana',
+    'vanguard',
+    'vc',
+    've',
+    'vegas',
+    'ventures',
+    'verisign',
+    'versicherung',
+    'vet',
+    'vg',
+    'vi',
+    'viajes',
+    'video',
+    'vig',
+    'viking',
+    'villas',
+    'vin',
+    'vip',
+    'virgin',
+    'visa',
+    'vision',
+    'viva',
+    'vivo',
+    'vlaanderen',
+    'vn',
+    'vodka',
+    'volvo',
+    'vote',
+    'voting',
+    'voto',
+    'voyage',
+    'vu',
+    'wales',
+    'walmart',
+    'walter',
+    'wang',
+    'wanggou',
+    'watch',
+    'watches',
+    'weather',
+    'weatherchannel',
+    'webcam',
+    'weber',
+    'website',
+    'wed',
+    'wedding',
+    'weibo',
+    'weir',
+    'wf',
+    'whoswho',
+    'wien',
+    'wiki',
+    'williamhill',
+    'win',
+    'windows',
+    'wine',
+    'winners',
+    'wme',
+    'wolterskluwer',
+    'woodside',
+    'work',
+    'works',
+    'world',
+    'wow',
+    'ws',
+    'wtc',
+    'wtf',
+    'xbox',
+    'xerox',
+    'xihuan',
+    'xin',
+    'xn--11b4c3d',
+    'xn--1ck2e1b',
+    'xn--1qqw23a',
+    'xn--2scrj9c',
+    'xn--30rr7y',
+    'xn--3bst00m',
+    'xn--3ds443g',
+    'xn--3e0b707e',
+    'xn--3hcrj9c',
+    'xn--3pxu8k',
+    'xn--42c2d9a',
+    'xn--45br5cyl',
+    'xn--45brj9c',
+    'xn--45q11c',
+    'xn--4dbrk0ce',
+    'xn--4gbrim',
+    'xn--54b7fta0cc',
+    'xn--55qw42g',
+    'xn--55qx5d',
+    'xn--5su34j936bgsg',
+    'xn--5tzm5g',
+    'xn--6frz82g',
+    'xn--6qq986b3xl',
+    'xn--80adxhks',
+    'xn--80ao21a',
+    'xn--80aqecdr1a',
+    'xn--80asehdb',
+    'xn--80aswg',
+    'xn--8y0a063a',
+    'xn--90a3ac',
+    'xn--90ae',
+    'xn--90ais',
+    'xn--9dbq2a',
+    'xn--9et52u',
+    'xn--9krt00a',
+    'xn--b4w605ferd',
+    'xn--bck1b9a5dre4c',
+    'xn--c1avg',
+    'xn--c2br7g',
+    'xn--cck2b3b',
+    'xn--cckwcxetd',
+    'xn--cg4bki',
+    'xn--clchc0ea0b2g2a9gcd',
+    'xn--czr694b',
+    'xn--czrs0t',
+    'xn--czru2d',
+    'xn--d1acj3b',
+    'xn--d1alf',
+    'xn--e1a4c',
+    'xn--eckvdtc9d',
+    'xn--efvy88h',
+    'xn--fct429k',
+    'xn--fhbei',
+    'xn--fiq228c5hs',
+    'xn--fiq64b',
+    'xn--fiqs8s',
+    'xn--fiqz9s',
+    'xn--fjq720a',
+    'xn--flw351e',
+    'xn--fpcrj9c3d',
+    'xn--fzc2c9e2c',
+    'xn--fzys8d69uvgm',
+    'xn--g2xx48c',
+    'xn--gckr3f0f',
+    'xn--gecrj9c',
+    'xn--gk3at1e',
+    'xn--h2breg3eve',
+    'xn--h2brj9c',
+    'xn--h2brj9c8c',
+    'xn--hxt814e',
+    'xn--i1b6b1a6a2e',
+    'xn--imr513n',
+    'xn--io0a7i',
+    'xn--j1aef',
+    'xn--j1amh',
+    'xn--j6w193g',
+    'xn--jlq480n2rg',
+    'xn--jvr189m',
+    'xn--kcrx77d1x4a',
+    'xn--kprw13d',
+    'xn--kpry57d',
+    'xn--kput3i',
+    'xn--l1acc',
+    'xn--lgbbat1ad8j',
+    'xn--mgb9awbf',
+    'xn--mgba3a3ejt',
+    'xn--mgba3a4f16a',
+    'xn--mgba7c0bbn0a',
+    'xn--mgbaam7a8h',
+    'xn--mgbab2bd',
+    'xn--mgbah1a3hjkrd',
+    'xn--mgbai9azgqp6j',
+    'xn--mgbayh7gpa',
+    'xn--mgbbh1a',
+    'xn--mgbbh1a71e',
+    'xn--mgbc0a9azcg',
+    'xn--mgbca7dzdo',
+    'xn--mgbcpq6gpa1a',
+    'xn--mgberp4a5d4ar',
+    'xn--mgbgu82a',
+    'xn--mgbi4ecexp',
+    'xn--mgbpl2fh',
+    'xn--mgbt3dhd',
+    'xn--mgbtx2b',
+    'xn--mgbx4cd0ab',
+    'xn--mix891f',
+    'xn--mk1bu44c',
+    'xn--mxtq1m',
+    'xn--ngbc5azd',
+    'xn--ngbe9e0a',
+    'xn--ngbrx',
+    'xn--node',
+    'xn--nqv7f',
+    'xn--nqv7fs00ema',
+    'xn--nyqy26a',
+    'xn--o3cw4h',
+    'xn--ogbpf8fl',
+    'xn--otu796d',
+    'xn--p1acf',
+    'xn--p1ai',
+    'xn--pgbs0dh',
+    'xn--pssy2u',
+    'xn--q7ce6a',
+    'xn--q9jyb4c',
+    'xn--qcka1pmc',
+    'xn--qxa6a',
+    'xn--qxam',
+    'xn--rhqv96g',
+    'xn--rovu88b',
+    'xn--rvc1e0am3e',
+    'xn--s9brj9c',
+    'xn--ses554g',
+    'xn--t60b56a',
+    'xn--tckwe',
+    'xn--tiq49xqyj',
+    'xn--unup4y',
+    'xn--vermgensberater-ctb',
+    'xn--vermgensberatung-pwb',
+    'xn--vhquv',
+    'xn--vuq861b',
+    'xn--w4r85el8fhu5dnra',
+    'xn--w4rs40l',
+    'xn--wgbh1c',
+    'xn--wgbl6a',
+    'xn--xhq521b',
+    'xn--xkc2al3hye2a',
+    'xn--xkc2dl3a5ee0h',
+    'xn--y9a3aq',
+    'xn--yfro4i67o',
+    'xn--ygbi2ammx',
+    'xn--zfr164b',
+    'xxx',
+    'xyz',
+    'yachts',
+    'yahoo',
+    'yamaxun',
+    'yandex',
+    'ye',
+    'yodobashi',
+    'yoga',
+    'yokohama',
+    'you',
+    'youtube',
+    'yt',
+    'yun',
+    'za',
+    'zappos',
+    'zara',
+    'zero',
+    'zip',
+    'zm',
+    'zone',
+    'zuerich',
+    'zw'
+]