summaryrefslogtreecommitdiff
path: root/src/portal/py
diff options
context:
space:
mode:
Diffstat (limited to 'src/portal/py')
-rw-r--r--src/portal/py/config.def.py4
-rw-r--r--src/portal/py/modules/__init__.py12
-rw-r--r--src/portal/py/modules/fanbox.py10
-rw-r--r--src/portal/py/modules/instagram.py18
-rw-r--r--src/portal/py/modules/patreon.py8
-rw-r--r--src/portal/py/modules/pixiv_app.py151
-rw-r--r--src/portal/py/modules/pixiv_web.py78
-rw-r--r--src/portal/py/modules/twitter.py61
-rw-r--r--src/portal/py/post.py30
-rw-r--r--src/portal/py/query_parser.py4
-rw-r--r--src/portal/py/test.py4
-rw-r--r--src/portal/py/tests/archive_query.py11
-rw-r--r--src/portal/py/tests/old_twitter_api.py4
-rw-r--r--src/portal/py/tests/rewrite_post.py24
-rw-r--r--src/portal/py/tests/test.py109
-rw-r--r--src/portal/py/tests/touch_all.py79
-rw-r--r--src/portal/py/tlds.py1445
17 files changed, 1741 insertions, 311 deletions
diff --git a/src/portal/py/config.def.py b/src/portal/py/config.def.py
index 8c0dab9..0c7ea30 100644
--- a/src/portal/py/config.def.py
+++ b/src/portal/py/config.def.py
@@ -6,7 +6,7 @@ TWITTER_ACCESS_TOKEN = ''
TWITTER_ACCESS_TOKEN_SECRET = ''
TWITTER_CONSUMER_TOKEN = ''
TWITTER_CONSUMER_TOKEN_SECRET = ''
-TWITTER_COOKIES_PATH = '{}/cookies_twitter.txt'.format(BASE_DIR)
+TWITTER_COOKIES_PATH = f'{BASE_DIR}/cookies_twitter.txt'
TWITTER_TIMEOUT = 5 # seconds
FANBOX_SESSID = ''
@@ -18,7 +18,7 @@ PIXIV_REFRESH_TOKEN = ''
INSTAGRAM_USERNAME = ''
INSTAGRAM_PASSWORD = ''
INSTAGRAM_SESSION_ID = ''
-INSTAGRAM_SETTINGS_PATH = Path('{}/ig_settings.json'.format(BASE_DIR))
+INSTAGRAM_SETTINGS_PATH = Path(f'{BASE_DIR}/ig_settings.json')
INSTAGRAM_USER_AGENT = ''
PATREON_USER_ID = ''
diff --git a/src/portal/py/modules/__init__.py b/src/portal/py/modules/__init__.py
index fee4c22..8a1e3cb 100644
--- a/src/portal/py/modules/__init__.py
+++ b/src/portal/py/modules/__init__.py
@@ -1,9 +1,9 @@
import config
from modules.youtube import YoutubeModule
-#from modules.twitter import TwitterScrapeModule
-#from modules.pixiv_app import PixivAppModule
-#from modules.pixiv_web import PixivWebModule
+from modules.twitter import TwitterScrapeModule
+from modules.pixiv_app import PixivAppModule
+from modules.pixiv_web import PixivWebModule
#from modules.fanbox import FanboxModule
#from modules.instagram import InstagramModule
#from modules.patreon import PatreonModule
@@ -13,9 +13,9 @@ ALL_MODULES = {
# 'twitter': (TwitterModule(
# config.TWITTER_ACCESS_TOKEN, config.TWITTER_ACCESS_TOKEN_SECRET,
# config.TWITTER_CONSUMER_TOKEN, config.TWITTER_CONSUMER_TOKEN_SECRET), []),
-# 'twitter': (TwitterScrapeModule(config.TWITTER_COOKIES_PATH), []),
-# 'pixiv_web': (PixivWebModule(config.PIXIV_SESSID, config.PIXIV_USERID), []),
-# 'pixiv_app': (PixivAppModule(config.PIXIV_REFRESH_TOKEN), []),
+ 'twitter': (TwitterScrapeModule(config.TWITTER_COOKIES_PATH), []),
+ 'pixiv_web': (PixivWebModule(config.PIXIV_SESSID, config.PIXIV_USERID), []),
+ 'pixiv_app': (PixivAppModule(config.PIXIV_REFRESH_TOKEN), []),
# 'fanbox': (FanboxModule(config.FANBOX_SESSID, config.FANBOX_CF_CLEARANCE), []),
# 'instagram': (InstagramModule(
# config.INSTAGRAM_USER_AGENT, config.INSTAGRAM_SETTINGS_PATH,
diff --git a/src/portal/py/modules/fanbox.py b/src/portal/py/modules/fanbox.py
index 9d233bb..01e2b8c 100644
--- a/src/portal/py/modules/fanbox.py
+++ b/src/portal/py/modules/fanbox.py
@@ -31,7 +31,7 @@ class FanboxBase(Search):
unique_id = f'fanbox:p:{post_id}'
kwargs['unique_id'] = unique_id
kwargs['raw_responses'] = {}
- kwargs['raw_responses']['api'] = json.dumps(data)
+ kwargs['raw_responses']['api'] = json.dumps(data, separators=(',', ':'), indent=None)
publish_date = reformat_pixiv_date(data['publishedDatetime']).timestamp()
current_date = get_current_utc_time().timestamp()
kwargs['dates'] = [
@@ -56,12 +56,12 @@ class FanboxBase(Search):
media = {}
if data['type'] == 'image':
text = data['body']['text']
- for i, value in enumerate(data['body']['images']):
- media[str(i)] = Image(url=Url(value['originalUrl'], value['extension']), thumbnail_url=Url(value['thumbnailUrl']))
+ for index, value in enumerate(data['body']['images']):
+ media[str(index)] = Image(url=Url(value['originalUrl'], value['extension']), thumbnail_url=Url(value['thumbnailUrl']))
elif data['type'] == 'file':
text = data['body']['text']
- for i, value in enumerate(data['body']['files']):
- media[str(i)] = File(url=Url(value['url'], value['extension']), name=value['name'])
+ for index, value in enumerate(data['body']['files']):
+ media[str(index)] = File(url=Url(value['url'], value['extension']), name=value['name'])
elif data['type'] == 'article':
for block in data['body']['blocks']:
if block['type'] == 'p':
diff --git a/src/portal/py/modules/instagram.py b/src/portal/py/modules/instagram.py
index 014ffc1..7d71f9f 100644
--- a/src/portal/py/modules/instagram.py
+++ b/src/portal/py/modules/instagram.py
@@ -22,7 +22,7 @@ class InstagramSearch(Search):
if num >= self.page + self.REACH_LIMIT:
return False
request_satisfied = False
- for i in range(self.page, num + 1):
+ for index in range(self.page, num + 1):
try:
media, self.cursor = self.module.cl.user_medias_paginated_v1(self.pk, 0, end_cursor=self.cursor)
#media, self.cursor = self.module.cl.user_clips_paginated_v1(self.pk, amount=12, end_cursor=self.cursor)
@@ -34,24 +34,24 @@ class InstagramSearch(Search):
return False
if not self.cursor:
self.completed = True
- self.pages[i] = []
+ self.pages[index] = []
for m in media:
kwargs = {}
kwargs['type'] = PostType.POST
- unique_id = 'instagram:p:{}'.format(m.id)
- #unique_id = 'instagram:r:{}'.format(m.id)
+ unique_id = f'instagram:p:{m.id}'
+ #unique_id = f'instagram:r:{m.id}'
kwargs['unique_id'] = unique_id
kwargs['raw_responses'] = {}
kwargs['raw_responses']['tile'] = m.json()
- kwargs['url'] = 'https://www.instagram.com/p/{}'.format(m.code)
- #kwargs['url'] = 'https://www.instagram.com/reel/{}'.format(m.code)
+ kwargs['url'] = f'https://www.instagram.com/p/{m.code}'
+ #kwargs['url'] = f'https://www.instagram.com/reel/{m.code}'
create_date = m.taken_at.timestamp()
current_date = get_current_utc_time().timestamp()
kwargs['dates'] = [
Date(DateType.CREATED, (create_date, create_date), DateMeta.NONE),
Date(DateType.RETRIEVED, (current_date, current_date), DateMeta.NONE)
]
- user = User(unique_id='instagram:u:{}'.format(m.user.pk))
+ user = User(unique_id=f'instagram:u:{m.user.pk}')
if m.user.username:
user.username = m.user.username
if m.user.full_name:
@@ -79,9 +79,9 @@ class InstagramSearch(Search):
kwargs['media'] = media
post = Post(**kwargs)
self.module.add_to_map(unique_id, post)
- self.pages[i].append(unique_id)
+ self.pages[index].append(unique_id)
self.page += 1
- if i == num:
+ if index == num:
request_satisfied = True
return request_satisfied
diff --git a/src/portal/py/modules/patreon.py b/src/portal/py/modules/patreon.py
index 0e4345a..bf2f0df 100644
--- a/src/portal/py/modules/patreon.py
+++ b/src/portal/py/modules/patreon.py
@@ -126,7 +126,7 @@ class PatreonUser(PatreonBase):
return False
request_satisfied = False
- for i in range(self.page, num + 1):
+ for index in range(self.page, num + 1):
url = f'{BASE_URL}/posts'
#'filter[accessible_by_user_id]': self.module.user_id,
params = {
@@ -187,17 +187,17 @@ class PatreonUser(PatreonBase):
else:
log.warn(f'Unhandled include type {inc['type']}.')
- self.pages[i] = []
+ self.pages[index] = []
for entry in obj['data']:
if entry['type'] != 'post':
log.warn(f'Unhandled entry type {entry['type']}.')
continue
post = self.create_post(entry, hash)
if post:
- self.pages[i].append(post.unique_id)
+ self.pages[index].append(post.unique_id)
self.page += 1
- if i == num:
+ if index == num:
request_satisfied = True
return request_satisfied
diff --git a/src/portal/py/modules/pixiv_app.py b/src/portal/py/modules/pixiv_app.py
index 02d5254..7bd341b 100644
--- a/src/portal/py/modules/pixiv_app.py
+++ b/src/portal/py/modules/pixiv_app.py
@@ -1,9 +1,9 @@
import json
from typing import Optional, Any
from base import USER_AGENT, Search, Module, ParsedJson
-from post import Url, PostType, DateType, User, Post, Image, Tag, TagType
+from post import Url, PostType, Date, DateType, DateMeta, User, Post, Image, Tag, TagType
from query_parser import QueryParser
-from pixivpy3 import AppPixivAPI
+from pixivpy3 import AppPixivAPI, models
from modules.common import get_current_utc_time, reformat_pixiv_date
class PixivAppBase(Search):
@@ -18,63 +18,77 @@ class PixivAppBase(Search):
def request_page(self, num: int) -> bool:
request_satisfied = False
- for i in range(self.page, num + 1):
+ for index in range(self.page, num + 1):
if not self.next_qs:
break
- if i in self.pages:
+ if index in self.pages:
continue
- self.pages[i] = []
- success, self.next_qs = self.api_request_page(i)
+ self.pages[index] = []
+ success, self.next_qs = self.api_request_page(index)
if not success:
self.errored = True
break
- if i == num:
+ if index == num:
request_satisfied = True
self.page += 1
return request_satisfied
- def create_media_url(self, url: str) -> Url:
- return Url(url, url.split('.')[-1])
-
def create_tag(self, data: ParsedJson) -> Tag:
- tag = Tag(type=TagType.GENERAL, name=data['name'])
+ kwargs = {}
+ kwargs['type'] = TagType.GENERAL
+ kwargs['name'] = data['name']
+ kwargs['alts'] = {}
if data['translated_name']:
- tag.alts['en'] = data['translated_name']
- return tag
+ kwargs['alts']['en'] = data['translated_name']
+ return Tag(**kwargs)
- def create_post(self, data: ParsedJson) -> Post:
- if len(data['meta_pages']) == 0:
- data['meta_pages'].append({
- 'image_urls': {
- 'medium': data['image_urls']['medium'],
- 'original': data['meta_single_page']['original_image_url']
- }
- })
+ # Other objects returned from pixivpy3 are likely also ModelT but it seems
+ # so inconsistent internally for now keep some params as ParsedJson.
+ def create_post(self, data: ParsedJson | models.ModelT, retrieved_date: Optional[float] = None) -> Post:
+ is_deleted = not data['visible']
kwargs = {}
+ kwargs['module'] = 'pixiv_app'
kwargs['type'] = PostType.POST
- unique_id = 'pixiv:i:{}'.format(data['id'])
+ unique_id = f'pixiv:i:{data['id']}'
kwargs['unique_id'] = unique_id
kwargs['raw_responses'] = {}
- kwargs['raw_responses']['api'] = json.dumps(data)
- kwargs['url'] = 'https://www.pixiv.net/en/artworks/{}'.format(data['id'])
- kwargs['dates'] = {
- DateType.CREATED: reformat_pixiv_date(data['create_date']).timestamp(),
- DateType.RETRIEVED: get_current_utc_time().timestamp()
- }
+ try:
+ kwargs['raw_responses']['api'] = json.dumps(data, separators=(',', ':'), indent=None)
+ except TypeError:
+ kwargs['raw_responses']['api'] = data.model_dump_json(indent=None)
+ kwargs['url'] = f'https://www.pixiv.net/en/artworks/{data['id']}'
+ if not retrieved_date:
+ retrieved_date = get_current_utc_time().timestamp()
+ kwargs['dates'] = [
+ Date(DateType.RETRIEVED, (retrieved_date, retrieved_date), DateMeta.NONE)
+ ]
+ if not is_deleted:
+ create_date = reformat_pixiv_date(data['create_date']).timestamp()
+ kwargs['dates'].append(Date(DateType.CREATED, (create_date, create_date), DateMeta.NONE))
user = User(unique_id=f'pixiv:u:{data['user']['id']}', username=data['user']['account'], display_name=data['user']['name'])
- user.profile_picture_url = data['user']['profile_image_urls']['medium']
+ if not is_deleted:
+ user.profile_picture_url = Url(data['user']['profile_image_urls']['medium'])
kwargs['author'] = user
kwargs['title'] = data['title']
kwargs['text'] = data['caption']
- kwargs['likes'] = data['total_bookmarks']
+ if not is_deleted: # Default to None instead of 0.
+ kwargs['bookmarks'] = data['total_bookmarks']
+ kwargs['views'] = data['total_view']
if 'total_comments' in data:
kwargs['comments'] = data['total_comments']
- kwargs['views'] = data['total_view']
- kwargs['tags'] = [self.create_tag(tag) for tag in data['tags']]
- media = {}
- for i, page in enumerate(data['meta_pages']):
- media[str(i)] = Image(url=self.create_media_url(page['image_urls']['original']), thumbnail_url=self.create_media_url(page['image_urls']['medium']))
- kwargs['media'] = media
+ kwargs['tags'] = [self.create_tag(t) for t in data['tags']]
+ if not is_deleted:
+ if len(data['meta_pages']) == 0:
+ data['meta_pages'].append({
+ 'image_urls': {
+ 'medium': data['image_urls']['medium'],
+ 'original': data['meta_single_page']['original_image_url']
+ }
+ })
+ media = {}
+ for index, page in enumerate(data['meta_pages']):
+ media[str(index)] = Image(url=Url(page['image_urls']['original']), thumbnail_url=Url(page['image_urls']['medium']))
+ kwargs['media'] = media
post = Post(**kwargs)
self.module.add_to_map(unique_id, post)
return post
@@ -82,16 +96,23 @@ class PixivAppBase(Search):
def create_user(self, data: ParsedJson) -> User:
unique_id = f'pixiv:u:{data['id']}'
user = User(unique_id=unique_id, username=data['account'], display_name=data['name'])
+ user.profile_picture_url = Url(data['profile_image_urls']['medium'])
self.module.add_to_map(unique_id, user)
return user
+ def remake_post(self, raw_responses: ParsedJson, original_date: float) -> Optional[Post]:
+ api = json.loads(raw_responses['api'])
+ return self.create_post(api, original_date)
+
class PixivAppSearch(PixivAppBase):
def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
self.next_qs: Optional[dict[str, Any]] = {}
self.next_qs['word'] = arg
- self.next_qs['sort'] = 'popular_desc'
- self.next_qs['search_ai_type'] = 1
+ #self.next_qs['sort'] = 'popular_desc'
+ # https://github.com/upbit/pixivpy/issues/352
+ # We want to include AI results for completeness, to be ignored further down the chain.
+ #self.next_qs['search_ai_type'] = 1
self.next_qs['start_date'] = '2019-01-01'
self.next_qs['end_date'] = '2019-12-31'
@@ -99,27 +120,10 @@ class PixivAppSearch(PixivAppBase):
if not self.next_qs:
return False, None
obj = self.module.api.search_illust(**self.next_qs)
- if 'illusts' not in obj:
- return False, None
- for post in obj['illusts']:
+ for post in obj.illusts:
self.pages[num].append(self.create_post(post).unique_id)
return True, self.module.api.parse_qs(obj['next_url'])
-class PixivAppIllust(PixivAppBase):
- def __init__(self, userdata: Any, arg: str) -> None:
- super().__init__(userdata)
- self.next_qs: Optional[dict[str, Any]] = {}
- self.next_qs['illust_id'] = arg
-
- def api_request_page(self, num: int) -> tuple[bool, Optional[dict[str, Any]]]:
- if not self.next_qs:
- return False, None
- obj = self.module.api.illust_detail(**self.next_qs)
- if 'illust' not in obj:
- return False, None
- self.pages[num].append(self.create_post(obj['illust']).unique_id)
- return True, None
-
class PixivAppUser(PixivAppBase):
def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
@@ -131,9 +135,7 @@ class PixivAppUser(PixivAppBase):
if not self.next_qs:
return False, None
obj = self.module.api.user_illusts(**self.next_qs)
- if 'illusts' not in obj:
- return False, None
- for post in obj['illusts']:
+ for post in obj.illusts:
self.pages[num].append(self.create_post(post).unique_id)
return True, self.module.api.parse_qs(obj['next_url'])
@@ -148,9 +150,7 @@ class PixivAppBookmarks(PixivAppBase):
if not self.next_qs:
return False, None
obj = self.module.api.user_bookmarks_illust(**self.next_qs)
- if 'illusts' not in obj:
- return False, None
- for post in obj['illusts']:
+ for post in obj.illusts:
self.pages[num].append(self.create_post(post).unique_id)
return True, self.module.api.parse_qs(obj['next_url'])
@@ -165,22 +165,39 @@ class PixivAppFollowing(PixivAppBase):
if not self.next_qs:
return False, None
obj = self.module.api.user_following(**self.next_qs)
- if 'user_previews' not in obj:
- return False, None
- for user in obj['user_previews']:
+ for user in obj.user_previews:
self.pages[num].append(self.create_user(user['user']).unique_id)
return True, self.module.api.parse_qs(obj['next_url'])
+class PixivAppIllust(PixivAppBase):
+ def __init__(self, userdata: Any, arg: str) -> None:
+ super().__init__(userdata)
+ self.next_qs: Optional[dict[str, Any]] = {}
+ self.next_qs['illust_id'] = arg
+
+ def api_request_page(self, num: int) -> tuple[bool, Optional[dict[str, Any]]]:
+ if not self.next_qs:
+ return False, None
+ obj = self.module.api.illust_detail(**self.next_qs)
+ self.pages[num].append(self.create_post(obj.illust).unique_id)
+ return True, None
+
class PixivAppModule(Module):
def __init__(self, refresh_token: str) -> None:
super().__init__()
self.parser: QueryParser = QueryParser(self, 'search')
self.parser.add_command('search', PixivAppSearch)
- self.parser.add_command('illust', PixivAppIllust)
self.parser.add_command('user', PixivAppUser)
self.parser.add_command('bookmarks', PixivAppBookmarks)
self.parser.add_command('following', PixivAppFollowing)
- self.api: AppPixivAPI = AppPixivAPI()
+ self.parser.add_command('illust', PixivAppIllust)
+ headers = {
+ "app-os": "ios",
+ "app-os-version": "16.7.2",
+ "app-version": "7.19.6",
+ "user-agent": "PixivIOSApp/7.19.6 (iOS 16.7.2; iPhone13,2)"
+ }
+ self.api: AppPixivAPI = AppPixivAPI(headers=headers)
self.api.auth(refresh_token=refresh_token)
def search(self, query: str, *extra_args: Any) -> Optional[Search]:
diff --git a/src/portal/py/modules/pixiv_web.py b/src/portal/py/modules/pixiv_web.py
index 3c04202..e7399dc 100644
--- a/src/portal/py/modules/pixiv_web.py
+++ b/src/portal/py/modules/pixiv_web.py
@@ -33,8 +33,8 @@ class PixivWebBase(Search):
def parse_pages(self, data: ParsedJson) -> dict[str, Media]:
media = {}
- for i, m in enumerate(data):
- media[str(i)] = Image(url=Url(m['urls']['original']), thumbnail_url=Url(m['urls']['small']))
+ for index, m in enumerate(data):
+ media[str(index)] = Image(url=Url(m['urls']['original']), thumbnail_url=Url(m['urls']['small']))
return media
def parse_ugoira(self, data: ParsedJson, thumbnail_url: str) -> Animation:
@@ -43,7 +43,7 @@ class PixivWebBase(Search):
animation.frames.append((frame['file'], frame['delay']))
return animation
- def seek_profile_picture_url(self, user_illusts: ParsedJson) -> Optional[Url]:
+ def search_profile_picture_url(self, user_illusts: ParsedJson) -> Optional[Url]:
for illust in user_illusts.values():
if illust is None:
continue
@@ -52,7 +52,7 @@ class PixivWebBase(Search):
return None
def create_tag(self, data: ParsedJson) -> Tag:
- kwargs= {}
+ kwargs = {}
kwargs['type'] = TagType.GENERAL
kwargs['name'] = data['tag']
kwargs['alts'] = {}
@@ -63,26 +63,27 @@ class PixivWebBase(Search):
kwargs['alts']['en'] = data['translation']['en']
return Tag(**kwargs)
- def create_post(self, data: ParsedJson, pages: Optional[ParsedJson] = None, ugoira: Optional[ParsedJson] = None) -> Optional[Post]:
+ def create_post(self, data: ParsedJson, pages: Optional[ParsedJson] = None, ugoira: Optional[ParsedJson] = None, retrieved_date: Optional[float] = None) -> Optional[Post]:
kwargs = {}
kwargs['type'] = PostType.POST
illust_id = data['illustId']
unique_id = f'pixiv:i:{illust_id}'
kwargs['unique_id'] = unique_id
kwargs['raw_responses'] = {}
- kwargs['raw_responses']['api'] = json.dumps(data)
+ kwargs['raw_responses']['api'] = json.dumps(data, separators=(',', ':'), indent=None)
kwargs['url'] = f'https://www.pixiv.net/{LANG}/artworks/{data['illustId']}'
create_date = reformat_pixiv_date(data['createDate']).timestamp()
- current_date = get_current_utc_time().timestamp()
+ if not retrieved_date:
+ retrieved_date = get_current_utc_time().timestamp()
kwargs['dates'] = [
Date(DateType.CREATED, (create_date, create_date), DateMeta.NONE),
- Date(DateType.RETRIEVED, (current_date, current_date), DateMeta.NONE)
+ Date(DateType.RETRIEVED, (retrieved_date, retrieved_date), DateMeta.NONE)
]
upload_date = reformat_pixiv_date(data['uploadDate']).timestamp()
if upload_date != create_date:
kwargs['dates'].append(Date(DateType.EDITED, (upload_date, upload_date), DateMeta.NONE))
user = User(unique_id=f'pixiv:u:{data['userId']}', username=data['userAccount'], display_name=data['userName'])
- profile_picture_url = self.seek_profile_picture_url(data['userIllusts'])
+ profile_picture_url = self.search_profile_picture_url(data['userIllusts'])
if profile_picture_url:
user.profile_picture_url = profile_picture_url
kwargs['author'] = user
@@ -124,19 +125,6 @@ class PixivWebBase(Search):
self.module.add_to_map(unique_id, post)
return post
- def remake_post(self, raw_responses: ParsedJson, original_date: float) -> Optional[Post]:
- api = json.loads(raw_responses['api'])
- pages = raw_responses.get('pages', None)
- if pages:
- pages = json.loads(pages)
- ugoira = raw_responses.get('ugoira', None)
- if ugoira:
- ugoira = json.loads(ugoira)
- post = self.create_post(api, pages, ugoira)
- if post:
- post.dates[DateType.RETRIEVED] = original_date
- return post
-
def request_illust(self, illust_id: int) -> Optional[Post]:
url = f'{BASE_URL}/illust/{illust_id}?lang={LANG}&version={VERSION}'
obj = self.check_api_response(self.module.do_request(Method.GET, url))
@@ -151,6 +139,16 @@ class PixivWebBase(Search):
self.module.add_to_map(unique_id, user)
return user
+ def remake_post(self, raw_responses: ParsedJson, original_date: float) -> Optional[Post]:
+ api = json.loads(raw_responses['api'])
+ pages = raw_responses.get('pages', None)
+ if pages:
+ pages = json.loads(pages)
+ ugoira = raw_responses.get('ugoira', None)
+ if ugoira:
+ ugoira = json.loads(ugoira)
+ return self.create_post(api, pages, ugoira, original_date)
+
class PixivWebSearch(PixivWebBase):
def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
@@ -178,23 +176,6 @@ class PixivWebSearch(PixivWebBase):
self.pages[num].append(post.unique_id)
return True
-class PixivWebIllust(PixivWebBase):
- def __init__(self, userdata: Any, arg: str) -> None:
- super().__init__(userdata)
- try:
- self.id: int = int(arg)
- except ValueError:
- self.errored = True
-
- def request_page(self, num: int) -> bool:
- post = self.request_illust(self.id)
- if not post:
- self.errored = True
- return False
- self.pages[num] = [post.unique_id]
- self.completed = True
- return True
-
class PixivWebUser(PixivWebBase):
def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
@@ -285,15 +266,32 @@ class PixivWebFollowing(PixivWebBase):
self.pages[num].append(self.create_user(user).unique_id)
return True
+class PixivWebIllust(PixivWebBase):
+ def __init__(self, userdata: Any, arg: str) -> None:
+ super().__init__(userdata)
+ try:
+ self.id: int = int(arg)
+ except ValueError:
+ self.errored = True
+
+ def request_page(self, num: int) -> bool:
+ post = self.request_illust(self.id)
+ if not post:
+ self.errored = True
+ return False
+ self.pages[num] = [post.unique_id]
+ self.completed = True
+ return True
+
class PixivWebModule(Module):
def __init__(self, sessid: str, user_id: str) -> None:
super().__init__()
self.parser: QueryParser = QueryParser(self, 'search')
self.parser.add_command('search', PixivWebSearch)
- self.parser.add_command('illust', PixivWebIllust)
self.parser.add_command('user', PixivWebUser)
self.parser.add_command('bookmarks', PixivWebBookmarks)
self.parser.add_command('following', PixivWebFollowing)
+ self.parser.add_command('illust', PixivWebIllust)
self.headers['User-Agent'] = USER_AGENT
self.headers['Accept-Encoding'] = 'gzip, deflate, br, zstd'
self.headers['Accept-Language'] = 'en-US,en;q=0.5'
diff --git a/src/portal/py/modules/twitter.py b/src/portal/py/modules/twitter.py
index 2f45ec1..93286c3 100644
--- a/src/portal/py/modules/twitter.py
+++ b/src/portal/py/modules/twitter.py
@@ -3,15 +3,15 @@ import http.cookiejar
from typing import Optional, Any, Iterator
from datetime import timezone
from base import USER_AGENT, Search, Module
-from post import Url, PostType, DateType, PostRef, Post, User, Media, Image, Video
+from post import Url, PostType, Date, DateType, DateMeta, PostRef, Post, User, Media, Image, Video
from query_parser import QueryParser
from modules.common import get_current_utc_time
# GraphQL API
-from snscrape.modules import twitter
from snscrape.base import ScraperException
+from snscrape.modules import twitter
from snscrape.modules.twitter import (Tweet, TweetRef, Tombstone, UserRef,
- TwitterSearchScraper, TwitterProfileScraper, TwitterUserScraper, TwitterTweetScraper)
+ TwitterSearchScraper, TwitterSearchScraperMode, TwitterProfileScraper, TwitterUserScraper, TwitterTweetScraper)
class TwitterScrapeBase(Search):
# Posts per emulated page because snscrape returns an iterator.
@@ -20,7 +20,7 @@ class TwitterScrapeBase(Search):
# Don't allow requests for a page more than this amount past the current page.
REACH_LIMIT = 4
- def __init__(self, userdata: Any):
+ def __init__(self, userdata: Any) -> None:
super().__init__()
self.module: TwitterScrapeModule = userdata
self.iterator: Iterator[Tweet | TweetRef | Tombstone]
@@ -28,38 +28,39 @@ class TwitterScrapeBase(Search):
def create_media_url(self, url: str) -> Url:
format = url.find('format=')
+ ext = None
if format >= 0:
- ext = url[format + 7:format + 10]
- else:
- question = url.rfind('?')
- if question >= 0:
- url = url[0:question]
- ext = url.split('.')[-1]
+ ext = url[format + 7:]
+ amp = ext.find('&')
+ if amp >= 0:
+ ext = ext[:amp]
return Url(url, ext)
def parse_media(self, data: list[twitter.Medium]) -> dict[str, Media]:
media = {}
- for i, m in enumerate(data):
+ for index, m in enumerate(data):
if isinstance(m, twitter.Photo):
- media[str(i)] = Image(url=self.create_media_url(m.fullUrl), thumbnail_url=self.create_media_url(m.previewUrl))
+ media[str(index)] = Image(url=self.create_media_url(m.fullUrl), thumbnail_url=self.create_media_url(m.previewUrl))
elif isinstance(m, twitter.Video) or isinstance(m, twitter.Gif):
videos = sorted(m.variants, key=lambda x: x.bitrate if x.bitrate else 0, reverse=True)
- media[str(i)] = Video(url=self.create_media_url(videos[0].url), thumbnail_url=self.create_media_url(m.thumbnailUrl))
+ media[str(index)] = Video(url=self.create_media_url(videos[0].url), thumbnail_url=self.create_media_url(m.thumbnailUrl))
return media
def create_post(self, tweet: Tweet | TweetRef | Tombstone) -> Post:
kwargs = {}
+ kwargs['module'] = 'twitter_scrape'
kwargs['unique_id'] = f'twitter:t:{tweet.id}'
if isinstance(tweet, TweetRef) or isinstance(tweet, Tombstone):
kwargs['type'] = PostType.TOMBSTONE
return Post(**kwargs)
kwargs['raw_responses'] = tweet.rawResponses
kwargs['url'] = tweet.url
- # @TODO: Replace correct?
- kwargs['dates'] = {
- DateType.CREATED: tweet.date.replace(tzinfo=timezone.utc).timestamp(),
- DateType.RETRIEVED: get_current_utc_time().timestamp()
- }
+ create_date = tweet.date.replace(tzinfo=timezone.utc).timestamp()
+ current_date = get_current_utc_time().timestamp()
+ kwargs['dates'] = [
+ Date(DateType.CREATED, (create_date, create_date), DateMeta.NONE),
+ Date(DateType.RETRIEVED, (current_date, current_date), DateMeta.NONE)
+ ]
author = User(unique_id=f'twitter:u:{tweet.user.id}')
if not isinstance(tweet.user, UserRef):
author.username = tweet.user.username
@@ -116,40 +117,40 @@ class TwitterScrapeBase(Search):
if num - self.page >= self.REACH_LIMIT:
return False
request_satisfied = False
- for i in range(self.page, num + 1):
+ for index in range(self.page, num + 1):
if self.completed:
break
- if i in self.pages:
+ if index in self.pages:
continue
- self.pages[i] = []
- if not self.step_iterator_for_page(i):
+ self.pages[index] = []
+ if not self.step_iterator_for_page(index):
break
- self.page = i
- if i == num:
+ self.page = index
+ if index == num:
request_satisfied = True
return request_satisfied
class TwitterScrapeSearch(TwitterScrapeBase):
- def __init__(self, userdata: Any, arg: str):
+ def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
- self.iterator = TwitterSearchScraper(arg, top=True, cookies=self.module.cookies).get_items()
+ self.iterator = TwitterSearchScraper(arg, mode=TwitterSearchScraperMode.TOP, cookies=self.module.cookies).get_items()
class TwitterScrapeUser(TwitterScrapeBase):
- def __init__(self, userdata: Any, arg: str):
+ def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
if arg.startswith('@'):
arg = arg[1:]
self.iterator = TwitterUserScraper(arg, cookies=self.module.cookies).get_items()
class TwitterScrapeProfile(TwitterScrapeBase):
- def __init__(self, userdata: Any, arg: str):
+ def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
if arg.startswith('@'):
arg = arg[1:]
self.iterator = TwitterProfileScraper(arg, cookies=self.module.cookies).get_items()
class TwitterScrapeTweet(TwitterScrapeBase):
- def __init__(self, userdata: Any, arg: str):
+ def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
if arg.startswith('https://'):
arg = arg[arg.rfind('/') + 1:]
@@ -159,7 +160,7 @@ class TwitterScrapeTweet(TwitterScrapeBase):
self.iterator = TwitterTweetScraper(arg, cookies=self.module.cookies).get_items()
class TwitterScrapeModule(Module):
- def __init__(self, cookies_path: str):
+ def __init__(self, cookies_path: str) -> None:
super().__init__()
self.parser: QueryParser = QueryParser(self, 'search')
self.parser.add_command('search', TwitterScrapeSearch)
diff --git a/src/portal/py/post.py b/src/portal/py/post.py
index 1a285c2..f173d0e 100644
--- a/src/portal/py/post.py
+++ b/src/portal/py/post.py
@@ -4,13 +4,6 @@ from typing import Optional, Any
from enum import IntEnum, IntFlag
from json import JSONEncoder
-'''
-List of Things:
- - Social media posts
- - Albums
- - Loose files
-'''
-
class PostType(IntEnum):
UNKNOWN = 0
POST = 1
@@ -54,27 +47,26 @@ def df(c: Any) -> Any:
return field(default_factory=lambda: c)
def default_url_ext_guess(url: str) -> str:
- # @TODO: List of known extensions.
+ slash = url.find('/')
+ if slash < 0:
+ return ''
+ url = url[slash:]
question = url.rfind('?')
if question >= 0:
- guess = url[0:question].rsplit('.')[-1]
- else:
- guess = url.rsplit('.')[-1]
- return guess
+ url = url[0:question]
+ s = url.split('.')
+ if len(s) > 1:
+ return s[-1]
+ return ''
@dataclasses.dataclass
class Url():
url: str
ext: str
-
def __init__(self, url: str, ext: str = None) -> None:
self.url = url
if not ext:
- ext_guess = default_url_ext_guess(url)
- if ext_guess:
- ext = ext_guess
- else:
- ext = 'unknown'
+ ext = default_url_ext_guess(url)
self.ext = ext
@dataclasses.dataclass
@@ -158,6 +150,7 @@ class PostRef():
'''
V4:
- Improved date.
+ - Track origin module.
- Ends when all current modules are ported and/or completed.
V5:
- Formatted body/text.
@@ -165,6 +158,7 @@ V5:
@dataclasses.dataclass
class Post():
version: int = 4
+ module: str = ''
type: PostType = PostType.UNKNOWN
unique_id: str = ''
raw_responses: dict[str, str] = df({})
diff --git a/src/portal/py/query_parser.py b/src/portal/py/query_parser.py
index cd6c1e6..bf276ce 100644
--- a/src/portal/py/query_parser.py
+++ b/src/portal/py/query_parser.py
@@ -20,7 +20,7 @@ class QueryParser():
sp = query.split(' ')
- for i, word in enumerate(sp):
+ for index, word in enumerate(sp):
col = word.find(':')
if col == -1:
continue
@@ -34,7 +34,7 @@ class QueryParser():
else:
arg = None
search = self.commands[command]
- if i == 0:
+ if index == 0:
if len(sp) > 1:
query = query.replace(word + ' ', '', 1)
else:
diff --git a/src/portal/py/test.py b/src/portal/py/test.py
index 0ca89a4..e862bf0 100644
--- a/src/portal/py/test.py
+++ b/src/portal/py/test.py
@@ -1,2 +1,4 @@
#import tests.test
-import tests.archive_query
+import tests.touch_all
+#import tests.rewrite_post
+#import tests.archive_query
diff --git a/src/portal/py/tests/archive_query.py b/src/portal/py/tests/archive_query.py
index 4cc1e1e..3de4329 100644
--- a/src/portal/py/tests/archive_query.py
+++ b/src/portal/py/tests/archive_query.py
@@ -15,8 +15,8 @@ mode = 'pixiv_web'
#mode = 'fanbox'
#mode = 'patreon'
#mode = 'instagram'
-#cmd = 'user'
-cmd = 'bookmarks'
+cmd = 'user'
+#cmd = 'bookmarks'
#cmd = 'search'
module = ALL_MODULES[mode][0]
if not module.init():
@@ -180,8 +180,8 @@ class QueryDownloadThread(threading.Thread):
with queue_cond:
queue_cond.notify()
-#RUNTIME_PATH = "./run"
-RUNTIME_PATH = "/mnt/store/files/tmp/run"
+#RUNTIME_PATH = './run'
+RUNTIME_PATH = '/mnt/store/files/tmp/run'
LOG_FILE = f'{RUNTIME_PATH}/archive4.log'
ARG_FILE = f'{RUNTIME_PATH}/completed_args4.log'
@@ -216,10 +216,7 @@ def download_args():
os.mkdir(output_dir)
os.mkdir(f'{output_dir}/raw_responses')
- #args = module.search(f'following:{22781328}')
- # @TODO:
args = []
- # twitter test: butcha_u, sep__rina, nagayori000, sattinittas,
threads = []
start = True
diff --git a/src/portal/py/tests/old_twitter_api.py b/src/portal/py/tests/old_twitter_api.py
index 40d0346..4427dfd 100644
--- a/src/portal/py/tests/old_twitter_api.py
+++ b/src/portal/py/tests/old_twitter_api.py
@@ -27,14 +27,14 @@ log = Logger(LOG_FILE)
compat_thread = QueryDownloadThread(log, None, None, sys.argv[1])
def compat_media_download(obj, post, output_base):
- for i, key in enumerate(post.media.keys()):
+ for index, key in enumerate(post.media.keys()):
media = post.media[key]
url = media.url
media_path = f'{output_base}_media{key}.{url.ext}'
if os.path.isfile(media_path):
obj.log.write(f'Skipping media {media_path}.')
continue
- origin_path = f'{origin_base}/{post.author.unique_id.replace(':', '_')}/{post.unique_id.replace(':', '_')}_media{i}.{url.ext}'
+ origin_path = f'{origin_base}/{post.author.unique_id.replace(':', '_')}/{post.unique_id.replace(':', '_')}_media{index}.{url.ext}'
if os.path.isfile(origin_path):
shutil.copyfile(origin_path, media_path)
#post.media[key].url.url = url.url.replace('name=orig', 'name=large')
diff --git a/src/portal/py/tests/rewrite_post.py b/src/portal/py/tests/rewrite_post.py
index 3bb152b..f00cfc7 100644
--- a/src/portal/py/tests/rewrite_post.py
+++ b/src/portal/py/tests/rewrite_post.py
@@ -2,20 +2,24 @@ import sys
import os
import gzip
import json
-import hashlib
sys.path.append('../py')
-import config
-#from post import PostEncoder, DateType
-#from modules.pixiv_web import PixivWebBase, PixivWebModule
-#from modules.twitter import TwitterScrapeBase, TwitterScrapeModule
-from logger import Logger
+from modules import ALL_MODULES
+from post import PostEncoder
+from modules.pixiv_app import PixivAppBase
-#module = PixivWebModule(config.PIXIV_SESSID)
-#search = PixivWebBase(module)
+module = ALL_MODULES['pixiv_app'][0]
+module.init()
+search = PixivAppBase(module)
-log = Logger('deleted_posts.log')
+target = sys.argv[1]
+with gzip.open(target, 'rb') as f:
+ obj = json.loads(f.read())
+ mtime = os.path.getmtime(target)
+ print(json.dumps(search.remake_post(obj['raw_responses'], mtime), cls=PostEncoder))
'''
+log = Logger('deleted_posts.log')
+
def rewrite_post(path):
print(f'Rewriting post @ {path}')
with gzip.open(path, 'rb') as f:
@@ -27,7 +31,6 @@ def rewrite_post(path):
#print(json.dumps(post, indent=4, cls=PostEncoder))
with gzip.open(path, 'wb') as f:
f.write(json.dumps(post, cls=PostEncoder).encode('utf-8'))
-'''
def dump_raw_response(path, target):
print(f'Extracting raw_response from {path}')
@@ -54,3 +57,4 @@ for user in os.listdir(target):
if file.split('.')[-1] == 'gz':
dump_raw_response(f'{target}/{user}/{file}', raw_responses)
#rewrite_post(f'{target}/{user}/{file}')
+'''
diff --git a/src/portal/py/tests/test.py b/src/portal/py/tests/test.py
index ef166d9..d3e5640 100644
--- a/src/portal/py/tests/test.py
+++ b/src/portal/py/tests/test.py
@@ -1,13 +1,4 @@
-import sys
-import json
-import httpx
-import os.path
-from base import Method
-from typing import Optional, Any
-from post import DateType, DateMeta, Date, PostEncoder, Post, MediaType
-from modules import ALL_MODULES
-from twitter.scraper import Scraper
-
+#from twitter.scraper import Scraper
#scraper = Scraper(cookies = {
# 'ct0': '',
# 'auth_token': ''
@@ -15,101 +6,3 @@ from twitter.scraper import Scraper
#
#media = scraper.media([1557919548904419328], limit=10)
#print(media)
-
-#output = '/mnt/store/files/ext/patreon_tmp/FPSBlyck'
-#
-#module = ALL_MODULES['patreon'][0]
-#module.init()
-#search = module.search('FPSBlyck')
-#page_num = 0
-#mark = False
-#while not mark:
-# page = search.get_page(page_num)
-# if not page:
-# break
-# page_num += 1
-# for unique_id in page:
-# if unique_id == 'patreon:p:29707872':
-# print('hit mark')
-# mark = True
-# post = module.get_item(unique_id)
-# print(json.dumps(post, indent=4, cls=PostEncoder))
-# for key, m in post.media.items():
-# if m.type == MediaType.FILE:
-# path = f'{output}/{post.unique_id.replace(':', '_')}_{m.name}'
-# if not os.path.isfile(path):
-# dl = module.get_download(post.unique_id, key)
-# retries = 3
-# r: Optional[httpx.Response] = None
-# while retries >= 0:
-# try:
-# r = httpx.get(dl['urls'][0].url, headers=dl['headers'], cookies=dl['cookies'], follow_redirects=True)
-# except:
-# retries -= 1
-# else:
-# break
-# if r:
-# with open(path, 'wb+') as f:
-# f.write(r.content)
-# else:
-# print('Download failed')
-# else:
-# print('Skipping file')
-
-
-#module = ALL_MODULES['pixiv_web'][0]
-#module.init()
-#search = module.search('illust:91352621')
-#page = search.get_page(0)
-#for unique_id in page:
-# post = module.get_item(unique_id)
-# print(json.dumps(post, indent=4, cls=PostEncoder))
-
-module = ALL_MODULES['instagram'][0]
-module.init()
-search = module.search('plottttwistttttt')
-page = search.get_page(0)
-for unique_id in page:
- post = module.get_item(unique_id)
- print(json.dumps(post, indent=4, cls=PostEncoder))
-
-#d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.EDITED_AT_UNKNOWN_TIME)
-#d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.NONE)
-#print(json.dumps(d, indent=4, cls=PostEncoder))
-#sys.exit(1)
-
-#module = ALL_MODULES['fanbox'][0]
-#module = ALL_MODULES['pixiv_web'][0]
-#module = ALL_MODULES['youtube'][0]
-#module = ALL_MODULES['twitter'][0]
-#module.init()
-
-#search = module.search('opium_00pium')
-#search = module.search('search:ddd')
-#page = search.get_page(0)
-#for unique_id in page:
-# post = module.get_item(unique_id)
-# print(json.dumps(post, indent=4, cls=PostEncoder))
-# break
-
-#post = module.get_item(page[0])
-#for key in post.media.keys():
-# download_params = module.get_download(post.unique_id, key)
-# for url in download_params['urls']:
-# #response = module.do_request(Method.GET, url.url)
-# r = httpx.get(url.url, headers=download_params['headers'])
-# with open(f'test_{key}.{url.ext}', 'wb+') as f:
-# f.write(r.content)
-
-#print(json.dumps(page, indent=4, cls=PostEncoder))
-#for i in range(0, len(post.media)):
-# download_params = module.get_download(post.unique_id, i)
-# r = httpx.get(download_params['url'], headers=download_params['headers'])
-# if r.status_code != 200:
-# print('error{}'.format(r.status_code))
-# else:
-# with open('test{}.png'.format(i), 'wb+') as f:
-# f.write(r.content)
-
-#print(json.dumps(page, indent=4, cls=PostEncoder))
-#print(page)
diff --git a/src/portal/py/tests/touch_all.py b/src/portal/py/tests/touch_all.py
new file mode 100644
index 0000000..6d6bfa1
--- /dev/null
+++ b/src/portal/py/tests/touch_all.py
@@ -0,0 +1,79 @@
+import json
+from typing import Optional, Any
+from post import PostEncoder
+from modules import ALL_MODULES
+
+'''
+post = module.get_item(page[0])
+for key in post.media.keys():
+ download_params = module.get_download(post.unique_id, key)
+ for url in download_params['urls']:
+ #response = module.do_request(Method.GET, url.url)
+ r = httpx.get(url.url, headers=download_params['headers'])
+ if r.status_code != 200:
+ print('error{}'.format(r.status_code))
+ else:
+ with open(f'test_{key}.{url.ext}', 'wb+') as f:
+ f.write(r.content)
+'''
+
+def print_page(module: Any, page: Optional[list[str]]) -> None:
+ if not page:
+ print('Page is None')
+ return
+ count = 0
+ for unique_id in page:
+ post = module.get_item(unique_id)
+ print(json.dumps(post, indent=4, cls=PostEncoder))
+ count = count + 1
+ if count == 2:
+ break
+
+''' Blocked on certain endpoints, snscrape is outdated.
+module = ALL_MODULES['twitter'][0]
+module.init()
+
+search = module.search('search:#小関麗奈誕生祭2024')
+print_page(module, search.get_page(0))
+search = module.search('user:xuuikie')
+print_page(module, search.get_page(0))
+search = module.search('profile:@barackobama')
+print_page(module, search.get_page(0))
+search = module.search('tweet:1905966372288434680')
+print_page(module, search.get_page(0))
+'''
+
+'''
+module = ALL_MODULES['pixiv_app'][0]
+module.init()
+search = module.search('search:サムス・アラン')
+print_page(module, search.get_page(0))
+search = module.search('user:757584')
+print_page(module, search.get_page(0))
+search = module.search('bookmarks:2034628')
+print_page(module, search.get_page(0))
+search = module.search('following:22781328')
+print_page(module, search.get_page(0))
+search = module.search('illust:91352621')
+print_page(module, search.get_page(0))
+search = module.search('illust:26497574') # Deleted.
+print_page(module, search.get_page(0))
+'''
+
+''' No on-demand loading (Will load entire user before returning).
+module = ALL_MODULES['pixiv_web'][0]
+module.init()
+
+search = module.search('search:サムス・アラン')
+print_page(module, search.get_page(0))
+search = module.search('user:757584')
+print_page(module, search.get_page(0))
+search = module.search('bookmarks:2034628')
+print_page(module, search.get_page(0))
+search = module.search('following:22781328')
+print_page(module, search.get_page(0))
+search = module.search('illust:91352621')
+print_page(module, search.get_page(0))
+search = module.search('illust:100455669') # Deleted.
+print_page(module, search.get_page(0))
+'''
diff --git a/src/portal/py/tlds.py b/src/portal/py/tlds.py
new file mode 100644
index 0000000..d0e4cdd
--- /dev/null
+++ b/src/portal/py/tlds.py
@@ -0,0 +1,1445 @@
+# https://data.iana.org/TLD/tlds-alpha-by-domain.txt
+# Version 2025051100, Last Updated Sun May 11 07:07:01 2025 UTC
+tlds = [
+ 'aaa',
+ 'aarp',
+ 'abb',
+ 'abbott',
+ 'abbvie',
+ 'abc',
+ 'able',
+ 'abogado',
+ 'abudhabi',
+ 'ac',
+ 'academy',
+ 'accenture',
+ 'accountant',
+ 'accountants',
+ 'aco',
+ 'actor',
+ 'ad',
+ 'ads',
+ 'adult',
+ 'ae',
+ 'aeg',
+ 'aero',
+ 'aetna',
+ 'af',
+ 'afl',
+ 'africa',
+ 'ag',
+ 'agakhan',
+ 'agency',
+ 'ai',
+ 'aig',
+ 'airbus',
+ 'airforce',
+ 'airtel',
+ 'akdn',
+ 'al',
+ 'alibaba',
+ 'alipay',
+ 'allfinanz',
+ 'allstate',
+ 'ally',
+ 'alsace',
+ 'alstom',
+ 'am',
+ 'amazon',
+ 'americanexpress',
+ 'americanfamily',
+ 'amex',
+ 'amfam',
+ 'amica',
+ 'amsterdam',
+ 'analytics',
+ 'android',
+ 'anquan',
+ 'anz',
+ 'ao',
+ 'aol',
+ 'apartments',
+ 'app',
+ 'apple',
+ 'aq',
+ 'aquarelle',
+ 'ar',
+ 'arab',
+ 'aramco',
+ 'archi',
+ 'army',
+ 'arpa',
+ 'art',
+ 'arte',
+ 'as',
+ 'asda',
+ 'asia',
+ 'associates',
+ 'at',
+ 'athleta',
+ 'attorney',
+ 'au',
+ 'auction',
+ 'audi',
+ 'audible',
+ 'audio',
+ 'auspost',
+ 'author',
+ 'auto',
+ 'autos',
+ 'aw',
+ 'aws',
+ 'ax',
+ 'axa',
+ 'az',
+ 'azure',
+ 'ba',
+ 'baby',
+ 'baidu',
+ 'banamex',
+ 'band',
+ 'bank',
+ 'bar',
+ 'barcelona',
+ 'barclaycard',
+ 'barclays',
+ 'barefoot',
+ 'bargains',
+ 'baseball',
+ 'basketball',
+ 'bauhaus',
+ 'bayern',
+ 'bb',
+ 'bbc',
+ 'bbt',
+ 'bbva',
+ 'bcg',
+ 'bcn',
+ 'bd',
+ 'be',
+ 'beats',
+ 'beauty',
+ 'beer',
+ 'berlin',
+ 'best',
+ 'bestbuy',
+ 'bet',
+ 'bf',
+ 'bg',
+ 'bh',
+ 'bharti',
+ 'bi',
+ 'bible',
+ 'bid',
+ 'bike',
+ 'bing',
+ 'bingo',
+ 'bio',
+ 'biz',
+ 'bj',
+ 'black',
+ 'blackfriday',
+ 'blockbuster',
+ 'blog',
+ 'bloomberg',
+ 'blue',
+ 'bm',
+ 'bms',
+ 'bmw',
+ 'bn',
+ 'bnpparibas',
+ 'bo',
+ 'boats',
+ 'boehringer',
+ 'bofa',
+ 'bom',
+ 'bond',
+ 'boo',
+ 'book',
+ 'booking',
+ 'bosch',
+ 'bostik',
+ 'boston',
+ 'bot',
+ 'boutique',
+ 'box',
+ 'br',
+ 'bradesco',
+ 'bridgestone',
+ 'broadway',
+ 'broker',
+ 'brother',
+ 'brussels',
+ 'bs',
+ 'bt',
+ 'build',
+ 'builders',
+ 'business',
+ 'buy',
+ 'buzz',
+ 'bv',
+ 'bw',
+ 'by',
+ 'bz',
+ 'bzh',
+ 'ca',
+ 'cab',
+ 'cafe',
+ 'cal',
+ 'call',
+ 'calvinklein',
+ 'cam',
+ 'camera',
+ 'camp',
+ 'canon',
+ 'capetown',
+ 'capital',
+ 'capitalone',
+ 'car',
+ 'caravan',
+ 'cards',
+ 'care',
+ 'career',
+ 'careers',
+ 'cars',
+ 'casa',
+ 'case',
+ 'cash',
+ 'casino',
+ 'cat',
+ 'catering',
+ 'catholic',
+ 'cba',
+ 'cbn',
+ 'cbre',
+ 'cc',
+ 'cd',
+ 'center',
+ 'ceo',
+ 'cern',
+ 'cf',
+ 'cfa',
+ 'cfd',
+ 'cg',
+ 'ch',
+ 'chanel',
+ 'channel',
+ 'charity',
+ 'chase',
+ 'chat',
+ 'cheap',
+ 'chintai',
+ 'christmas',
+ 'chrome',
+ 'church',
+ 'ci',
+ 'cipriani',
+ 'circle',
+ 'cisco',
+ 'citadel',
+ 'citi',
+ 'citic',
+ 'city',
+ 'ck',
+ 'cl',
+ 'claims',
+ 'cleaning',
+ 'click',
+ 'clinic',
+ 'clinique',
+ 'clothing',
+ 'cloud',
+ 'club',
+ 'clubmed',
+ 'cm',
+ 'cn',
+ 'co',
+ 'coach',
+ 'codes',
+ 'coffee',
+ 'college',
+ 'cologne',
+ 'com',
+ 'commbank',
+ 'community',
+ 'company',
+ 'compare',
+ 'computer',
+ 'comsec',
+ 'condos',
+ 'construction',
+ 'consulting',
+ 'contact',
+ 'contractors',
+ 'cooking',
+ 'cool',
+ 'coop',
+ 'corsica',
+ 'country',
+ 'coupon',
+ 'coupons',
+ 'courses',
+ 'cpa',
+ 'cr',
+ 'credit',
+ 'creditcard',
+ 'creditunion',
+ 'cricket',
+ 'crown',
+ 'crs',
+ 'cruise',
+ 'cruises',
+ 'cu',
+ 'cuisinella',
+ 'cv',
+ 'cw',
+ 'cx',
+ 'cy',
+ 'cymru',
+ 'cyou',
+ 'cz',
+ 'dad',
+ 'dance',
+ 'data',
+ 'date',
+ 'dating',
+ 'datsun',
+ 'day',
+ 'dclk',
+ 'dds',
+ 'de',
+ 'deal',
+ 'dealer',
+ 'deals',
+ 'degree',
+ 'delivery',
+ 'dell',
+ 'deloitte',
+ 'delta',
+ 'democrat',
+ 'dental',
+ 'dentist',
+ 'desi',
+ 'design',
+ 'dev',
+ 'dhl',
+ 'diamonds',
+ 'diet',
+ 'digital',
+ 'direct',
+ 'directory',
+ 'discount',
+ 'discover',
+ 'dish',
+ 'diy',
+ 'dj',
+ 'dk',
+ 'dm',
+ 'dnp',
+ 'do',
+ 'docs',
+ 'doctor',
+ 'dog',
+ 'domains',
+ 'dot',
+ 'download',
+ 'drive',
+ 'dtv',
+ 'dubai',
+ 'dunlop',
+ 'dupont',
+ 'durban',
+ 'dvag',
+ 'dvr',
+ 'dz',
+ 'earth',
+ 'eat',
+ 'ec',
+ 'eco',
+ 'edeka',
+ 'edu',
+ 'education',
+ 'ee',
+ 'eg',
+ 'email',
+ 'emerck',
+ 'energy',
+ 'engineer',
+ 'engineering',
+ 'enterprises',
+ 'epson',
+ 'equipment',
+ 'er',
+ 'ericsson',
+ 'erni',
+ 'es',
+ 'esq',
+ 'estate',
+ 'et',
+ 'eu',
+ 'eurovision',
+ 'eus',
+ 'events',
+ 'exchange',
+ 'expert',
+ 'exposed',
+ 'express',
+ 'extraspace',
+ 'fage',
+ 'fail',
+ 'fairwinds',
+ 'faith',
+ 'family',
+ 'fan',
+ 'fans',
+ 'farm',
+ 'farmers',
+ 'fashion',
+ 'fast',
+ 'fedex',
+ 'feedback',
+ 'ferrari',
+ 'ferrero',
+ 'fi',
+ 'fidelity',
+ 'fido',
+ 'film',
+ 'final',
+ 'finance',
+ 'financial',
+ 'fire',
+ 'firestone',
+ 'firmdale',
+ 'fish',
+ 'fishing',
+ 'fit',
+ 'fitness',
+ 'fj',
+ 'fk',
+ 'flickr',
+ 'flights',
+ 'flir',
+ 'florist',
+ 'flowers',
+ 'fly',
+ 'fm',
+ 'fo',
+ 'foo',
+ 'food',
+ 'football',
+ 'ford',
+ 'forex',
+ 'forsale',
+ 'forum',
+ 'foundation',
+ 'fox',
+ 'fr',
+ 'free',
+ 'fresenius',
+ 'frl',
+ 'frogans',
+ 'frontier',
+ 'ftr',
+ 'fujitsu',
+ 'fun',
+ 'fund',
+ 'furniture',
+ 'futbol',
+ 'fyi',
+ 'ga',
+ 'gal',
+ 'gallery',
+ 'gallo',
+ 'gallup',
+ 'game',
+ 'games',
+ 'gap',
+ 'garden',
+ 'gay',
+ 'gb',
+ 'gbiz',
+ 'gd',
+ 'gdn',
+ 'ge',
+ 'gea',
+ 'gent',
+ 'genting',
+ 'george',
+ 'gf',
+ 'gg',
+ 'ggee',
+ 'gh',
+ 'gi',
+ 'gift',
+ 'gifts',
+ 'gives',
+ 'giving',
+ 'gl',
+ 'glass',
+ 'gle',
+ 'global',
+ 'globo',
+ 'gm',
+ 'gmail',
+ 'gmbh',
+ 'gmo',
+ 'gmx',
+ 'gn',
+ 'godaddy',
+ 'gold',
+ 'goldpoint',
+ 'golf',
+ 'goo',
+ 'goodyear',
+ 'goog',
+ 'google',
+ 'gop',
+ 'got',
+ 'gov',
+ 'gp',
+ 'gq',
+ 'gr',
+ 'grainger',
+ 'graphics',
+ 'gratis',
+ 'green',
+ 'gripe',
+ 'grocery',
+ 'group',
+ 'gs',
+ 'gt',
+ 'gu',
+ 'gucci',
+ 'guge',
+ 'guide',
+ 'guitars',
+ 'guru',
+ 'gw',
+ 'gy',
+ 'hair',
+ 'hamburg',
+ 'hangout',
+ 'haus',
+ 'hbo',
+ 'hdfc',
+ 'hdfcbank',
+ 'health',
+ 'healthcare',
+ 'help',
+ 'helsinki',
+ 'here',
+ 'hermes',
+ 'hiphop',
+ 'hisamitsu',
+ 'hitachi',
+ 'hiv',
+ 'hk',
+ 'hkt',
+ 'hm',
+ 'hn',
+ 'hockey',
+ 'holdings',
+ 'holiday',
+ 'homedepot',
+ 'homegoods',
+ 'homes',
+ 'homesense',
+ 'honda',
+ 'horse',
+ 'hospital',
+ 'host',
+ 'hosting',
+ 'hot',
+ 'hotels',
+ 'hotmail',
+ 'house',
+ 'how',
+ 'hr',
+ 'hsbc',
+ 'ht',
+ 'hu',
+ 'hughes',
+ 'hyatt',
+ 'hyundai',
+ 'ibm',
+ 'icbc',
+ 'ice',
+ 'icu',
+ 'id',
+ 'ie',
+ 'ieee',
+ 'ifm',
+ 'ikano',
+ 'il',
+ 'im',
+ 'imamat',
+ 'imdb',
+ 'immo',
+ 'immobilien',
+ 'in',
+ 'inc',
+ 'industries',
+ 'infiniti',
+ 'info',
+ 'ing',
+ 'ink',
+ 'institute',
+ 'insurance',
+ 'insure',
+ 'int',
+ 'international',
+ 'intuit',
+ 'investments',
+ 'io',
+ 'ipiranga',
+ 'iq',
+ 'ir',
+ 'irish',
+ 'is',
+ 'ismaili',
+ 'ist',
+ 'istanbul',
+ 'it',
+ 'itau',
+ 'itv',
+ 'jaguar',
+ 'java',
+ 'jcb',
+ 'je',
+ 'jeep',
+ 'jetzt',
+ 'jewelry',
+ 'jio',
+ 'jll',
+ 'jm',
+ 'jmp',
+ 'jnj',
+ 'jo',
+ 'jobs',
+ 'joburg',
+ 'jot',
+ 'joy',
+ 'jp',
+ 'jpmorgan',
+ 'jprs',
+ 'juegos',
+ 'juniper',
+ 'kaufen',
+ 'kddi',
+ 'ke',
+ 'kerryhotels',
+ 'kerryproperties',
+ 'kfh',
+ 'kg',
+ 'kh',
+ 'ki',
+ 'kia',
+ 'kids',
+ 'kim',
+ 'kindle',
+ 'kitchen',
+ 'kiwi',
+ 'km',
+ 'kn',
+ 'koeln',
+ 'komatsu',
+ 'kosher',
+ 'kp',
+ 'kpmg',
+ 'kpn',
+ 'kr',
+ 'krd',
+ 'kred',
+ 'kuokgroup',
+ 'kw',
+ 'ky',
+ 'kyoto',
+ 'kz',
+ 'la',
+ 'lacaixa',
+ 'lamborghini',
+ 'lamer',
+ 'land',
+ 'landrover',
+ 'lanxess',
+ 'lasalle',
+ 'lat',
+ 'latino',
+ 'latrobe',
+ 'law',
+ 'lawyer',
+ 'lb',
+ 'lc',
+ 'lds',
+ 'lease',
+ 'leclerc',
+ 'lefrak',
+ 'legal',
+ 'lego',
+ 'lexus',
+ 'lgbt',
+ 'li',
+ 'lidl',
+ 'life',
+ 'lifeinsurance',
+ 'lifestyle',
+ 'lighting',
+ 'like',
+ 'lilly',
+ 'limited',
+ 'limo',
+ 'lincoln',
+ 'link',
+ 'live',
+ 'living',
+ 'lk',
+ 'llc',
+ 'llp',
+ 'loan',
+ 'loans',
+ 'locker',
+ 'locus',
+ 'lol',
+ 'london',
+ 'lotte',
+ 'lotto',
+ 'love',
+ 'lpl',
+ 'lplfinancial',
+ 'lr',
+ 'ls',
+ 'lt',
+ 'ltd',
+ 'ltda',
+ 'lu',
+ 'lundbeck',
+ 'luxe',
+ 'luxury',
+ 'lv',
+ 'ly',
+ 'ma',
+ 'madrid',
+ 'maif',
+ 'maison',
+ 'makeup',
+ 'man',
+ 'management',
+ 'mango',
+ 'map',
+ 'market',
+ 'marketing',
+ 'markets',
+ 'marriott',
+ 'marshalls',
+ 'mattel',
+ 'mba',
+ 'mc',
+ 'mckinsey',
+ 'md',
+ 'me',
+ 'med',
+ 'media',
+ 'meet',
+ 'melbourne',
+ 'meme',
+ 'memorial',
+ 'men',
+ 'menu',
+ 'merckmsd',
+ 'mg',
+ 'mh',
+ 'miami',
+ 'microsoft',
+ 'mil',
+ 'mini',
+ 'mint',
+ 'mit',
+ 'mitsubishi',
+ 'mk',
+ 'ml',
+ 'mlb',
+ 'mls',
+ 'mm',
+ 'mma',
+ 'mn',
+ 'mo',
+ 'mobi',
+ 'mobile',
+ 'moda',
+ 'moe',
+ 'moi',
+ 'mom',
+ 'monash',
+ 'money',
+ 'monster',
+ 'mormon',
+ 'mortgage',
+ 'moscow',
+ 'moto',
+ 'motorcycles',
+ 'mov',
+ 'movie',
+ 'mp',
+ 'mq',
+ 'mr',
+ 'ms',
+ 'msd',
+ 'mt',
+ 'mtn',
+ 'mtr',
+ 'mu',
+ 'museum',
+ 'music',
+ 'mv',
+ 'mw',
+ 'mx',
+ 'my',
+ 'mz',
+ 'na',
+ 'nab',
+ 'nagoya',
+ 'name',
+ 'navy',
+ 'nba',
+ 'nc',
+ 'ne',
+ 'nec',
+ 'net',
+ 'netbank',
+ 'netflix',
+ 'network',
+ 'neustar',
+ 'new',
+ 'news',
+ 'next',
+ 'nextdirect',
+ 'nexus',
+ 'nf',
+ 'nfl',
+ 'ng',
+ 'ngo',
+ 'nhk',
+ 'ni',
+ 'nico',
+ 'nike',
+ 'nikon',
+ 'ninja',
+ 'nissan',
+ 'nissay',
+ 'nl',
+ 'no',
+ 'nokia',
+ 'norton',
+ 'now',
+ 'nowruz',
+ 'nowtv',
+ 'np',
+ 'nr',
+ 'nra',
+ 'nrw',
+ 'ntt',
+ 'nu',
+ 'nyc',
+ 'nz',
+ 'obi',
+ 'observer',
+ 'office',
+ 'okinawa',
+ 'olayan',
+ 'olayangroup',
+ 'ollo',
+ 'om',
+ 'omega',
+ 'one',
+ 'ong',
+ 'onl',
+ 'online',
+ 'ooo',
+ 'open',
+ 'oracle',
+ 'orange',
+ 'org',
+ 'organic',
+ 'origins',
+ 'osaka',
+ 'otsuka',
+ 'ott',
+ 'ovh',
+ 'pa',
+ 'page',
+ 'panasonic',
+ 'paris',
+ 'pars',
+ 'partners',
+ 'parts',
+ 'party',
+ 'pay',
+ 'pccw',
+ 'pe',
+ 'pet',
+ 'pf',
+ 'pfizer',
+ 'pg',
+ 'ph',
+ 'pharmacy',
+ 'phd',
+ 'philips',
+ 'phone',
+ 'photo',
+ 'photography',
+ 'photos',
+ 'physio',
+ 'pics',
+ 'pictet',
+ 'pictures',
+ 'pid',
+ 'pin',
+ 'ping',
+ 'pink',
+ 'pioneer',
+ 'pizza',
+ 'pk',
+ 'pl',
+ 'place',
+ 'play',
+ 'playstation',
+ 'plumbing',
+ 'plus',
+ 'pm',
+ 'pn',
+ 'pnc',
+ 'pohl',
+ 'poker',
+ 'politie',
+ 'porn',
+ 'post',
+ 'pr',
+ 'pramerica',
+ 'praxi',
+ 'press',
+ 'prime',
+ 'pro',
+ 'prod',
+ 'productions',
+ 'prof',
+ 'progressive',
+ 'promo',
+ 'properties',
+ 'property',
+ 'protection',
+ 'pru',
+ 'prudential',
+ 'ps',
+ 'pt',
+ 'pub',
+ 'pw',
+ 'pwc',
+ 'py',
+ 'qa',
+ 'qpon',
+ 'quebec',
+ 'quest',
+ 'racing',
+ 'radio',
+ 're',
+ 'read',
+ 'realestate',
+ 'realtor',
+ 'realty',
+ 'recipes',
+ 'red',
+ 'redstone',
+ 'redumbrella',
+ 'rehab',
+ 'reise',
+ 'reisen',
+ 'reit',
+ 'reliance',
+ 'ren',
+ 'rent',
+ 'rentals',
+ 'repair',
+ 'report',
+ 'republican',
+ 'rest',
+ 'restaurant',
+ 'review',
+ 'reviews',
+ 'rexroth',
+ 'rich',
+ 'richardli',
+ 'ricoh',
+ 'ril',
+ 'rio',
+ 'rip',
+ 'ro',
+ 'rocks',
+ 'rodeo',
+ 'rogers',
+ 'room',
+ 'rs',
+ 'rsvp',
+ 'ru',
+ 'rugby',
+ 'ruhr',
+ 'run',
+ 'rw',
+ 'rwe',
+ 'ryukyu',
+ 'sa',
+ 'saarland',
+ 'safe',
+ 'safety',
+ 'sakura',
+ 'sale',
+ 'salon',
+ 'samsclub',
+ 'samsung',
+ 'sandvik',
+ 'sandvikcoromant',
+ 'sanofi',
+ 'sap',
+ 'sarl',
+ 'sas',
+ 'save',
+ 'saxo',
+ 'sb',
+ 'sbi',
+ 'sbs',
+ 'sc',
+ 'scb',
+ 'schaeffler',
+ 'schmidt',
+ 'scholarships',
+ 'school',
+ 'schule',
+ 'schwarz',
+ 'science',
+ 'scot',
+ 'sd',
+ 'se',
+ 'search',
+ 'seat',
+ 'secure',
+ 'security',
+ 'seek',
+ 'select',
+ 'sener',
+ 'services',
+ 'seven',
+ 'sew',
+ 'sex',
+ 'sexy',
+ 'sfr',
+ 'sg',
+ 'sh',
+ 'shangrila',
+ 'sharp',
+ 'shell',
+ 'shia',
+ 'shiksha',
+ 'shoes',
+ 'shop',
+ 'shopping',
+ 'shouji',
+ 'show',
+ 'si',
+ 'silk',
+ 'sina',
+ 'singles',
+ 'site',
+ 'sj',
+ 'sk',
+ 'ski',
+ 'skin',
+ 'sky',
+ 'skype',
+ 'sl',
+ 'sling',
+ 'sm',
+ 'smart',
+ 'smile',
+ 'sn',
+ 'sncf',
+ 'so',
+ 'soccer',
+ 'social',
+ 'softbank',
+ 'software',
+ 'sohu',
+ 'solar',
+ 'solutions',
+ 'song',
+ 'sony',
+ 'soy',
+ 'spa',
+ 'space',
+ 'sport',
+ 'spot',
+ 'sr',
+ 'srl',
+ 'ss',
+ 'st',
+ 'stada',
+ 'staples',
+ 'star',
+ 'statebank',
+ 'statefarm',
+ 'stc',
+ 'stcgroup',
+ 'stockholm',
+ 'storage',
+ 'store',
+ 'stream',
+ 'studio',
+ 'study',
+ 'style',
+ 'su',
+ 'sucks',
+ 'supplies',
+ 'supply',
+ 'support',
+ 'surf',
+ 'surgery',
+ 'suzuki',
+ 'sv',
+ 'swatch',
+ 'swiss',
+ 'sx',
+ 'sy',
+ 'sydney',
+ 'systems',
+ 'sz',
+ 'tab',
+ 'taipei',
+ 'talk',
+ 'taobao',
+ 'target',
+ 'tatamotors',
+ 'tatar',
+ 'tattoo',
+ 'tax',
+ 'taxi',
+ 'tc',
+ 'tci',
+ 'td',
+ 'tdk',
+ 'team',
+ 'tech',
+ 'technology',
+ 'tel',
+ 'temasek',
+ 'tennis',
+ 'teva',
+ 'tf',
+ 'tg',
+ 'th',
+ 'thd',
+ 'theater',
+ 'theatre',
+ 'tiaa',
+ 'tickets',
+ 'tienda',
+ 'tips',
+ 'tires',
+ 'tirol',
+ 'tj',
+ 'tjmaxx',
+ 'tjx',
+ 'tk',
+ 'tkmaxx',
+ 'tl',
+ 'tm',
+ 'tmall',
+ 'tn',
+ 'to',
+ 'today',
+ 'tokyo',
+ 'tools',
+ 'top',
+ 'toray',
+ 'toshiba',
+ 'total',
+ 'tours',
+ 'town',
+ 'toyota',
+ 'toys',
+ 'tr',
+ 'trade',
+ 'trading',
+ 'training',
+ 'travel',
+ 'travelers',
+ 'travelersinsurance',
+ 'trust',
+ 'trv',
+ 'tt',
+ 'tube',
+ 'tui',
+ 'tunes',
+ 'tushu',
+ 'tv',
+ 'tvs',
+ 'tw',
+ 'tz',
+ 'ua',
+ 'ubank',
+ 'ubs',
+ 'ug',
+ 'uk',
+ 'unicom',
+ 'university',
+ 'uno',
+ 'uol',
+ 'ups',
+ 'us',
+ 'uy',
+ 'uz',
+ 'va',
+ 'vacations',
+ 'vana',
+ 'vanguard',
+ 'vc',
+ 've',
+ 'vegas',
+ 'ventures',
+ 'verisign',
+ 'versicherung',
+ 'vet',
+ 'vg',
+ 'vi',
+ 'viajes',
+ 'video',
+ 'vig',
+ 'viking',
+ 'villas',
+ 'vin',
+ 'vip',
+ 'virgin',
+ 'visa',
+ 'vision',
+ 'viva',
+ 'vivo',
+ 'vlaanderen',
+ 'vn',
+ 'vodka',
+ 'volvo',
+ 'vote',
+ 'voting',
+ 'voto',
+ 'voyage',
+ 'vu',
+ 'wales',
+ 'walmart',
+ 'walter',
+ 'wang',
+ 'wanggou',
+ 'watch',
+ 'watches',
+ 'weather',
+ 'weatherchannel',
+ 'webcam',
+ 'weber',
+ 'website',
+ 'wed',
+ 'wedding',
+ 'weibo',
+ 'weir',
+ 'wf',
+ 'whoswho',
+ 'wien',
+ 'wiki',
+ 'williamhill',
+ 'win',
+ 'windows',
+ 'wine',
+ 'winners',
+ 'wme',
+ 'wolterskluwer',
+ 'woodside',
+ 'work',
+ 'works',
+ 'world',
+ 'wow',
+ 'ws',
+ 'wtc',
+ 'wtf',
+ 'xbox',
+ 'xerox',
+ 'xihuan',
+ 'xin',
+ 'xn--11b4c3d',
+ 'xn--1ck2e1b',
+ 'xn--1qqw23a',
+ 'xn--2scrj9c',
+ 'xn--30rr7y',
+ 'xn--3bst00m',
+ 'xn--3ds443g',
+ 'xn--3e0b707e',
+ 'xn--3hcrj9c',
+ 'xn--3pxu8k',
+ 'xn--42c2d9a',
+ 'xn--45br5cyl',
+ 'xn--45brj9c',
+ 'xn--45q11c',
+ 'xn--4dbrk0ce',
+ 'xn--4gbrim',
+ 'xn--54b7fta0cc',
+ 'xn--55qw42g',
+ 'xn--55qx5d',
+ 'xn--5su34j936bgsg',
+ 'xn--5tzm5g',
+ 'xn--6frz82g',
+ 'xn--6qq986b3xl',
+ 'xn--80adxhks',
+ 'xn--80ao21a',
+ 'xn--80aqecdr1a',
+ 'xn--80asehdb',
+ 'xn--80aswg',
+ 'xn--8y0a063a',
+ 'xn--90a3ac',
+ 'xn--90ae',
+ 'xn--90ais',
+ 'xn--9dbq2a',
+ 'xn--9et52u',
+ 'xn--9krt00a',
+ 'xn--b4w605ferd',
+ 'xn--bck1b9a5dre4c',
+ 'xn--c1avg',
+ 'xn--c2br7g',
+ 'xn--cck2b3b',
+ 'xn--cckwcxetd',
+ 'xn--cg4bki',
+ 'xn--clchc0ea0b2g2a9gcd',
+ 'xn--czr694b',
+ 'xn--czrs0t',
+ 'xn--czru2d',
+ 'xn--d1acj3b',
+ 'xn--d1alf',
+ 'xn--e1a4c',
+ 'xn--eckvdtc9d',
+ 'xn--efvy88h',
+ 'xn--fct429k',
+ 'xn--fhbei',
+ 'xn--fiq228c5hs',
+ 'xn--fiq64b',
+ 'xn--fiqs8s',
+ 'xn--fiqz9s',
+ 'xn--fjq720a',
+ 'xn--flw351e',
+ 'xn--fpcrj9c3d',
+ 'xn--fzc2c9e2c',
+ 'xn--fzys8d69uvgm',
+ 'xn--g2xx48c',
+ 'xn--gckr3f0f',
+ 'xn--gecrj9c',
+ 'xn--gk3at1e',
+ 'xn--h2breg3eve',
+ 'xn--h2brj9c',
+ 'xn--h2brj9c8c',
+ 'xn--hxt814e',
+ 'xn--i1b6b1a6a2e',
+ 'xn--imr513n',
+ 'xn--io0a7i',
+ 'xn--j1aef',
+ 'xn--j1amh',
+ 'xn--j6w193g',
+ 'xn--jlq480n2rg',
+ 'xn--jvr189m',
+ 'xn--kcrx77d1x4a',
+ 'xn--kprw13d',
+ 'xn--kpry57d',
+ 'xn--kput3i',
+ 'xn--l1acc',
+ 'xn--lgbbat1ad8j',
+ 'xn--mgb9awbf',
+ 'xn--mgba3a3ejt',
+ 'xn--mgba3a4f16a',
+ 'xn--mgba7c0bbn0a',
+ 'xn--mgbaam7a8h',
+ 'xn--mgbab2bd',
+ 'xn--mgbah1a3hjkrd',
+ 'xn--mgbai9azgqp6j',
+ 'xn--mgbayh7gpa',
+ 'xn--mgbbh1a',
+ 'xn--mgbbh1a71e',
+ 'xn--mgbc0a9azcg',
+ 'xn--mgbca7dzdo',
+ 'xn--mgbcpq6gpa1a',
+ 'xn--mgberp4a5d4ar',
+ 'xn--mgbgu82a',
+ 'xn--mgbi4ecexp',
+ 'xn--mgbpl2fh',
+ 'xn--mgbt3dhd',
+ 'xn--mgbtx2b',
+ 'xn--mgbx4cd0ab',
+ 'xn--mix891f',
+ 'xn--mk1bu44c',
+ 'xn--mxtq1m',
+ 'xn--ngbc5azd',
+ 'xn--ngbe9e0a',
+ 'xn--ngbrx',
+ 'xn--node',
+ 'xn--nqv7f',
+ 'xn--nqv7fs00ema',
+ 'xn--nyqy26a',
+ 'xn--o3cw4h',
+ 'xn--ogbpf8fl',
+ 'xn--otu796d',
+ 'xn--p1acf',
+ 'xn--p1ai',
+ 'xn--pgbs0dh',
+ 'xn--pssy2u',
+ 'xn--q7ce6a',
+ 'xn--q9jyb4c',
+ 'xn--qcka1pmc',
+ 'xn--qxa6a',
+ 'xn--qxam',
+ 'xn--rhqv96g',
+ 'xn--rovu88b',
+ 'xn--rvc1e0am3e',
+ 'xn--s9brj9c',
+ 'xn--ses554g',
+ 'xn--t60b56a',
+ 'xn--tckwe',
+ 'xn--tiq49xqyj',
+ 'xn--unup4y',
+ 'xn--vermgensberater-ctb',
+ 'xn--vermgensberatung-pwb',
+ 'xn--vhquv',
+ 'xn--vuq861b',
+ 'xn--w4r85el8fhu5dnra',
+ 'xn--w4rs40l',
+ 'xn--wgbh1c',
+ 'xn--wgbl6a',
+ 'xn--xhq521b',
+ 'xn--xkc2al3hye2a',
+ 'xn--xkc2dl3a5ee0h',
+ 'xn--y9a3aq',
+ 'xn--yfro4i67o',
+ 'xn--ygbi2ammx',
+ 'xn--zfr164b',
+ 'xxx',
+ 'xyz',
+ 'yachts',
+ 'yahoo',
+ 'yamaxun',
+ 'yandex',
+ 'ye',
+ 'yodobashi',
+ 'yoga',
+ 'yokohama',
+ 'you',
+ 'youtube',
+ 'yt',
+ 'yun',
+ 'za',
+ 'zappos',
+ 'zara',
+ 'zero',
+ 'zip',
+ 'zm',
+ 'zone',
+ 'zuerich',
+ 'zw'
+]
generated by cgit v1.2.3-101-g0448 (git 2.54.0) at 2026-07-25 23:13:31 +0000