Everything before initial synced list

Signed-off-by: Andrew Opalach <andrew@akon.city>

33 files changed, 506 insertions, 254 deletions

diff --git a/src/portal/cpy/build.sh b/src/portal/cpy/build.sh
index 45a47f5..15ff4f5 100755
--- a/src/portal/cpy/build.sh
+++ b/src/portal/cpy/build.sh
@@ -1,3 +1,3 @@
 #! /usr/bin/env sh
 export PYTHONDONTWRITEBYTECODE=1
-python3.12 setup.py build_ext -i
+python3 setup.py build_ext -i
diff --git a/src/portal/cpy/portal.pyx b/src/portal/cpy/portal.pyx
index 3ddde80..ec33ac6 100644
--- a/src/portal/cpy/portal.pyx
+++ b/src/portal/cpy/portal.pyx
@@ -24,7 +24,7 @@ def encode_str(str):
 def decode_str(str):
     return str.decode('UTF-8')
 
-class LogOutputHook(object):
+class OutputHook(object):
     def __init__(self):
         self.fd = sys.__stdout__
 
@@ -35,7 +35,7 @@ class LogOutputHook(object):
         if len(msg) > 0 and msg != '\n':
             log_info(encode_str(msg))
 
-hook = LogOutputHook()
+hook = OutputHook()
 sys.stdout = hook
 sys.stderr = hook
 
@@ -147,8 +147,8 @@ cdef public int add_post_internal(bridge.camu_search *search, int id, unsigned i
         bridge.camu_search_add_post(search, num, &cpost)
         return 0
     py_str_to_str(&cpost.url, encode_str(py_post.url))
-    for type, date in py_post.dates.items():
-        post.camu_post_add_date(&cpost, type.value, date)
+    for date in py_post.dates:
+        post.camu_post_add_date(&cpost, date.type, date.range[0], date.range[1], date.meta)
     py_str_to_str(&cpost.author.unique_id, encode_str(py_post.author.unique_id))
     py_str_to_wstr(&cpost.author.username, encode_str(py_post.author.username))
     py_str_to_wstr(&cpost.author.display_name, encode_str(py_post.author.display_name))
diff --git a/src/portal/cpy/post.pxd b/src/portal/cpy/post.pxd
index 3f66508..aae9839 100644
--- a/src/portal/cpy/post.pxd
+++ b/src/portal/cpy/post.pxd
@@ -7,7 +7,9 @@ cdef extern from "../src/post.h":
 
     cdef struct camu_post_date:
         u8 type
-        f32 timestamp
+        f32 range_start
+        f32 range_end
+        u32 meta
 
     ctypedef struct camu_dates_array:
         u32 size
@@ -57,5 +59,5 @@ cdef extern from "../src/post.h":
         camu_post_ref in_reply_to
 
     void camu_post_reset(camu_post *post)
-    void camu_post_add_date(camu_post *post, u8 type, f32 timestamp)
+    void camu_post_add_date(camu_post *post, u8 type, f32 range_start, f32 range_end, u32 meta)
     void camu_post_add_media(camu_post *post, u8 type, str *key, str *url, str *ext, str *thumbnail_url, str *thumbnail_ext)
diff --git a/src/portal/meson.build b/src/portal/meson.build
index 0df4366..c3bbeb4 100644
--- a/src/portal/meson.build
+++ b/src/portal/meson.build
@@ -5,8 +5,10 @@ portal_src = [
   'src/packet_ext.c'
 ]
 portal_deps = []
+portal_args = ['-DCAMU_HAVE_PORTAL']
 
 python3_embed = import('python').find_installation('python3.12').dependency(embed: true)
 portal_deps += [python3_embed]
 
-portal = declare_dependency(sources: portal_src, dependencies: portal_deps)
+portal = declare_dependency(sources: portal_src, dependencies: portal_deps,
+    compile_args: portal_args)
diff --git a/src/portal/patches/snscrape_auth.diff b/src/portal/patches/snscrape_auth.diff
index 67962a9..efdc6e0 100644
--- a/src/portal/patches/snscrape_auth.diff
+++ b/src/portal/patches/snscrape_auth.diff
@@ -121,7 +121,7 @@ index d1719a8..43d9076 100644
  		r._snscrapeObj = obj
  		if apiType is _TwitterAPIType.GRAPHQL and 'errors' in obj:
  			msg = 'Twitter responded with an error: ' + ', '.join(f'{e["name"]}: {e["message"]}' for e in obj['errors'])
-+			blockFor = 30.0
++			blockFor = 10.0
 +			_logger.warn(f'Got errors waiting for {blockFor} seconds')
 +			time.sleep(blockFor)
 +			return False, msg
diff --git a/src/portal/py/base.py b/src/portal/py/base.py
index cc883af..7869521 100644
--- a/src/portal/py/base.py
+++ b/src/portal/py/base.py
@@ -1,21 +1,24 @@
 import log
 import time
+import json
 import httpx
+import binascii
 import threading
 from enum import Enum
+from blake3 import blake3
 from collections import OrderedDict
 from typing import Optional, Self, Any
 from post import Post, User
 
-USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0'
+type ParsedJson = Any
+
+USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; rv:131.0) Gecko/20100101 Firefox/131.0'
 
 class Method(Enum):
     HEAD = "HEAD"
     GET = "GET"
     POST = "POST"
 
-type ParsedJson = Any
-
 class Search():
     def __init__(self):
         self.pages: dict[int, list[str]] = {}
@@ -59,8 +62,9 @@ class Module():
     def __init__(self):
         self.headers: dict[str, str] = {}
         self.cookies: dict[str, str] = {}
+        self.mutex: threading.Lock = threading.Lock()
+        self.raw_responses: dict[str, str] = {}
         self.unique_id_map: OrderedDict = OrderedDict()
-        self.map_mutex: threading.Lock = threading.Lock()
         self.session: httpx.Client = httpx.Client(follow_redirects=True, timeout=20.0, http2=True)
 
     def init(self) -> bool:
@@ -84,6 +88,7 @@ class Module():
                 response = None
                 continue
             if response.status_code == 404 or response.status_code == 403: # Shortcut 404, 403
+                log.warn(f'Returning None, {response.status_code}')
                 return None
             if int(response.status_code / 100) != 2: # Non-200 response
                 log.error(f'Non-200 status code ({response.status_code}), retrying in 5 seconds...')
@@ -101,8 +106,16 @@ class Module():
                 break
         return response
 
+    def add_raw_response(self, data: ParsedJson) -> str:
+        with self.mutex:
+            dump = json.dumps(data)
+            hash = blake3(dump.encode('utf-8')).digest()
+            hash_str = binascii.hexlify(hash).decode('ascii')
+            self.raw_responses[hash_str] = dump
+            return hash_str
+
     def add_to_map(self, unique_id: str, item: Post | User):
-        with self.map_mutex:
+        with self.mutex:
             self.unique_id_map[unique_id] = item
             if len(self.unique_id_map) > 10240:
                 self.unique_id_map.popitem(last=False)
@@ -111,7 +124,7 @@ class Module():
         return None
 
     def get_item(self, unique_id: str) -> Optional[Post | User]:
-        with self.map_mutex:
+        with self.mutex:
             return self.unique_id_map.get(unique_id, None)
 
     def get_download(self, unique_id: str, key: str) -> Optional[dict[str, Any]]:
diff --git a/src/portal/py/modules/__init__.py b/src/portal/py/modules/__init__.py
index 994b25a..1a9e9ca 100644
--- a/src/portal/py/modules/__init__.py
+++ b/src/portal/py/modules/__init__.py
@@ -1,11 +1,11 @@
 import config
 
 from modules.youtube import YoutubeModule
-from modules.twitter import TwitterScrapeModule
-from modules.pixiv_app import PixivAppModule
-from modules.pixiv_web import PixivWebModule
+#from modules.twitter import TwitterScrapeModule
+#from modules.pixiv_app import PixivAppModule
+#from modules.pixiv_web import PixivWebModule
 from modules.fanbox import FanboxModule
-from modules.instagram import InstagramModule
+#from modules.instagram import InstagramModule
 from modules.patreon import PatreonModule
 
 ALL_MODULES = {
@@ -13,12 +13,12 @@ ALL_MODULES = {
 #    'twitter': (TwitterModule(
 #        config.TWITTER_ACCESS_TOKEN, config.TWITTER_ACCESS_TOKEN_SECRET,
 #        config.TWITTER_CONSUMER_TOKEN, config.TWITTER_CONSUMER_TOKEN_SECRET), []),
-    'twitter': (TwitterScrapeModule(config.TWITTER_COOKIES_PATH), []),
-    'pixiv_web': (PixivWebModule(config.PIXIV_SESSID, config.PIXIV_USERID), []),
+#    'twitter': (TwitterScrapeModule(config.TWITTER_COOKIES_PATH), []),
+#    'pixiv_web': (PixivWebModule(config.PIXIV_SESSID, config.PIXIV_USERID), []),
 #    'pixiv_app': (PixivAppModule(config.PIXIV_REFRESH_TOKEN), []),
 #    'fanbox': (FanboxModule(config.FANBOX_SESSID), []),
 #    'instagram': (InstagramModule(
 #        config.INSTAGRAM_USER_AGENT, config.INSTAGRAM_SETTINGS_PATH,
-#        config.INSTAGRAM_SESSION_ID), [])
-    'patreon': (PatreonModule(config.PATREON_SESSION_ID, config.PATREON_UUID, config.PATREON_USER_ID), [])
+#        config.INSTAGRAM_SESSION_ID), []),
+#    'patreon': (PatreonModule(config.PATREON_SESSION_ID, config.PATREON_UUID, config.PATREON_USER_ID), [])
 }
diff --git a/src/portal/py/modules/common.py b/src/portal/py/modules/common.py
index 62c3d5d..3b87884 100644
--- a/src/portal/py/modules/common.py
+++ b/src/portal/py/modules/common.py
@@ -1,6 +1,6 @@
 from datetime import datetime, timezone
 
-def parse_pixiv_date(date_str: str) -> datetime:
+def reformat_pixiv_date(date_str: str) -> datetime:
     rindex = date_str.rfind(':')
     date_str = date_str[:rindex] + date_str[rindex + 1:]
     return datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S%z")
diff --git a/src/portal/py/modules/fanbox.py b/src/portal/py/modules/fanbox.py
index 4323d6b..bf55b9d 100644
--- a/src/portal/py/modules/fanbox.py
+++ b/src/portal/py/modules/fanbox.py
@@ -3,9 +3,9 @@ import httpx
 from typing import Optional, Any
 from json.decoder import JSONDecodeError
 from base import USER_AGENT, Search, Module, Method, ParsedJson
-from post import MediaUrl, PostType, DateType, Post, User, Image, File, Tag
+from post import Url, PostType, DateType, Post, User, Image, File, Tag
 from query_parser import QueryParser
-from modules.common import parse_pixiv_date, get_current_utc_time
+from modules.common import reformat_pixiv_date, get_current_utc_time
 
 BASE_URL = "https://api.fanbox.cc"
 
@@ -23,8 +23,8 @@ class FanboxBase(Search):
             return None
         return obj['body']
 
-    def create_media_url(self, url: str) -> MediaUrl:
-        return MediaUrl(url, url.split('.')[-1])
+    def create_media_url(self, url: str) -> Url:
+        return Url(url, url.split('.')[-1])
 
     def create_post(self, data: ParsedJson) -> Optional[Post]:
         kwargs = {}
@@ -33,12 +33,12 @@ class FanboxBase(Search):
         kwargs['unique_id'] = unique_id
         kwargs['raw_responses'] = {}
         kwargs['raw_responses']['api'] = json.dumps(data)
-        publish_date = parse_pixiv_date(data['publishedDatetime']).timestamp()
+        publish_date = reformat_pixiv_date(data['publishedDatetime']).timestamp()
         kwargs['dates'] = {
             DateType.CREATED: publish_date,
             DateType.RETRIEVED: get_current_utc_time().timestamp()
         }
-        update_date = parse_pixiv_date(data['updatedDatetime']).timestamp()
+        update_date = reformat_pixiv_date(data['updatedDatetime']).timestamp()
         if update_date != publish_date:
             kwargs['dates'][DateType.EDITED] = update_date
         user = User(unique_id=f'fanbox:u:{data['user']['userId']}', username=data['user']['name'])
@@ -57,11 +57,11 @@ class FanboxBase(Search):
             if data['type'] == 'image':
                 text = data['body']['text']
                 for i, value in enumerate(data['body']['images']):
-                    media[str(i)] = Image(url=MediaUrl(value['originalUrl'], value['extension']), thumbnail_url=self.create_media_url(value['thumbnailUrl']))
+                    media[str(i)] = Image(url=Url(value['originalUrl'], value['extension']), thumbnail_url=self.create_media_url(value['thumbnailUrl']))
             elif data['type'] == 'file':
                 text = data['body']['text']
                 for i, value in enumerate(data['body']['files']):
-                    media[str(i)] = File(url=MediaUrl(value['url'], value['extension']), name=value['name'])
+                    media[str(i)] = File(url=Url(value['url'], value['extension']), name=value['name'])
             elif data['type'] == 'article':
                 for block in data['body']['blocks']:
                     if block['type'] == 'p':
@@ -71,9 +71,9 @@ class FanboxBase(Search):
                     elif block['type'] == 'url_embed':
                         text += f'embed::{block['urlEmbedId']}[]' + '\n'
                 for key, value in data['body']['imageMap'].items():
-                    media[key] = Image(url=MediaUrl(value['originalUrl'], value['extension']), thumbnail_url=self.create_media_url(value['thumbnailUrl']))
+                    media[key] = Image(url=Url(value['originalUrl'], value['extension']), thumbnail_url=self.create_media_url(value['thumbnailUrl']))
                 for key, value in data['body']['fileMap'].items():
-                    media[key] = File(url=MediaUrl(value['url'], value['extension']), name=value['name'])
+                    media[key] = File(url=Url(value['url'], value['extension']), name=value['name'])
             kwargs['text'] = text
             kwargs['media'] = media
         post = Post(**kwargs)
@@ -138,7 +138,7 @@ class FanboxUser(FanboxBase):
             self.errored = True
             return False
         self.pages[num] = []
-        for item in obj['items']:
+        for item in obj:
             post = self.request_post(item['id'])
             if post:
                 self.pages[num].append(post.unique_id)
@@ -169,12 +169,20 @@ class FanboxModule(Module):
         self.parser.add_command('post', FanboxPost)
         self.parser.add_command('user', FanboxUser)
         self.parser.add_command('supporting', FanboxSupporting)
-        self.headers['Accept-Encoding'] = 'gzip, deflate, br'
+        self.headers['Accept-Encoding'] = 'gzip, deflate, br, zstd'
         self.headers['Accept-Language'] = 'en-US,en;q=0.5'
+        self.headers['Alt-Used'] = 'api.fanbox.cc'
         self.headers['Origin'] =  'https://www.fanbox.cc'
         self.headers['Referer'] =  'https://www.fanbox.cc/'
         self.headers['User-Agent'] = USER_AGENT
         self.cookies['FANBOXSESSID'] = sessid
+        self.cookies['privacy_policy_agreement'] = '7'
+        self.cookies['privacy_policy_notification'] = '0'
+        self.cookies['p_ab_id'] = '0'
+        self.cookies['p_ab_id_2'] = '6'
+        self.cookies['p_ab_d_id'] = '251960935'
+        self.cookies['cf_clearance'] = ''
+        self.cookies['__cf_bm'] = ''
 
     def search(self, query: str, *extra_args: Any) -> Optional[Search]:
         return self.parser.parse_query(query)
diff --git a/src/portal/py/modules/instagram.py b/src/portal/py/modules/instagram.py
index d107566..28f7def 100644
--- a/src/portal/py/modules/instagram.py
+++ b/src/portal/py/modules/instagram.py
@@ -1,10 +1,9 @@
 import os
 import log
-import email.utils
 from typing import Optional, Any
 from pathlib import Path
 from base import USER_AGENT, Module, Search
-from post import MediaUrl, PostType, DateType, Post, User, Image, Video
+from post import Url, PostType, DateType, Post, User, Image, Video
 from modules.common import get_current_utc_time
 from instagrapi import Client
 from instagrapi.exceptions import UserNotFound, LoginRequired, ChallengeRequired
@@ -19,13 +18,13 @@ class InstagramSearch(Search):
         self.page: int = 0
         self.cursor: Any = None
 
-    def create_media_url(self, url: str) -> MediaUrl:
+    def create_media_url(self, url: str) -> Url:
         question = url.find('?')
         if question >= 0:
             ext = url[:question].split('.')[-1]
         else:
             ext = url.split('.')[-1]
-        return MediaUrl(url, ext)
+        return Url(url, ext)
 
     def request_page(self, num: int) -> bool:
         if num >= self.page + self.REACH_LIMIT:
@@ -34,6 +33,7 @@ class InstagramSearch(Search):
         for i in range(self.page, num + 1):
             try:
                 media, self.cursor = self.module.cl.user_medias_paginated_v1(self.pk, 0, end_cursor=self.cursor)
+                #media, self.cursor = self.module.cl.user_clips_paginated_v1(self.pk, amount=12, end_cursor=self.cursor)
             except LoginRequired:
                 self.errored = True
                 return False
@@ -47,10 +47,12 @@ class InstagramSearch(Search):
                 kwargs = {}
                 kwargs['type'] = PostType.POST
                 unique_id = 'instagram:p:{}'.format(m.id)
+                #unique_id = 'instagram:r:{}'.format(m.id)
                 kwargs['unique_id'] = unique_id
                 kwargs['raw_responses'] = {}
                 kwargs['raw_responses']['tile'] = m.json()
                 kwargs['url'] = 'https://www.instagram.com/p/{}'.format(m.code)
+                #kwargs['url'] = 'https://www.instagram.com/reel/{}'.format(m.code)
                 kwargs['dates'] = {
                     DateType.CREATED: m.taken_at.timestamp(),
                     DateType.RETRIEVED: get_current_utc_time().timestamp()
diff --git a/src/portal/py/modules/patreon.py b/src/portal/py/modules/patreon.py
index 40c3255..f33ec18 100644
--- a/src/portal/py/modules/patreon.py
+++ b/src/portal/py/modules/patreon.py
@@ -1,9 +1,12 @@
-import json
+import log
 import httpx
+import sys
 from json import JSONDecodeError
 from typing import Optional, Any
+from datetime import datetime
 from base import USER_AGENT, Search, Module, Method, ParsedJson
-from post import MediaUrl, PostType, DateType, Post, User, Media, Image, Animation, Tag, TagType
+from modules.common import get_current_utc_time
+from post import Date, DateType, DateMeta, Post, User, Url, Media, File
 from query_parser import QueryParser
 
 BASE_URL = 'https://www.patreon.com/api'
@@ -26,7 +29,7 @@ class PatreonBase(Search):
             return None
         return obj
 
-    def search_user(self, query: str) -> Optional[User]:
+    def search_user(self, query: str) -> tuple[Optional[User], Optional[str]]:
         url = f'{BASE_URL}/search'
         params = {
             'q': query,
@@ -37,32 +40,76 @@ class PatreonBase(Search):
         }
         obj = self.check_api_response(self.module.do_request(Method.GET, url, params=params))
         if not obj:
-            return None
+            return None, None
         for result in obj['data']:
+            username = result['attributes']['creator_name']
+            display_name = result['attributes']['name']
+            # Only accept exact matches.
+            if username.lower() != query.lower() and display_name.lower() != query.lower():
+                continue
             prefix = result['id'].find('campaign_')
             if prefix >= 0:
                 campaign = result['id'][prefix + len('campaign_'):]
                 unique_id = 'patreon:c:' + campaign
-                user = User(unique_id=unique_id, username=result['attributes']['creator_name'], display_name=result['attributes']['name'])
-                return user
-        return None
+                user = User(unique_id=unique_id, username=username, display_name=display_name)
+                return user, campaign
+        return None, None
 
 class PatreonUser(PatreonBase):
     def __init__(self, userdata: Any, arg: str):
         super().__init__(userdata)
         self.username: str = arg
         self.user: Optional[User] = None
-        self.campaign: str
+        self.campaign: Optional[str] = None
+        self.cursor: Optional[str] = None
+        self.attachments: dict[str, Media] = {}
+
+    # post_type
+    #  image_file: https://www.patreon.com/posts/emma-again-29391136
+    #  text_only: https://www.patreon.com/posts/hello-update-29391184
+
+    def create_post(self, data: ParsedJson, hash: str) -> Optional[Post]:
+        kwargs = {}
+        unique_id = f'patreon:p:{data['id']}'
+        kwargs['unique_id'] = unique_id
+        kwargs['raw_responses'] = {}
+        kwargs['raw_responses']['api'] = hash
+        kwargs['url'] = data['attributes']['url']
+        published_at = datetime.strptime(data['attributes']['published_at'], "%Y-%m-%dT%H:%M:%S.%f%z").timestamp()
+        current_time = get_current_utc_time().timestamp()
+        kwargs['dates'] = [
+            Date(DateType.CREATED, (published_at, published_at), DateMeta.NONE),
+            Date(DateType.RETRIEVED, (current_time, current_time), DateMeta.NONE),
+        ]
+        kwargs['author'] = self.user
+        kwargs['title'] = data['attributes']['title']
+        kwargs['likes'] = data['attributes']['like_count']
+        kwargs['comments'] = data['attributes']['comment_count']
+        kwargs['text'] = data['attributes']['content']
+        post_type = data['attributes']['post_type']
+        media = {}
+        #if 'access_rules' in data['relationships']:
+        #    if not (len(data['relationships']['access_rules']['data']) == 1 and data['relationships']['access_rules']['data'][0]['id'] == '9816126'):
+        #        return None
+        if 'media' in data['relationships']:
+            for m in data['relationships']['media']['data']:
+                media[m['id']] = self.attachments[m['id']]
+        if 'attachments' in data['relationships']:
+            for a in data['relationships']['attachments']['data']:
+                media[a['id']] = self.attachments[a['id']]
+        kwargs['media'] = media
+        post = Post(**kwargs)
+        self.module.add_to_map(unique_id, post)
+        return post
 
     def request_page(self, num: int) -> bool:
         if num >= self.page + self.REACH_LIMIT:
             return False
         if not self.user:
-            user = self.search_user(self.username)
-            if not user:
+            self.user, self.campaign = self.search_user(self.username)
+            if not self.user:
                 self.errored = True
                 return False
-            self.campaign = user.unique_id[len('patreon:c:'):]
         request_satisfied = False
         for i in range(self.page, num + 1):
             url = f'{BASE_URL}/posts'
@@ -81,7 +128,7 @@ class PatreonUser(PatreonBase):
                 'filter[contains_exclusive_posts]': 'true',
                 'filter[is_draft]': 'false',
                 'filter[accessible_by_user_id]': self.module.user_id,
-                'sort': '-published_at',
+                'sort': 'published_at',
                 'json-api-version': '1.0',
                 'json-api-use-default-includes': 'false'
             }
@@ -91,8 +138,27 @@ class PatreonUser(PatreonBase):
             if not obj:
                 self.errored = True
                 return False
-            print(json.dumps(obj, indent=4))
-        return True
+            hash = self.module.add_raw_response(obj)
+            self.cursor = obj['meta']['pagination']['cursors']['next']
+            self.pages[i] = []
+            for inc in obj['included']:
+                if inc['type'] == 'attachment':
+                    self.attachments[inc['id']] = File(url=Url(inc['attributes']['url']), name=inc['attributes']['name'])
+                elif inc['type'] == 'media':
+                    self.attachments[inc['id']] = Media(url=Url(inc['attributes']['download_url']), thumbnail_url=Url(inc['attributes']['image_urls']['thumbnail']))
+                else:
+                    log.error(f'Unhandled inc type {inc['type']}.')
+            for entry in obj['data']:
+                if entry['type'] != 'post':
+                    log.error(f'Unhandled entry type {entry['type']}.')
+                    continue
+                post = self.create_post(entry, hash)
+                if post:
+                    self.pages[i].append(post.unique_id)
+            self.page += 1
+            if i == num:
+                request_satisfied = True
+        return request_satisfied
 
 class PatreonModule(Module):
     def __init__(self, session_id: str, uuid: str, user_id: str):
@@ -100,7 +166,8 @@ class PatreonModule(Module):
         self.user_id: str = user_id
         self.parser: QueryParser = QueryParser(self, 'user')
         self.parser.add_command('user', PatreonUser)
-        self.headers['Accept-Encoding'] = 'gzip, deflate, br'
+        self.headers['User-Agent'] = USER_AGENT
+        self.headers['Accept-Encoding'] = 'gzip, deflate, br, zstd'
         self.headers['Accept-Language'] = 'en-US,en;q=0.5'
         self.headers['x-patreon-uuid'] = uuid
         self.cookies['patreon_locale_code'] = 'en-US'
@@ -116,9 +183,4 @@ class PatreonModule(Module):
         post = self.unique_id_map[unique_id]
         if key not in post.media:
             return None
-        return {
-            'urls': [post.media[key].url],
-            'headers': {
-                'User-Agent': USER_AGENT
-            }
-        }
+        return { 'urls': [post.media[key].url], 'headers': self.headers, 'cookies': self.cookies }
diff --git a/src/portal/py/modules/pixiv_app.py b/src/portal/py/modules/pixiv_app.py
index c84030d..ef5886d 100644
--- a/src/portal/py/modules/pixiv_app.py
+++ b/src/portal/py/modules/pixiv_app.py
@@ -1,10 +1,10 @@
 import json
 from typing import Optional, Any
 from base import USER_AGENT, Search, Module, ParsedJson
-from post import MediaUrl, PostType, DateType, User, Post, Image, Tag, TagType
+from post import Url, PostType, DateType, User, Post, Image, Tag, TagType
 from query_parser import QueryParser
-from pixivpy3 import *
-from modules.common import get_current_utc_time, parse_pixiv_date
+from pixivpy3 import AppPixivAPI
+from modules.common import get_current_utc_time, reformat_pixiv_date
 
 class PixivAppBase(Search):
     def __init__(self, userdata: Any):
@@ -33,8 +33,8 @@ class PixivAppBase(Search):
             self.page += 1
         return request_satisfied
 
-    def create_media_url(self, url: str) -> MediaUrl:
-        return MediaUrl(url, url.split('.')[-1])
+    def create_media_url(self, url: str) -> Url:
+        return Url(url, url.split('.')[-1])
 
     def create_tag(self, data: ParsedJson) -> Tag:
         tag = Tag(type=TagType.GENERAL, name=data['name'])
@@ -58,7 +58,7 @@ class PixivAppBase(Search):
         kwargs['raw_responses']['api'] = json.dumps(data)
         kwargs['url'] = 'https://www.pixiv.net/en/artworks/{}'.format(data['id'])
         kwargs['dates'] = {
-            DateType.CREATED: parse_pixiv_date(data['create_date']).timestamp(),
+            DateType.CREATED: reformat_pixiv_date(data['create_date']).timestamp(),
             DateType.RETRIEVED: get_current_utc_time().timestamp()
         }
         user = User(unique_id=f'pixiv:u:{data['user']['id']}', username=data['user']['account'], display_name=data['user']['name'])
diff --git a/src/portal/py/modules/pixiv_web.py b/src/portal/py/modules/pixiv_web.py
index be68b7e..7dbb8bb 100644
--- a/src/portal/py/modules/pixiv_web.py
+++ b/src/portal/py/modules/pixiv_web.py
@@ -5,9 +5,9 @@ import urllib.parse
 from typing import Optional, Any
 from json.decoder import JSONDecodeError
 from base import USER_AGENT, Search, Module, Method, ParsedJson
-from post import MediaUrl, PostType, DateType, Post, User, Media, Image, Animation, Tag, TagType
+from post import Url, PostType, DateType, Post, User, Media, Image, Animation, Tag, TagType
 from query_parser import QueryParser
-from modules.common import parse_pixiv_date, get_current_utc_time
+from modules.common import reformat_pixiv_date, get_current_utc_time
 
 BASE_URL = 'https://www.pixiv.net/ajax'
 LANG = 'en'
@@ -30,8 +30,8 @@ class PixivWebBase(Search):
             return None
         return obj['body']
 
-    def create_media_url(self, url: str) -> MediaUrl:
-        return MediaUrl(url, url.split('.')[-1])
+    def create_media_url(self, url: str) -> Url:
+        return Url(url, url.split('.')[-1])
 
     def parse_pages(self, data: ParsedJson) -> dict[str, Media]:
         media = {}
@@ -45,7 +45,7 @@ class PixivWebBase(Search):
             animation.frames.append((frame['file'], frame['delay']))
         return animation
 
-    def seek_profile_picture_url(self, user_illusts: ParsedJson) -> Optional[MediaUrl]:
+    def seek_profile_picture_url(self, user_illusts: ParsedJson) -> Optional[Url]:
         for illust in user_illusts.values():
             if illust is None:
                 continue
@@ -71,14 +71,15 @@ class PixivWebBase(Search):
         kwargs['raw_responses'] = {}
         kwargs['raw_responses']['api'] = json.dumps(data)
         kwargs['url'] = f'https://www.pixiv.net/{LANG}/artworks/{data['illustId']}'
-        create_date = parse_pixiv_date(data['createDate']).timestamp()
+        create_date = reformat_pixiv_date(data['createDate']).timestamp()
+        current_date = get_current_utc_time().timestamp()
         kwargs['dates'] = {
-            DateType.CREATED: create_date,
-            DateType.RETRIEVED: get_current_utc_time().timestamp()
+            DateType.CREATED: (create_date, create_date, 0),
+            DateType.RETRIEVED: (current_date, current_date, 0)
         }
-        upload_date = parse_pixiv_date(data['uploadDate']).timestamp()
+        upload_date = reformat_pixiv_date(data['uploadDate']).timestamp()
         if upload_date != create_date:
-            kwargs['dates'][DateType.EDITED] = upload_date
+            kwargs['dates'][DateType.EDITED] = (upload_date, upload_date, 0)
         user = User(unique_id=f'pixiv:u:{data['userId']}', username=data['userAccount'], display_name=data['userName'])
         profile_picture_url = self.seek_profile_picture_url(data['userIllusts'])
         if profile_picture_url:
@@ -242,10 +243,10 @@ class PixivWebModule(Module):
         self.parser.add_command('illust', PixivWebIllust)
         self.parser.add_command('user', PixivWebUser)
         self.parser.add_command('following', PixivWebFollowing)
-        self.headers['Accept-Encoding'] = 'gzip, deflate, br' # add 'zstd' when httpx update
+        self.headers['User-Agent'] = USER_AGENT
+        self.headers['Accept-Encoding'] = 'gzip, deflate, br, zstd'
         self.headers['Accept-Language'] = 'en-US,en;q=0.5'
         self.headers['Referer'] =  'https://www.pixiv.net/'
-        self.headers['User-Agent'] = USER_AGENT
         self.headers['x-user-id'] = user_id
         self.cookies['PHPSESSID'] = sessid
 
diff --git a/src/portal/py/modules/twitter.py b/src/portal/py/modules/twitter.py
index 2996d0e..386bc5b 100644
--- a/src/portal/py/modules/twitter.py
+++ b/src/portal/py/modules/twitter.py
@@ -3,7 +3,7 @@ import http.cookiejar
 from typing import Optional, Any, Iterator
 from datetime import timezone
 from base import USER_AGENT, Search, Module
-from post import MediaUrl, PostType, DateType, PostRef, Post, User, Media, Image, Video
+from post import Url, PostType, DateType, PostRef, Post, User, Media, Image, Video
 from query_parser import QueryParser
 from modules.common import get_current_utc_time
 
@@ -26,7 +26,7 @@ class TwitterScrapeBase(Search):
         self.iterator: Iterator[Tweet | TweetRef | Tombstone]
         self.page: int = 0
 
-    def create_media_url(self, url: str) -> MediaUrl:
+    def create_media_url(self, url: str) -> Url:
         format = url.find('format=')
         if format >= 0:
             ext = url[format + 7:format + 10]
@@ -35,7 +35,7 @@ class TwitterScrapeBase(Search):
             if question >= 0:
                 url = url[0:question]
             ext = url.split('.')[-1]
-        return MediaUrl(url, ext)
+        return Url(url, ext)
 
     def parse_media(self, data: list[twitter.Medium]) -> dict[str, Media]:
         media = {}
diff --git a/src/portal/py/modules/youtube.py b/src/portal/py/modules/youtube.py
index d14dfe8..f809ff6 100644
--- a/src/portal/py/modules/youtube.py
+++ b/src/portal/py/modules/youtube.py
@@ -1,7 +1,8 @@
 import log
+import json
 from typing import Optional, Any
 from base import Search, Module, ParsedJson
-from post import MediaUrl, Post, PostType, Media, Video
+from post import Url, Post, PostType, Media, Video
 from query_parser import QueryParser
 from yt_dlp import YoutubeDL
 
@@ -25,65 +26,53 @@ ydl_opts = {
     'quiet': False,
     'logger': YDLLogger(),
     'cachedir': False,
-#    'cookiefile': '',
+    'socket_timeout': 10
+#    'cookiefile': ''
 }
 
-ydl = YoutubeDL(ydl_opts)
-
-def get_playback_url(data, video=True):
-    if 'entries' in data:
-        if len(data['entries']) == 0:
-            return None
-        data = data['entries'][0]
+# https://github.com/yt-dlp/yt-dlp/issues/4103
 
-    if 'formats' not in data:
-        if 'url' in data:
-            return data['url']
-        return None
+ydl = YoutubeDL(ydl_opts)
 
-    # Filter out hls temporarily.
-    data['formats'] = list(filter(lambda f: not f['protocol'].startswith('m3u8'), data['formats']))
+def get_playback_url(data, video=True) -> Optional[str]:
+    data['formats'] = list(filter(
+        lambda f: not f['protocol'].startswith('m3u8'), data['formats']))
 
     if len(data['formats']) == 0:
         return None
 
-    url = data['formats'][0]['url']
+    result = data['formats'][-1]['url']
 
-    # audio_ext?
-    has_audio = list(filter(lambda f: 'acodec' not in f or f['acodec'] != 'none', data['formats']))
+    has_audio = list(filter(lambda f:
+        (f['acodec'] and f['acodec'] != 'none') or
+        (f['audio_ext'] and f['audio_ext'] != 'none') or
+        (f['abr']), data['formats']))
 
     if video:
-        if len(has_audio) > 0:
-            data['formats'] = has_audio
-        try:
-            data['formats'] = list(filter(lambda f: 'quality' in f, data['formats']))
-            url = max(data['formats'], key=lambda f: f['quality'])['url']
-        except:
-            pass
+        data['formats'] = list(filter(lambda f:
+            (f['vcodec'] and f['vcodec'] != 'none') or
+            (f['video_ext'] and f['video_ext'] != 'none') or
+            (f['vbr']), has_audio))
     else:
-        if len(has_audio) == 0:
-            return None
-        data['formats'] = has_audio
-        try:
-            audio_only = list(filter(lambda f: f['vcodec'] == 'none', data['formats']))
-            if len(audio_only) > 0:
-                data['formats'] = audio_only
-            else:
-                data['formats'] = list(filter(lambda f: f['ext'] in ['mp4'], data['formats']))
-        except:
-            pass
-        try:
-            data['formats'] = list(filter(lambda f: 'abr' in f, data['formats']))
-            url = max(data['formats'], key=lambda f: f['abr'])['url']
-        except:
-            pass
+        data['formats'] = list(filter(lambda f:
+            (f['vcodec'] and f['vcodec'] == 'none') and
+            (f['video_ext'] and f['video_ext'] == 'none') and
+            (not f['vbr']), has_audio))
+
+    if len(data['formats']) > 0:
+        selection = max(data['formats'], key=lambda f: 0 if 'quality' not in f else f['quality'])
+        log.info(json.dumps(selection, indent=4))
+        result = selection['url']
+    else:
+        log.warn('Defaulting in Youtube get_playback_url().')
 
-    return url
+    return result
 
 class YoutubeBase(Search):
     def __init__(self, userdata: Any):
         super().__init__()
         self.module: YoutubeModule = userdata
+        self.guessed_index: int = 0
 
     def get_info(self) -> Optional[ParsedJson]:
         return None
@@ -92,11 +81,25 @@ class YoutubeBase(Search):
         info = self.get_info()
         if not info:
             return False
-        url = get_playback_url(info)
+        if 'entries' in info:
+            if len(info['entries']) == 0:
+                return False
+            if len(info['entries']) <= self.guessed_index:
+                self.guessed_index = 0
+            info = info['entries'][self.guessed_index]
+        if 'formats' not in info and 'url' in info:
+            self.link = info['url']
+            info = self.get_info()
+            if not info:
+                return False
+        if 'direct' in info and info['direct']:
+            url = info['url']
+        else:
+            url = get_playback_url(info, video=False)
         if not url:
             return False
-        media: dict[str, Media] = { '0': Video(url=MediaUrl(url=url)) }
         unique_id = f'youtube:v:{info['id']}'
+        media: dict[str, Media] = { '0': Video(url=Url(url)) }
         self.module.add_to_map(unique_id, Post(type=PostType.POST, unique_id=unique_id, url='', title=info['title'], text='', media=media))
         self.pages[num] = [unique_id]
         return True
@@ -107,7 +110,13 @@ class YoutubeSearch(YoutubeBase):
         self.query: str = arg
 
     def get_info(self) -> Optional[ParsedJson]:
-        return ydl.extract_info(f'ytsearch1:{self.query}', download=False)
+        ydl.params['extract_flat'] =  False
+        try:
+            info = ydl.extract_info(f'ytsearch1:{self.query}', download=False)
+        except Exception as e:
+            log.error(repr(e))
+            return None
+        return info
 
 class YoutubeLink(YoutubeBase):
     def __init__(self, userdata: Any, arg: str):
@@ -115,7 +124,24 @@ class YoutubeLink(YoutubeBase):
         self.link: str = arg
 
     def get_info(self) -> Optional[ParsedJson]:
-        return ydl.extract_info(self.link, download=False)
+        index = self.link.find('&index=')
+        if index >= 0:
+            sub = self.link[index + 7:]
+            end = sub.find('&')
+            if end >= 0:
+                sub = sub[:end]
+            try:
+                self.guessed_index = int(sub, 10) - 1
+            except ValueError:
+                pass
+            log.info(f'Guessed index: {self.guessed_index}')
+        ydl.params['extract_flat'] = 'in_playlist'
+        try:
+            info = ydl.extract_info(self.link, download=False)
+        except Exception as e:
+            log.error(repr(e))
+            return None
+        return info
 
 class YoutubeModule(Module):
     def __init__(self):
diff --git a/src/portal/py/post.py b/src/portal/py/post.py
index 8149e4c..76e82b9 100644
--- a/src/portal/py/post.py
+++ b/src/portal/py/post.py
@@ -1,22 +1,38 @@
 import dataclasses
 from dataclasses import field
 from typing import Optional, Any
-from enum import Enum
+from enum import IntEnum, IntFlag
 from json import JSONEncoder
 
-class PostType(int, Enum):
+'''
+List of Things:
+ - Social media posts
+ - Albums
+ - Loose files
+'''
+
+class PostType(IntEnum):
     UNKNOWN = 0
     POST = 1
     REPOST = 2
     PREVIEW = 3
     TOMBSTONE = 4
 
-class DateType(int, Enum):
-    CREATED = 0
-    EDITED = 1
-    RETRIEVED = 2
+class DateType(IntEnum):
+    UNKNOWN = 0
+    CREATED = 1
+    EDITED = 2
+    RETRIEVED = 3
+
+class DateMeta(IntFlag):
+    NONE = 0
+    ESTIMATE = 1
+    GUESS = 1 << 1
+    LOOSE_PRECISION = 1 << 2
+    EDITED_AT_UNKNOWN_TIME = 1 << 3
+    ASSUMED_TYPE = 1 << 4
 
-class MediaType(int, Enum):
+class MediaType(IntEnum):
     UNKNOWN = 0
     FILE = 1
     AUDIO = 2
@@ -25,7 +41,7 @@ class MediaType(int, Enum):
     VIDEO_SPLIT = 5
     ANIMATION = 6
 
-class TagType(int, Enum):
+class TagType(IntEnum):
     UNKNOWN = 0
     GENERAL = 1
     ARTIST = 2
@@ -37,15 +53,33 @@ class TagType(int, Enum):
 def df(c: Any) -> Any:
     return field(default_factory=lambda: c)
 
+def default_url_ext_guess(url):
+    # TODO: List of known extensions.
+    question = url.rfind('?')
+    if question >= 0:
+        guess = url[0:question].rsplit('.')[-1]
+    else:
+        guess = url.rsplit('.')[-1]
+    return guess
+
 @dataclasses.dataclass
-class MediaUrl():
-    url: str = ''
-    ext: str = 'unknown'
+class Url():
+    url: str
+    ext: str
+
+    def __init__(self, url: str, ext: str = 'unknown'):
+        self.url = url
+        ext_guess = default_url_ext_guess(url)
+        if ext_guess:
+            self.ext = ext_guess
+        else:
+            self.ext = ext
 
 @dataclasses.dataclass
 class Media():
     type: MediaType = MediaType.UNKNOWN
-    url: MediaUrl = df(MediaUrl())
+    url: Url = df(Url(''))
+    thumbnail_url: Url = df(Url(''))
 
 @dataclasses.dataclass
 class File(Media):
@@ -59,24 +93,20 @@ class Audio(Media):
 @dataclasses.dataclass
 class Image(Media):
     type: MediaType = MediaType.IMAGE
-    thumbnail_url: MediaUrl = df(MediaUrl())
 
 @dataclasses.dataclass
 class Video(Media):
     type: MediaType = MediaType.VIDEO
-    thumbnail_url: MediaUrl = df(MediaUrl())
 
 @dataclasses.dataclass
 class VideoSplit(Media):
     type: MediaType = MediaType.VIDEO_SPLIT
-    audio_url: MediaUrl = df(MediaUrl())
-    subtitle_url: MediaUrl = df(MediaUrl())
-    thumbnail_url: MediaUrl = df(MediaUrl())
+    audio_url: Url = df(Url(''))
+    subtitle_url: Url = df(Url(''))
 
 @dataclasses.dataclass
 class Animation(Media):
     type: MediaType = MediaType.ANIMATION
-    thumbnail_url: MediaUrl = df(MediaUrl())
     frames: list[tuple[str, int]] = df([])
 
 @dataclasses.dataclass
@@ -85,31 +115,59 @@ class Tag():
     name: str = ''
     alts: dict[str, str] = df({})
 
+'''
+Date(DateType.CREATED, (1396411200.0, 1396756800.0), DateMeta.ESTIMATE)
+ - Post is estimated to have been created at some point between April 2nd and 6th 2014.
+
+Date(DateType.EDITED, (1396411200.0, 1550247862.0), DateMeta.EDITED_AT_UNKNOWN_TIME)
+ - We have no clue when the post was edited, but it had to have been between the time it was created and now.
+ - EDITED_AT_UNKNOWN_TIME should be used when either of CREATED and/or RETRIEVED are used as placeholders.
+   It can be combined with GUESS or ESTIMATE if, for example, there is some indication about the end of the range.
+
+Date(DateType.RETRIEVED, (1550247862.0, 1550247862.0), DateMeta.NONE)
+ - Time of processing this post. Will likely be slightly after it was actually retrived but that is negligible.
+ - RETRIEVED can still have a range and meta if applicable.
+
+Date(DateType.CREATED, (1483228800.0, 1546300800.0), DateMeta.LOOSE_PRECISION)
+ - Date is exactly between "01 Jan 2017 12:00:00 AM UTC" and "01 Jan 2019 12:00:00 AM UTC".
+   Combined with LOOSE_PRECISION, the application will assume a meaning of "between 2017 and 2019".
+
+Date(DateType.CREATED, (1145059200.0, 1145059200.0), DateMeta.ASSUMED_TYPE)
+ - The date of April 15th 2006 is associated with the resource. We are assuming that's the creation date.
+'''
+@dataclasses.dataclass
+class Date():
+    type: DateType = DateType.UNKNOWN
+    range: tuple[float, float] = (0.0, 0.0)
+    meta: DateMeta = DateMeta.NONE
+    note: str = ''
+
 @dataclasses.dataclass
 class User():
     unique_id: str = ''
     username: str = ''
     display_name: str = ''
-    profile_picture_url: MediaUrl = df(MediaUrl())
+    profile_picture_url: Url = df(Url(''))
 
 @dataclasses.dataclass
 class PostRef():
     unique_id: str = ''
 
-# V4 Ideas:
-# - Date estimate and range
-#  - Edited but unknown when
-#  - Generally an estimate/guess
-# - Formated text/body
-
+'''
+V4:
+ - Improved date.
+ - Ends when all current modules are ported and/or completed.
+V5:
+ - Formatted body/text.
+'''
 @dataclasses.dataclass
 class Post():
-    version: int = 3
+    version: int = 4
     type: PostType = PostType.UNKNOWN
     unique_id: str = ''
     raw_responses: dict[str, str] = df({})
     url: str = ''
-    dates: dict[DateType, float] = df({})
+    dates: list[Date] = df([])
     author: User = df(User())
     title: str = ''
     text: str = ''
@@ -128,8 +186,4 @@ class Post():
 
 class PostEncoder(JSONEncoder):
     def default(self, o):
-        if isinstance(o, Enum):
-            return o.value
-        if dataclasses.is_dataclass(o):
-            return dataclasses.asdict(o)
         return o.__dict__
diff --git a/src/portal/py/test.py b/src/portal/py/test.py
new file mode 100644
index 0000000..c5687b1
--- /dev/null
+++ b/src/portal/py/test.py
@@ -0,0 +1 @@
+import tests.test
diff --git a/src/portal/py/tests/__init__.py b/src/portal/py/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/portal/py/tests/__init__.py
diff --git a/src/portal/tests/archive_query.py b/src/portal/py/tests/archive_query.py
index 9f69c1f..1097634 100644
--- a/src/portal/tests/archive_query.py
+++ b/src/portal/py/tests/archive_query.py
@@ -5,13 +5,13 @@ import json
 import threading
 import gzip
 from datetime import datetime, timezone
-sys.path.append('../py')
+sys.path.append('../')
 from base import Method
 from post import PostEncoder, PostType, DateType, User
 from modules import ALL_MODULES
 from logger import Logger
 
-mode = 'instagram'
+mode = 'fanbox'
 cmd = 'user'
 module = ALL_MODULES[mode][0]
 if not module.init():
@@ -66,8 +66,8 @@ class QueryDownloadThread(threading.Thread):
             else:
                 with open(path, open_method) as f:
                     f.write(content)
-        except:
-            self.log.write(f'ERROR: Writing file failed ({path}).')
+        except Exception as e:
+            self.log.write(f'ERROR: Writing file failed: {path} ({repr(e)}).')
             return False
         return True
 
@@ -168,18 +168,19 @@ class QueryDownloadThread(threading.Thread):
         with queue_cond:
             queue_cond.notify()
 
+#RUNTIME_PATH = "./run"
 RUNTIME_PATH = "/mnt/store/files/tmp/run"
-LOG_FILE = f'{RUNTIME_PATH}/archive.log'
-ARG_FILE = f'{RUNTIME_PATH}/completed_args.log'
+LOG_FILE = f'{RUNTIME_PATH}/archive3.log'
+ARG_FILE = f'{RUNTIME_PATH}/completed_args3.log'
 
 def read_completed_args(path):
     args = []
     if os.path.isfile(path):
         with open(path, 'r') as f:
-            for l in f.read().splitlines():
-                if l[0] == '-':
+            for line in f.read().splitlines():
+                if line[0] == '-':
                     continue
-                s = l.split(':')
+                s = line.split(':')
                 if s[3] == 'c':
                     args.append(f'{s[0]}:{s[1]}:{s[2]}')
     return args
@@ -203,7 +204,7 @@ def download_args():
         os.mkdir(output_dir)
 
     #args = module.search(f'following:{22781328}')
-    args = ['byjoshuajamal']
+    args = ['anoh223']
 
     threads = []
     start = True
diff --git a/src/portal/tests/logger.py b/src/portal/py/tests/logger.py
index 80e22da..80e22da 100644
--- a/src/portal/tests/logger.py
+++ b/src/portal/py/tests/logger.py
diff --git a/src/portal/tests/old_twitter_api.py b/src/portal/py/tests/old_twitter_api.py
index 40d0346..40d0346 100644
--- a/src/portal/tests/old_twitter_api.py
+++ b/src/portal/py/tests/old_twitter_api.py
diff --git a/src/portal/tests/rewrite_post.py b/src/portal/py/tests/rewrite_post.py
index 3bb152b..3bb152b 100644
--- a/src/portal/tests/rewrite_post.py
+++ b/src/portal/py/tests/rewrite_post.py
diff --git a/src/portal/py/tests/test.py b/src/portal/py/tests/test.py
new file mode 100644
index 0000000..bee7265
--- /dev/null
+++ b/src/portal/py/tests/test.py
@@ -0,0 +1,103 @@
+import json
+import httpx
+import os.path
+from base import Method
+from typing import Optional, Any
+from post import DateType, DateMeta, Date, PostEncoder, Post, MediaType
+from modules import ALL_MODULES
+#from twitter.scraper import Scraper
+
+#scraper = Scraper(cookies = {
+#  'ct0': '5a3c98b7b50a74dc556fa497f497932e8958b5b2ef7ba74440ce3f341bb4753f663d3fe7633222d62a52d4ce121673234b927d97eeef806129d42bbfba7ab4d4cfbd2749c20fe6f8da480d0a1e6629e1',
+#  'auth_token': 'ea66b4c2ebd04e22de855dce6f1366221635bf2e'
+#})
+#
+#media = scraper.media([1557919548904419328], limit=10)
+#print(media)
+
+#output = '/mnt/store/files/ext/patreon_tmp/FPSBlyck'
+#
+#module = ALL_MODULES['patreon'][0]
+#module.init()
+#search = module.search('FPSBlyck')
+#page_num = 0
+#mark = False
+#while not mark:
+#    page = search.get_page(page_num)
+#    if not page:
+#        break
+#    page_num += 1
+#    for unique_id in page:
+#        if unique_id == 'patreon:p:29707872':
+#            print('hit mark')
+#            mark = True
+#        post = module.get_item(unique_id)
+#        print(json.dumps(post, indent=4, cls=PostEncoder))
+#        for key, m in post.media.items():
+#            if m.type == MediaType.FILE:
+#                path = f'{output}/{post.unique_id.replace(':', '_')}_{m.name}'
+#                if not os.path.isfile(path):
+#                    dl = module.get_download(post.unique_id, key)
+#                    retries = 3
+#                    r: Optional[httpx.Response] = None
+#                    while retries >= 0:
+#                        try:
+#                            r = httpx.get(dl['urls'][0].url, headers=dl['headers'], cookies=dl['cookies'], follow_redirects=True)
+#                        except:
+#                            retries -= 1
+#                        else:
+#                            break
+#                    if r:
+#                        with open(path, 'wb+') as f:
+#                            f.write(r.content)
+#                    else:
+#                        print('Download failed')
+#                else:
+#                    print('Skipping file')
+
+ALL_MODULES['youtube'][0].init()
+search = ALL_MODULES['youtube'][0].search('yoyoyoy')
+page = search.get_page(0)
+print(page)
+#print(json.dumps(page, indent=4, cls=PostEncoder))
+
+#d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.EDITED_AT_UNKNOWN_TIME)
+#d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.NONE)
+#print(json.dumps(d, indent=4, cls=PostEncoder))
+#sys.exit(1)
+
+#module = ALL_MODULES['fanbox'][0]
+#module = ALL_MODULES['pixiv_web'][0]
+#module = ALL_MODULES['youtube'][0]
+#module = ALL_MODULES['twitter'][0]
+#module.init()
+
+#search = module.search('opium_00pium')
+#search = module.search('search:ddd')
+#page = search.get_page(0)
+#for unique_id in page:
+#    post = module.get_item(unique_id)
+#    print(json.dumps(post, indent=4, cls=PostEncoder))
+#    break
+
+#post = module.get_item(page[0])
+#for key in post.media.keys():
+#  download_params = module.get_download(post.unique_id, key)
+#  for url in download_params['urls']:
+#      #response = module.do_request(Method.GET, url.url)
+#      r = httpx.get(url.url, headers=download_params['headers'])
+#      with open(f'test_{key}.{url.ext}', 'wb+') as f:
+#          f.write(r.content)
+
+#print(json.dumps(page, indent=4, cls=PostEncoder))
+#for i in range(0, len(post.media)):
+#    download_params = module.get_download(post.unique_id, i)
+#    r = httpx.get(download_params['url'], headers=download_params['headers'])
+#    if r.status_code != 200:
+#        print('error{}'.format(r.status_code))
+#    else:
+#        with open('test{}.png'.format(i), 'wb+') as f:
+#            f.write(r.content)
+
+#print(json.dumps(page, indent=4, cls=PostEncoder))
+#print(page)
diff --git a/src/portal/tests/unescape_json.py b/src/portal/py/tests/unescape_json.py
index eaf5194..eaf5194 100644
--- a/src/portal/tests/unescape_json.py
+++ b/src/portal/py/tests/unescape_json.py
diff --git a/src/portal/requirements.txt b/src/portal/requirements.txt
index d241387..f4982dc 100644
--- a/src/portal/requirements.txt
+++ b/src/portal/requirements.txt
@@ -1,2 +1,4 @@
-httpx[http2,brotli]
+httpx[http2,brotli,zstd]
+blake3
 pillow
+notcurses
diff --git a/src/portal/scripts/create_vendor.sh b/src/portal/scripts/create_vendor.sh
index f4d551e..6fdf9e0 100755
--- a/src/portal/scripts/create_vendor.sh
+++ b/src/portal/scripts/create_vendor.sh
@@ -4,7 +4,7 @@ python3 -m venv ./venv
 source ./venv/bin/activate
 
 cd vendor/
-rm -r vendor/
+rm -rf vendor/
 
 pip3 install -r ../requirements.txt --upgrade --target=./vendor
 
@@ -12,7 +12,11 @@ cd yt-dlp/
 pip3 install . --upgrade --target=../vendor
 cd ../
 
-cd snscrape/
+#cd snscrape/
+#pip3 install . --upgrade --target=../vendor
+#cd ../
+
+cd twitter-api-client/
 pip3 install . --upgrade --target=../vendor
 cd ../
 
@@ -24,5 +28,13 @@ cd instagrapi/
 pip3 install . --upgrade --target=../vendor
 cd ../
 
+cd CyberDropDownloader/
+pip3 install . --upgrade --target=../vendor
+cd ../
+
+#cd gallery-dl/
+#pip3 install . --upgrade --target=../vendor
+#cd ../
+
 cd ../
 rm -r venv
diff --git a/src/portal/scripts/run_py.sh b/src/portal/scripts/run_py.sh
index ddbe869..4414f93 100755
--- a/src/portal/scripts/run_py.sh
+++ b/src/portal/scripts/run_py.sh
@@ -1,4 +1,5 @@
 #! /usr/bin/env sh
+export CURL_CA_BUNDLE=/etc/ssl/certs/ca-bundle.crt
 export PYTHONDONTWRITEBYTECODE=1
 export PYTHONPATH=$HOME/c/camu/src/portal/vendor/vendor
-python3 $@
+$@
diff --git a/src/portal/src/packet_ext.c b/src/portal/src/packet_ext.c
index 580c307..e2909ef 100644
--- a/src/portal/src/packet_ext.c
+++ b/src/portal/src/packet_ext.c
@@ -16,7 +16,9 @@ void aki_packet_write_camu_post(struct aki_packet *packet, struct camu_post *pos
     struct camu_post_date *date;
     al_array_foreach_ptr(post->dates, i, date) {
         AKI_PACKET_WRITE_TYPE(packet, u8, date->type);
-        AKI_PACKET_WRITE_TYPE(packet, f32, date->timestamp);
+        AKI_PACKET_WRITE_TYPE(packet, f32, date->range_start);
+        AKI_PACKET_WRITE_TYPE(packet, f32, date->range_end);
+        AKI_PACKET_WRITE_TYPE(packet, u32, date->meta);
     }
     aki_packet_write_str(packet, &post->author.unique_id);
     aki_packet_write_wstr(packet, &post->author.username);
@@ -64,7 +66,9 @@ void aki_packet_read_camu_post(struct aki_packet *packet, struct camu_post *post
     for (u32 i = 0; i < dates_size; i++) {
         struct camu_post_date date;
         AKI_PACKET_READ_TYPE(packet, u8, date.type);
-        AKI_PACKET_READ_TYPE(packet, f32, date.timestamp);
+        AKI_PACKET_READ_TYPE(packet, f32, date.range_start);
+        AKI_PACKET_READ_TYPE(packet, f32, date.range_end);
+        AKI_PACKET_READ_TYPE(packet, u32, date.meta);
         al_array_push(post->dates, date);
     }
     aki_packet_read_str(packet, &post->author.unique_id);
diff --git a/src/portal/src/post.c b/src/portal/src/post.c
index c5eb4b5..4cb0bcc 100644
--- a/src/portal/src/post.c
+++ b/src/portal/src/post.c
@@ -56,9 +56,9 @@ void camu_post_add_media(struct camu_post *post, u8 type, str *key, str *url, st
     }));
 }
 
-void camu_post_add_date(struct camu_post *post, u8 type, f32 timestamp)
+void camu_post_add_date(struct camu_post *post, u8 type, f32 range_start, f32 range_end, u32 meta)
 {
     al_array_push(post->dates, ((struct camu_post_date){
-        .type = type, .timestamp = timestamp
+        .type = type, .range_start = range_start, .range_end = range_end, .meta = meta
     }));
 }
diff --git a/src/portal/src/post.h b/src/portal/src/post.h
index 11fa9b6..a946968 100644
--- a/src/portal/src/post.h
+++ b/src/portal/src/post.h
@@ -20,6 +20,13 @@ enum {
 };
 
 enum {
+    CAMU_DATE_ESTIMATE = 1,
+    CAMU_DATE_GUESS = 1 << 1,
+    CAMU_DATE_EDITED_AT_UNKNOWN_TIME = 1 << 2,
+    CAMU_DATE_ASSUMED_TYPE = 1 << 3
+};
+
+enum {
     CAMU_MEDIA_UNKNOWN = 0,
     CAMU_MEDIA_FILE,
     CAMU_MEDIA_AUDIO,
@@ -46,7 +53,9 @@ typedef struct {
 
 struct camu_post_date {
     u8 type;
-    f32 timestamp;
+    f32 range_start;
+    f32 range_end;
+    u32 meta;
 };
 
 typedef array(struct camu_post_date) camu_dates_array;
@@ -98,5 +107,5 @@ void camu_post_reset(struct camu_post *post);
 void camu_post_clone(struct camu_post *dest, struct camu_post *src);
 
 // Python internal.
-void camu_post_add_date(struct camu_post *post, u8 type, f32 timestamp);
+void camu_post_add_date(struct camu_post *post, u8 type, f32 range_start, f32 range_end, u32 meta);
 void camu_post_add_media(struct camu_post *post, u8 type, str *key, str *url, str *ext, str *thumbnail_url, str *thumbnail_ext);
diff --git a/src/portal/src/search.c b/src/portal/src/search.c
index 7334c41..bcb0643 100644
--- a/src/portal/src/search.c
+++ b/src/portal/src/search.c
@@ -38,7 +38,7 @@ static struct camu_result_page *page_at_index(struct camu_search *search, u32 nu
     al_array_foreach_ptr(search->pages, i, page) {
         if (page->num == num) return page;
     }
-    al_array_push(search->pages, (struct camu_result_page){0});
+    al_array_push(search->pages, (struct camu_result_page){ 0 });
     page = &al_array_last(search->pages);
     page->num = num;
     al_array_init(page->posts);
@@ -47,10 +47,10 @@ static struct camu_result_page *page_at_index(struct camu_search *search, u32 nu
 }
 
 
-void camu_portal_init(struct camu_portal *portal, struct camu_post_cache *cache)
+void camu_portal_init(struct camu_portal_bridge *bridge, struct camu_post_cache *cache)
 {
-    portal->cache = cache;
-    al_array_init(portal->searches);
+    bridge->cache = cache;
+    al_array_init(bridge->searches);
 }
 
 static void camu_search_init_internal(struct camu_search *search)
@@ -59,7 +59,7 @@ static void camu_search_init_internal(struct camu_search *search)
     al_array_init(search->pages);
 }
 
-s32 camu_portal_create_search(struct camu_portal *portal, str *module, str *query)
+s32 camu_portal_create_search(struct camu_portal_bridge *bridge, str *module, str *query)
 {
     s32 id = portal_bridge_search(module, query);
     if (id >= 0) {
@@ -68,31 +68,31 @@ s32 camu_portal_create_search(struct camu_portal *portal, str *module, str *quer
         search->id = id;
         al_str_clone(&search->module, module);
         al_str_clone(&search->query, query);
-        search->portal = portal;
-        al_array_push(portal->searches, search);
+        search->bridge = bridge;
+        al_array_push(bridge->searches, search);
     }
     al_log_info("portal", "New search %x (%.*s).", id, AL_STR_PRINTF(query));
     return id;
 }
 
-struct camu_search *camu_portal_get_search(struct camu_portal *portal, s32 id)
+struct camu_search *camu_portal_get_search(struct camu_portal_bridge *bridge, s32 id)
 {
     struct camu_search *search;
-    al_array_foreach(portal->searches, i, search) {
+    al_array_foreach(bridge->searches, i, search) {
         if (search->id == id) return search;
     }
     return NULL;
 }
 
-void camu_portal_discard_search(struct camu_portal *portal, s32 id)
+void camu_portal_discard_search(struct camu_portal_bridge *bridge, s32 id)
 {
-    (void)portal;
+    (void)bridge;
     (void)id;
 }
 
-void camu_portal_close(struct camu_portal *portal)
+void camu_portal_close(struct camu_portal_bridge *bridge)
 {
-    (void)portal;
+    (void)bridge;
 }
 
 bool camu_search_get_page(struct camu_search *search, u32 num)
@@ -105,11 +105,11 @@ bool camu_search_get_page(struct camu_search *search, u32 num)
     if (portal_bridge_get_page(search, search->id, num) == -1) {
         return false;
     }
-    struct camu_portal *portal = search->portal;
-    if (portal->cache) {
+    struct camu_portal_bridge *bridge = search->bridge;
+    if (bridge->cache) {
         struct camu_post *post;
         al_array_foreach_ptr(al_array_at(search->pages, num).posts, i, post) {
-            camu_post_cache_push(portal->cache, post);
+            camu_post_cache_push(bridge->cache, post);
         }
     }
 out:
diff --git a/src/portal/src/search.h b/src/portal/src/search.h
index 4962170..1f43ded 100644
--- a/src/portal/src/search.h
+++ b/src/portal/src/search.h
@@ -15,22 +15,22 @@ struct camu_search {
     str query;
     u32 page;
     array(struct camu_result_page) pages;
-    struct camu_portal *portal;
+    struct camu_portal_bridge *bridge;
 };
 
-struct camu_portal {
-    struct camu_post_cache *cache;
+struct camu_portal_bridge {
     array(struct camu_search *) searches;
+    struct camu_post_cache *cache;
 };
 
 bool camu_python_init(void);
 void camu_python_close(void);
 
-void camu_portal_init(struct camu_portal *portal, struct camu_post_cache *cache);
-s32 camu_portal_create_search(struct camu_portal *portal, str *module_str, str *search_str);
-struct camu_search *camu_portal_get_search(struct camu_portal *portal, s32 id);
-void camu_portal_discard_search(struct camu_portal *portal, s32 id);
-void camu_portal_close(struct camu_portal *portal);
+void camu_portal_init(struct camu_portal_bridge *bridge, struct camu_post_cache *cache);
+s32 camu_portal_create_search(struct camu_portal_bridge *bridge, str *module_str, str *search_str);
+struct camu_search *camu_portal_get_search(struct camu_portal_bridge *bridge, s32 id);
+void camu_portal_discard_search(struct camu_portal_bridge *bridge, s32 id);
+void camu_portal_close(struct camu_portal_bridge *bridge);
 
 bool camu_search_get_page(struct camu_search *search, u32 num);
 
diff --git a/src/portal/tests/test.py b/src/portal/tests/test.py
deleted file mode 100644
index a5139f6..0000000
--- a/src/portal/tests/test.py
+++ /dev/null
@@ -1,51 +0,0 @@
-import json
-import httpx
-import sys
-sys.path.append('../py')
-from base import Method
-from post import DateType, PostEncoder, Media
-from modules import ALL_MODULES
-
-#ALL_MODULES['instagram'][0].init()
-#search = ALL_MODULES['instagram'][0].search('opium_00pium')
-#page = search.get_page(0)
-#print(page)
-#print(json.dumps(page, indent=4, cls=PostEncoder))
-
-d = Media()
-print(d)
-sys.exit(1)
-
-#module = ALL_MODULES['fanbox'][0]
-#module = ALL_MODULES['pixiv_web'][0]
-module = ALL_MODULES['instagram'][0]
-#module = ALL_MODULES['twitter'][0]
-module.init()
-
-search = module.search('opium_00pium')
-page = search.get_page(1)
-for unique_id in page:
-    post = module.get_item(unique_id)
-    print(json.dumps(post, indent=4, cls=PostEncoder))
-
-#post = module.get_item(page[0])
-#for key in post.media.keys():
-#  download_params = module.get_download(post.unique_id, key)
-#  for url in download_params['urls']:
-#      #response = module.do_request(Method.GET, url.url)
-#      r = httpx.get(url.url, headers=download_params['headers'])
-#      with open(f'test_{key}.{url.ext}', 'wb+') as f:
-#          f.write(r.content)
-
-#print(json.dumps(page, indent=4, cls=PostEncoder))
-#for i in range(0, len(post.media)):
-#    download_params = module.get_download(post.unique_id, i)
-#    r = httpx.get(download_params['url'], headers=download_params['headers'])
-#    if r.status_code != 200:
-#        print('error{}'.format(r.status_code))
-#    else:
-#        with open('test{}.png'.format(i), 'wb+') as f:
-#            f.write(r.content)
-
-#print(json.dumps(page, indent=4, cls=PostEncoder))
-#print(page)