Add camu

- Subprojects temporarily omitted

Signed-off-by: Andrew Opalach <andrew@akon.city>

31 files changed, 1912 insertions, 0 deletions

diff --git a/src/shoki/cpy/lib.pxd b/src/shoki/cpy/lib.pxd
new file mode 100644
index 0000000..5749141
--- /dev/null
+++ b/src/shoki/cpy/lib.pxd
@@ -0,0 +1,13 @@
+include "types.pxd"
+
+cdef extern from "<al/lib.h>":
+     void al_free(void *ptr)
+     void al_malloc(size_t size)
+     void al_strlen(char *s)
+     void al_bzero(void *s, size_t n)
+
+cdef extern from "<al/log.h>":
+     s32 al_log_info(const char *ns, const char *, ...)
+     s32 al_log_warn(const char *ns, const char *, ...)
+     s32 al_log_error(const char *ns, const char *, ...)
+     s32 al_log_debug(const char *ns, const char *, ...)
diff --git a/src/shoki/cpy/post.pxd b/src/shoki/cpy/post.pxd
new file mode 100644
index 0000000..0b0a97e
--- /dev/null
+++ b/src/shoki/cpy/post.pxd
@@ -0,0 +1,58 @@
+include "str.pxd"
+
+cdef extern from "../../shoki/src/post.h":
+    ctypedef struct sho_optional_int:
+        s64 i
+        bool set
+
+    cdef struct sho_post_date:
+        u8 type
+        s64 timestamp
+
+    cdef struct sho_post_media:
+        u8 type
+        str url
+        str thumbnail_url
+
+    ctypedef struct sho_dates_array:
+        u32 size
+        u32 alloc
+        sho_post_date *data
+
+    ctypedef struct sho_media_array:
+        u32 size
+        u32 alloc
+        sho_post_media *data
+
+    cdef struct sho_post_user:
+        str unique_id
+        str username
+        str display_name
+        str profile_image_url
+
+    cdef struct sho_post_ref:
+        str unique_id
+
+    cdef struct sho_post:
+        u16 version
+        u8 type
+        str unique_id
+        str raw_responses
+        str url
+        sho_dates_array dates
+        sho_post_user author
+        str title
+        str text
+        sho_optional_int likes
+        sho_optional_int reposts
+        sho_optional_int quotes
+        sho_optional_int comments
+        sho_optional_int views
+        sho_media_array media
+        sho_post_ref post
+        sho_post_ref quoted
+        sho_post_ref in_reply_to
+
+    void sho_post_reset(sho_post *post)
+    void sho_post_add_media(sho_post *post, u8 type, str *url, str *thumbnail_url)
+    void sho_post_add_date(sho_post *post, u8 type, s64 timestamp)
diff --git a/src/shoki/cpy/result.pxd b/src/shoki/cpy/result.pxd
new file mode 100644
index 0000000..1bebb55
--- /dev/null
+++ b/src/shoki/cpy/result.pxd
@@ -0,0 +1,8 @@
+include "post.pxd"
+
+cdef extern from "../../shoki/src/search.h":
+     cdef struct sho_result:
+          pass
+
+     void sho_result_add_post(sho_result *res, s32 page, sho_post *post)
+     void sho_result_add_to_list(sho_result *res, s32 page, str *unique_id)
diff --git a/src/shoki/cpy/setup.py b/src/shoki/cpy/setup.py
new file mode 100644
index 0000000..1b491f1
--- /dev/null
+++ b/src/shoki/cpy/setup.py
@@ -0,0 +1,6 @@
+from Cython.Build import cythonize
+from Cython.Compiler import Options
+from setuptools import Extension
+Options.embed = True
+alabaster_inc = "../../../subprojects/libalabaster/include"
+cythonize([Extension("shoki", ["shoki.pyx"], include_dirs=[alabaster_inc], extra_link_args=[])], language_level=3)
diff --git a/src/shoki/cpy/shoki.pyx b/src/shoki/cpy/shoki.pyx
new file mode 100644
index 0000000..6d1dd56
--- /dev/null
+++ b/src/shoki/cpy/shoki.pyx
@@ -0,0 +1,132 @@
+cimport lib
+cimport str
+cimport post
+cimport result
+
+import sys
+sys.path.append('./py')
+
+cdef public int log_info(char *msg) except -1:
+    return lib.al_log_info("shoki", "%s", msg)
+
+cdef public int log_warn(char *msg) except -1:
+    return lib.al_log_warn("shoki", "%s", msg)
+
+cdef public int log_error(char *msg) except -1:
+    return lib.al_log_error("shoki", "%s", msg)
+
+cdef public int log_debug(char *msg) except -1:
+    return lib.al_log_debug("shoki", "%s", msg)
+
+def encode_str(str):
+    return str.encode('UTF-8')
+
+def decode_str(str):
+    return str.decode('UTF-8')
+
+class OutputHook(object):
+    def __init__(self):
+        self.fd = sys.__stdout__
+
+    def __getattr__(self, attr):
+        return getattr(self.fd, attr)
+
+    def write(self, msg):
+        if len(msg) > 0 and msg != '\n':
+            log_debug(encode_str(msg))
+
+hook = OutputHook()
+sys.stdout = hook
+sys.stderr = hook
+
+class ShokiLogger():
+    def debug(self, msg):
+        log_debug(encode_str(msg))
+
+    def info(self, msg):
+        log_info(encode_str(msg))
+
+    def warning(self, msg):
+        log_warn(encode_str(msg))
+
+    def error(self, msg):
+        log_error(encode_str(msg))
+
+import log
+log.set_logger(ShokiLogger())
+
+def exception_handler(exception_type, exception, traceback):
+    log.error(repr(exception))
+sys.excepthook = exception_handler
+
+import client
+from post import PostType, DateType
+
+shoki = client.ShokiClient()
+for name, p in client.ALL_PROVIDERS.items():
+    if p[0].init():
+        shoki.register_provider(name, p)
+
+cdef public int sho_client_search(str.str *provider, str.str *query) except -1:
+    cdef char *c_str0 = str.al_str_to_c_str(provider)
+    cdef char *c_str1 = str.al_str_to_c_str(query)
+    ret = shoki.search(decode_str(c_str0), decode_str(c_str1))
+    lib.al_free(c_str0)
+    lib.al_free(c_str1)
+    return ret
+
+cdef public int py_str_to_str(str.str *s, char *py_c_str) except -1:
+    str.al_str_from(s, py_c_str)
+    return 0
+
+cdef public int sho_client_get_page(result.sho_result *res, int search_id, int num) except -1:
+    cdef post.sho_post cpost
+    cdef str.str url
+    cdef str.str thumbnail_url
+    cdef str.str unique_id
+    page = shoki.get_page(search_id, num)
+    if not page: return -1
+    for py_post in page['posts']:
+        post.sho_post_reset(&cpost)
+        cpost.version = py_post.version
+        cpost.type = py_post.type.value
+        py_str_to_str(&cpost.unique_id, encode_str(py_post.unique_id))
+        py_str_to_str(&cpost.url, encode_str(py_post.url))
+        for date in py_post.dates:
+            post.sho_post_add_date(&cpost, date.value, py_post.dates[date])
+        py_str_to_str(&cpost.author.unique_id, encode_str(py_post.author.unique_id))
+        py_str_to_str(&cpost.author.username, encode_str(py_post.author.username))
+        py_str_to_str(&cpost.author.display_name, encode_str(py_post.author.display_name))
+        py_str_to_str(&cpost.author.profile_image_url, encode_str(py_post.author.profile_image_url))
+        if cpost.type == PostType.REPOST.value:
+            py_str_to_str(&cpost.post.unique_id, encode_str(py_post.post.unique_id))
+            result.sho_result_add_post(res, num, &cpost)
+            continue
+        py_str_to_str(&cpost.title, encode_str(py_post.title))
+        py_str_to_str(&cpost.text, encode_str(py_post.text))
+        cpost.likes.set = py_post.likes != None
+        if cpost.likes.set:
+            cpost.likes.i = py_post.likes
+        cpost.reposts.set = py_post.reposts != None
+        if cpost.reposts.set:
+            cpost.reposts.i = py_post.reposts
+        cpost.quotes.set = py_post.quotes != None
+        if cpost.quotes.set:
+            cpost.quotes.i = py_post.quotes
+        cpost.comments.set = py_post.comments != None
+        if cpost.comments.set:
+            cpost.comments.i = py_post.comments
+        cpost.views.set = py_post.views != None
+        if cpost.views.set:
+            cpost.views.i = py_post.views
+        for media in py_post.media:
+            py_str_to_str(&url, encode_str(media.url))
+            py_str_to_str(&thumbnail_url, encode_str(media.thumbnail_url))
+            post.sho_post_add_media(&cpost, 0, &url, &thumbnail_url)
+        py_str_to_str(&cpost.quoted.unique_id, encode_str(py_post.quoted.unique_id))
+        py_str_to_str(&cpost.in_reply_to.unique_id, encode_str(py_post.in_reply_to.unique_id))
+        result.sho_result_add_post(res, num, &cpost)
+    for py_unique_id in page['list']:
+        py_str_to_str(&unique_id, encode_str(py_unique_id))
+        result.sho_result_add_to_list(res, num, &unique_id)
+    return 0
diff --git a/src/shoki/cpy/str.pxd b/src/shoki/cpy/str.pxd
new file mode 100644
index 0000000..a12ced4
--- /dev/null
+++ b/src/shoki/cpy/str.pxd
@@ -0,0 +1,11 @@
+include "types.pxd"
+
+cdef extern from "<al/str.h>":
+     ctypedef struct str:
+         u32 len
+         s32 alloc
+         char *data
+
+     void al_str_from(str *s, const char *str)
+     char *al_str_to_c_str(str *s)
+
diff --git a/src/shoki/cpy/types.pxd b/src/shoki/cpy/types.pxd
new file mode 100644
index 0000000..fec35a5
--- /dev/null
+++ b/src/shoki/cpy/types.pxd
@@ -0,0 +1,11 @@
+ctypedef unsigned char u8;
+ctypedef char s8;
+ctypedef unsigned short u16;
+ctypedef short s16;
+ctypedef unsigned int u32;
+ctypedef int s32;
+ctypedef unsigned long u64;
+ctypedef long s64;
+ctypedef float f32;
+ctypedef double f64;
+ctypedef bint bool
diff --git a/src/shoki/meson.build b/src/shoki/meson.build
new file mode 100644
index 0000000..557d497
--- /dev/null
+++ b/src/shoki/meson.build
@@ -0,0 +1,14 @@
+shoki_src = [
+  'src/search.c',
+  'src/post.c',
+  'src/packet_ext.c',
+  'src/post_cache.c'
+]
+shoki_inc = []
+shoki_deps = [common_deps]
+
+python = dependency('python-3.11-embed', required: true)
+shoki_deps += [python]
+
+shoki = declare_dependency(sources: shoki_src,
+  include_directories: shoki_inc, dependencies: shoki_deps)
diff --git a/src/shoki/py/__init__.py b/src/shoki/py/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/shoki/py/__init__.py
diff --git a/src/shoki/py/client.py b/src/shoki/py/client.py
new file mode 100644
index 0000000..3986060
--- /dev/null
+++ b/src/shoki/py/client.py
@@ -0,0 +1,51 @@
+import log
+import config
+from providers.twitter import TwitterProvider
+from providers.youtube import YoutubeProvider
+
+ALL_PROVIDERS = {
+    'twitter': (TwitterProvider(
+        config.TWITTER_ACCESS_TOKEN, config.TWITTER_ACCESS_TOKEN_SECRET,
+        config.TWITTER_CONSUMER_TOKEN, config.TWITTER_CONSUMER_TOKEN_SECRET,
+        config.TWITTER_COOKIES_PATH), []),
+    'youtube': (YoutubeProvider(), [])
+}
+
+class ShokiClient():
+    def __init__(self):
+        self.providers = {}
+        self.searches = {}
+        self.inc = 0
+
+    def register_provider(self, name, p):
+        log.info('Registered provider \'{}\'.'.format(name))
+        self.providers[name] = p
+
+    def get_providers(self):
+        return self.providers.keys()
+
+    def search(self, provider, query):
+        if provider not in self.providers:
+            return -1
+        search_id = self.inc
+        p = self.providers[provider]
+        try:
+            search = p[0].search(query, *p[1])
+            self.searches[search_id] = search
+            self.inc += 1
+        except Exception as e:
+            log.error(repr(e))
+            return -1
+        return search_id
+
+    def get_page(self, search_id, num):
+        if search_id not in self.searches:
+            return None
+        try:
+            return self.searches[search_id].get_page(num)
+        except Exception as e:
+            log.error(repr(e))
+            return None
+
+    def get_playback_data(self, unique_id):
+        return ''
diff --git a/src/shoki/py/config.def.py b/src/shoki/py/config.def.py
new file mode 100644
index 0000000..237d658
--- /dev/null
+++ b/src/shoki/py/config.def.py
@@ -0,0 +1,19 @@
+from pathlib import Path
+
+BASE_DIR = ''
+
+TWITTER_ACCESS_TOKEN = ''
+TWITTER_ACCESS_TOKEN_SECRET = ''
+TWITTER_CONSUMER_TOKEN = ''
+TWITTER_CONSUMER_TOKEN_SECRET = ''
+TWITTER_COOKIES_PATH = '{}/cookies_twitter.txt'.format(BASE_DIR)
+TWITTER_TIMEOUT = 5 # seconds
+
+FANBOX_SESSID = ''
+
+PIXIV_REFRESH_TOKEN = ''
+
+INSTAGRAM_USERNAME = ''
+INSTAGRAM_PASSWORD = ''
+INSTAGRAM_SETTINGS_PATH = Path('{}/ig_settings.json'.format(BASE_DIR))
+INSTAGRAM_USER_AGENT = 'Instagram 146.0.0.27.125 (iPhone12,1; iOS 13_3; en_US; en-US; scale=2.00; 1656x3584; 190542906)'
diff --git a/src/shoki/py/log.py b/src/shoki/py/log.py
new file mode 100644
index 0000000..c36c54e
--- /dev/null
+++ b/src/shoki/py/log.py
@@ -0,0 +1,30 @@
+class DefaultLogger():
+    def debug(self, msg):
+        print(msg)
+
+    def info(self, msg):
+        print(msg)
+
+    def warning(self, msg):
+        print(msg)
+
+    def error(self, msg):
+        print(msg)
+
+LOGGER = DefaultLogger()
+
+def set_logger(logger):
+    global LOGGER
+    LOGGER = logger
+
+def debug(msg):
+    LOGGER.debug(msg)
+
+def info(msg):
+    LOGGER.info(msg)
+
+def warning(msg):
+    LOGGER.warning(msg)
+
+def error(msg):
+    LOGGER.error(msg)
diff --git a/src/shoki/py/post.py b/src/shoki/py/post.py
new file mode 100644
index 0000000..d489298
--- /dev/null
+++ b/src/shoki/py/post.py
@@ -0,0 +1,70 @@
+import dataclasses
+from dataclasses import field
+from typing import Optional
+from json import JSONEncoder
+from enum import Enum
+
+class PostType(Enum):
+    UNKNOWN = -1
+    POST = 0
+    REPOST = 1
+
+class DateType(Enum):
+    CREATED = 0
+    EDITED = 1
+    ARCHIVED = 2
+
+@dataclasses.dataclass
+class Media():
+    url: str = ''
+    base64_data: str = ''
+
+@dataclasses.dataclass
+class Image(Media):
+    type: str = 'image'
+    thumbnail_url: str = ''
+
+@dataclasses.dataclass
+class Video(Media):
+    type: str = 'video'
+    thumbnail_url: str = ''
+
+@dataclasses.dataclass
+class Audio(Media):
+    type: str = 'audio'
+
+@dataclasses.dataclass
+class User():
+    unique_id: str = ''
+    username: str = ''
+    display_name: str = ''
+    profile_image_url: str = ''
+
+@dataclasses.dataclass
+class PostRef():
+    unique_id: str = ''
+
+@dataclasses.dataclass
+class Post():
+    version: int = 2
+    type: 'PostType' = PostType.UNKNOWN
+    unique_id: str = ''
+    raw_responses: dict[str, str] = field(default_factory=lambda: {})
+    url: str = ''
+    dates: dict['DateType', int] = field(default_factory=lambda: {})
+    author: 'User' = field(default_factory=lambda: User())
+    title: str = ''
+    text: str = ''
+    likes: Optional[int] = None
+    reposts: Optional[int] = None
+    quotes: Optional[int] = None
+    comments: Optional[int] = None
+    views: Optional[int] = None
+    media: list['Media'] = field(default_factory=lambda: [])
+    post: 'PostRef' = field(default_factory=lambda: PostRef())
+    quoted: 'PostRef' = field(default_factory=lambda: PostRef())
+    in_reply_to: 'PostRef' = field(default_factory=lambda: PostRef())
+
+class PostEncoder(JSONEncoder):
+    def default(self, obj):
+        return obj.__dict__
diff --git a/src/shoki/py/provider.py b/src/shoki/py/provider.py
new file mode 100644
index 0000000..a719d01
--- /dev/null
+++ b/src/shoki/py/provider.py
@@ -0,0 +1,39 @@
+import httpx
+import json
+import log
+
+class Provider():
+    def __init__(self):
+        self.cookies = {}
+        self.headers = {}
+        self.unique_id_map = {}
+
+    def init(self):
+        return True
+
+    # TODO: retries.
+    def get(self, url, params = None):
+        log.debug('HTTP request {} {}.'.format(url, params))
+        r = httpx.get(url, params=params, cookies=self.cookies, headers=self.headers)
+        # TODO: Non-200 = err, support redirects (if not already done in httpx)?
+        if r.status_code == 403 or r.status_code == 404:
+            log.error('HTTP {}.'.format(r.status_code))
+            return None
+        log.debug('HTTP {}.'.format(r.status_code))
+        return r
+
+    def get_json(self, url, params = None):
+        try:
+            r = self.get(url, params)
+            if not r:
+                return None
+            return json.loads(r.content)
+        except json.JSONDecodeError as e:
+            log.error(repr(e))
+            return None
+
+    def get_obj_from_unique_id(self, unique_id):
+        return None
+
+    def search(self, query, *extra_args):
+        return None
diff --git a/src/shoki/py/providers/__init__.py b/src/shoki/py/providers/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/shoki/py/providers/__init__.py
diff --git a/src/shoki/py/providers/fanbox.py b/src/shoki/py/providers/fanbox.py
new file mode 100644
index 0000000..7739be3
--- /dev/null
+++ b/src/shoki/py/providers/fanbox.py
@@ -0,0 +1,130 @@
+import urllib.parse
+import log
+from plugin import Plugin
+from search import Search
+from post import Post, Image
+
+BASE_URL = "https://api.fanbox.cc/"
+
+PAGES_PER_QUERY = 2
+
+class FanboxSearch(Search):
+    def __init__(self, plugin, creator_id):
+        super().__init__()
+        self.p = plugin
+        self.creator_id = creator_id
+        self.pagination_pages = []
+        self.pagination_complete = False
+
+    def paginate(self):
+        res = self.p.get_json(BASE_URL + 'post.paginateCreator', { 'creatorId': self.creator_id })
+        if not res or 'body' not in res or len(res['body']) == 0:
+            return False
+        for p in res['body']:
+            params = urllib.parse.parse_qs(urllib.parse.urlparse(p).query)
+            page = {
+                'url': p,
+                'params': params,
+                'data': None,
+                'limit': -1 if 'limit' not in params else int(params['limit'][0])
+            }
+            self.pagination_pages.append(page)
+        return True
+
+    def load_pagination_page(self, p):
+        if not p['data']:
+            res = self.p.get_json(BASE_URL + 'post.listCreator', p['params'])
+            if not res:
+                return False
+            p['data'] = res['body']['items']
+        # The get_post_at_index logic assumes limit will always be exact.
+        p['limit'] = len(p['data'])
+        return True
+
+    def get_post_at_index(self, index):
+        if not self.pagination_complete:
+            if not self.paginate():
+                return None
+            self.pagination_complete = True
+        page = None
+        count = 0
+        for p in self.pagination_pages:
+            if p['limit'] == -1:
+                if not self.load_pagination_page(p):
+                    return None
+            ncount = count + p['limit']
+            if ncount > index:
+                page = p
+                break
+            count = ncount
+        if not page or not self.load_pagination_page(page) or count + page['limit'] <= index:
+            return None
+        return page['data'][index - count]
+
+    def add_images_to_map(self, body):
+        if 'imageMap' not in body:
+            return
+        for k in body['imageMap'].keys():
+            self.p.image_map[k] = body['imageMap'][k]['originalUrl']
+
+    def load_page(self, index):
+        self.pages[index] = []
+        s = index * PAGES_PER_QUERY
+        for i in range(s, s + PAGES_PER_QUERY):
+            p = self.get_post_at_index(i)
+            # Break to return what we have, which could be less than expected.
+            if not p:
+                self.completed = True
+                break
+            unique_id = 'fanbox:{}'.format(p['id'])
+            if unique_id not in self.p.unique_id_map:
+                res = self.p.get_json(
+                    BASE_URL + 'post.info', { 'postId': p['id'] }
+                )
+                if not res or 'body' not in res:
+                    # TODO: Log this.
+                    continue
+                if res['body']['type'] == 'article':
+                    self.add_images_to_map(res['body']['body'])
+                self.p.unique_id_map[unique_id] = res['body']
+            # TODO: Should store Post object here instead.
+            body = self.p.unique_id_map[unique_id]
+            text = ''
+            media = []
+            # If body is None, that post is restricted (not subbed or from a higher tier than your sub)
+            if body['body']:
+                if body['type'] == 'image':
+                    text = body['body']['text']
+                    for i in body['body']['images']:
+                        media.append(Image(url=i['originalUrl'], thumbnail_url=''))
+                elif body['type'] == 'article':
+                    for b in body['body']['blocks']:
+                        if b['type'] == 'p':
+                            text += b['text'] + '\n'
+                        elif b['type'] == 'image':
+                            media.append(Image(url=self.p.image_map[b['imageId']], thumbnail_url=''))
+                        else:
+                            log.warn('Unhandled block type.')
+                else:
+                    log.warn('Unhandled post type.')
+            # TODO: Raw response
+            self.pages[index].append(Post(
+                unique_id=unique_id, url='', title=body['title'], text=text, media=media))
+        return True
+
+class FanboxPlugin(Plugin):
+    def __init__(self, sessid):
+        super().__init__()
+        self.image_map = {}
+        self.headers.update({
+            'accept': 'application/json, text/plain, */*',
+            'accept-encoding': 'gzip, deflate, br',
+            'accept-language': 'ja,en-US;q=0.9,en;q=0.8',
+            'origin': 'https://www.fanbox.cc',
+            'referer': 'https://www.fanbox.cc/',
+            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36'
+        })
+        self.cookies['FANBOXSESSID'] = sessid
+
+    def search(self, query, *extra_args):
+        return FanboxSearch(self, query)
diff --git a/src/shoki/py/providers/instagram.py b/src/shoki/py/providers/instagram.py
new file mode 100644
index 0000000..0146ba3
--- /dev/null
+++ b/src/shoki/py/providers/instagram.py
@@ -0,0 +1,96 @@
+import os
+import config
+import log
+from plugin import Plugin
+from search import Search
+from instagrapi import Client
+from instagrapi.exceptions import UserNotFound, LoginRequired, ChallengeRequired
+from post import User, Post, Image, Video
+
+REACH_LIMIT = 3
+
+class InstagramSearch(Search):
+    def __init__(self, cl, pk):
+        super().__init__()
+        self.cl = cl
+        self.pk = pk
+        self.pagination_page = 0
+        self.pagination_cursor = None
+
+    def load_page(self, index):
+        if index >= self.pagination_page + REACH_LIMIT:
+            return False
+        request_satisfied = False
+        for _ in range(self.pagination_page, index + 1):
+            if self.pagination_page > 0 and not self.pagination_cursor:
+                self.completed = True
+                return False
+            try:
+                media, self.pagination_cursor = self.cl.user_medias_paginated_v1(
+                    self.pk, 0, end_cursor=self.pagination_cursor)
+            except LoginRequired:
+                return False
+            except ChallengeRequired:
+                return False
+            self.pages[self.pagination_page] = []
+            for m in media:
+                unique_id = 'instagram:p:{}'.format(m.id)
+                media = []
+                if m.media_type == 8: # Album
+                    for r in m.resources:
+                        if r.media_type == 1: # Photo
+                            media.append(Image(url=r.thumbnail_url))
+                        elif r.media_type == 2: # Video
+                            media.append(Video(url=r.video_url, thumbnail_url=r.thumbnail_url))
+                elif m.media_type == 2: # Video
+                    #print(m.product_type) # feed, igtv, or clips
+                    media.append(Video(url=m.video_url, thumbnail_url=m.thumbnail_url))
+                elif m.media_type == 1: # Photo
+                    media.append(Image(url=m.thumbnail_url))
+                kwargs = {}
+                kwargs['unique_id'] = unique_id
+                kwargs['raw_responses'] = { 'tile': m.json() }
+                kwargs['url'] = 'https://www.instagram.com/p/{}'.format(m.code)
+                user = User(unique_id='instagram:u:{}'.format(m.user.pk))
+                if m.user.username:
+                    user.username = m.user.username
+                if m.user.full_name:
+                    user.display_name = m.user.full_name
+                kwargs['author'] = user
+                kwargs['title'] = m.title
+                kwargs['text'] = m.caption_text
+                kwargs['media'] = media
+                self.pages[self.pagination_page].append(Post(**kwargs))
+            if self.pagination_page == index:
+                request_satisfied = True
+            self.pagination_page += 1
+        return request_satisfied
+
+class InstagramPlugin(Plugin):
+    def __init__(self):
+        super().__init__()
+        self.cl = Client()
+        self.cl.set_user_agent(config.INSTAGRAM_USER_AGENT)
+
+    def init(self):
+        if os.path.exists(config.INSTAGRAM_SETTINGS_PATH):
+            self.cl.load_settings(config.INSTAGRAM_SETTINGS_PATH)
+            if self.cl.login(config.INSTAGRAM_USERNAME, config.INSTAGRAM_PASSWORD):
+                return True
+        if not self.cl.login(config.INSTAGRAM_USERNAME, config.INSTAGRAM_PASSWORD):
+            return False
+        self.cl.dump_settings(config.INSTAGRAM_SETTINGS_PATH)
+        return True
+
+    def search(self, query, *extra_args):
+        try:
+            user = self.cl.user_info_by_username_v1(query)
+        except UserNotFound:
+            return None
+        except LoginRequired:
+            log.error('Login required.')
+            return None
+        except ChallengeRequired:
+            log.error('Challenge required.')
+            return None
+        return InstagramSearch(self.cl, user.pk)
diff --git a/src/shoki/py/providers/pixiv.py b/src/shoki/py/providers/pixiv.py
new file mode 100644
index 0000000..9276d61
--- /dev/null
+++ b/src/shoki/py/providers/pixiv.py
@@ -0,0 +1,113 @@
+import json
+import requests
+from post import User, Image, Post
+from search import Search
+from plugin import Plugin
+from query_parser import QueryParser
+from pixivpy3 import *
+
+class PixivBase(Search):
+    def __init__(self, userdata, arg):
+        super().__init__()
+        self.plugin = userdata
+        self.next_page = 0
+
+    def api_load_page(self, index):
+        return False, None
+
+    def load_page(self, index):
+        request_satisfied = False
+        for i in range(self.next_page, index + 1):
+            if not self.next_qs:
+                break
+            self.pages[i] = []
+            res, self.next_qs = self.api_load_page(i)
+            if not res:
+                break
+            if i == index:
+                request_satisfied = True
+            self.next_page += 1
+        return request_satisfied
+
+    def make_post(self, post):
+        if len(post['meta_pages']) == 0:
+            post['meta_pages'].append({
+                'image_urls': {
+                    "medium": post['image_urls']['medium'],
+                    "original": post['meta_single_page']['original_image_url']
+                }
+            })
+        kwargs = {}
+        kwargs['unique_id'] = 'pixiv:i:{}'.format(post['id'])
+        kwargs['raw_responses'] = { 'api': json.dumps(post) }
+        kwargs['url'] = 'https://www.pixiv.net/en/artworks/{}'.format(post['id'])
+        kwargs['author'] = User(
+            unique_id='pixiv:u:{}'.format(post['user']['id']),
+            username=post['user']['account'], display_name=post['user']['name'])
+        kwargs['title'] = post['title']
+        kwargs['text'] = post['caption']
+        kwargs['likes'] = post['total_bookmarks']
+        if 'total_comments' in post:
+            kwargs['comments'] = post['total_comments']
+        media = []
+        for img in post['meta_pages']:
+            media.append(Image(
+                url=img['image_urls']['original'], thumbnail_url=img['image_urls']['medium']))
+        kwargs['media'] = media
+        return Post(**kwargs)
+
+class PixivFollowing(PixivBase):
+    def __init__(self, userdata, arg):
+        super().__init__(userdata, arg)
+        self.next_qs = { "user_id": arg, "restrict": "public" }
+
+    def api_load_page(self, index):
+        obj = self.plugin.api.user_following(**self.next_qs)
+        if 'user_previews' not in obj:
+            return False, None
+        for u in obj['user_previews']:
+            user = u['user']
+            self.pages[index].append(User(
+                unique_id='pixiv:u:{}'.format(user['id']), username=user['account'],
+                display_name=user['name']))
+        return True, self.plugin.api.parse_qs(obj['next_url'])
+
+class PixivUser(PixivBase):
+    def __init__(self, userdata, arg):
+        super().__init__(userdata, arg)
+        self.next_qs = { 'user_id': arg, 'type': 'illust' }
+
+    def api_load_page(self, index):
+        obj = self.plugin.api.user_illusts(**self.next_qs)
+        if 'illusts' not in obj:
+            return False, None
+        for post in obj['illusts']:
+            self.pages[index].append(self.make_post(post))
+        return True, self.plugin.api.parse_qs(obj['next_url'])
+
+class PixivBookmarks(PixivBase):
+    def __init__(self, userdata, arg):
+        super().__init__(userdata, arg)
+        self.next_qs = { 'user_id': arg, 'restrict': 'public' }
+
+    def api_load_page(self, index):
+        obj = self.plugin.api.user_bookmarks_illust(**self.next_qs)
+        if 'illusts' not in obj:
+            return False, None
+        for post in obj['illusts']:
+            self.pages[index].append(self.make_post(post))
+        return True, self.plugin.api.parse_qs(obj['next_url'])
+
+class PixivPlugin(Plugin):
+    def __init__(self, refresh_token):
+        super().__init__()
+        self.parser = QueryParser(self, 'following')
+        self.parser.add_command('following', PixivFollowing)
+        self.parser.add_command('user', PixivUser)
+        self.parser.add_command('bookmarks', PixivBookmarks)
+        self.api = AppPixivAPI()
+        self.api.requests = requests.Session()
+        self.api.auth(refresh_token=refresh_token)
+
+    def search(self, query, *extra_args):
+        return self.parser.parse_query(query)
diff --git a/src/shoki/py/providers/searx.py b/src/shoki/py/providers/searx.py
new file mode 100644
index 0000000..b536f28
--- /dev/null
+++ b/src/shoki/py/providers/searx.py
@@ -0,0 +1,93 @@
+import sys
+sys.path.append('../../../subprojects/searxng')
+import json
+from searx import settings
+from searx.utils import  gen_useragent
+from searx.engines import load_engine, register_engine
+from plugin import Plugin
+from search import Search
+from post import Post, Image
+
+# TODO: Put these on the Plugin object.
+ENGINES = {}
+
+ENGINE_INITIAL_INDEX = {
+    'google images': 0,
+    'google': 1,
+    'bing images': 0
+}
+
+class SearxSearch(Search):
+    def __init__(self, plugin, query, engine):
+        super().__init__()
+        global ENGINES
+        self.query = query
+        self.plugin = plugin
+        self.key = engine.replace(' ', '_')
+        self.engine = ENGINES[engine]
+        self.index = ENGINE_INITIAL_INDEX[engine]
+
+    def load_page(self, index):
+        self.pages[index] = []
+        res = self.plugin.send_http_request(self.engine.request(self.query, {
+            'language': 'en-US',
+            'safesearch': 0,
+            'time_range': None,
+            'pageno': self.index + index,
+            'cookies': self.plugin.cookies,
+            'headers': self.plugin.headers,
+            'data': None
+        }))
+        if not res:
+            return False
+        try:
+            ret = self.engine.response(res)
+        except:
+            return False
+        if len(ret) == 0:
+            # TODO: Test.
+            self.completed = True
+            return False
+        for i, p in enumerate(ret):
+            if 'url' not in p:
+                continue
+            # Don't add this to the global map because it's not actually unique.
+            unique_id = '{}:{}_{}:{}'.format(self.key, self.query, i, index)
+            media = []
+            if 'img_src' in p:
+                media.append(Image(url=p['img_src'], thumbnail_url=''))
+            elif 'image_url' in p:
+                media.append(Image(url=p['image_url'], thumbnail_url=''))
+            kwargs = {}
+            kwargs['unique_id'] = unique_id
+            kwargs['raw_responses'] = { 'result': json.dumps(p) }
+            kwargs['url'] = p['url']
+            kwargs['title'] = p['title']
+            kwargs['text'] = p['content']
+            kwargs['media'] = media
+            self.pages[index].append(Post(**kwargs))
+        return True
+
+class SearxPlugin(Plugin):
+    def __init__(self):
+        super().__init__()
+        global ENGINES
+        self.headers = {
+            'User-Agent': gen_useragent(),
+            'Accept-Language': 'en-US,en;q=0.5',
+        }
+        for engine_data in settings['engines']:
+            engine = load_engine(engine_data)
+            if engine:
+                register_engine(engine)
+                ENGINES[engine.name] = engine
+
+    def send_http_request(self, params):
+        self.cookies.update(params['cookies'])
+        self.headers.update(params['headers'])
+        return self.get(params['url'], params=params['data'])
+
+    def search(self, query, *extra_args):
+        if not extra_args:
+            return None
+        return SearxSearch(self, query, extra_args[0])
diff --git a/src/shoki/py/providers/twitter.py b/src/shoki/py/providers/twitter.py
new file mode 100644
index 0000000..2fc0faf
--- /dev/null
+++ b/src/shoki/py/providers/twitter.py
@@ -0,0 +1,425 @@
+import json
+import config
+import http.cookiejar
+from datetime import timezone
+from socketserver import UDPServer
+
+import log
+from provider import Provider
+from search import Search
+from post import PostType, DateType, PostRef, Post, User, Image, Video
+from query_parser import QueryParser
+
+# Official Twitter API
+#from twitter import Twitter2, TwitterError, OAuth
+
+# GraphQL Scrape API
+import snscrape.modules.twitter
+from snscrape.base import ScraperException
+from snscrape.modules.twitter import (TwitterSearchScraper, TwitterProfileScraper,
+    TwitterUserScraper, TwitterTweetScraper, TweetRef)
+
+# Don't attempt to load a page 4 or more pages past the current page.
+REACH_LIMIT = 4
+
+# Quite incomplete API based backend. Should *not* be used for archiving.
+# HOLD: I can't test this without paying $100 for the X API WTFFF.
+'''
+class TwitterBase(Search):
+    ALL_PARAMS = {
+        'tweet.fields': 'attachments,author_id,context_annotations,conversation_id,created_at,entities,geo,id,in_reply_to_user_id,lang,public_metrics,possibly_sensitive,referenced_tweets,reply_settings,source,text,withheld',
+        'user.fields': 'created_at,description,entities,id,location,name,pinned_tweet_id,profile_image_url,protected,public_metrics,url,username,verified,withheld',
+        'media.fields': 'duration_ms,height,media_key,preview_image_url,type,url,width,public_metrics',
+        'place.fields': 'contained_within,country,country_code,full_name,geo,id,name,place_type',
+        'poll.fields': 'duration_minutes,end_datetime,id,options,voting_status',
+    }
+    ALL_EXPRESSIONS = 'author_id,referenced_tweets.id,referenced_tweets.id.author_id,entities.mentions.username,attachments.poll_ids,attachments.media_keys,in_reply_to_user_id,geo.place_id'
+
+    SOME_PARAMS = {
+        'tweet.fields': 'attachments,author_id,text,entities,referenced_tweets',
+        'user.fields': 'id,name,profile_image_url,url,username',
+        'media.fields': 'duration_ms,height,media_key,preview_image_url,type,url,width,public_metrics',
+        'place.fields': '',
+        'poll.fields': '',
+    }
+    SOME_EXPRESSIONS = 'author_id,referenced_tweets.id,referenced_tweets.id.author_id,entities.mentions.username,attachments.media_keys,in_reply_to_user_id'
+
+    def __init__(self, userdata, arg):
+        super().__init__()
+        self.provider = userdata
+        self.params = self.SOME_PARAMS.copy()
+        self.media_map = {}
+        self.user_id = None
+        self.pagination_page = 0
+        self.pagination_token = None
+        self.arg = arg
+
+    def add_attachemnts(self, l, r, data):
+        if 'media_keys' not in data:
+            return
+        for m in data['media_keys']:
+            if m in r:
+                continue
+            r.append(m)
+            if m in self.media_map:
+                l.append(Image(url=self.media_map[m], thumbnail_url=''))
+
+    def add_entity_urls(self, data):
+        if 'urls' in data:
+            for u in data['urls']:
+                if 'media_key' in u and u['media_key'] not in self.media_map:
+                    self.media_map[u['media_key']] = u['expanded_url']
+
+    def make_post(self, tweet):
+        print(json.dumps(tweet, indent=4))
+        media = []
+        repeats = []
+        if 'referenced_tweets' in t:
+            print(len(referenced_tweets))
+            #for rt in t['referenced_tweets']:
+            #    if rt['id'] in self.tweet_map:
+            #        rrt = self.tweet_map[rt['id']]
+            #        if 'entities' in rrt:
+            #            self.add_entity_urls(rrt['entities'])
+            #        if 'attachments' in rrt:
+            #            self.add_attachemnts(media, repeats, rrt['attachments'])
+        if 'attachments' in t:
+            self.add_attachemnts(media, repeats, t['attachments'])
+        unique_id = 'twitter:t:{}'.format(t['id'])
+        url = 'https://twitter.com/{}/status/{}'.format(t['author_id'], t['id'])
+        kwargs = {}
+        kwargs['unique_id'] = unique_id
+        kwargs['raw_responses'] = { 'tweet' : json.dumps(t) }
+        kwargs['url'] = url
+        kwargs['author'] = User(unique_id='twitter:u:{}'.format(t['author_id']))
+        kwargs['title'] = ''
+        kwargs['text'] = t['text']
+        kwargs['media'] = media
+        return Post(**kwargs)
+
+    def add_items_from_search(self, data):
+        if 'includes' in data:
+            includes = data['includes']
+            if 'media' in includes:
+                for i in includes['media']:
+                    if 'url' in i:
+                        self.media_map[i['media_key']] = i['url']
+            #if 'tweets' in includes:
+            #    for t in includes['tweets']:
+            #        self.tweet_map[t['id']] = t
+        if 'data' not in data or not data['data']:
+            return False
+        l = data['data'] if type(data['data']) == list else [data['data']]
+        for t in l:
+            if 'entities' in t:
+                self.add_entity_urls(t['entities'])
+            post = self.make_post(t)
+            self.pages[self.pagination_page].append(post)
+        return True
+
+    def add_user_ids_from_search(self, data):
+        if not data['data']:
+            return False
+        for u in data['data']:
+            self.pages[self.pagination_page].append(User(
+                'twitter:u:{}'.format(u['id']), username=u['username'], display_name=u['name']))
+        return True
+
+    def should_process_index(self, index):
+        if index - self.pagination_page >= REACH_LIMIT:
+            return False
+        if self.pagination_page > 0 and not self.pagination_token:
+            self.completed = True
+            return False
+        if self.pagination_token:
+            self.params['pagination_token'] = self.pagination_token
+        self.pages[self.pagination_page] = []
+        return True
+
+    def handle_data(self, data, user_ids=False):
+        if 'next_token' not in data['meta'].keys():
+            self.pagination_token = None
+        else:
+            self.pagination_token = data['meta']['next_token']
+        if user_ids:
+            if not self.add_user_ids_from_search(data):
+                return False
+        else:
+            if not self.add_items_from_search(data):
+                return False
+        self.pagination_page += 1
+        return True
+
+    def get_user_id(self, username):
+        if self.user_id:
+            return True
+        try:
+            data = self.provider.t.users.by.username._username(
+                _username=username, _timeout=config.TWITTER_TIMEOUT)
+        except TwitterError as e:
+            log.error(repr(e))
+            return False
+        self.user_id = data['data']['id']
+        return True
+
+class TwitterSearch(TwitterBase):
+    def load_page(self, index):
+        for _ in range(self.pagination_page, index + 1):
+            if not self.should_process_index(index):
+                return False
+            try:
+                data = self.provider.t.tweets.search.recent(
+                    query=self.arg, expansions=self.SOME_EXPRESSIONS, params=self.params,
+                    sort_order='relevancy', max_results=25, _timeout=config.TWITTER_TIMEOUT)
+            except TwitterError as e:
+                log.error(repr(e))
+                return False
+            return self.handle_data(data)
+
+class TwitterUser(TwitterBase):
+    def __init__(self, userdata, arg):
+        if arg.startswith('@'):
+            arg = arg[1:]
+        super().__init__(userdata, arg)
+
+    def load_page(self, index):
+        for _ in range(self.pagination_page, index + 1):
+            if not self.should_process_index(index):
+                return False
+            if not self.get_user_id(self.arg):
+                return False
+            try:
+                data = self.provider.t.users._id.tweets(
+                    _id=self.user_id, expansions=self.SOME_EXPRESSIONS, params=self.params,
+                    max_results=25, _timeout=config.TWITTER_TIMEOUT)
+            except TwitterError as e:
+                log.error(repr(e))
+                return False
+            return self.handle_data(data)
+
+class TwitterTimeline(TwitterBase):
+    def __init__(self, userdata, arg):
+        if not arg:
+            arg = 'pizzabelly'
+        super().__init__(userdata, arg)
+        self.params['exclude'] = 'replies'
+
+    def load_page(self, index):
+        for _ in range(self.pagination_page, index + 1):
+            if not self.should_process_index(index):
+                return False
+            if not self.get_user_id(self.arg):
+                return False
+            try:
+                data = self.provider.t.users._id.timelines.reverse_chronological(
+                    _id=self.user_id, expansions=self.SOME_EXPRESSIONS, params=self.params,
+                    max_results=25, _timeout=config.TWITTER_TIMEOUT)
+            except TwitterError as e:
+                log.error(repr(e))
+                return False
+            return self.handle_data(data)
+
+class TwitterLikes(TwitterBase):
+    def __init__(self, userdata, arg):
+        if arg.startswith('@'):
+            arg = arg[1:]
+        super().__init__(userdata, arg)
+
+    def load_page(self, index):
+        for _ in range(self.pagination_page, index + 1):
+            if not self.should_process_index(index):
+                return False
+            if not self.get_user_id(self.arg):
+                return False
+            try:
+                data = self.provider.t.users._id.liked_tweets(
+                    _id=self.user_id, expansions=self.SOME_EXPRESSIONS, params=self.params,
+                    max_results=25, _timeout=config.TWITTER_TIMEOUT)
+            except TwitterError as e:
+                log.error(repr(e))
+                return False
+            return self.handle_data(data)
+
+class TwitterTweet(TwitterBase):
+    def load_page(self, index):
+        for _ in range(self.pagination_page, index + 1):
+            if not self.should_process_index(index):
+                return False
+            try:
+                data = self.provider.t.tweets(
+                    ids=self.arg, expansions=self.SOME_EXPRESSIONS, params=self.params,
+                    _timeout=config.TWITTER_TIMEOUT)
+            except TwitterError as e:
+                log.error(repr(e))
+                return False
+            return self.handle_data(data)
+
+class TwitterFollowing(TwitterBase):
+    def __init__(self, userdata, arg):
+        if arg.startswith('@'):
+            arg = arg[1:]
+        super().__init__(userdata, arg)
+        del self.params['media.fields']
+        del self.params['place.fields']
+        del self.params['poll.fields']
+
+    def load_page(self, index):
+        for _ in range(self.pagination_page, index + 1):
+            if not self.should_process_index(index):
+                return False
+            if not self.get_user_id(self.arg):
+                return False
+            try:
+                data = self.provider.t.users._id.following(
+                    _id=self.user_id, params=self.params, max_results=25, _timeout=config.TWITTER_TIMEOUT)
+            except TwitterError as e:
+                log.error(repr(e))
+                return False
+            return self.handle_data(data, True)
+'''
+
+class TwitterScrapeBase(Search):
+    # Posts per emulated page because snscrape returns an iterator.
+    POSTS_PER_PAGE = 8
+
+    def __init__(self, userdata, arg):
+        super().__init__()
+        self.provider = userdata
+        self.cookies = self.provider.cookies
+        self.arg = arg
+        self.iterator = None
+        self.iter_page = 0
+        self.sent_hashes = []
+
+    def parse_media(self, l):
+        media = []
+        for m in l:
+            if isinstance(m, snscrape.modules.twitter.Photo):
+                media.append(Image(url=m.fullUrl, thumbnail_url=m.previewUrl))
+            elif isinstance(m, snscrape.modules.twitter.Video):
+                videos = sorted(m.variants, key=lambda x: x.bitrate if x.bitrate else 0, reverse=True)
+                media.append(Video(url=videos[0].url, thumbnail_url=m.thumbnailUrl))
+            elif isinstance(m, snscrape.modules.twitter.Gif):
+                gifs = sorted(m.variants, key=lambda x: x.bitrate if x.bitrate else 0, reverse=True)
+                media.append(Video(url=gifs[0].url, thumbnail_url=m.thumbnailUrl))
+        return media
+
+    def make_post(self, tweet):
+        kwargs = {}
+        kwargs['unique_id'] = 'twitter:t:{}'.format(tweet.id)
+        if isinstance(tweet, TweetRef):
+            return Post(**kwargs)
+        kwargs['raw_responses'] = tweet.rawResponses
+        h = tweet.rawResponses['hash']
+        if h not in self.sent_hashes:
+            self.sent_hashes.append(h)
+        else:
+            del kwargs['raw_responses']['api']
+        kwargs['url'] = tweet.url
+        kwargs['dates'] = {
+            DateType.CREATED: tweet.date.replace(tzinfo=timezone.utc).timestamp()
+        }
+        author = User(unique_id='twitter:u:{}'.format(tweet.user.id), username='@' + tweet.user.username,
+            profile_image_url=tweet.user.profileImageUrl)
+        if tweet.user.displayname:
+            author.display_name = tweet.user.displayname
+        kwargs['author'] = author
+        if tweet.retweetedTweet:
+            kwargs['type'] = PostType.REPOST
+            kwargs['post'] = PostRef(unique_id='twitter:t:{}'.format(tweet.retweetedTweet.id))
+            return Post(**kwargs)
+        kwargs['type'] = PostType.POST
+        kwargs['text'] = tweet.rawContent
+        kwargs['likes'] = tweet.likeCount
+        kwargs['reposts'] = tweet.retweetCount
+        kwargs['quotes'] = tweet.quoteCount
+        kwargs['comments'] = tweet.replyCount
+        kwargs['views'] = tweet.viewCount
+        if tweet.media:
+            kwargs['media'] = self.parse_media(tweet.media)
+        if tweet.inReplyToTweetId:
+            kwargs['in_reply_to'] = PostRef(unique_id='twitter:t:{}'.format(tweet.inReplyToTweetId))
+        if tweet.quotedTweet:
+            kwargs['quoted'] = PostRef(unique_id='twitter:t:{}'.format(tweet.quotedTweet.id))
+        return Post(**kwargs)
+
+    def add_post(self, i, post):
+        self.pages[i]['posts'].append(post)
+
+    def add_post_to_list(self, i, post):
+        self.add_post(i, post)
+        self.pages[i]['list'].append(post.unique_id)
+
+    def load_page(self, index):
+        if not self.iterator or index - self.iter_page >= REACH_LIMIT:
+            return False
+        if index in self.pages:
+            return True
+        # It's possible to load some "pages", but not enough to satisfy the requested index.
+        request_satisfied = False
+        for i in range(self.iter_page, index + 1):
+            if self.completed:
+                break
+            self.pages[i] = { 'posts': [], 'list': [] }
+            for _ in range(0, self.POSTS_PER_PAGE):
+                try:
+                    tweet = next(self.iterator)
+                except StopIteration:
+                    self.completed = True
+                    break
+                except ScraperException as e:
+                    log.error(repr(e))
+                    continue
+                self.add_post_to_list(i, self.make_post(tweet))
+                if tweet.retweetedTweet:
+                    self.add_post(i, self.make_post(tweet.retweetedTweet))
+                if tweet.quotedTweet:
+                    self.add_post(i, self.make_post(tweet.quotedTweet))
+            if i == index:
+                request_satisfied = True
+            self.iter_page += 1
+        return request_satisfied
+
+class TwitterScrapeSearch(TwitterScrapeBase):
+    def __init__(self, userdata, arg):
+        super().__init__(userdata, arg)
+        self.iterator = TwitterSearchScraper(self.arg, top=True, cookies=self.cookies).get_items()
+
+class TwitterScrapeUser(TwitterScrapeBase):
+    def __init__(self, userdata, arg):
+        super().__init__(userdata, arg)
+        if (self.arg.startswith('@')):
+            self.arg = self.arg[1:]
+        self.iterator = TwitterProfileScraper(self.arg, cookies=self.cookies).get_items()
+        #self.iterator = TwitterUserScraper(self.arg, cookies=self.cookies).get_items()
+
+class TwitterScrapeTweet(TwitterScrapeBase):
+    def __init__(self, userdata, arg):
+        super().__init__(userdata, arg)
+        self.single = True
+        if (self.arg.startswith('https://')):
+            self.arg = self.arg[self.arg.rfind('/') + 1:]
+            question = self.arg.find('?')
+            if question >= 0:
+                self.arg = self.arg[:question]
+        self.iterator = TwitterTweetScraper(self.arg, cookies=self.cookies).get_items()
+
+class TwitterProvider(Provider):
+    def __init__(self, access_key, access_secret, consumer_key, consumer_secret, cookies_path):
+        super().__init__()
+        self.parser = QueryParser(self, 'search')
+        self.parser.add_command('search', TwitterScrapeSearch)
+        self.parser.add_command('user', TwitterScrapeUser)
+        self.parser.add_command('tweet', TwitterScrapeTweet)
+        #self.parser.add_command('timeline', TwitterTimeline)
+        #self.parser.add_command('likes', TwitterLikes)
+        #self.parser.add_command('following', TwitterFollowing)
+        #self.t = Twitter2(auth=OAuth(access_key, access_secret, consumer_key, consumer_secret), retry=True)
+        # Cookies are used by the scrape backend.
+        cookie_jar = http.cookiejar.MozillaCookieJar()
+        cookie_jar.load(filename=cookies_path, ignore_expires=True)
+        for c in cookie_jar:
+            self.cookies[c.name] = c.value
+
+    def search(self, query, *extra_args):
+        return self.parser.parse_query(query)
diff --git a/src/shoki/py/providers/youtube.py b/src/shoki/py/providers/youtube.py
new file mode 100644
index 0000000..f17211e
--- /dev/null
+++ b/src/shoki/py/providers/youtube.py
@@ -0,0 +1,115 @@
+import log
+import yt_dlp
+from provider import Provider
+from search import Search
+from post import Post, PostType, Image, Video
+from query_parser import QueryParser
+
+class YDLLogger():
+    def debug(self, msg):
+        if msg.startswith('[debug] '):
+            log.debug(msg)
+        else:
+            log.info(msg)
+
+    def info(self, msg):
+        pass
+
+    def warning(self, msg):
+        log.warning(msg)
+
+    def error(self, msg):
+        log.error(msg)
+
+ydl_opts = {
+    'quiet': False,
+    'logger': YDLLogger(),
+    'cachedir': False,
+#    'cookiefile': '',
+}
+
+ydl = yt_dlp.YoutubeDL(ydl_opts)
+
+def get_playback_url(data, video=True):
+    if 'entries' in data:
+        data = data['entries'][0]
+
+    url = data['formats'][0]['url']
+
+    '''
+    data['formats'] = list(filter(lambda f: 'container' not in f, data['formats']))
+
+    data['formats'] = list(filter(
+        lambda f: not ('container' in f and (f['container'] in ['mp4_dash', 'webm_dash'])), data['formats']
+    ))
+    '''
+
+    data['formats'] = list(filter(lambda f: 'manifest_url' not in f, data['formats']))
+
+    has_audio = list(filter(lambda f: 'acodec' not in f or f['acodec'] != 'none', data['formats']))
+
+    if video:
+        if len(has_audio) > 0:
+            data['formats'] = has_audio
+        try:
+            data['formats'] = list(filter(lambda f: 'quality' in f, data['formats']))
+            url = max(data['formats'], key=lambda f: f['quality'])['url']
+        except:
+            pass
+    else:
+        if len(has_audio) == 0:
+            return None
+        data['formats'] = has_audio
+        try:
+            audio_only = list(filter(lambda f: f['vcodec'] == 'none', data['formats']))
+            if len(audio_only) > 0:
+                data['formats'] = audio_only
+            else:
+                data['formats'] = list(filter(lambda f: f['ext'] in ['mp4'], data['formats']))
+        except:
+            pass
+        try:
+            data['formats'] = list(filter(lambda f: 'abr' in f, data['formats']))
+            url = max(data['formats'], key=lambda f: f['abr'])['url']
+        except:
+            pass
+
+    return url
+
+class YoutubeBase(Search):
+    def __init__(self, userdata, arg):
+        super().__init__()
+        self.provider = userdata
+        self.query = arg
+
+    def load_page(self, index):
+        self.pages[index] = { 'posts': [], 'list': [] }
+        res = self.get_info()
+        if res:
+            media = [Video(url=get_playback_url(res))]
+            self.pages[index]['posts'].append(Post(type=PostType.POST, unique_id='', url='', title='', text='', media=media))
+        return True
+
+class YoutubeSearch(YoutubeBase):
+    def __init__(self, userdata, arg):
+        super().__init__(userdata, arg)
+
+    def get_info(self):
+        return ydl.extract_info('ytsearch1:{}'.format(self.query), download=False)
+
+class YoutubeLink(YoutubeBase):
+    def __init__(self, userdata, arg):
+        super().__init__(userdata, arg)
+
+    def get_info(self):
+        return ydl.extract_info(self.query, download=False)
+
+class YoutubeProvider(Provider):
+    def __init__(self):
+        super().__init__()
+        self.parser = QueryParser(self, 'search')
+        self.parser.add_command('search', YoutubeSearch)
+        self.parser.add_command('link', YoutubeLink)
+
+    def search(self, query, *extra_args):
+        return self.parser.parse_query(query)
diff --git a/src/shoki/py/query_parser.py b/src/shoki/py/query_parser.py
new file mode 100644
index 0000000..ee23bef
--- /dev/null
+++ b/src/shoki/py/query_parser.py
@@ -0,0 +1,39 @@
+class QueryParser():
+    def __init__(self, userdata, default_command):
+        self.commands = {}
+        self.escaped_commands = {}
+        self.userdata = userdata
+        self.default_command = default_command
+
+    def add_command(self, name, search):
+        self.commands[name] = search
+        self.escaped_commands['\\' + name] = name
+
+    def parse_query(self, query):
+        arg = query
+        search = self.commands[self.default_command]
+
+        sp = query.split(' ')
+
+        for i, word in enumerate(sp):
+            col = word.find(':')
+            if col == -1:
+                continue
+            command = word[0:col]
+            if command in self.escaped_commands:
+                query = query.replace(command, self.escaped_commands[command], 1)
+                continue
+            if command in self.commands:
+                if col + 1 < len(word):
+                    arg = word[col+1:]
+                else:
+                    arg = None
+                search = self.commands[command]
+                if i == 0:
+                    if len(sp) > 1:
+                        query = query.replace(word + ' ', '', 1)
+                    else:
+                        query = query.replace(word, '', 1)
+                else:
+                    query = query.replace(' ' + word + ' ', '', 1)
+        return search(self.userdata, arg)
diff --git a/src/shoki/py/search.py b/src/shoki/py/search.py
new file mode 100644
index 0000000..4134851
--- /dev/null
+++ b/src/shoki/py/search.py
@@ -0,0 +1,35 @@
+class Search():
+    def __init__(self):
+        self.pages = {}
+        self.iter_page = 0
+        self.iter_index = 0
+        self.single = False
+        self.completed = False
+
+    def __iter__(self):
+        self.iter_page = 0
+        self.iter_index = 0
+        return self
+
+    def __next__(self):
+        if self.iter_index >= len(self.pages[self.iter_page]):
+            self.iter_page += 1
+            self.iter_index = 0
+        if self.iter_page not in self.pages:
+            if not self.load_page(self.iter_page):
+                raise StopIteration
+        return self.pages[self.iter_page][self.iter_index]
+
+    def load_page(self, index):
+        return False
+
+    def get_page(self, index):
+        if index not in self.pages:
+            if self.completed:
+                if self.single:
+                    return self.pages[0]
+                else:
+                    return None
+            if not self.load_page(index):
+                return None
+        return self.pages[index]
diff --git a/src/shoki/src/packet_ext.c b/src/shoki/src/packet_ext.c
new file mode 100644
index 0000000..26ba9ee
--- /dev/null
+++ b/src/shoki/src/packet_ext.c
@@ -0,0 +1,91 @@
+#include "search.h"
+#include "packet_ext.h"
+
+static void aki_packet_write_sho_optional_int(struct aki_packet *packet, sho_optional_int *o)
+{
+    AKI_PACKET_WRITE_TYPE(packet, s64, o->i);
+    AKI_PACKET_WRITE_TYPE(packet, bool, o->set);
+}
+
+void aki_packet_write_sho_post(struct aki_packet *packet, struct sho_post *post)
+{
+    AKI_PACKET_WRITE_TYPE(packet, u16, post->version);
+    AKI_PACKET_WRITE_TYPE(packet, u8, post->type);
+    aki_packet_write_string(packet, &post->unique_id);
+    aki_packet_write_string(packet, &post->url);
+    AKI_PACKET_WRITE_TYPE(packet, u32, post->dates.size);
+    struct sho_post_date *date;
+    al_array_foreach_ptr(post->dates, i, date) {
+        AKI_PACKET_WRITE_TYPE(packet, u8, date->type);
+        AKI_PACKET_WRITE_TYPE(packet, s64, date->timestamp);
+    }
+    aki_packet_write_string(packet, &post->author.unique_id);
+    aki_packet_write_string(packet, &post->author.username);
+    aki_packet_write_string(packet, &post->author.display_name);
+    aki_packet_write_string(packet, &post->author.profile_image_url);
+    aki_packet_write_string(packet, &post->title);
+    aki_packet_write_string(packet, &post->text);
+    aki_packet_write_sho_optional_int(packet, &post->likes);
+    aki_packet_write_sho_optional_int(packet, &post->reposts);
+    aki_packet_write_sho_optional_int(packet, &post->quotes);
+    aki_packet_write_sho_optional_int(packet, &post->comments);
+    aki_packet_write_sho_optional_int(packet, &post->views);
+    AKI_PACKET_WRITE_TYPE(packet, u32, post->media.size);
+    struct sho_post_media *media;
+    al_array_foreach_ptr(post->media, i, media) {
+        AKI_PACKET_WRITE_TYPE(packet, u8, media->type);
+        aki_packet_write_string(packet, &media->url);
+        aki_packet_write_string(packet, &media->thumbnail_url);
+    }
+    aki_packet_write_string(packet, &post->post.unique_id);
+    aki_packet_write_string(packet, &post->quoted.unique_id);
+    aki_packet_write_string(packet, &post->in_reply_to.unique_id);
+}
+
+static void aki_packet_read_sho_optional_int(struct aki_packet *packet, sho_optional_int *o)
+{
+    AKI_PACKET_READ_TYPE(packet, s64, o->i);
+    AKI_PACKET_READ_TYPE(packet, bool, o->set);
+}
+
+void aki_packet_read_sho_post(struct aki_packet *packet, struct sho_post *post)
+{
+    sho_post_reset(post);
+    AKI_PACKET_READ_TYPE(packet, u16, post->version);
+    AKI_PACKET_READ_TYPE(packet, u8, post->type);
+    aki_packet_read_string(packet, &post->unique_id);
+    aki_packet_read_string(packet, &post->url);
+    u32 dates_size;
+    AKI_PACKET_READ_TYPE(packet, u32, dates_size);
+    al_array_reserve(post->dates, dates_size);
+    for (u32 i = 0; i < dates_size; i++) {
+        struct sho_post_date date;
+        AKI_PACKET_READ_TYPE(packet, u8, date.type);
+        AKI_PACKET_READ_TYPE(packet, s64, date.timestamp);
+        al_array_push(post->dates, date);
+    }
+    aki_packet_read_string(packet, &post->author.unique_id);
+    aki_packet_read_string(packet, &post->author.username);
+    aki_packet_read_string(packet, &post->author.display_name);
+    aki_packet_read_string(packet, &post->author.profile_image_url);
+    aki_packet_read_string(packet, &post->title);
+    aki_packet_read_string(packet, &post->text);
+    aki_packet_read_sho_optional_int(packet, &post->likes);
+    aki_packet_read_sho_optional_int(packet, &post->reposts);
+    aki_packet_read_sho_optional_int(packet, &post->quotes);
+    aki_packet_read_sho_optional_int(packet, &post->comments);
+    aki_packet_read_sho_optional_int(packet, &post->views);
+    u32 media_size;
+    AKI_PACKET_READ_TYPE(packet, u32, media_size);
+    al_array_reserve(post->media, media_size);
+    for (u32 i = 0; i < media_size; i++) {
+        struct sho_post_media media;
+        AKI_PACKET_READ_TYPE(packet, u8, media.type);
+        aki_packet_read_string(packet, &media.url);
+        aki_packet_read_string(packet, &media.thumbnail_url);
+        al_array_push(post->media, media);
+    }
+    aki_packet_read_string(packet, &post->post.unique_id);
+    aki_packet_read_string(packet, &post->quoted.unique_id);
+    aki_packet_read_string(packet, &post->in_reply_to.unique_id);
+}
diff --git a/src/shoki/src/packet_ext.h b/src/shoki/src/packet_ext.h
new file mode 100644
index 0000000..ff0fda1
--- /dev/null
+++ b/src/shoki/src/packet_ext.h
@@ -0,0 +1,8 @@
+#pragma once
+
+#include <aki/common.h>
+
+#include "post.h"
+
+void aki_packet_write_sho_post(struct aki_packet *packet, struct sho_post *post);
+void aki_packet_read_sho_post(struct aki_packet *packet, struct sho_post *post);
diff --git a/src/shoki/src/post.c b/src/shoki/src/post.c
new file mode 100644
index 0000000..22fb7ef
--- /dev/null
+++ b/src/shoki/src/post.c
@@ -0,0 +1,55 @@
+#include "post.h"
+
+void sho_post_reset(struct sho_post *post)
+{
+    al_bzero(post, sizeof(struct sho_post));
+    al_array_init(post->dates);
+    al_array_init(post->media);
+}
+
+void sho_post_add_media(struct sho_post *post, u8 type, str *url, str *thumbnail_url)
+{
+    al_array_push(post->media, ((struct sho_post_media){
+        .type = type, .url = *url, .thumbnail_url = *thumbnail_url
+    }));
+}
+
+void sho_post_add_date(struct sho_post *post, u8 type, s64 timestamp)
+{
+    al_array_push(post->dates, ((struct sho_post_date){
+        .type = type, .timestamp = timestamp
+    }));
+}
+
+void sho_post_clone(struct sho_post *dest, struct sho_post *src)
+{
+    sho_post_reset(dest);
+    dest->version = src->version;
+    dest->type = src->type;
+    al_str_clone(&dest->unique_id, &src->unique_id);
+    al_str_clone(&dest->url, &src->url);
+    al_array_clone(dest->dates, src->dates);
+    al_str_clone(&dest->author.unique_id, &src->author.unique_id);
+    al_str_clone(&dest->author.username, &src->author.username);
+    al_str_clone(&dest->author.display_name, &src->author.display_name);
+    al_str_clone(&dest->author.profile_image_url, &src->author.profile_image_url);
+    al_str_clone(&dest->title, &src->title);
+    al_str_clone(&dest->text, &src->text);
+    dest->likes = src->likes;
+    dest->reposts = src->reposts;
+    dest->quotes = src->quotes;
+    dest->comments = src->comments;
+    dest->views = src->views;
+    struct sho_post_media *media;
+    struct sho_post_media media_copy;
+    al_array_foreach_ptr(src->media, i, media) {
+        al_bzero(&media_copy, sizeof(struct sho_post_media));
+        media_copy.type = media->type;
+        al_str_clone(&media_copy.url, &media->url);
+        al_str_clone(&media_copy.thumbnail_url, &media->thumbnail_url);
+        al_array_push(dest->media, media_copy);
+    }
+    al_str_clone(&dest->post.unique_id, &src->post.unique_id);
+    al_str_clone(&dest->quoted.unique_id, &src->quoted.unique_id);
+    al_str_clone(&dest->in_reply_to.unique_id, &src->in_reply_to.unique_id);
+}
diff --git a/src/shoki/src/post.h b/src/shoki/src/post.h
new file mode 100644
index 0000000..6814b0f
--- /dev/null
+++ b/src/shoki/src/post.h
@@ -0,0 +1,78 @@
+#pragma once
+
+#include <al/types.h>
+#include <al/str.h>
+#include <al/array.h>
+
+enum {
+    SHOKI_MEDIA_IMAGE = 0,
+    SHOKI_MEDIA_VIDEO,
+    SHOKI_MEDIA_AUDIO
+};
+
+enum {
+    SHOKI_POST_POST = 0,
+    SHOKI_POST_REPOST
+};
+
+enum {
+    SHOKI_DATE_CREATED = 0,
+    SHOKI_DATE_EDITED,
+    SHOKI_DATE_ARCHIVED
+};
+
+typedef struct {
+    s64 i;
+    bool set;
+} sho_optional_int;
+
+struct sho_post_date {
+    u8 type;
+    s64 timestamp;
+};
+
+struct sho_post_media {
+    u8 type;
+    str url;
+    str thumbnail_url;
+};
+
+typedef array(struct sho_post_date) sho_dates_array;
+typedef array(struct sho_post_media) sho_media_array;
+
+struct sho_post_user {
+    str unique_id;
+    str username;
+    str display_name;
+    str profile_image_url;
+};
+
+struct sho_post_ref {
+    str unique_id;
+};
+
+struct sho_post {
+    u16 version;
+    u8 type;
+    str unique_id;
+    str raw_responses;
+    str url;
+    sho_dates_array dates;
+    struct sho_post_user author;
+    str title;
+    str text;
+    sho_optional_int likes;
+    sho_optional_int reposts;
+    sho_optional_int quotes;
+    sho_optional_int comments;
+    sho_optional_int views;
+    sho_media_array media;
+    struct sho_post_ref post; // reposted post.
+    struct sho_post_ref quoted;
+    struct sho_post_ref in_reply_to;
+};
+
+void sho_post_reset(struct sho_post *post);
+void sho_post_add_media(struct sho_post *post, u8 type, str *url, str *thumbnail_url);
+void sho_post_add_date(struct sho_post *post, u8 type, s64 timestamp);
+void sho_post_clone(struct sho_post *dest, struct sho_post *src);
diff --git a/src/shoki/src/post_cache.c b/src/shoki/src/post_cache.c
new file mode 100644
index 0000000..a8f02eb
--- /dev/null
+++ b/src/shoki/src/post_cache.c
@@ -0,0 +1,30 @@
+#include "post_cache.h"
+
+void sho_post_cache_init(struct sho_post_cache *cache)
+{
+    al_array_init(cache->cache);
+}
+
+void sho_post_cache_push(struct sho_post_cache *cache, struct sho_post *post)
+{
+    struct sho_post *check = sho_post_cache_get(cache, &post->unique_id);
+    if (!check) {
+        struct sho_post *npost = al_alloc_object(struct sho_post);
+        sho_post_clone(npost, post);
+        al_array_push(cache->cache, npost);
+    }
+}
+
+struct sho_post *sho_post_cache_get(struct sho_post_cache *cache, str *unique_id)
+{
+    struct sho_post *post;
+    al_array_foreach(cache->cache, i, post) {
+        if (al_str_eq(&post->unique_id, unique_id)) {
+            return post;
+        }
+        if (al_str_eq(&post->author.unique_id, unique_id)) {
+            return post;
+        }
+    }
+    return NULL;
+}
diff --git a/src/shoki/src/post_cache.h b/src/shoki/src/post_cache.h
new file mode 100644
index 0000000..80cd0b7
--- /dev/null
+++ b/src/shoki/src/post_cache.h
@@ -0,0 +1,13 @@
+#pragma once
+
+#include <al/array.h>
+
+#include "post.h"
+
+struct sho_post_cache {
+    array(struct sho_post *) cache;
+};
+
+void sho_post_cache_init(struct sho_post_cache *cache);
+void sho_post_cache_push(struct sho_post_cache *cache, struct sho_post *post);
+struct sho_post *sho_post_cache_get(struct sho_post_cache *cache, str *unique_id);
diff --git a/src/shoki/src/search.c b/src/shoki/src/search.c
new file mode 100644
index 0000000..b4b2157
--- /dev/null
+++ b/src/shoki/src/search.c
@@ -0,0 +1,99 @@
+#include <al/log.h>
+
+#include "../cpy/shoki.c"
+
+#include "search.h"
+
+bool sho_python_init(void)
+{
+    PyPreConfig pre;
+    PyPreConfig_InitPythonConfig(&pre);
+    pre.utf8_mode = 1;
+    Py_PreInitialize(&pre);
+    if (PyImport_AppendInittab("shoki", PyInit_shoki) == -1) {
+        al_log_error("shoki", "Could not extend in-built modules table.");
+        sho_python_close();
+        return false;
+    }
+    Py_Initialize();
+    PyObject *module = PyImport_ImportModule("shoki");
+    if (!module) {
+        PyErr_Print();
+        al_log_error("shoki", "Could not import module.");
+        sho_python_close();
+        return false;
+    }
+    return true;
+}
+
+void sho_result_init(struct sho_result *res)
+{
+    al_array_init(res->pages);
+}
+
+static struct sho_result_page *page_at_index(struct sho_result *res, s32 num)
+{
+    struct sho_result_page *page;
+    al_array_foreach_ptr(res->pages, i, page) {
+        if (page->num == num) {
+            return page;
+        }
+    }
+    return NULL;
+}
+
+void sho_result_add_post(struct sho_result *res, s32 num, struct sho_post *post)
+{
+    struct sho_result_page *page = page_at_index(res, num);
+    if (!page) {
+        al_array_push(res->pages, (struct sho_result_page){0});
+        page = &al_array_last(res->pages);
+        page->num = num;
+        al_array_init(page->posts);
+        al_array_init(page->list);
+    }
+    al_array_push(page->posts, *post);
+}
+
+void sho_result_add_to_list(struct sho_result *res, s32 num, str *unique_id)
+{
+    struct sho_result_page *page = page_at_index(res, num);
+    al_array_push(page->list, *unique_id);
+}
+
+void sho_search_init(struct sho_search *search)
+{
+    search->id = -1;
+    search->page = 0;
+    sho_result_init(&search->result);
+}
+
+s32 sho_search_more_results(struct sho_search *search, str *provider, str *query)
+{
+    if (search->id < 0) {
+        search->id = sho_client_search(provider, query);
+    } else {
+        search->page++;
+    }
+    if (search->id < 0) return -1;
+    al_log_info("shoki", "Loading page %i (%.*s).", search->page, AL_STR_PRINTF(query));
+    if (sho_client_get_page(&search->result, search->id, search->page) < 0) {
+        return -1;
+    }
+    return search->page;
+}
+
+void sho_search_free(struct sho_search *search)
+{
+    struct sho_result_page *page;
+    al_array_foreach_ptr(search->result.pages, i, page) {
+        al_array_free(page->posts);
+        al_array_free(page->list);
+    }
+    al_array_free(search->result.pages);
+}
+
+void sho_python_close(void)
+{
+    if (Py_IsInitialized()) Py_Finalize();
+}
diff --git a/src/shoki/src/search.h b/src/shoki/src/search.h
new file mode 100644
index 0000000..97e40ae
--- /dev/null
+++ b/src/shoki/src/search.h
@@ -0,0 +1,30 @@
+#pragma once
+
+#include "post.h"
+
+struct sho_result_page {
+    s32 num;
+    array(struct sho_post) posts;
+    array(str) list;
+};
+
+struct sho_result {
+    array(struct sho_result_page) pages;
+};
+
+struct sho_search {
+    s32 id;
+    s32 page;
+    struct sho_result result;
+};
+
+bool sho_python_init(void);
+void sho_python_close(void);
+
+void sho_result_init(struct sho_result *res);
+void sho_result_add_post(struct sho_result *res, s32 num, struct sho_post *post);
+void sho_result_add_to_list(struct sho_result *res, s32 num, str *unique_id);
+
+void sho_search_init(struct sho_search *search);
+s32 sho_search_more_results(struct sho_search *search, str *provider, str *query);
+void sho_search_free(struct sho_search *s);