From a48a68cfb04b2020737c0adfb0e7667451a22b5c Mon Sep 17 00:00:00 2001 From: Andrew Opalach Date: Mon, 6 Nov 2023 11:54:08 -0500 Subject: Add camu - Subprojects temporarily omitted Signed-off-by: Andrew Opalach --- src/shoki/cpy/lib.pxd | 13 ++ src/shoki/cpy/post.pxd | 58 +++++ src/shoki/cpy/result.pxd | 8 + src/shoki/cpy/setup.py | 6 + src/shoki/cpy/shoki.pyx | 132 +++++++++++ src/shoki/cpy/str.pxd | 11 + src/shoki/cpy/types.pxd | 11 + src/shoki/meson.build | 14 ++ src/shoki/py/__init__.py | 0 src/shoki/py/client.py | 51 +++++ src/shoki/py/config.def.py | 19 ++ src/shoki/py/log.py | 30 +++ src/shoki/py/post.py | 70 ++++++ src/shoki/py/provider.py | 39 ++++ src/shoki/py/providers/__init__.py | 0 src/shoki/py/providers/fanbox.py | 130 +++++++++++ src/shoki/py/providers/instagram.py | 96 ++++++++ src/shoki/py/providers/pixiv.py | 113 ++++++++++ src/shoki/py/providers/searx.py | 93 ++++++++ src/shoki/py/providers/twitter.py | 425 ++++++++++++++++++++++++++++++++++++ src/shoki/py/providers/youtube.py | 115 ++++++++++ src/shoki/py/query_parser.py | 39 ++++ src/shoki/py/search.py | 35 +++ src/shoki/src/packet_ext.c | 91 ++++++++ src/shoki/src/packet_ext.h | 8 + src/shoki/src/post.c | 55 +++++ src/shoki/src/post.h | 78 +++++++ src/shoki/src/post_cache.c | 30 +++ src/shoki/src/post_cache.h | 13 ++ src/shoki/src/search.c | 99 +++++++++ src/shoki/src/search.h | 30 +++ 31 files changed, 1912 insertions(+) create mode 100644 src/shoki/cpy/lib.pxd create mode 100644 src/shoki/cpy/post.pxd create mode 100644 src/shoki/cpy/result.pxd create mode 100644 src/shoki/cpy/setup.py create mode 100644 src/shoki/cpy/shoki.pyx create mode 100644 src/shoki/cpy/str.pxd create mode 100644 src/shoki/cpy/types.pxd create mode 100644 src/shoki/meson.build create mode 100644 src/shoki/py/__init__.py create mode 100644 src/shoki/py/client.py create mode 100644 src/shoki/py/config.def.py create mode 100644 src/shoki/py/log.py create mode 100644 src/shoki/py/post.py create mode 100644 src/shoki/py/provider.py create mode 100644 src/shoki/py/providers/__init__.py create mode 100644 src/shoki/py/providers/fanbox.py create mode 100644 src/shoki/py/providers/instagram.py create mode 100644 src/shoki/py/providers/pixiv.py create mode 100644 src/shoki/py/providers/searx.py create mode 100644 src/shoki/py/providers/twitter.py create mode 100644 src/shoki/py/providers/youtube.py create mode 100644 src/shoki/py/query_parser.py create mode 100644 src/shoki/py/search.py create mode 100644 src/shoki/src/packet_ext.c create mode 100644 src/shoki/src/packet_ext.h create mode 100644 src/shoki/src/post.c create mode 100644 src/shoki/src/post.h create mode 100644 src/shoki/src/post_cache.c create mode 100644 src/shoki/src/post_cache.h create mode 100644 src/shoki/src/search.c create mode 100644 src/shoki/src/search.h (limited to 'src/shoki') diff --git a/src/shoki/cpy/lib.pxd b/src/shoki/cpy/lib.pxd new file mode 100644 index 0000000..5749141 --- /dev/null +++ b/src/shoki/cpy/lib.pxd @@ -0,0 +1,13 @@ +include "types.pxd" + +cdef extern from "": + void al_free(void *ptr) + void al_malloc(size_t size) + void al_strlen(char *s) + void al_bzero(void *s, size_t n) + +cdef extern from "": + s32 al_log_info(const char *ns, const char *, ...) + s32 al_log_warn(const char *ns, const char *, ...) + s32 al_log_error(const char *ns, const char *, ...) + s32 al_log_debug(const char *ns, const char *, ...) diff --git a/src/shoki/cpy/post.pxd b/src/shoki/cpy/post.pxd new file mode 100644 index 0000000..0b0a97e --- /dev/null +++ b/src/shoki/cpy/post.pxd @@ -0,0 +1,58 @@ +include "str.pxd" + +cdef extern from "../../shoki/src/post.h": + ctypedef struct sho_optional_int: + s64 i + bool set + + cdef struct sho_post_date: + u8 type + s64 timestamp + + cdef struct sho_post_media: + u8 type + str url + str thumbnail_url + + ctypedef struct sho_dates_array: + u32 size + u32 alloc + sho_post_date *data + + ctypedef struct sho_media_array: + u32 size + u32 alloc + sho_post_media *data + + cdef struct sho_post_user: + str unique_id + str username + str display_name + str profile_image_url + + cdef struct sho_post_ref: + str unique_id + + cdef struct sho_post: + u16 version + u8 type + str unique_id + str raw_responses + str url + sho_dates_array dates + sho_post_user author + str title + str text + sho_optional_int likes + sho_optional_int reposts + sho_optional_int quotes + sho_optional_int comments + sho_optional_int views + sho_media_array media + sho_post_ref post + sho_post_ref quoted + sho_post_ref in_reply_to + + void sho_post_reset(sho_post *post) + void sho_post_add_media(sho_post *post, u8 type, str *url, str *thumbnail_url) + void sho_post_add_date(sho_post *post, u8 type, s64 timestamp) diff --git a/src/shoki/cpy/result.pxd b/src/shoki/cpy/result.pxd new file mode 100644 index 0000000..1bebb55 --- /dev/null +++ b/src/shoki/cpy/result.pxd @@ -0,0 +1,8 @@ +include "post.pxd" + +cdef extern from "../../shoki/src/search.h": + cdef struct sho_result: + pass + + void sho_result_add_post(sho_result *res, s32 page, sho_post *post) + void sho_result_add_to_list(sho_result *res, s32 page, str *unique_id) diff --git a/src/shoki/cpy/setup.py b/src/shoki/cpy/setup.py new file mode 100644 index 0000000..1b491f1 --- /dev/null +++ b/src/shoki/cpy/setup.py @@ -0,0 +1,6 @@ +from Cython.Build import cythonize +from Cython.Compiler import Options +from setuptools import Extension +Options.embed = True +alabaster_inc = "../../../subprojects/libalabaster/include" +cythonize([Extension("shoki", ["shoki.pyx"], include_dirs=[alabaster_inc], extra_link_args=[])], language_level=3) diff --git a/src/shoki/cpy/shoki.pyx b/src/shoki/cpy/shoki.pyx new file mode 100644 index 0000000..6d1dd56 --- /dev/null +++ b/src/shoki/cpy/shoki.pyx @@ -0,0 +1,132 @@ +cimport lib +cimport str +cimport post +cimport result + +import sys +sys.path.append('./py') + +cdef public int log_info(char *msg) except -1: + return lib.al_log_info("shoki", "%s", msg) + +cdef public int log_warn(char *msg) except -1: + return lib.al_log_warn("shoki", "%s", msg) + +cdef public int log_error(char *msg) except -1: + return lib.al_log_error("shoki", "%s", msg) + +cdef public int log_debug(char *msg) except -1: + return lib.al_log_debug("shoki", "%s", msg) + +def encode_str(str): + return str.encode('UTF-8') + +def decode_str(str): + return str.decode('UTF-8') + +class OutputHook(object): + def __init__(self): + self.fd = sys.__stdout__ + + def __getattr__(self, attr): + return getattr(self.fd, attr) + + def write(self, msg): + if len(msg) > 0 and msg != '\n': + log_debug(encode_str(msg)) + +hook = OutputHook() +sys.stdout = hook +sys.stderr = hook + +class ShokiLogger(): + def debug(self, msg): + log_debug(encode_str(msg)) + + def info(self, msg): + log_info(encode_str(msg)) + + def warning(self, msg): + log_warn(encode_str(msg)) + + def error(self, msg): + log_error(encode_str(msg)) + +import log +log.set_logger(ShokiLogger()) + +def exception_handler(exception_type, exception, traceback): + log.error(repr(exception)) +sys.excepthook = exception_handler + +import client +from post import PostType, DateType + +shoki = client.ShokiClient() +for name, p in client.ALL_PROVIDERS.items(): + if p[0].init(): + shoki.register_provider(name, p) + +cdef public int sho_client_search(str.str *provider, str.str *query) except -1: + cdef char *c_str0 = str.al_str_to_c_str(provider) + cdef char *c_str1 = str.al_str_to_c_str(query) + ret = shoki.search(decode_str(c_str0), decode_str(c_str1)) + lib.al_free(c_str0) + lib.al_free(c_str1) + return ret + +cdef public int py_str_to_str(str.str *s, char *py_c_str) except -1: + str.al_str_from(s, py_c_str) + return 0 + +cdef public int sho_client_get_page(result.sho_result *res, int search_id, int num) except -1: + cdef post.sho_post cpost + cdef str.str url + cdef str.str thumbnail_url + cdef str.str unique_id + page = shoki.get_page(search_id, num) + if not page: return -1 + for py_post in page['posts']: + post.sho_post_reset(&cpost) + cpost.version = py_post.version + cpost.type = py_post.type.value + py_str_to_str(&cpost.unique_id, encode_str(py_post.unique_id)) + py_str_to_str(&cpost.url, encode_str(py_post.url)) + for date in py_post.dates: + post.sho_post_add_date(&cpost, date.value, py_post.dates[date]) + py_str_to_str(&cpost.author.unique_id, encode_str(py_post.author.unique_id)) + py_str_to_str(&cpost.author.username, encode_str(py_post.author.username)) + py_str_to_str(&cpost.author.display_name, encode_str(py_post.author.display_name)) + py_str_to_str(&cpost.author.profile_image_url, encode_str(py_post.author.profile_image_url)) + if cpost.type == PostType.REPOST.value: + py_str_to_str(&cpost.post.unique_id, encode_str(py_post.post.unique_id)) + result.sho_result_add_post(res, num, &cpost) + continue + py_str_to_str(&cpost.title, encode_str(py_post.title)) + py_str_to_str(&cpost.text, encode_str(py_post.text)) + cpost.likes.set = py_post.likes != None + if cpost.likes.set: + cpost.likes.i = py_post.likes + cpost.reposts.set = py_post.reposts != None + if cpost.reposts.set: + cpost.reposts.i = py_post.reposts + cpost.quotes.set = py_post.quotes != None + if cpost.quotes.set: + cpost.quotes.i = py_post.quotes + cpost.comments.set = py_post.comments != None + if cpost.comments.set: + cpost.comments.i = py_post.comments + cpost.views.set = py_post.views != None + if cpost.views.set: + cpost.views.i = py_post.views + for media in py_post.media: + py_str_to_str(&url, encode_str(media.url)) + py_str_to_str(&thumbnail_url, encode_str(media.thumbnail_url)) + post.sho_post_add_media(&cpost, 0, &url, &thumbnail_url) + py_str_to_str(&cpost.quoted.unique_id, encode_str(py_post.quoted.unique_id)) + py_str_to_str(&cpost.in_reply_to.unique_id, encode_str(py_post.in_reply_to.unique_id)) + result.sho_result_add_post(res, num, &cpost) + for py_unique_id in page['list']: + py_str_to_str(&unique_id, encode_str(py_unique_id)) + result.sho_result_add_to_list(res, num, &unique_id) + return 0 diff --git a/src/shoki/cpy/str.pxd b/src/shoki/cpy/str.pxd new file mode 100644 index 0000000..a12ced4 --- /dev/null +++ b/src/shoki/cpy/str.pxd @@ -0,0 +1,11 @@ +include "types.pxd" + +cdef extern from "": + ctypedef struct str: + u32 len + s32 alloc + char *data + + void al_str_from(str *s, const char *str) + char *al_str_to_c_str(str *s) + diff --git a/src/shoki/cpy/types.pxd b/src/shoki/cpy/types.pxd new file mode 100644 index 0000000..fec35a5 --- /dev/null +++ b/src/shoki/cpy/types.pxd @@ -0,0 +1,11 @@ +ctypedef unsigned char u8; +ctypedef char s8; +ctypedef unsigned short u16; +ctypedef short s16; +ctypedef unsigned int u32; +ctypedef int s32; +ctypedef unsigned long u64; +ctypedef long s64; +ctypedef float f32; +ctypedef double f64; +ctypedef bint bool diff --git a/src/shoki/meson.build b/src/shoki/meson.build new file mode 100644 index 0000000..557d497 --- /dev/null +++ b/src/shoki/meson.build @@ -0,0 +1,14 @@ +shoki_src = [ + 'src/search.c', + 'src/post.c', + 'src/packet_ext.c', + 'src/post_cache.c' +] +shoki_inc = [] +shoki_deps = [common_deps] + +python = dependency('python-3.11-embed', required: true) +shoki_deps += [python] + +shoki = declare_dependency(sources: shoki_src, + include_directories: shoki_inc, dependencies: shoki_deps) diff --git a/src/shoki/py/__init__.py b/src/shoki/py/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/shoki/py/client.py b/src/shoki/py/client.py new file mode 100644 index 0000000..3986060 --- /dev/null +++ b/src/shoki/py/client.py @@ -0,0 +1,51 @@ +import log +import config +from providers.twitter import TwitterProvider +from providers.youtube import YoutubeProvider + +ALL_PROVIDERS = { + 'twitter': (TwitterProvider( + config.TWITTER_ACCESS_TOKEN, config.TWITTER_ACCESS_TOKEN_SECRET, + config.TWITTER_CONSUMER_TOKEN, config.TWITTER_CONSUMER_TOKEN_SECRET, + config.TWITTER_COOKIES_PATH), []), + 'youtube': (YoutubeProvider(), []) +} + +class ShokiClient(): + def __init__(self): + self.providers = {} + self.searches = {} + self.inc = 0 + + def register_provider(self, name, p): + log.info('Registered provider \'{}\'.'.format(name)) + self.providers[name] = p + + def get_providers(self): + return self.providers.keys() + + def search(self, provider, query): + if provider not in self.providers: + return -1 + search_id = self.inc + p = self.providers[provider] + try: + search = p[0].search(query, *p[1]) + self.searches[search_id] = search + self.inc += 1 + except Exception as e: + log.error(repr(e)) + return -1 + return search_id + + def get_page(self, search_id, num): + if search_id not in self.searches: + return None + try: + return self.searches[search_id].get_page(num) + except Exception as e: + log.error(repr(e)) + return None + + def get_playback_data(self, unique_id): + return '' diff --git a/src/shoki/py/config.def.py b/src/shoki/py/config.def.py new file mode 100644 index 0000000..237d658 --- /dev/null +++ b/src/shoki/py/config.def.py @@ -0,0 +1,19 @@ +from pathlib import Path + +BASE_DIR = '' + +TWITTER_ACCESS_TOKEN = '' +TWITTER_ACCESS_TOKEN_SECRET = '' +TWITTER_CONSUMER_TOKEN = '' +TWITTER_CONSUMER_TOKEN_SECRET = '' +TWITTER_COOKIES_PATH = '{}/cookies_twitter.txt'.format(BASE_DIR) +TWITTER_TIMEOUT = 5 # seconds + +FANBOX_SESSID = '' + +PIXIV_REFRESH_TOKEN = '' + +INSTAGRAM_USERNAME = '' +INSTAGRAM_PASSWORD = '' +INSTAGRAM_SETTINGS_PATH = Path('{}/ig_settings.json'.format(BASE_DIR)) +INSTAGRAM_USER_AGENT = 'Instagram 146.0.0.27.125 (iPhone12,1; iOS 13_3; en_US; en-US; scale=2.00; 1656x3584; 190542906)' diff --git a/src/shoki/py/log.py b/src/shoki/py/log.py new file mode 100644 index 0000000..c36c54e --- /dev/null +++ b/src/shoki/py/log.py @@ -0,0 +1,30 @@ +class DefaultLogger(): + def debug(self, msg): + print(msg) + + def info(self, msg): + print(msg) + + def warning(self, msg): + print(msg) + + def error(self, msg): + print(msg) + +LOGGER = DefaultLogger() + +def set_logger(logger): + global LOGGER + LOGGER = logger + +def debug(msg): + LOGGER.debug(msg) + +def info(msg): + LOGGER.info(msg) + +def warning(msg): + LOGGER.warning(msg) + +def error(msg): + LOGGER.error(msg) diff --git a/src/shoki/py/post.py b/src/shoki/py/post.py new file mode 100644 index 0000000..d489298 --- /dev/null +++ b/src/shoki/py/post.py @@ -0,0 +1,70 @@ +import dataclasses +from dataclasses import field +from typing import Optional +from json import JSONEncoder +from enum import Enum + +class PostType(Enum): + UNKNOWN = -1 + POST = 0 + REPOST = 1 + +class DateType(Enum): + CREATED = 0 + EDITED = 1 + ARCHIVED = 2 + +@dataclasses.dataclass +class Media(): + url: str = '' + base64_data: str = '' + +@dataclasses.dataclass +class Image(Media): + type: str = 'image' + thumbnail_url: str = '' + +@dataclasses.dataclass +class Video(Media): + type: str = 'video' + thumbnail_url: str = '' + +@dataclasses.dataclass +class Audio(Media): + type: str = 'audio' + +@dataclasses.dataclass +class User(): + unique_id: str = '' + username: str = '' + display_name: str = '' + profile_image_url: str = '' + +@dataclasses.dataclass +class PostRef(): + unique_id: str = '' + +@dataclasses.dataclass +class Post(): + version: int = 2 + type: 'PostType' = PostType.UNKNOWN + unique_id: str = '' + raw_responses: dict[str, str] = field(default_factory=lambda: {}) + url: str = '' + dates: dict['DateType', int] = field(default_factory=lambda: {}) + author: 'User' = field(default_factory=lambda: User()) + title: str = '' + text: str = '' + likes: Optional[int] = None + reposts: Optional[int] = None + quotes: Optional[int] = None + comments: Optional[int] = None + views: Optional[int] = None + media: list['Media'] = field(default_factory=lambda: []) + post: 'PostRef' = field(default_factory=lambda: PostRef()) + quoted: 'PostRef' = field(default_factory=lambda: PostRef()) + in_reply_to: 'PostRef' = field(default_factory=lambda: PostRef()) + +class PostEncoder(JSONEncoder): + def default(self, obj): + return obj.__dict__ diff --git a/src/shoki/py/provider.py b/src/shoki/py/provider.py new file mode 100644 index 0000000..a719d01 --- /dev/null +++ b/src/shoki/py/provider.py @@ -0,0 +1,39 @@ +import httpx +import json +import log + +class Provider(): + def __init__(self): + self.cookies = {} + self.headers = {} + self.unique_id_map = {} + + def init(self): + return True + + # TODO: retries. + def get(self, url, params = None): + log.debug('HTTP request {} {}.'.format(url, params)) + r = httpx.get(url, params=params, cookies=self.cookies, headers=self.headers) + # TODO: Non-200 = err, support redirects (if not already done in httpx)? + if r.status_code == 403 or r.status_code == 404: + log.error('HTTP {}.'.format(r.status_code)) + return None + log.debug('HTTP {}.'.format(r.status_code)) + return r + + def get_json(self, url, params = None): + try: + r = self.get(url, params) + if not r: + return None + return json.loads(r.content) + except json.JSONDecodeError as e: + log.error(repr(e)) + return None + + def get_obj_from_unique_id(self, unique_id): + return None + + def search(self, query, *extra_args): + return None diff --git a/src/shoki/py/providers/__init__.py b/src/shoki/py/providers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/shoki/py/providers/fanbox.py b/src/shoki/py/providers/fanbox.py new file mode 100644 index 0000000..7739be3 --- /dev/null +++ b/src/shoki/py/providers/fanbox.py @@ -0,0 +1,130 @@ +import urllib.parse +import log +from plugin import Plugin +from search import Search +from post import Post, Image + +BASE_URL = "https://api.fanbox.cc/" + +PAGES_PER_QUERY = 2 + +class FanboxSearch(Search): + def __init__(self, plugin, creator_id): + super().__init__() + self.p = plugin + self.creator_id = creator_id + self.pagination_pages = [] + self.pagination_complete = False + + def paginate(self): + res = self.p.get_json(BASE_URL + 'post.paginateCreator', { 'creatorId': self.creator_id }) + if not res or 'body' not in res or len(res['body']) == 0: + return False + for p in res['body']: + params = urllib.parse.parse_qs(urllib.parse.urlparse(p).query) + page = { + 'url': p, + 'params': params, + 'data': None, + 'limit': -1 if 'limit' not in params else int(params['limit'][0]) + } + self.pagination_pages.append(page) + return True + + def load_pagination_page(self, p): + if not p['data']: + res = self.p.get_json(BASE_URL + 'post.listCreator', p['params']) + if not res: + return False + p['data'] = res['body']['items'] + # The get_post_at_index logic assumes limit will always be exact. + p['limit'] = len(p['data']) + return True + + def get_post_at_index(self, index): + if not self.pagination_complete: + if not self.paginate(): + return None + self.pagination_complete = True + page = None + count = 0 + for p in self.pagination_pages: + if p['limit'] == -1: + if not self.load_pagination_page(p): + return None + ncount = count + p['limit'] + if ncount > index: + page = p + break + count = ncount + if not page or not self.load_pagination_page(page) or count + page['limit'] <= index: + return None + return page['data'][index - count] + + def add_images_to_map(self, body): + if 'imageMap' not in body: + return + for k in body['imageMap'].keys(): + self.p.image_map[k] = body['imageMap'][k]['originalUrl'] + + def load_page(self, index): + self.pages[index] = [] + s = index * PAGES_PER_QUERY + for i in range(s, s + PAGES_PER_QUERY): + p = self.get_post_at_index(i) + # Break to return what we have, which could be less than expected. + if not p: + self.completed = True + break + unique_id = 'fanbox:{}'.format(p['id']) + if unique_id not in self.p.unique_id_map: + res = self.p.get_json( + BASE_URL + 'post.info', { 'postId': p['id'] } + ) + if not res or 'body' not in res: + # TODO: Log this. + continue + if res['body']['type'] == 'article': + self.add_images_to_map(res['body']['body']) + self.p.unique_id_map[unique_id] = res['body'] + # TODO: Should store Post object here instead. + body = self.p.unique_id_map[unique_id] + text = '' + media = [] + # If body is None, that post is restricted (not subbed or from a higher tier than your sub) + if body['body']: + if body['type'] == 'image': + text = body['body']['text'] + for i in body['body']['images']: + media.append(Image(url=i['originalUrl'], thumbnail_url='')) + elif body['type'] == 'article': + for b in body['body']['blocks']: + if b['type'] == 'p': + text += b['text'] + '\n' + elif b['type'] == 'image': + media.append(Image(url=self.p.image_map[b['imageId']], thumbnail_url='')) + else: + log.warn('Unhandled block type.') + else: + log.warn('Unhandled post type.') + # TODO: Raw response + self.pages[index].append(Post( + unique_id=unique_id, url='', title=body['title'], text=text, media=media)) + return True + +class FanboxPlugin(Plugin): + def __init__(self, sessid): + super().__init__() + self.image_map = {} + self.headers.update({ + 'accept': 'application/json, text/plain, */*', + 'accept-encoding': 'gzip, deflate, br', + 'accept-language': 'ja,en-US;q=0.9,en;q=0.8', + 'origin': 'https://www.fanbox.cc', + 'referer': 'https://www.fanbox.cc/', + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36' + }) + self.cookies['FANBOXSESSID'] = sessid + + def search(self, query, *extra_args): + return FanboxSearch(self, query) diff --git a/src/shoki/py/providers/instagram.py b/src/shoki/py/providers/instagram.py new file mode 100644 index 0000000..0146ba3 --- /dev/null +++ b/src/shoki/py/providers/instagram.py @@ -0,0 +1,96 @@ +import os +import config +import log +from plugin import Plugin +from search import Search +from instagrapi import Client +from instagrapi.exceptions import UserNotFound, LoginRequired, ChallengeRequired +from post import User, Post, Image, Video + +REACH_LIMIT = 3 + +class InstagramSearch(Search): + def __init__(self, cl, pk): + super().__init__() + self.cl = cl + self.pk = pk + self.pagination_page = 0 + self.pagination_cursor = None + + def load_page(self, index): + if index >= self.pagination_page + REACH_LIMIT: + return False + request_satisfied = False + for _ in range(self.pagination_page, index + 1): + if self.pagination_page > 0 and not self.pagination_cursor: + self.completed = True + return False + try: + media, self.pagination_cursor = self.cl.user_medias_paginated_v1( + self.pk, 0, end_cursor=self.pagination_cursor) + except LoginRequired: + return False + except ChallengeRequired: + return False + self.pages[self.pagination_page] = [] + for m in media: + unique_id = 'instagram:p:{}'.format(m.id) + media = [] + if m.media_type == 8: # Album + for r in m.resources: + if r.media_type == 1: # Photo + media.append(Image(url=r.thumbnail_url)) + elif r.media_type == 2: # Video + media.append(Video(url=r.video_url, thumbnail_url=r.thumbnail_url)) + elif m.media_type == 2: # Video + #print(m.product_type) # feed, igtv, or clips + media.append(Video(url=m.video_url, thumbnail_url=m.thumbnail_url)) + elif m.media_type == 1: # Photo + media.append(Image(url=m.thumbnail_url)) + kwargs = {} + kwargs['unique_id'] = unique_id + kwargs['raw_responses'] = { 'tile': m.json() } + kwargs['url'] = 'https://www.instagram.com/p/{}'.format(m.code) + user = User(unique_id='instagram:u:{}'.format(m.user.pk)) + if m.user.username: + user.username = m.user.username + if m.user.full_name: + user.display_name = m.user.full_name + kwargs['author'] = user + kwargs['title'] = m.title + kwargs['text'] = m.caption_text + kwargs['media'] = media + self.pages[self.pagination_page].append(Post(**kwargs)) + if self.pagination_page == index: + request_satisfied = True + self.pagination_page += 1 + return request_satisfied + +class InstagramPlugin(Plugin): + def __init__(self): + super().__init__() + self.cl = Client() + self.cl.set_user_agent(config.INSTAGRAM_USER_AGENT) + + def init(self): + if os.path.exists(config.INSTAGRAM_SETTINGS_PATH): + self.cl.load_settings(config.INSTAGRAM_SETTINGS_PATH) + if self.cl.login(config.INSTAGRAM_USERNAME, config.INSTAGRAM_PASSWORD): + return True + if not self.cl.login(config.INSTAGRAM_USERNAME, config.INSTAGRAM_PASSWORD): + return False + self.cl.dump_settings(config.INSTAGRAM_SETTINGS_PATH) + return True + + def search(self, query, *extra_args): + try: + user = self.cl.user_info_by_username_v1(query) + except UserNotFound: + return None + except LoginRequired: + log.error('Login required.') + return None + except ChallengeRequired: + log.error('Challenge required.') + return None + return InstagramSearch(self.cl, user.pk) diff --git a/src/shoki/py/providers/pixiv.py b/src/shoki/py/providers/pixiv.py new file mode 100644 index 0000000..9276d61 --- /dev/null +++ b/src/shoki/py/providers/pixiv.py @@ -0,0 +1,113 @@ +import json +import requests +from post import User, Image, Post +from search import Search +from plugin import Plugin +from query_parser import QueryParser +from pixivpy3 import * + +class PixivBase(Search): + def __init__(self, userdata, arg): + super().__init__() + self.plugin = userdata + self.next_page = 0 + + def api_load_page(self, index): + return False, None + + def load_page(self, index): + request_satisfied = False + for i in range(self.next_page, index + 1): + if not self.next_qs: + break + self.pages[i] = [] + res, self.next_qs = self.api_load_page(i) + if not res: + break + if i == index: + request_satisfied = True + self.next_page += 1 + return request_satisfied + + def make_post(self, post): + if len(post['meta_pages']) == 0: + post['meta_pages'].append({ + 'image_urls': { + "medium": post['image_urls']['medium'], + "original": post['meta_single_page']['original_image_url'] + } + }) + kwargs = {} + kwargs['unique_id'] = 'pixiv:i:{}'.format(post['id']) + kwargs['raw_responses'] = { 'api': json.dumps(post) } + kwargs['url'] = 'https://www.pixiv.net/en/artworks/{}'.format(post['id']) + kwargs['author'] = User( + unique_id='pixiv:u:{}'.format(post['user']['id']), + username=post['user']['account'], display_name=post['user']['name']) + kwargs['title'] = post['title'] + kwargs['text'] = post['caption'] + kwargs['likes'] = post['total_bookmarks'] + if 'total_comments' in post: + kwargs['comments'] = post['total_comments'] + media = [] + for img in post['meta_pages']: + media.append(Image( + url=img['image_urls']['original'], thumbnail_url=img['image_urls']['medium'])) + kwargs['media'] = media + return Post(**kwargs) + +class PixivFollowing(PixivBase): + def __init__(self, userdata, arg): + super().__init__(userdata, arg) + self.next_qs = { "user_id": arg, "restrict": "public" } + + def api_load_page(self, index): + obj = self.plugin.api.user_following(**self.next_qs) + if 'user_previews' not in obj: + return False, None + for u in obj['user_previews']: + user = u['user'] + self.pages[index].append(User( + unique_id='pixiv:u:{}'.format(user['id']), username=user['account'], + display_name=user['name'])) + return True, self.plugin.api.parse_qs(obj['next_url']) + +class PixivUser(PixivBase): + def __init__(self, userdata, arg): + super().__init__(userdata, arg) + self.next_qs = { 'user_id': arg, 'type': 'illust' } + + def api_load_page(self, index): + obj = self.plugin.api.user_illusts(**self.next_qs) + if 'illusts' not in obj: + return False, None + for post in obj['illusts']: + self.pages[index].append(self.make_post(post)) + return True, self.plugin.api.parse_qs(obj['next_url']) + +class PixivBookmarks(PixivBase): + def __init__(self, userdata, arg): + super().__init__(userdata, arg) + self.next_qs = { 'user_id': arg, 'restrict': 'public' } + + def api_load_page(self, index): + obj = self.plugin.api.user_bookmarks_illust(**self.next_qs) + if 'illusts' not in obj: + return False, None + for post in obj['illusts']: + self.pages[index].append(self.make_post(post)) + return True, self.plugin.api.parse_qs(obj['next_url']) + +class PixivPlugin(Plugin): + def __init__(self, refresh_token): + super().__init__() + self.parser = QueryParser(self, 'following') + self.parser.add_command('following', PixivFollowing) + self.parser.add_command('user', PixivUser) + self.parser.add_command('bookmarks', PixivBookmarks) + self.api = AppPixivAPI() + self.api.requests = requests.Session() + self.api.auth(refresh_token=refresh_token) + + def search(self, query, *extra_args): + return self.parser.parse_query(query) diff --git a/src/shoki/py/providers/searx.py b/src/shoki/py/providers/searx.py new file mode 100644 index 0000000..b536f28 --- /dev/null +++ b/src/shoki/py/providers/searx.py @@ -0,0 +1,93 @@ +import sys +sys.path.append('../../../subprojects/searxng') +import json +from searx import settings +from searx.utils import gen_useragent +from searx.engines import load_engine, register_engine +from plugin import Plugin +from search import Search +from post import Post, Image + +# TODO: Put these on the Plugin object. +ENGINES = {} + +ENGINE_INITIAL_INDEX = { + 'google images': 0, + 'google': 1, + 'bing images': 0 +} + +class SearxSearch(Search): + def __init__(self, plugin, query, engine): + super().__init__() + global ENGINES + self.query = query + self.plugin = plugin + self.key = engine.replace(' ', '_') + self.engine = ENGINES[engine] + self.index = ENGINE_INITIAL_INDEX[engine] + + def load_page(self, index): + self.pages[index] = [] + res = self.plugin.send_http_request(self.engine.request(self.query, { + 'language': 'en-US', + 'safesearch': 0, + 'time_range': None, + 'pageno': self.index + index, + 'cookies': self.plugin.cookies, + 'headers': self.plugin.headers, + 'data': None + })) + if not res: + return False + try: + ret = self.engine.response(res) + except: + return False + if len(ret) == 0: + # TODO: Test. + self.completed = True + return False + for i, p in enumerate(ret): + if 'url' not in p: + continue + # Don't add this to the global map because it's not actually unique. + unique_id = '{}:{}_{}:{}'.format(self.key, self.query, i, index) + media = [] + if 'img_src' in p: + media.append(Image(url=p['img_src'], thumbnail_url='')) + elif 'image_url' in p: + media.append(Image(url=p['image_url'], thumbnail_url='')) + kwargs = {} + kwargs['unique_id'] = unique_id + kwargs['raw_responses'] = { 'result': json.dumps(p) } + kwargs['url'] = p['url'] + kwargs['title'] = p['title'] + kwargs['text'] = p['content'] + kwargs['media'] = media + self.pages[index].append(Post(**kwargs)) + return True + +class SearxPlugin(Plugin): + def __init__(self): + super().__init__() + global ENGINES + self.headers = { + 'User-Agent': gen_useragent(), + 'Accept-Language': 'en-US,en;q=0.5', + } + for engine_data in settings['engines']: + engine = load_engine(engine_data) + if engine: + register_engine(engine) + ENGINES[engine.name] = engine + + def send_http_request(self, params): + self.cookies.update(params['cookies']) + self.headers.update(params['headers']) + return self.get(params['url'], params=params['data']) + + def search(self, query, *extra_args): + if not extra_args: + return None + return SearxSearch(self, query, extra_args[0]) diff --git a/src/shoki/py/providers/twitter.py b/src/shoki/py/providers/twitter.py new file mode 100644 index 0000000..2fc0faf --- /dev/null +++ b/src/shoki/py/providers/twitter.py @@ -0,0 +1,425 @@ +import json +import config +import http.cookiejar +from datetime import timezone +from socketserver import UDPServer + +import log +from provider import Provider +from search import Search +from post import PostType, DateType, PostRef, Post, User, Image, Video +from query_parser import QueryParser + +# Official Twitter API +#from twitter import Twitter2, TwitterError, OAuth + +# GraphQL Scrape API +import snscrape.modules.twitter +from snscrape.base import ScraperException +from snscrape.modules.twitter import (TwitterSearchScraper, TwitterProfileScraper, + TwitterUserScraper, TwitterTweetScraper, TweetRef) + +# Don't attempt to load a page 4 or more pages past the current page. +REACH_LIMIT = 4 + +# Quite incomplete API based backend. Should *not* be used for archiving. +# HOLD: I can't test this without paying $100 for the X API WTFFF. +''' +class TwitterBase(Search): + ALL_PARAMS = { + 'tweet.fields': 'attachments,author_id,context_annotations,conversation_id,created_at,entities,geo,id,in_reply_to_user_id,lang,public_metrics,possibly_sensitive,referenced_tweets,reply_settings,source,text,withheld', + 'user.fields': 'created_at,description,entities,id,location,name,pinned_tweet_id,profile_image_url,protected,public_metrics,url,username,verified,withheld', + 'media.fields': 'duration_ms,height,media_key,preview_image_url,type,url,width,public_metrics', + 'place.fields': 'contained_within,country,country_code,full_name,geo,id,name,place_type', + 'poll.fields': 'duration_minutes,end_datetime,id,options,voting_status', + } + ALL_EXPRESSIONS = 'author_id,referenced_tweets.id,referenced_tweets.id.author_id,entities.mentions.username,attachments.poll_ids,attachments.media_keys,in_reply_to_user_id,geo.place_id' + + SOME_PARAMS = { + 'tweet.fields': 'attachments,author_id,text,entities,referenced_tweets', + 'user.fields': 'id,name,profile_image_url,url,username', + 'media.fields': 'duration_ms,height,media_key,preview_image_url,type,url,width,public_metrics', + 'place.fields': '', + 'poll.fields': '', + } + SOME_EXPRESSIONS = 'author_id,referenced_tweets.id,referenced_tweets.id.author_id,entities.mentions.username,attachments.media_keys,in_reply_to_user_id' + + def __init__(self, userdata, arg): + super().__init__() + self.provider = userdata + self.params = self.SOME_PARAMS.copy() + self.media_map = {} + self.user_id = None + self.pagination_page = 0 + self.pagination_token = None + self.arg = arg + + def add_attachemnts(self, l, r, data): + if 'media_keys' not in data: + return + for m in data['media_keys']: + if m in r: + continue + r.append(m) + if m in self.media_map: + l.append(Image(url=self.media_map[m], thumbnail_url='')) + + def add_entity_urls(self, data): + if 'urls' in data: + for u in data['urls']: + if 'media_key' in u and u['media_key'] not in self.media_map: + self.media_map[u['media_key']] = u['expanded_url'] + + def make_post(self, tweet): + print(json.dumps(tweet, indent=4)) + media = [] + repeats = [] + if 'referenced_tweets' in t: + print(len(referenced_tweets)) + #for rt in t['referenced_tweets']: + # if rt['id'] in self.tweet_map: + # rrt = self.tweet_map[rt['id']] + # if 'entities' in rrt: + # self.add_entity_urls(rrt['entities']) + # if 'attachments' in rrt: + # self.add_attachemnts(media, repeats, rrt['attachments']) + if 'attachments' in t: + self.add_attachemnts(media, repeats, t['attachments']) + unique_id = 'twitter:t:{}'.format(t['id']) + url = 'https://twitter.com/{}/status/{}'.format(t['author_id'], t['id']) + kwargs = {} + kwargs['unique_id'] = unique_id + kwargs['raw_responses'] = { 'tweet' : json.dumps(t) } + kwargs['url'] = url + kwargs['author'] = User(unique_id='twitter:u:{}'.format(t['author_id'])) + kwargs['title'] = '' + kwargs['text'] = t['text'] + kwargs['media'] = media + return Post(**kwargs) + + def add_items_from_search(self, data): + if 'includes' in data: + includes = data['includes'] + if 'media' in includes: + for i in includes['media']: + if 'url' in i: + self.media_map[i['media_key']] = i['url'] + #if 'tweets' in includes: + # for t in includes['tweets']: + # self.tweet_map[t['id']] = t + if 'data' not in data or not data['data']: + return False + l = data['data'] if type(data['data']) == list else [data['data']] + for t in l: + if 'entities' in t: + self.add_entity_urls(t['entities']) + post = self.make_post(t) + self.pages[self.pagination_page].append(post) + return True + + def add_user_ids_from_search(self, data): + if not data['data']: + return False + for u in data['data']: + self.pages[self.pagination_page].append(User( + 'twitter:u:{}'.format(u['id']), username=u['username'], display_name=u['name'])) + return True + + def should_process_index(self, index): + if index - self.pagination_page >= REACH_LIMIT: + return False + if self.pagination_page > 0 and not self.pagination_token: + self.completed = True + return False + if self.pagination_token: + self.params['pagination_token'] = self.pagination_token + self.pages[self.pagination_page] = [] + return True + + def handle_data(self, data, user_ids=False): + if 'next_token' not in data['meta'].keys(): + self.pagination_token = None + else: + self.pagination_token = data['meta']['next_token'] + if user_ids: + if not self.add_user_ids_from_search(data): + return False + else: + if not self.add_items_from_search(data): + return False + self.pagination_page += 1 + return True + + def get_user_id(self, username): + if self.user_id: + return True + try: + data = self.provider.t.users.by.username._username( + _username=username, _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + self.user_id = data['data']['id'] + return True + +class TwitterSearch(TwitterBase): + def load_page(self, index): + for _ in range(self.pagination_page, index + 1): + if not self.should_process_index(index): + return False + try: + data = self.provider.t.tweets.search.recent( + query=self.arg, expansions=self.SOME_EXPRESSIONS, params=self.params, + sort_order='relevancy', max_results=25, _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + return self.handle_data(data) + +class TwitterUser(TwitterBase): + def __init__(self, userdata, arg): + if arg.startswith('@'): + arg = arg[1:] + super().__init__(userdata, arg) + + def load_page(self, index): + for _ in range(self.pagination_page, index + 1): + if not self.should_process_index(index): + return False + if not self.get_user_id(self.arg): + return False + try: + data = self.provider.t.users._id.tweets( + _id=self.user_id, expansions=self.SOME_EXPRESSIONS, params=self.params, + max_results=25, _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + return self.handle_data(data) + +class TwitterTimeline(TwitterBase): + def __init__(self, userdata, arg): + if not arg: + arg = 'pizzabelly' + super().__init__(userdata, arg) + self.params['exclude'] = 'replies' + + def load_page(self, index): + for _ in range(self.pagination_page, index + 1): + if not self.should_process_index(index): + return False + if not self.get_user_id(self.arg): + return False + try: + data = self.provider.t.users._id.timelines.reverse_chronological( + _id=self.user_id, expansions=self.SOME_EXPRESSIONS, params=self.params, + max_results=25, _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + return self.handle_data(data) + +class TwitterLikes(TwitterBase): + def __init__(self, userdata, arg): + if arg.startswith('@'): + arg = arg[1:] + super().__init__(userdata, arg) + + def load_page(self, index): + for _ in range(self.pagination_page, index + 1): + if not self.should_process_index(index): + return False + if not self.get_user_id(self.arg): + return False + try: + data = self.provider.t.users._id.liked_tweets( + _id=self.user_id, expansions=self.SOME_EXPRESSIONS, params=self.params, + max_results=25, _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + return self.handle_data(data) + +class TwitterTweet(TwitterBase): + def load_page(self, index): + for _ in range(self.pagination_page, index + 1): + if not self.should_process_index(index): + return False + try: + data = self.provider.t.tweets( + ids=self.arg, expansions=self.SOME_EXPRESSIONS, params=self.params, + _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + return self.handle_data(data) + +class TwitterFollowing(TwitterBase): + def __init__(self, userdata, arg): + if arg.startswith('@'): + arg = arg[1:] + super().__init__(userdata, arg) + del self.params['media.fields'] + del self.params['place.fields'] + del self.params['poll.fields'] + + def load_page(self, index): + for _ in range(self.pagination_page, index + 1): + if not self.should_process_index(index): + return False + if not self.get_user_id(self.arg): + return False + try: + data = self.provider.t.users._id.following( + _id=self.user_id, params=self.params, max_results=25, _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + return self.handle_data(data, True) +''' + +class TwitterScrapeBase(Search): + # Posts per emulated page because snscrape returns an iterator. + POSTS_PER_PAGE = 8 + + def __init__(self, userdata, arg): + super().__init__() + self.provider = userdata + self.cookies = self.provider.cookies + self.arg = arg + self.iterator = None + self.iter_page = 0 + self.sent_hashes = [] + + def parse_media(self, l): + media = [] + for m in l: + if isinstance(m, snscrape.modules.twitter.Photo): + media.append(Image(url=m.fullUrl, thumbnail_url=m.previewUrl)) + elif isinstance(m, snscrape.modules.twitter.Video): + videos = sorted(m.variants, key=lambda x: x.bitrate if x.bitrate else 0, reverse=True) + media.append(Video(url=videos[0].url, thumbnail_url=m.thumbnailUrl)) + elif isinstance(m, snscrape.modules.twitter.Gif): + gifs = sorted(m.variants, key=lambda x: x.bitrate if x.bitrate else 0, reverse=True) + media.append(Video(url=gifs[0].url, thumbnail_url=m.thumbnailUrl)) + return media + + def make_post(self, tweet): + kwargs = {} + kwargs['unique_id'] = 'twitter:t:{}'.format(tweet.id) + if isinstance(tweet, TweetRef): + return Post(**kwargs) + kwargs['raw_responses'] = tweet.rawResponses + h = tweet.rawResponses['hash'] + if h not in self.sent_hashes: + self.sent_hashes.append(h) + else: + del kwargs['raw_responses']['api'] + kwargs['url'] = tweet.url + kwargs['dates'] = { + DateType.CREATED: tweet.date.replace(tzinfo=timezone.utc).timestamp() + } + author = User(unique_id='twitter:u:{}'.format(tweet.user.id), username='@' + tweet.user.username, + profile_image_url=tweet.user.profileImageUrl) + if tweet.user.displayname: + author.display_name = tweet.user.displayname + kwargs['author'] = author + if tweet.retweetedTweet: + kwargs['type'] = PostType.REPOST + kwargs['post'] = PostRef(unique_id='twitter:t:{}'.format(tweet.retweetedTweet.id)) + return Post(**kwargs) + kwargs['type'] = PostType.POST + kwargs['text'] = tweet.rawContent + kwargs['likes'] = tweet.likeCount + kwargs['reposts'] = tweet.retweetCount + kwargs['quotes'] = tweet.quoteCount + kwargs['comments'] = tweet.replyCount + kwargs['views'] = tweet.viewCount + if tweet.media: + kwargs['media'] = self.parse_media(tweet.media) + if tweet.inReplyToTweetId: + kwargs['in_reply_to'] = PostRef(unique_id='twitter:t:{}'.format(tweet.inReplyToTweetId)) + if tweet.quotedTweet: + kwargs['quoted'] = PostRef(unique_id='twitter:t:{}'.format(tweet.quotedTweet.id)) + return Post(**kwargs) + + def add_post(self, i, post): + self.pages[i]['posts'].append(post) + + def add_post_to_list(self, i, post): + self.add_post(i, post) + self.pages[i]['list'].append(post.unique_id) + + def load_page(self, index): + if not self.iterator or index - self.iter_page >= REACH_LIMIT: + return False + if index in self.pages: + return True + # It's possible to load some "pages", but not enough to satisfy the requested index. + request_satisfied = False + for i in range(self.iter_page, index + 1): + if self.completed: + break + self.pages[i] = { 'posts': [], 'list': [] } + for _ in range(0, self.POSTS_PER_PAGE): + try: + tweet = next(self.iterator) + except StopIteration: + self.completed = True + break + except ScraperException as e: + log.error(repr(e)) + continue + self.add_post_to_list(i, self.make_post(tweet)) + if tweet.retweetedTweet: + self.add_post(i, self.make_post(tweet.retweetedTweet)) + if tweet.quotedTweet: + self.add_post(i, self.make_post(tweet.quotedTweet)) + if i == index: + request_satisfied = True + self.iter_page += 1 + return request_satisfied + +class TwitterScrapeSearch(TwitterScrapeBase): + def __init__(self, userdata, arg): + super().__init__(userdata, arg) + self.iterator = TwitterSearchScraper(self.arg, top=True, cookies=self.cookies).get_items() + +class TwitterScrapeUser(TwitterScrapeBase): + def __init__(self, userdata, arg): + super().__init__(userdata, arg) + if (self.arg.startswith('@')): + self.arg = self.arg[1:] + self.iterator = TwitterProfileScraper(self.arg, cookies=self.cookies).get_items() + #self.iterator = TwitterUserScraper(self.arg, cookies=self.cookies).get_items() + +class TwitterScrapeTweet(TwitterScrapeBase): + def __init__(self, userdata, arg): + super().__init__(userdata, arg) + self.single = True + if (self.arg.startswith('https://')): + self.arg = self.arg[self.arg.rfind('/') + 1:] + question = self.arg.find('?') + if question >= 0: + self.arg = self.arg[:question] + self.iterator = TwitterTweetScraper(self.arg, cookies=self.cookies).get_items() + +class TwitterProvider(Provider): + def __init__(self, access_key, access_secret, consumer_key, consumer_secret, cookies_path): + super().__init__() + self.parser = QueryParser(self, 'search') + self.parser.add_command('search', TwitterScrapeSearch) + self.parser.add_command('user', TwitterScrapeUser) + self.parser.add_command('tweet', TwitterScrapeTweet) + #self.parser.add_command('timeline', TwitterTimeline) + #self.parser.add_command('likes', TwitterLikes) + #self.parser.add_command('following', TwitterFollowing) + #self.t = Twitter2(auth=OAuth(access_key, access_secret, consumer_key, consumer_secret), retry=True) + # Cookies are used by the scrape backend. + cookie_jar = http.cookiejar.MozillaCookieJar() + cookie_jar.load(filename=cookies_path, ignore_expires=True) + for c in cookie_jar: + self.cookies[c.name] = c.value + + def search(self, query, *extra_args): + return self.parser.parse_query(query) diff --git a/src/shoki/py/providers/youtube.py b/src/shoki/py/providers/youtube.py new file mode 100644 index 0000000..f17211e --- /dev/null +++ b/src/shoki/py/providers/youtube.py @@ -0,0 +1,115 @@ +import log +import yt_dlp +from provider import Provider +from search import Search +from post import Post, PostType, Image, Video +from query_parser import QueryParser + +class YDLLogger(): + def debug(self, msg): + if msg.startswith('[debug] '): + log.debug(msg) + else: + log.info(msg) + + def info(self, msg): + pass + + def warning(self, msg): + log.warning(msg) + + def error(self, msg): + log.error(msg) + +ydl_opts = { + 'quiet': False, + 'logger': YDLLogger(), + 'cachedir': False, +# 'cookiefile': '', +} + +ydl = yt_dlp.YoutubeDL(ydl_opts) + +def get_playback_url(data, video=True): + if 'entries' in data: + data = data['entries'][0] + + url = data['formats'][0]['url'] + + ''' + data['formats'] = list(filter(lambda f: 'container' not in f, data['formats'])) + + data['formats'] = list(filter( + lambda f: not ('container' in f and (f['container'] in ['mp4_dash', 'webm_dash'])), data['formats'] + )) + ''' + + data['formats'] = list(filter(lambda f: 'manifest_url' not in f, data['formats'])) + + has_audio = list(filter(lambda f: 'acodec' not in f or f['acodec'] != 'none', data['formats'])) + + if video: + if len(has_audio) > 0: + data['formats'] = has_audio + try: + data['formats'] = list(filter(lambda f: 'quality' in f, data['formats'])) + url = max(data['formats'], key=lambda f: f['quality'])['url'] + except: + pass + else: + if len(has_audio) == 0: + return None + data['formats'] = has_audio + try: + audio_only = list(filter(lambda f: f['vcodec'] == 'none', data['formats'])) + if len(audio_only) > 0: + data['formats'] = audio_only + else: + data['formats'] = list(filter(lambda f: f['ext'] in ['mp4'], data['formats'])) + except: + pass + try: + data['formats'] = list(filter(lambda f: 'abr' in f, data['formats'])) + url = max(data['formats'], key=lambda f: f['abr'])['url'] + except: + pass + + return url + +class YoutubeBase(Search): + def __init__(self, userdata, arg): + super().__init__() + self.provider = userdata + self.query = arg + + def load_page(self, index): + self.pages[index] = { 'posts': [], 'list': [] } + res = self.get_info() + if res: + media = [Video(url=get_playback_url(res))] + self.pages[index]['posts'].append(Post(type=PostType.POST, unique_id='', url='', title='', text='', media=media)) + return True + +class YoutubeSearch(YoutubeBase): + def __init__(self, userdata, arg): + super().__init__(userdata, arg) + + def get_info(self): + return ydl.extract_info('ytsearch1:{}'.format(self.query), download=False) + +class YoutubeLink(YoutubeBase): + def __init__(self, userdata, arg): + super().__init__(userdata, arg) + + def get_info(self): + return ydl.extract_info(self.query, download=False) + +class YoutubeProvider(Provider): + def __init__(self): + super().__init__() + self.parser = QueryParser(self, 'search') + self.parser.add_command('search', YoutubeSearch) + self.parser.add_command('link', YoutubeLink) + + def search(self, query, *extra_args): + return self.parser.parse_query(query) diff --git a/src/shoki/py/query_parser.py b/src/shoki/py/query_parser.py new file mode 100644 index 0000000..ee23bef --- /dev/null +++ b/src/shoki/py/query_parser.py @@ -0,0 +1,39 @@ +class QueryParser(): + def __init__(self, userdata, default_command): + self.commands = {} + self.escaped_commands = {} + self.userdata = userdata + self.default_command = default_command + + def add_command(self, name, search): + self.commands[name] = search + self.escaped_commands['\\' + name] = name + + def parse_query(self, query): + arg = query + search = self.commands[self.default_command] + + sp = query.split(' ') + + for i, word in enumerate(sp): + col = word.find(':') + if col == -1: + continue + command = word[0:col] + if command in self.escaped_commands: + query = query.replace(command, self.escaped_commands[command], 1) + continue + if command in self.commands: + if col + 1 < len(word): + arg = word[col+1:] + else: + arg = None + search = self.commands[command] + if i == 0: + if len(sp) > 1: + query = query.replace(word + ' ', '', 1) + else: + query = query.replace(word, '', 1) + else: + query = query.replace(' ' + word + ' ', '', 1) + return search(self.userdata, arg) diff --git a/src/shoki/py/search.py b/src/shoki/py/search.py new file mode 100644 index 0000000..4134851 --- /dev/null +++ b/src/shoki/py/search.py @@ -0,0 +1,35 @@ +class Search(): + def __init__(self): + self.pages = {} + self.iter_page = 0 + self.iter_index = 0 + self.single = False + self.completed = False + + def __iter__(self): + self.iter_page = 0 + self.iter_index = 0 + return self + + def __next__(self): + if self.iter_index >= len(self.pages[self.iter_page]): + self.iter_page += 1 + self.iter_index = 0 + if self.iter_page not in self.pages: + if not self.load_page(self.iter_page): + raise StopIteration + return self.pages[self.iter_page][self.iter_index] + + def load_page(self, index): + return False + + def get_page(self, index): + if index not in self.pages: + if self.completed: + if self.single: + return self.pages[0] + else: + return None + if not self.load_page(index): + return None + return self.pages[index] diff --git a/src/shoki/src/packet_ext.c b/src/shoki/src/packet_ext.c new file mode 100644 index 0000000..26ba9ee --- /dev/null +++ b/src/shoki/src/packet_ext.c @@ -0,0 +1,91 @@ +#include "search.h" +#include "packet_ext.h" + +static void aki_packet_write_sho_optional_int(struct aki_packet *packet, sho_optional_int *o) +{ + AKI_PACKET_WRITE_TYPE(packet, s64, o->i); + AKI_PACKET_WRITE_TYPE(packet, bool, o->set); +} + +void aki_packet_write_sho_post(struct aki_packet *packet, struct sho_post *post) +{ + AKI_PACKET_WRITE_TYPE(packet, u16, post->version); + AKI_PACKET_WRITE_TYPE(packet, u8, post->type); + aki_packet_write_string(packet, &post->unique_id); + aki_packet_write_string(packet, &post->url); + AKI_PACKET_WRITE_TYPE(packet, u32, post->dates.size); + struct sho_post_date *date; + al_array_foreach_ptr(post->dates, i, date) { + AKI_PACKET_WRITE_TYPE(packet, u8, date->type); + AKI_PACKET_WRITE_TYPE(packet, s64, date->timestamp); + } + aki_packet_write_string(packet, &post->author.unique_id); + aki_packet_write_string(packet, &post->author.username); + aki_packet_write_string(packet, &post->author.display_name); + aki_packet_write_string(packet, &post->author.profile_image_url); + aki_packet_write_string(packet, &post->title); + aki_packet_write_string(packet, &post->text); + aki_packet_write_sho_optional_int(packet, &post->likes); + aki_packet_write_sho_optional_int(packet, &post->reposts); + aki_packet_write_sho_optional_int(packet, &post->quotes); + aki_packet_write_sho_optional_int(packet, &post->comments); + aki_packet_write_sho_optional_int(packet, &post->views); + AKI_PACKET_WRITE_TYPE(packet, u32, post->media.size); + struct sho_post_media *media; + al_array_foreach_ptr(post->media, i, media) { + AKI_PACKET_WRITE_TYPE(packet, u8, media->type); + aki_packet_write_string(packet, &media->url); + aki_packet_write_string(packet, &media->thumbnail_url); + } + aki_packet_write_string(packet, &post->post.unique_id); + aki_packet_write_string(packet, &post->quoted.unique_id); + aki_packet_write_string(packet, &post->in_reply_to.unique_id); +} + +static void aki_packet_read_sho_optional_int(struct aki_packet *packet, sho_optional_int *o) +{ + AKI_PACKET_READ_TYPE(packet, s64, o->i); + AKI_PACKET_READ_TYPE(packet, bool, o->set); +} + +void aki_packet_read_sho_post(struct aki_packet *packet, struct sho_post *post) +{ + sho_post_reset(post); + AKI_PACKET_READ_TYPE(packet, u16, post->version); + AKI_PACKET_READ_TYPE(packet, u8, post->type); + aki_packet_read_string(packet, &post->unique_id); + aki_packet_read_string(packet, &post->url); + u32 dates_size; + AKI_PACKET_READ_TYPE(packet, u32, dates_size); + al_array_reserve(post->dates, dates_size); + for (u32 i = 0; i < dates_size; i++) { + struct sho_post_date date; + AKI_PACKET_READ_TYPE(packet, u8, date.type); + AKI_PACKET_READ_TYPE(packet, s64, date.timestamp); + al_array_push(post->dates, date); + } + aki_packet_read_string(packet, &post->author.unique_id); + aki_packet_read_string(packet, &post->author.username); + aki_packet_read_string(packet, &post->author.display_name); + aki_packet_read_string(packet, &post->author.profile_image_url); + aki_packet_read_string(packet, &post->title); + aki_packet_read_string(packet, &post->text); + aki_packet_read_sho_optional_int(packet, &post->likes); + aki_packet_read_sho_optional_int(packet, &post->reposts); + aki_packet_read_sho_optional_int(packet, &post->quotes); + aki_packet_read_sho_optional_int(packet, &post->comments); + aki_packet_read_sho_optional_int(packet, &post->views); + u32 media_size; + AKI_PACKET_READ_TYPE(packet, u32, media_size); + al_array_reserve(post->media, media_size); + for (u32 i = 0; i < media_size; i++) { + struct sho_post_media media; + AKI_PACKET_READ_TYPE(packet, u8, media.type); + aki_packet_read_string(packet, &media.url); + aki_packet_read_string(packet, &media.thumbnail_url); + al_array_push(post->media, media); + } + aki_packet_read_string(packet, &post->post.unique_id); + aki_packet_read_string(packet, &post->quoted.unique_id); + aki_packet_read_string(packet, &post->in_reply_to.unique_id); +} diff --git a/src/shoki/src/packet_ext.h b/src/shoki/src/packet_ext.h new file mode 100644 index 0000000..ff0fda1 --- /dev/null +++ b/src/shoki/src/packet_ext.h @@ -0,0 +1,8 @@ +#pragma once + +#include + +#include "post.h" + +void aki_packet_write_sho_post(struct aki_packet *packet, struct sho_post *post); +void aki_packet_read_sho_post(struct aki_packet *packet, struct sho_post *post); diff --git a/src/shoki/src/post.c b/src/shoki/src/post.c new file mode 100644 index 0000000..22fb7ef --- /dev/null +++ b/src/shoki/src/post.c @@ -0,0 +1,55 @@ +#include "post.h" + +void sho_post_reset(struct sho_post *post) +{ + al_bzero(post, sizeof(struct sho_post)); + al_array_init(post->dates); + al_array_init(post->media); +} + +void sho_post_add_media(struct sho_post *post, u8 type, str *url, str *thumbnail_url) +{ + al_array_push(post->media, ((struct sho_post_media){ + .type = type, .url = *url, .thumbnail_url = *thumbnail_url + })); +} + +void sho_post_add_date(struct sho_post *post, u8 type, s64 timestamp) +{ + al_array_push(post->dates, ((struct sho_post_date){ + .type = type, .timestamp = timestamp + })); +} + +void sho_post_clone(struct sho_post *dest, struct sho_post *src) +{ + sho_post_reset(dest); + dest->version = src->version; + dest->type = src->type; + al_str_clone(&dest->unique_id, &src->unique_id); + al_str_clone(&dest->url, &src->url); + al_array_clone(dest->dates, src->dates); + al_str_clone(&dest->author.unique_id, &src->author.unique_id); + al_str_clone(&dest->author.username, &src->author.username); + al_str_clone(&dest->author.display_name, &src->author.display_name); + al_str_clone(&dest->author.profile_image_url, &src->author.profile_image_url); + al_str_clone(&dest->title, &src->title); + al_str_clone(&dest->text, &src->text); + dest->likes = src->likes; + dest->reposts = src->reposts; + dest->quotes = src->quotes; + dest->comments = src->comments; + dest->views = src->views; + struct sho_post_media *media; + struct sho_post_media media_copy; + al_array_foreach_ptr(src->media, i, media) { + al_bzero(&media_copy, sizeof(struct sho_post_media)); + media_copy.type = media->type; + al_str_clone(&media_copy.url, &media->url); + al_str_clone(&media_copy.thumbnail_url, &media->thumbnail_url); + al_array_push(dest->media, media_copy); + } + al_str_clone(&dest->post.unique_id, &src->post.unique_id); + al_str_clone(&dest->quoted.unique_id, &src->quoted.unique_id); + al_str_clone(&dest->in_reply_to.unique_id, &src->in_reply_to.unique_id); +} diff --git a/src/shoki/src/post.h b/src/shoki/src/post.h new file mode 100644 index 0000000..6814b0f --- /dev/null +++ b/src/shoki/src/post.h @@ -0,0 +1,78 @@ +#pragma once + +#include +#include +#include + +enum { + SHOKI_MEDIA_IMAGE = 0, + SHOKI_MEDIA_VIDEO, + SHOKI_MEDIA_AUDIO +}; + +enum { + SHOKI_POST_POST = 0, + SHOKI_POST_REPOST +}; + +enum { + SHOKI_DATE_CREATED = 0, + SHOKI_DATE_EDITED, + SHOKI_DATE_ARCHIVED +}; + +typedef struct { + s64 i; + bool set; +} sho_optional_int; + +struct sho_post_date { + u8 type; + s64 timestamp; +}; + +struct sho_post_media { + u8 type; + str url; + str thumbnail_url; +}; + +typedef array(struct sho_post_date) sho_dates_array; +typedef array(struct sho_post_media) sho_media_array; + +struct sho_post_user { + str unique_id; + str username; + str display_name; + str profile_image_url; +}; + +struct sho_post_ref { + str unique_id; +}; + +struct sho_post { + u16 version; + u8 type; + str unique_id; + str raw_responses; + str url; + sho_dates_array dates; + struct sho_post_user author; + str title; + str text; + sho_optional_int likes; + sho_optional_int reposts; + sho_optional_int quotes; + sho_optional_int comments; + sho_optional_int views; + sho_media_array media; + struct sho_post_ref post; // reposted post. + struct sho_post_ref quoted; + struct sho_post_ref in_reply_to; +}; + +void sho_post_reset(struct sho_post *post); +void sho_post_add_media(struct sho_post *post, u8 type, str *url, str *thumbnail_url); +void sho_post_add_date(struct sho_post *post, u8 type, s64 timestamp); +void sho_post_clone(struct sho_post *dest, struct sho_post *src); diff --git a/src/shoki/src/post_cache.c b/src/shoki/src/post_cache.c new file mode 100644 index 0000000..a8f02eb --- /dev/null +++ b/src/shoki/src/post_cache.c @@ -0,0 +1,30 @@ +#include "post_cache.h" + +void sho_post_cache_init(struct sho_post_cache *cache) +{ + al_array_init(cache->cache); +} + +void sho_post_cache_push(struct sho_post_cache *cache, struct sho_post *post) +{ + struct sho_post *check = sho_post_cache_get(cache, &post->unique_id); + if (!check) { + struct sho_post *npost = al_alloc_object(struct sho_post); + sho_post_clone(npost, post); + al_array_push(cache->cache, npost); + } +} + +struct sho_post *sho_post_cache_get(struct sho_post_cache *cache, str *unique_id) +{ + struct sho_post *post; + al_array_foreach(cache->cache, i, post) { + if (al_str_eq(&post->unique_id, unique_id)) { + return post; + } + if (al_str_eq(&post->author.unique_id, unique_id)) { + return post; + } + } + return NULL; +} diff --git a/src/shoki/src/post_cache.h b/src/shoki/src/post_cache.h new file mode 100644 index 0000000..80cd0b7 --- /dev/null +++ b/src/shoki/src/post_cache.h @@ -0,0 +1,13 @@ +#pragma once + +#include + +#include "post.h" + +struct sho_post_cache { + array(struct sho_post *) cache; +}; + +void sho_post_cache_init(struct sho_post_cache *cache); +void sho_post_cache_push(struct sho_post_cache *cache, struct sho_post *post); +struct sho_post *sho_post_cache_get(struct sho_post_cache *cache, str *unique_id); diff --git a/src/shoki/src/search.c b/src/shoki/src/search.c new file mode 100644 index 0000000..b4b2157 --- /dev/null +++ b/src/shoki/src/search.c @@ -0,0 +1,99 @@ +#include + +#include "../cpy/shoki.c" + +#include "search.h" + +bool sho_python_init(void) +{ + PyPreConfig pre; + PyPreConfig_InitPythonConfig(&pre); + pre.utf8_mode = 1; + Py_PreInitialize(&pre); + if (PyImport_AppendInittab("shoki", PyInit_shoki) == -1) { + al_log_error("shoki", "Could not extend in-built modules table."); + sho_python_close(); + return false; + } + Py_Initialize(); + PyObject *module = PyImport_ImportModule("shoki"); + if (!module) { + PyErr_Print(); + al_log_error("shoki", "Could not import module."); + sho_python_close(); + return false; + } + return true; +} + +void sho_result_init(struct sho_result *res) +{ + al_array_init(res->pages); +} + +static struct sho_result_page *page_at_index(struct sho_result *res, s32 num) +{ + struct sho_result_page *page; + al_array_foreach_ptr(res->pages, i, page) { + if (page->num == num) { + return page; + } + } + return NULL; +} + +void sho_result_add_post(struct sho_result *res, s32 num, struct sho_post *post) +{ + struct sho_result_page *page = page_at_index(res, num); + if (!page) { + al_array_push(res->pages, (struct sho_result_page){0}); + page = &al_array_last(res->pages); + page->num = num; + al_array_init(page->posts); + al_array_init(page->list); + } + al_array_push(page->posts, *post); +} + +void sho_result_add_to_list(struct sho_result *res, s32 num, str *unique_id) +{ + struct sho_result_page *page = page_at_index(res, num); + al_array_push(page->list, *unique_id); +} + +void sho_search_init(struct sho_search *search) +{ + search->id = -1; + search->page = 0; + sho_result_init(&search->result); +} + +s32 sho_search_more_results(struct sho_search *search, str *provider, str *query) +{ + if (search->id < 0) { + search->id = sho_client_search(provider, query); + } else { + search->page++; + } + if (search->id < 0) return -1; + al_log_info("shoki", "Loading page %i (%.*s).", search->page, AL_STR_PRINTF(query)); + if (sho_client_get_page(&search->result, search->id, search->page) < 0) { + return -1; + } + return search->page; +} + +void sho_search_free(struct sho_search *search) +{ + struct sho_result_page *page; + al_array_foreach_ptr(search->result.pages, i, page) { + al_array_free(page->posts); + al_array_free(page->list); + } + al_array_free(search->result.pages); +} + +void sho_python_close(void) +{ + if (Py_IsInitialized()) Py_Finalize(); +} diff --git a/src/shoki/src/search.h b/src/shoki/src/search.h new file mode 100644 index 0000000..97e40ae --- /dev/null +++ b/src/shoki/src/search.h @@ -0,0 +1,30 @@ +#pragma once + +#include "post.h" + +struct sho_result_page { + s32 num; + array(struct sho_post) posts; + array(str) list; +}; + +struct sho_result { + array(struct sho_result_page) pages; +}; + +struct sho_search { + s32 id; + s32 page; + struct sho_result result; +}; + +bool sho_python_init(void); +void sho_python_close(void); + +void sho_result_init(struct sho_result *res); +void sho_result_add_post(struct sho_result *res, s32 num, struct sho_post *post); +void sho_result_add_to_list(struct sho_result *res, s32 num, str *unique_id); + +void sho_search_init(struct sho_search *search); +s32 sho_search_more_results(struct sho_search *search, str *provider, str *query); +void sho_search_free(struct sho_search *s); -- cgit v1.2.3-101-g0448