From 02f3d3565602146bbbfce85b2719246f24036cb9 Mon Sep 17 00:00:00 2001 From: Andrew Opalach Date: Tue, 9 Apr 2024 11:24:01 -0400 Subject: Massive restructure and many changes - The server-side list concept is still a wip Signed-off-by: Andrew Opalach --- src/portal/cpy/bridge.pxd | 8 ++ src/portal/cpy/build.sh | 3 + src/portal/cpy/lib.pxd | 13 +++ src/portal/cpy/portal.pyx | 203 ++++++++++++++++++++++++++++++++++++++++++++++ src/portal/cpy/post.pxd | 61 ++++++++++++++ src/portal/cpy/setup.py | 6 ++ src/portal/cpy/str.pxd | 18 ++++ src/portal/cpy/types.pxd | 23 ++++++ 8 files changed, 335 insertions(+) create mode 100644 src/portal/cpy/bridge.pxd create mode 100755 src/portal/cpy/build.sh create mode 100644 src/portal/cpy/lib.pxd create mode 100644 src/portal/cpy/portal.pyx create mode 100644 src/portal/cpy/post.pxd create mode 100644 src/portal/cpy/setup.py create mode 100644 src/portal/cpy/str.pxd create mode 100644 src/portal/cpy/types.pxd (limited to 'src/portal/cpy') diff --git a/src/portal/cpy/bridge.pxd b/src/portal/cpy/bridge.pxd new file mode 100644 index 0000000..ac14811 --- /dev/null +++ b/src/portal/cpy/bridge.pxd @@ -0,0 +1,8 @@ +include "post.pxd" + +cdef extern from "../src/search.h": + cdef struct camu_search: + pass + + void camu_search_add_post(camu_search *search, u32 page, camu_post *post) + void camu_search_add_to_list(camu_search *search, u32 page, str *unique_id) diff --git a/src/portal/cpy/build.sh b/src/portal/cpy/build.sh new file mode 100755 index 0000000..45a47f5 --- /dev/null +++ b/src/portal/cpy/build.sh @@ -0,0 +1,3 @@ +#! /usr/bin/env sh +export PYTHONDONTWRITEBYTECODE=1 +python3.12 setup.py build_ext -i diff --git a/src/portal/cpy/lib.pxd b/src/portal/cpy/lib.pxd new file mode 100644 index 0000000..5749141 --- /dev/null +++ b/src/portal/cpy/lib.pxd @@ -0,0 +1,13 @@ +include "types.pxd" + +cdef extern from "": + void al_free(void *ptr) + void al_malloc(size_t size) + void al_strlen(char *s) + void al_bzero(void *s, size_t n) + +cdef extern from "": + s32 al_log_info(const char *ns, const char *, ...) + s32 al_log_warn(const char *ns, const char *, ...) + s32 al_log_error(const char *ns, const char *, ...) + s32 al_log_debug(const char *ns, const char *, ...) diff --git a/src/portal/cpy/portal.pyx b/src/portal/cpy/portal.pyx new file mode 100644 index 0000000..3ddde80 --- /dev/null +++ b/src/portal/cpy/portal.pyx @@ -0,0 +1,203 @@ +cimport lib +cimport str +cimport post +cimport bridge + +import sys +sys.path.append('./py') + +cdef public int log_info(char *msg) except -1: + return lib.al_log_info("portal", "%s", msg) + +cdef public int log_warn(char *msg) except -1: + return lib.al_log_warn("portal", "%s", msg) + +cdef public int log_error(char *msg) except -1: + return lib.al_log_error("portal", "%s", msg) + +cdef public int log_debug(char *msg) except -1: + return lib.al_log_debug("portal", "%s", msg) + +def encode_str(str): + return str.encode('UTF-8') + +def decode_str(str): + return str.decode('UTF-8') + +class LogOutputHook(object): + def __init__(self): + self.fd = sys.__stdout__ + + def __getattr__(self, attr): + return getattr(self.fd, attr) + + def write(self, msg): + if len(msg) > 0 and msg != '\n': + log_info(encode_str(msg)) + +hook = LogOutputHook() +sys.stdout = hook +sys.stderr = hook + +import log + +class PortalLogger(): + def debug(self, msg): + log_debug(encode_str(msg)) + + def info(self, msg): + log_info(encode_str(msg)) + + def warn(self, msg): + log_warn(encode_str(msg)) + + def error(self, msg): + log_error(encode_str(msg)) + +log.set_logger(PortalLogger()) + +def exception_handler(exception_type, exception, traceback): + log.error(repr(exception)) +sys.excepthook = exception_handler + +from random import randrange + +class Portal(): + def __init__(self, modules): + self.modules = modules + self.searches = {} + + def print_modules(self): + for name in ALL_MODULES.keys(): + log.info('Registered module \'{}\'.'.format(name)) + + def new_id(self): + while True: + id = randrange(16 ** 5) + if id not in self.searches: + return id + + def search(self, module, query): + if module not in self.modules: + return -1 + mod = self.modules[module] + search = mod[0].search(query, *mod[1]) + if not search: + return -1 + id = self.new_id() + self.searches[id] = search + return id + + def get_page(self, id, num): + if id not in self.searches: + return None + try: + return self.searches[id].get_page(num) + except Exception as e: + log.error(repr(e)) + return None + + def get_item(self, id, unique_id): + if id not in self.searches: + return None + try: + return self.searches[id].module.get_item(unique_id) + except Exception as e: + log.error(repr(e)) + return None + +from modules import ALL_MODULES + +portal = Portal(ALL_MODULES) +portal.print_modules() + +from post import PostType, DateType + +cdef public int portal_bridge_search(str.str *module, str.str *query) except -1: + cdef char *c_str0 = str.al_str_to_c_str(module) + cdef char *c_str1 = str.al_str_to_c_str(query) + ret = portal.search(decode_str(c_str0), decode_str(c_str1)) + lib.al_free(c_str0) + lib.al_free(c_str1) + return ret + +cdef public int py_str_to_str(str.str *s, char *py_c_str) except -1: + str.al_str_from(s, py_c_str) + return 0 + +cdef public int py_str_to_wstr(str.wstr *w, char *py_c_str) except -1: + str.al_wstr_from_cstr(w, py_c_str) + return 0 + +cdef public int add_post_internal(bridge.camu_search *search, int id, unsigned int num, char *py_unique_id) except -1: + cdef post.camu_post cpost + cdef str.str key + cdef str.str url + cdef str.str ext + cdef str.str thumbnail_url + cdef str.str thumbnail_ext + py_post = portal.get_item(id, decode_str(py_unique_id)) + if not py_post: + return 0 + post.camu_post_reset(&cpost) + cpost.version = py_post.version + cpost.type = py_post.type.value + py_str_to_str(&cpost.unique_id, encode_str(py_post.unique_id)) + if cpost.type == PostType.TOMBSTONE.value: + bridge.camu_search_add_post(search, num, &cpost) + return 0 + py_str_to_str(&cpost.url, encode_str(py_post.url)) + for type, date in py_post.dates.items(): + post.camu_post_add_date(&cpost, type.value, date) + py_str_to_str(&cpost.author.unique_id, encode_str(py_post.author.unique_id)) + py_str_to_wstr(&cpost.author.username, encode_str(py_post.author.username)) + py_str_to_wstr(&cpost.author.display_name, encode_str(py_post.author.display_name)) + py_str_to_str(&cpost.author.profile_picture_url, encode_str(py_post.author.profile_picture_url.url)) + if cpost.type != PostType.REPOST.value: + py_str_to_wstr(&cpost.title, encode_str(py_post.title)) + py_str_to_wstr(&cpost.text, encode_str(py_post.text)) + cpost.likes.set = py_post.likes != None + if cpost.likes.set: + cpost.likes.i = py_post.likes + cpost.bookmarks.set = py_post.bookmarks != None + if cpost.bookmarks.set: + cpost.bookmarks.i = py_post.bookmarks + cpost.reposts.set = py_post.reposts != None + if cpost.reposts.set: + cpost.reposts.i = py_post.reposts + cpost.quotes.set = py_post.quotes != None + if cpost.quotes.set: + cpost.quotes.i = py_post.quotes + cpost.comments.set = py_post.comments != None + if cpost.comments.set: + cpost.comments.i = py_post.comments + cpost.views.set = py_post.views != None + if cpost.views.set: + cpost.views.i = py_post.views + for k, media in py_post.media.items(): + py_str_to_str(&key, encode_str(k)) + py_str_to_str(&url, encode_str(media.url.url)) + py_str_to_str(&ext, encode_str(media.url.ext)) + py_str_to_str(&thumbnail_url, encode_str(media.thumbnail_url.url)) + py_str_to_str(&thumbnail_ext, encode_str(media.thumbnail_url.ext)) + post.camu_post_add_media(&cpost, media.type.value, &key, &url, &ext, &thumbnail_url, &thumbnail_ext) + py_str_to_str(&cpost.post.unique_id, encode_str(py_post.post.unique_id)) + py_str_to_str(&cpost.quoted.unique_id, encode_str(py_post.quoted.unique_id)) + py_str_to_str(&cpost.in_reply_to.unique_id, encode_str(py_post.in_reply_to.unique_id)) + if py_post.post.unique_id: + add_post_internal(search, id, num, encode_str(py_post.post.unique_id)) + if py_post.quoted.unique_id: + add_post_internal(search, id, num, encode_str(py_post.quoted.unique_id)) + bridge.camu_search_add_post(search, num, &cpost) + return 0 + +cdef public int portal_bridge_get_page(bridge.camu_search *search, int id, unsigned int num) except -1: + page = portal.get_page(id, num) + if not page: + return -1 + cdef str.str unique_id + for py_unique_id in page: + py_str_to_str(&unique_id, encode_str(py_unique_id)) + bridge.camu_search_add_to_list(search, num, &unique_id) + add_post_internal(search, id, num, encode_str(py_unique_id)) + return 0 diff --git a/src/portal/cpy/post.pxd b/src/portal/cpy/post.pxd new file mode 100644 index 0000000..3f66508 --- /dev/null +++ b/src/portal/cpy/post.pxd @@ -0,0 +1,61 @@ +include "str.pxd" + +cdef extern from "../src/post.h": + ctypedef struct optional_int: + s64 i + bool set + + cdef struct camu_post_date: + u8 type + f32 timestamp + + ctypedef struct camu_dates_array: + u32 size + u32 alloc + camu_post_date *data + + cdef struct camu_post_media: + u8 type + str key + str url + str ext + str thumbnail_url + str thumbnail_ext + + ctypedef struct camu_media_array: + u32 size + u32 alloc + camu_post_media *data + + cdef struct camu_post_user: + str unique_id + wstr username + wstr display_name + str profile_picture_url + + cdef struct camu_post_ref: + str unique_id + + cdef struct camu_post: + u16 version + u8 type + str unique_id + str url + camu_dates_array dates + camu_post_user author + wstr title + wstr text + optional_int likes + optional_int bookmarks + optional_int reposts + optional_int quotes + optional_int comments + optional_int views + camu_media_array media + camu_post_ref post + camu_post_ref quoted + camu_post_ref in_reply_to + + void camu_post_reset(camu_post *post) + void camu_post_add_date(camu_post *post, u8 type, f32 timestamp) + void camu_post_add_media(camu_post *post, u8 type, str *key, str *url, str *ext, str *thumbnail_url, str *thumbnail_ext) diff --git a/src/portal/cpy/setup.py b/src/portal/cpy/setup.py new file mode 100644 index 0000000..1a3e39c --- /dev/null +++ b/src/portal/cpy/setup.py @@ -0,0 +1,6 @@ +from setuptools import Extension +from Cython.Build import cythonize +from Cython.Compiler import Options +Options.embed = True +alabaster_inc = "../../../subprojects/libalabaster/include" +cythonize([Extension("portal", ["portal.pyx"], include_dirs=[alabaster_inc], extra_link_args=[])], language_level=3) diff --git a/src/portal/cpy/str.pxd b/src/portal/cpy/str.pxd new file mode 100644 index 0000000..3ef1e90 --- /dev/null +++ b/src/portal/cpy/str.pxd @@ -0,0 +1,18 @@ +include "types.pxd" + +cdef extern from "": + ctypedef struct str: + u32 len + u32 alloc + char *data + + void al_str_from(str *s, const char *str) + char *al_str_to_c_str(str *s) + +cdef extern from "": + ctypedef struct wstr: + u32 len + u32 alloc + wchar_t *data + + void al_wstr_from_cstr(wstr *s, const char *str) diff --git a/src/portal/cpy/types.pxd b/src/portal/cpy/types.pxd new file mode 100644 index 0000000..09cb224 --- /dev/null +++ b/src/portal/cpy/types.pxd @@ -0,0 +1,23 @@ +from libc.stdint cimport uint8_t +from libc.stdint cimport int8_t +from libc.stdint cimport uint16_t +from libc.stdint cimport int16_t +from libc.stdint cimport uint32_t +from libc.stdint cimport int32_t +from libc.stdint cimport uint64_t +from libc.stdint cimport int64_t +from libc.stddef cimport wchar_t + +ctypedef bint bool + +ctypedef uint8_t u8 +ctypedef int8_t s8 +ctypedef uint16_t u16 +ctypedef int16_t s16 +ctypedef uint32_t u32 +ctypedef int32_t s32 +ctypedef uint64_t u64 +ctypedef int64_t s64 + +ctypedef float f32 +ctypedef double f64 -- cgit v1.2.3-101-g0448