diff options
Diffstat (limited to 'src/portal')
47 files changed, 7446 insertions, 0 deletions
diff --git a/src/portal/cpy/bridge.pxd b/src/portal/cpy/bridge.pxd new file mode 100644 index 0000000..ac14811 --- /dev/null +++ b/src/portal/cpy/bridge.pxd @@ -0,0 +1,8 @@ +include "post.pxd" + +cdef extern from "../src/search.h": + cdef struct camu_search: + pass + + void camu_search_add_post(camu_search *search, u32 page, camu_post *post) + void camu_search_add_to_list(camu_search *search, u32 page, str *unique_id) diff --git a/src/portal/cpy/build.sh b/src/portal/cpy/build.sh new file mode 100755 index 0000000..45a47f5 --- /dev/null +++ b/src/portal/cpy/build.sh @@ -0,0 +1,3 @@ +#! /usr/bin/env sh +export PYTHONDONTWRITEBYTECODE=1 +python3.12 setup.py build_ext -i diff --git a/src/portal/cpy/lib.pxd b/src/portal/cpy/lib.pxd new file mode 100644 index 0000000..5749141 --- /dev/null +++ b/src/portal/cpy/lib.pxd @@ -0,0 +1,13 @@ +include "types.pxd" + +cdef extern from "<al/lib.h>": + void al_free(void *ptr) + void al_malloc(size_t size) + void al_strlen(char *s) + void al_bzero(void *s, size_t n) + +cdef extern from "<al/log.h>": + s32 al_log_info(const char *ns, const char *, ...) + s32 al_log_warn(const char *ns, const char *, ...) + s32 al_log_error(const char *ns, const char *, ...) + s32 al_log_debug(const char *ns, const char *, ...) diff --git a/src/portal/cpy/portal.pyx b/src/portal/cpy/portal.pyx new file mode 100644 index 0000000..3ddde80 --- /dev/null +++ b/src/portal/cpy/portal.pyx @@ -0,0 +1,203 @@ +cimport lib +cimport str +cimport post +cimport bridge + +import sys +sys.path.append('./py') + +cdef public int log_info(char *msg) except -1: + return lib.al_log_info("portal", "%s", msg) + +cdef public int log_warn(char *msg) except -1: + return lib.al_log_warn("portal", "%s", msg) + +cdef public int log_error(char *msg) except -1: + return lib.al_log_error("portal", "%s", msg) + +cdef public int log_debug(char *msg) except -1: + return lib.al_log_debug("portal", "%s", msg) + +def encode_str(str): + return str.encode('UTF-8') + +def decode_str(str): + return str.decode('UTF-8') + +class LogOutputHook(object): + def __init__(self): + self.fd = sys.__stdout__ + + def __getattr__(self, attr): + return getattr(self.fd, attr) + + def write(self, msg): + if len(msg) > 0 and msg != '\n': + log_info(encode_str(msg)) + +hook = LogOutputHook() +sys.stdout = hook +sys.stderr = hook + +import log + +class PortalLogger(): + def debug(self, msg): + log_debug(encode_str(msg)) + + def info(self, msg): + log_info(encode_str(msg)) + + def warn(self, msg): + log_warn(encode_str(msg)) + + def error(self, msg): + log_error(encode_str(msg)) + +log.set_logger(PortalLogger()) + +def exception_handler(exception_type, exception, traceback): + log.error(repr(exception)) +sys.excepthook = exception_handler + +from random import randrange + +class Portal(): + def __init__(self, modules): + self.modules = modules + self.searches = {} + + def print_modules(self): + for name in ALL_MODULES.keys(): + log.info('Registered module \'{}\'.'.format(name)) + + def new_id(self): + while True: + id = randrange(16 ** 5) + if id not in self.searches: + return id + + def search(self, module, query): + if module not in self.modules: + return -1 + mod = self.modules[module] + search = mod[0].search(query, *mod[1]) + if not search: + return -1 + id = self.new_id() + self.searches[id] = search + return id + + def get_page(self, id, num): + if id not in self.searches: + return None + try: + return self.searches[id].get_page(num) + except Exception as e: + log.error(repr(e)) + return None + + def get_item(self, id, unique_id): + if id not in self.searches: + return None + try: + return self.searches[id].module.get_item(unique_id) + except Exception as e: + log.error(repr(e)) + return None + +from modules import ALL_MODULES + +portal = Portal(ALL_MODULES) +portal.print_modules() + +from post import PostType, DateType + +cdef public int portal_bridge_search(str.str *module, str.str *query) except -1: + cdef char *c_str0 = str.al_str_to_c_str(module) + cdef char *c_str1 = str.al_str_to_c_str(query) + ret = portal.search(decode_str(c_str0), decode_str(c_str1)) + lib.al_free(c_str0) + lib.al_free(c_str1) + return ret + +cdef public int py_str_to_str(str.str *s, char *py_c_str) except -1: + str.al_str_from(s, py_c_str) + return 0 + +cdef public int py_str_to_wstr(str.wstr *w, char *py_c_str) except -1: + str.al_wstr_from_cstr(w, py_c_str) + return 0 + +cdef public int add_post_internal(bridge.camu_search *search, int id, unsigned int num, char *py_unique_id) except -1: + cdef post.camu_post cpost + cdef str.str key + cdef str.str url + cdef str.str ext + cdef str.str thumbnail_url + cdef str.str thumbnail_ext + py_post = portal.get_item(id, decode_str(py_unique_id)) + if not py_post: + return 0 + post.camu_post_reset(&cpost) + cpost.version = py_post.version + cpost.type = py_post.type.value + py_str_to_str(&cpost.unique_id, encode_str(py_post.unique_id)) + if cpost.type == PostType.TOMBSTONE.value: + bridge.camu_search_add_post(search, num, &cpost) + return 0 + py_str_to_str(&cpost.url, encode_str(py_post.url)) + for type, date in py_post.dates.items(): + post.camu_post_add_date(&cpost, type.value, date) + py_str_to_str(&cpost.author.unique_id, encode_str(py_post.author.unique_id)) + py_str_to_wstr(&cpost.author.username, encode_str(py_post.author.username)) + py_str_to_wstr(&cpost.author.display_name, encode_str(py_post.author.display_name)) + py_str_to_str(&cpost.author.profile_picture_url, encode_str(py_post.author.profile_picture_url.url)) + if cpost.type != PostType.REPOST.value: + py_str_to_wstr(&cpost.title, encode_str(py_post.title)) + py_str_to_wstr(&cpost.text, encode_str(py_post.text)) + cpost.likes.set = py_post.likes != None + if cpost.likes.set: + cpost.likes.i = py_post.likes + cpost.bookmarks.set = py_post.bookmarks != None + if cpost.bookmarks.set: + cpost.bookmarks.i = py_post.bookmarks + cpost.reposts.set = py_post.reposts != None + if cpost.reposts.set: + cpost.reposts.i = py_post.reposts + cpost.quotes.set = py_post.quotes != None + if cpost.quotes.set: + cpost.quotes.i = py_post.quotes + cpost.comments.set = py_post.comments != None + if cpost.comments.set: + cpost.comments.i = py_post.comments + cpost.views.set = py_post.views != None + if cpost.views.set: + cpost.views.i = py_post.views + for k, media in py_post.media.items(): + py_str_to_str(&key, encode_str(k)) + py_str_to_str(&url, encode_str(media.url.url)) + py_str_to_str(&ext, encode_str(media.url.ext)) + py_str_to_str(&thumbnail_url, encode_str(media.thumbnail_url.url)) + py_str_to_str(&thumbnail_ext, encode_str(media.thumbnail_url.ext)) + post.camu_post_add_media(&cpost, media.type.value, &key, &url, &ext, &thumbnail_url, &thumbnail_ext) + py_str_to_str(&cpost.post.unique_id, encode_str(py_post.post.unique_id)) + py_str_to_str(&cpost.quoted.unique_id, encode_str(py_post.quoted.unique_id)) + py_str_to_str(&cpost.in_reply_to.unique_id, encode_str(py_post.in_reply_to.unique_id)) + if py_post.post.unique_id: + add_post_internal(search, id, num, encode_str(py_post.post.unique_id)) + if py_post.quoted.unique_id: + add_post_internal(search, id, num, encode_str(py_post.quoted.unique_id)) + bridge.camu_search_add_post(search, num, &cpost) + return 0 + +cdef public int portal_bridge_get_page(bridge.camu_search *search, int id, unsigned int num) except -1: + page = portal.get_page(id, num) + if not page: + return -1 + cdef str.str unique_id + for py_unique_id in page: + py_str_to_str(&unique_id, encode_str(py_unique_id)) + bridge.camu_search_add_to_list(search, num, &unique_id) + add_post_internal(search, id, num, encode_str(py_unique_id)) + return 0 diff --git a/src/portal/cpy/post.pxd b/src/portal/cpy/post.pxd new file mode 100644 index 0000000..3f66508 --- /dev/null +++ b/src/portal/cpy/post.pxd @@ -0,0 +1,61 @@ +include "str.pxd" + +cdef extern from "../src/post.h": + ctypedef struct optional_int: + s64 i + bool set + + cdef struct camu_post_date: + u8 type + f32 timestamp + + ctypedef struct camu_dates_array: + u32 size + u32 alloc + camu_post_date *data + + cdef struct camu_post_media: + u8 type + str key + str url + str ext + str thumbnail_url + str thumbnail_ext + + ctypedef struct camu_media_array: + u32 size + u32 alloc + camu_post_media *data + + cdef struct camu_post_user: + str unique_id + wstr username + wstr display_name + str profile_picture_url + + cdef struct camu_post_ref: + str unique_id + + cdef struct camu_post: + u16 version + u8 type + str unique_id + str url + camu_dates_array dates + camu_post_user author + wstr title + wstr text + optional_int likes + optional_int bookmarks + optional_int reposts + optional_int quotes + optional_int comments + optional_int views + camu_media_array media + camu_post_ref post + camu_post_ref quoted + camu_post_ref in_reply_to + + void camu_post_reset(camu_post *post) + void camu_post_add_date(camu_post *post, u8 type, f32 timestamp) + void camu_post_add_media(camu_post *post, u8 type, str *key, str *url, str *ext, str *thumbnail_url, str *thumbnail_ext) diff --git a/src/portal/cpy/setup.py b/src/portal/cpy/setup.py new file mode 100644 index 0000000..1a3e39c --- /dev/null +++ b/src/portal/cpy/setup.py @@ -0,0 +1,6 @@ +from setuptools import Extension +from Cython.Build import cythonize +from Cython.Compiler import Options +Options.embed = True +alabaster_inc = "../../../subprojects/libalabaster/include" +cythonize([Extension("portal", ["portal.pyx"], include_dirs=[alabaster_inc], extra_link_args=[])], language_level=3) diff --git a/src/portal/cpy/str.pxd b/src/portal/cpy/str.pxd new file mode 100644 index 0000000..3ef1e90 --- /dev/null +++ b/src/portal/cpy/str.pxd @@ -0,0 +1,18 @@ +include "types.pxd" + +cdef extern from "<al/str.h>": + ctypedef struct str: + u32 len + u32 alloc + char *data + + void al_str_from(str *s, const char *str) + char *al_str_to_c_str(str *s) + +cdef extern from "<al/wstr.h>": + ctypedef struct wstr: + u32 len + u32 alloc + wchar_t *data + + void al_wstr_from_cstr(wstr *s, const char *str) diff --git a/src/portal/cpy/types.pxd b/src/portal/cpy/types.pxd new file mode 100644 index 0000000..09cb224 --- /dev/null +++ b/src/portal/cpy/types.pxd @@ -0,0 +1,23 @@ +from libc.stdint cimport uint8_t +from libc.stdint cimport int8_t +from libc.stdint cimport uint16_t +from libc.stdint cimport int16_t +from libc.stdint cimport uint32_t +from libc.stdint cimport int32_t +from libc.stdint cimport uint64_t +from libc.stdint cimport int64_t +from libc.stddef cimport wchar_t + +ctypedef bint bool + +ctypedef uint8_t u8 +ctypedef int8_t s8 +ctypedef uint16_t u16 +ctypedef int16_t s16 +ctypedef uint32_t u32 +ctypedef int32_t s32 +ctypedef uint64_t u64 +ctypedef int64_t s64 + +ctypedef float f32 +ctypedef double f64 diff --git a/src/portal/meson.build b/src/portal/meson.build new file mode 100644 index 0000000..0df4366 --- /dev/null +++ b/src/portal/meson.build @@ -0,0 +1,12 @@ +portal_src = [ + 'src/post.c', + 'src/search.c', + 'src/post_cache.c', + 'src/packet_ext.c' +] +portal_deps = [] + +python3_embed = import('python').find_installation('python3.12').dependency(embed: true) +portal_deps += [python3_embed] + +portal = declare_dependency(sources: portal_src, dependencies: portal_deps) diff --git a/src/portal/patches/instagrapi_no_android.diff b/src/portal/patches/instagrapi_no_android.diff new file mode 100644 index 0000000..6f4fd34 --- /dev/null +++ b/src/portal/patches/instagrapi_no_android.diff @@ -0,0 +1,17 @@ +diff --git a/instagrapi/mixins/private.py b/instagrapi/mixins/private.py +index 9db04b0..2b17d75 100644 +--- a/instagrapi/mixins/private.py ++++ b/instagrapi/mixins/private.py +@@ -129,9 +129,9 @@ class PrivateRequestMixin: + # X-IG-WWW-Claim: hmac.AR3zruvyGTlwHvVd2ACpGCWLluOppXX4NAVDV-iYslo9CaDd + "X-Bloks-Is-Layout-RTL": "false", + "X-Bloks-Is-Panorama-Enabled": "true", +- "X-IG-Device-ID": self.uuid, +- "X-IG-Family-Device-ID": self.phone_id, +- "X-IG-Android-ID": self.android_device_id, ++ #"X-IG-Device-ID": self.uuid, ++ #"X-IG-Family-Device-ID": self.phone_id, ++ #"X-IG-Android-ID": self.android_device_id, + "X-IG-Timezone-Offset": str(self.timezone_offset), + "X-IG-Connection-Type": "WIFI", + "X-IG-Capabilities": "3brTvx0=", # "3brTvwE=" in instabot diff --git a/src/portal/patches/pixivpy_reauth.diff b/src/portal/patches/pixivpy_reauth.diff new file mode 100644 index 0000000..d9ce0d6 --- /dev/null +++ b/src/portal/patches/pixivpy_reauth.diff @@ -0,0 +1,96 @@ +diff --git a/pixivpy3/aapi.py b/pixivpy3/aapi.py +index 1492385..ffc88c0 100644 +--- a/pixivpy3/aapi.py ++++ b/pixivpy3/aapi.py +@@ -76,12 +76,11 @@ class AppPixivAPI(BasePixivAPI): + ) -> Response: + headers_ = CaseInsensitiveDict(headers or {}) + if self.hosts != "https://app-api.pixiv.net": +- headers_["host"] = "app-api.pixiv.net" +- if "user-agent" not in headers_: +- # Set User-Agent if not provided +- headers_["app-os"] = "ios" +- headers_["app-os-version"] = "14.6" +- headers_["user-agent"] = "PixivIOSApp/7.13.3 (iOS 14.6; iPhone13,2)" ++ headers_["Host"] = "app-api.pixiv.net" ++ headers_["App-OS"] = "ios" ++ headers_["App-OS-Version"] = "16.7.2" ++ headers_["App-Version"] = "7.19.6" ++ headers_["User-Agent"] = "PixivIOSApp/7.19.6 (iOS 16.7.2; iPhone13,2)" + + if not req_auth: + return self.requests_call(method, url, headers_, params, data) +diff --git a/pixivpy3/api.py b/pixivpy3/api.py +index c26dda2..d0e1c0f 100644 +--- a/pixivpy3/api.py ++++ b/pixivpy3/api.py +@@ -26,6 +26,7 @@ class BasePixivAPI: + self.user_id: int | str = 0 + self.access_token: str | None = None + self.refresh_token: str | None = None ++ self.token_expiration: int | None = None + self.hosts = "https://app-api.pixiv.net" + + # self.requests = requests.Session() +@@ -62,6 +63,11 @@ class BasePixivAPI: + stream: bool = False, + ) -> Response: + """requests http/https call for Pixiv API""" ++ if self.token_expiration is not None and datetime.utcnow().timestamp() >= self.token_expiration: ++ # Token expired refresh auth ++ print('Token expired, refreshing auth') ++ self.token_expiration = None ++ self.auth() + merged_headers = self.additional_headers.copy() + if headers: + # Use the headers in the parameter to override the +@@ -118,23 +124,22 @@ class BasePixivAPI: + headers: ParamDict = None, + ) -> ParsedJson: + """Login with password, or use the refresh_token to acquire a new bearer token""" +- local_time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S+00:00") ++ local_time = datetime.utcnow() ++ local_time_str = local_time.strftime("%Y-%m-%dT%H:%M:%S+00:00") + headers_ = CaseInsensitiveDict(headers or {}) +- headers_["x-client-time"] = local_time +- headers_["x-client-hash"] = hashlib.md5((local_time + self.hash_secret).encode("utf-8")).hexdigest() +- # Allow mock UA due to #171: https://github.com/upbit/pixivpy/issues/171 +- if "user-agent" not in headers_: +- headers_["app-os"] = "ios" +- headers_["app-os-version"] = "14.6" +- headers_["user-agent"] = "PixivIOSApp/7.13.3 (iOS 14.6; iPhone13,2)" +- ++ headers_["X-Client-Time"] = local_time_str ++ headers_["X-Client-Hash"] = hashlib.md5((local_time_str + self.hash_secret).encode("utf-8")).hexdigest() ++ headers_["App-OS"] = "ios" ++ headers_["App-OS-Version"] = "16.7.2" ++ headers_["App-Version"] = "7.19.6" ++ headers_["User-Agent"] = "PixivIOSApp/7.19.6 (iOS 16.7.2; iPhone13,2)" + # noinspection PyUnresolvedReferences + if not hasattr(self, "hosts") or self.hosts == "https://app-api.pixiv.net": + auth_hosts = "https://oauth.secure.pixiv.net" + else: + # noinspection PyUnresolvedReferences + auth_hosts = self.hosts # BAPI解析成IP的场景 +- headers_["host"] = "oauth.secure.pixiv.net" ++ headers_["Host"] = "oauth.secure.pixiv.net" + url = "%s/auth/token" % auth_hosts + data = { + "get_secure_url": 1, +@@ -149,6 +154,8 @@ class BasePixivAPI: + elif refresh_token or self.refresh_token: + data["grant_type"] = "refresh_token" + data["refresh_token"] = refresh_token or self.refresh_token ++ if self.access_token: ++ data["access_token"] = self.access_token + else: + raise PixivError("[ERROR] auth() but no password or refresh_token is set.") + +@@ -174,6 +181,7 @@ class BasePixivAPI: + self.user_id = token.response.user.id + self.access_token = token.response.access_token + self.refresh_token = token.response.refresh_token ++ self.token_expiration = local_time.timestamp() + token.response.expires_in + except json.JSONDecodeError: + raise PixivError( + "Get access_token error! Response: %s" % token, diff --git a/src/portal/patches/searxng_disable_engines_default.diff b/src/portal/patches/searxng_disable_engines_default.diff new file mode 100644 index 0000000..923519c --- /dev/null +++ b/src/portal/patches/searxng_disable_engines_default.diff @@ -0,0 +1,3675 @@ +diff --git a/searx/settings.yml b/searx/settings.yml +index 6946b89d..92378207 100644 +--- a/searx/settings.yml ++++ b/searx/settings.yml +@@ -294,468 +294,6 @@ categories_as_tabs: + social media: + + engines: +- - name: 9gag +- engine: 9gag +- shortcut: 9g +- disabled: true +- +- - name: annas archive +- engine: annas_archive +- disabled: true +- shortcut: aa +- +- # - name: annas articles +- # engine: annas_archive +- # shortcut: aaa +- # # https://docs.searxng.org/dev/engines/online/annas_archive.html +- # aa_content: 'journal_article' # book_any .. magazine, standards_document +- # aa_ext: 'pdf' # pdf, epub, .. +- # aa_sort: 'newest' # newest, oldest, largest, smallest +- +- - name: apk mirror +- engine: apkmirror +- timeout: 4.0 +- shortcut: apkm +- disabled: true +- +- - name: apple app store +- engine: apple_app_store +- shortcut: aps +- disabled: true +- +- # Requires Tor +- - name: ahmia +- engine: ahmia +- categories: onions +- enable_http: true +- shortcut: ah +- +- - name: anaconda +- engine: xpath +- paging: true +- first_page_num: 0 +- search_url: https://anaconda.org/search?q={query}&page={pageno} +- results_xpath: //tbody/tr +- url_xpath: ./td/h5/a[last()]/@href +- title_xpath: ./td/h5 +- content_xpath: ./td[h5]/text() +- categories: it +- timeout: 6.0 +- shortcut: conda +- disabled: true +- +- - name: arch linux wiki +- engine: archlinux +- shortcut: al +- +- - name: artic +- engine: artic +- shortcut: arc +- timeout: 4.0 +- +- - name: arxiv +- engine: arxiv +- shortcut: arx +- timeout: 4.0 +- +- # tmp suspended: dh key too small +- # - name: base +- # engine: base +- # shortcut: bs +- +- - name: bandcamp +- engine: bandcamp +- shortcut: bc +- categories: music +- +- - name: wikipedia +- engine: wikipedia +- shortcut: wp +- # add "list" to the array to get results in the results list +- display_type: ["infobox"] +- base_url: 'https://{language}.wikipedia.org/' +- categories: [general] +- +- - name: bilibili +- engine: bilibili +- shortcut: bil +- disabled: true +- +- - name: bing +- engine: bing +- shortcut: bi +- disabled: true +- +- - name: bing images +- engine: bing_images +- shortcut: bii +- +- - name: bing news +- engine: bing_news +- shortcut: bin +- +- - name: bing videos +- engine: bing_videos +- shortcut: biv +- +- - name: bitbucket +- engine: xpath +- paging: true +- search_url: https://bitbucket.org/repo/all/{pageno}?name={query} +- url_xpath: //article[@class="repo-summary"]//a[@class="repo-link"]/@href +- title_xpath: //article[@class="repo-summary"]//a[@class="repo-link"] +- content_xpath: //article[@class="repo-summary"]/p +- categories: [it, repos] +- timeout: 4.0 +- disabled: true +- shortcut: bb +- about: +- website: https://bitbucket.org/ +- wikidata_id: Q2493781 +- official_api_documentation: https://developer.atlassian.com/bitbucket +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: btdigg +- engine: btdigg +- shortcut: bt +- disabled: true +- +- - name: ccc-tv +- engine: xpath +- paging: false +- search_url: https://media.ccc.de/search/?q={query} +- url_xpath: //div[@class="caption"]/h3/a/@href +- title_xpath: //div[@class="caption"]/h3/a/text() +- content_xpath: //div[@class="caption"]/h4/@title +- categories: videos +- disabled: true +- shortcut: c3tv +- about: +- website: https://media.ccc.de/ +- wikidata_id: Q80729951 +- official_api_documentation: https://github.com/voc/voctoweb +- use_official_api: false +- require_api_key: false +- results: HTML +- # We don't set language: de here because media.ccc.de is not just +- # for a German audience. It contains many English videos and many +- # German videos have English subtitles. +- +- - name: openverse +- engine: openverse +- categories: images +- shortcut: opv +- +- - name: chefkoch +- engine: chefkoch +- shortcut: chef +- # to show premium or plus results too: +- # skip_premium: false +- +- # - name: core.ac.uk +- # engine: core +- # categories: science +- # shortcut: cor +- # # get your API key from: https://core.ac.uk/api-keys/register/ +- # api_key: 'unset' +- +- - name: crossref +- engine: crossref +- shortcut: cr +- timeout: 30 +- disabled: true +- +- - name: crowdview +- engine: json_engine +- shortcut: cv +- categories: general +- paging: false +- search_url: https://crowdview-next-js.onrender.com/api/search-v3?query={query} +- results_query: results +- url_query: link +- title_query: title +- content_query: snippet +- disabled: true +- about: +- website: https://crowdview.ai/ +- +- - name: yep +- engine: json_engine +- shortcut: yep +- categories: general +- disabled: true +- paging: false +- content_html_to_text: true +- title_html_to_text: true +- search_url: https://api.yep.com/fs/1/?type=web&q={query}&no_correct=false&limit=100 +- results_query: 1/results +- title_query: title +- url_query: url +- content_query: snippet +- about: +- website: https://yep.com +- use_official_api: false +- require_api_key: false +- results: JSON +- +- - name: curlie +- engine: xpath +- shortcut: cl +- categories: general +- disabled: true +- paging: true +- lang_all: '' +- search_url: https://curlie.org/search?q={query}&lang={lang}&start={pageno}&stime=92452189 +- page_size: 20 +- results_xpath: //div[@id="site-list-content"]/div[@class="site-item"] +- url_xpath: ./div[@class="title-and-desc"]/a/@href +- title_xpath: ./div[@class="title-and-desc"]/a/div +- content_xpath: ./div[@class="title-and-desc"]/div[@class="site-descr"] +- about: +- website: https://curlie.org/ +- wikidata_id: Q60715723 +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: currency +- engine: currency_convert +- categories: general +- shortcut: cc +- +- - name: deezer +- engine: deezer +- shortcut: dz +- disabled: true +- +- - name: deviantart +- engine: deviantart +- shortcut: da +- timeout: 3.0 +- +- - name: ddg definitions +- engine: duckduckgo_definitions +- shortcut: ddd +- weight: 2 +- disabled: true +- tests: *tests_infobox +- +- # cloudflare protected +- # - name: digbt +- # engine: digbt +- # shortcut: dbt +- # timeout: 6.0 +- # disabled: true +- +- - name: docker hub +- engine: docker_hub +- shortcut: dh +- categories: [it, packages] +- +- - name: erowid +- engine: xpath +- paging: true +- first_page_num: 0 +- page_size: 30 +- search_url: https://www.erowid.org/search.php?q={query}&s={pageno} +- url_xpath: //dl[@class="results-list"]/dt[@class="result-title"]/a/@href +- title_xpath: //dl[@class="results-list"]/dt[@class="result-title"]/a/text() +- content_xpath: //dl[@class="results-list"]/dd[@class="result-details"] +- categories: [] +- shortcut: ew +- disabled: true +- about: +- website: https://www.erowid.org/ +- wikidata_id: Q1430691 +- official_api_documentation: +- use_official_api: false +- require_api_key: false +- results: HTML +- +- # - name: elasticsearch +- # shortcut: es +- # engine: elasticsearch +- # base_url: http://localhost:9200 +- # username: elastic +- # password: changeme +- # index: my-index +- # # available options: match, simple_query_string, term, terms, custom +- # query_type: match +- # # if query_type is set to custom, provide your query here +- # #custom_query_json: {"query":{"match_all": {}}} +- # #show_metadata: false +- # disabled: true +- +- - name: wikidata +- engine: wikidata +- shortcut: wd +- timeout: 3.0 +- weight: 2 +- # add "list" to the array to get results in the results list +- display_type: ["infobox"] +- tests: *tests_infobox +- categories: [general] +- +- - name: duckduckgo +- engine: duckduckgo +- shortcut: ddg +- +- - name: duckduckgo images +- engine: duckduckgo_images +- shortcut: ddi +- timeout: 3.0 +- disabled: true +- +- - name: duckduckgo weather +- engine: duckduckgo_weather +- shortcut: ddw +- disabled: true +- +- - name: apple maps +- engine: apple_maps +- shortcut: apm +- disabled: true +- timeout: 5.0 +- +- - name: emojipedia +- engine: emojipedia +- timeout: 4.0 +- shortcut: em +- disabled: true +- +- - name: tineye +- engine: tineye +- shortcut: tin +- timeout: 9.0 +- disabled: true +- +- - name: etymonline +- engine: xpath +- paging: true +- search_url: https://etymonline.com/search?page={pageno}&q={query} +- url_xpath: //a[contains(@class, "word__name--")]/@href +- title_xpath: //a[contains(@class, "word__name--")] +- content_xpath: //section[contains(@class, "word__defination")] +- first_page_num: 1 +- shortcut: et +- categories: [dictionaries] +- about: +- website: https://www.etymonline.com/ +- wikidata_id: Q1188617 +- official_api_documentation: +- use_official_api: false +- require_api_key: false +- results: HTML +- +- # - name: ebay +- # engine: ebay +- # shortcut: eb +- # base_url: 'https://www.ebay.com' +- # disabled: true +- # timeout: 5 +- +- - name: 1x +- engine: www1x +- shortcut: 1x +- timeout: 3.0 +- disabled: true +- +- - name: fdroid +- engine: fdroid +- shortcut: fd +- disabled: true +- +- - name: flickr +- categories: images +- shortcut: fl +- # You can use the engine using the official stable API, but you need an API +- # key, see: https://www.flickr.com/services/apps/create/ +- # engine: flickr +- # api_key: 'apikey' # required! +- # Or you can use the html non-stable engine, activated by default +- engine: flickr_noapi +- +- - name: free software directory +- engine: mediawiki +- shortcut: fsd +- categories: [it, software wikis] +- base_url: https://directory.fsf.org/ +- search_type: title +- timeout: 5.0 +- disabled: true +- about: +- website: https://directory.fsf.org/ +- wikidata_id: Q2470288 +- +- # - name: freesound +- # engine: freesound +- # shortcut: fnd +- # disabled: true +- # timeout: 15.0 +- # API key required, see: https://freesound.org/docs/api/overview.html +- # api_key: MyAPIkey +- +- - name: frinkiac +- engine: frinkiac +- shortcut: frk +- disabled: true +- +- - name: genius +- engine: genius +- shortcut: gen +- +- - name: gentoo +- engine: gentoo +- shortcut: ge +- timeout: 10.0 +- +- - name: gitlab +- engine: json_engine +- paging: true +- search_url: https://gitlab.com/api/v4/projects?search={query}&page={pageno} +- url_query: web_url +- title_query: name_with_namespace +- content_query: description +- page_size: 20 +- categories: [it, repos] +- shortcut: gl +- timeout: 10.0 +- disabled: true +- about: +- website: https://about.gitlab.com/ +- wikidata_id: Q16639197 +- official_api_documentation: https://docs.gitlab.com/ee/api/ +- use_official_api: false +- require_api_key: false +- results: JSON +- +- - name: github +- engine: github +- shortcut: gh +- +- # This a Gitea service. If you would like to use a different instance, +- # change codeberg.org to URL of the desired Gitea host. Or you can create a +- # new engine by copying this and changing the name, shortcut and search_url. +- +- - name: codeberg +- engine: json_engine +- search_url: https://codeberg.org/api/v1/repos/search?q={query}&limit=10 +- url_query: html_url +- title_query: name +- content_query: description +- categories: [it, repos] +- shortcut: cb +- disabled: true +- about: +- website: https://codeberg.org/ +- wikidata_id: +- official_api_documentation: https://try.gitea.io/api/swagger +- use_official_api: false +- require_api_key: false +- results: JSON +- + - name: google + engine: google + shortcut: go +@@ -774,1372 +312,1837 @@ engines: + # result_container: + # - ['one_title_contains', 'Salvador'] + +- - name: google news +- engine: google_news +- shortcut: gon +- # additional_tests: +- # android: *test_android +- +- - name: google videos +- engine: google_videos +- shortcut: gov +- # additional_tests: +- # android: *test_android +- +- - name: google scholar +- engine: google_scholar +- shortcut: gos +- +- - name: google play apps +- engine: google_play +- categories: [files, apps] +- shortcut: gpa +- play_categ: apps +- disabled: true +- +- - name: google play movies +- engine: google_play +- categories: videos +- shortcut: gpm +- play_categ: movies +- disabled: true +- +- - name: material icons +- engine: material_icons +- categories: images +- shortcut: mi +- disabled: true +- +- - name: gpodder +- engine: json_engine +- shortcut: gpod +- timeout: 4.0 +- paging: false +- search_url: https://gpodder.net/search.json?q={query} +- url_query: url +- title_query: title +- content_query: description +- page_size: 19 +- categories: music +- disabled: true +- about: +- website: https://gpodder.net +- wikidata_id: Q3093354 +- official_api_documentation: https://gpoddernet.readthedocs.io/en/latest/api/ +- use_official_api: false +- requires_api_key: false +- results: JSON +- +- - name: habrahabr +- engine: xpath +- paging: true +- search_url: https://habr.com/en/search/page{pageno}/?q={query} +- results_xpath: //article[contains(@class, "tm-articles-list__item")] +- url_xpath: .//a[@class="tm-title__link"]/@href +- title_xpath: .//a[@class="tm-title__link"] +- content_xpath: .//div[contains(@class, "article-formatted-body")] +- categories: it +- timeout: 4.0 +- disabled: true +- shortcut: habr +- about: +- website: https://habr.com/ +- wikidata_id: Q4494434 +- official_api_documentation: https://habr.com/en/docs/help/api/ +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: hoogle +- engine: xpath +- paging: true +- search_url: https://hoogle.haskell.org/?hoogle={query}&start={pageno} +- results_xpath: '//div[@class="result"]' +- title_xpath: './/div[@class="ans"]//a' +- url_xpath: './/div[@class="ans"]//a/@href' +- content_xpath: './/div[@class="from"]' +- page_size: 20 +- categories: [it, packages] +- shortcut: ho +- about: +- website: https://hoogle.haskell.org/ +- wikidata_id: Q34010 +- official_api_documentation: https://hackage.haskell.org/api +- use_official_api: false +- require_api_key: false +- results: JSON +- +- - name: imdb +- engine: imdb +- shortcut: imdb +- timeout: 6.0 +- disabled: true +- +- - name: ina +- engine: ina +- shortcut: in +- timeout: 6.0 +- disabled: true +- +- - name: invidious +- engine: invidious +- # Instanes will be selected randomly, see https://api.invidious.io/ for +- # instances that are stable (good uptime) and close to you. +- base_url: +- - https://invidious.io.lol +- - https://invidious.fdn.fr +- - https://yt.artemislena.eu +- - https://invidious.tiekoetter.com +- - https://invidious.flokinet.to +- - https://vid.puffyan.us +- - https://invidious.privacydev.net +- - https://inv.tux.pizza +- shortcut: iv +- timeout: 3.0 +- disabled: true +- +- - name: jisho +- engine: jisho +- shortcut: js +- timeout: 3.0 +- disabled: true +- +- - name: kickass +- engine: kickass +- shortcut: kc +- timeout: 4.0 +- disabled: true +- +- - name: lemmy communities +- engine: lemmy +- lemmy_type: Communities +- shortcut: leco +- +- - name: lemmy users +- engine: lemmy +- network: lemmy communities +- lemmy_type: Users +- shortcut: leus +- +- - name: lemmy posts +- engine: lemmy +- network: lemmy communities +- lemmy_type: Posts +- shortcut: lepo +- +- - name: lemmy comments +- engine: lemmy +- network: lemmy communities +- lemmy_type: Comments +- shortcut: lecom +- +- - name: library genesis +- engine: xpath +- search_url: https://libgen.fun/search.php?req={query} +- url_xpath: //a[contains(@href,"get.php?md5")]/@href +- title_xpath: //a[contains(@href,"book/")]/text()[1] +- content_xpath: //td/a[1][contains(@href,"=author")]/text() +- categories: files +- timeout: 7.0 +- disabled: true +- shortcut: lg +- about: +- website: https://libgen.fun/ +- wikidata_id: Q22017206 +- official_api_documentation: +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: z-library +- engine: zlibrary +- shortcut: zlib +- categories: files +- timeout: 7.0 +- +- - name: library of congress +- engine: loc +- shortcut: loc +- categories: images +- +- - name: lingva +- engine: lingva +- shortcut: lv +- # set lingva instance in url, by default it will use the official instance +- # url: https://lingva.ml +- +- - name: lobste.rs +- engine: xpath +- search_url: https://lobste.rs/search?utf8=%E2%9C%93&q={query}&what=stories&order=relevance +- results_xpath: //li[contains(@class, "story")] +- url_xpath: .//a[@class="u-url"]/@href +- title_xpath: .//a[@class="u-url"] +- content_xpath: .//a[@class="domain"] +- categories: it +- shortcut: lo +- timeout: 5.0 +- disabled: true +- about: +- website: https://lobste.rs/ +- wikidata_id: Q60762874 +- official_api_documentation: +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: azlyrics +- shortcut: lyrics +- engine: xpath +- timeout: 4.0 +- disabled: true +- categories: [music, lyrics] +- paging: true +- search_url: https://search.azlyrics.com/search.php?q={query}&w=lyrics&p={pageno} +- url_xpath: //td[@class="text-left visitedlyr"]/a/@href +- title_xpath: //span/b/text() +- content_xpath: //td[@class="text-left visitedlyr"]/a/small +- about: +- website: https://azlyrics.com +- wikidata_id: Q66372542 +- official_api_documentation: +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: metacpan +- engine: metacpan +- shortcut: cpan +- disabled: true +- number_of_results: 20 +- +- # - name: meilisearch +- # engine: meilisearch +- # shortcut: mes +- # enable_http: true +- # base_url: http://localhost:7700 +- # index: my-index +- +- - name: mixcloud +- engine: mixcloud +- shortcut: mc +- +- # MongoDB engine +- # Required dependency: pymongo +- # - name: mymongo +- # engine: mongodb +- # shortcut: md +- # exact_match_only: false +- # host: '127.0.0.1' +- # port: 27017 +- # enable_http: true +- # results_per_page: 20 +- # database: 'business' +- # collection: 'reviews' # name of the db collection +- # key: 'name' # key in the collection to search for +- +- - name: mwmbl +- engine: mwmbl +- # api_url: https://api.mwmbl.org +- shortcut: mwm +- disabled: true +- +- - name: npm +- engine: json_engine +- paging: true +- first_page_num: 0 +- search_url: https://api.npms.io/v2/search?q={query}&size=25&from={pageno} +- results_query: results +- url_query: package/links/npm +- title_query: package/name +- content_query: package/description +- page_size: 25 +- categories: [it, packages] +- disabled: true +- timeout: 5.0 +- shortcut: npm +- about: +- website: https://npms.io/ +- wikidata_id: Q7067518 +- official_api_documentation: https://api-docs.npms.io/ +- use_official_api: false +- require_api_key: false +- results: JSON +- +- - name: nyaa +- engine: nyaa +- shortcut: nt +- disabled: true +- +- - name: mankier +- engine: json_engine +- search_url: https://www.mankier.com/api/v2/mans/?q={query} +- results_query: results +- url_query: url +- title_query: name +- content_query: description +- categories: it +- shortcut: man +- about: +- website: https://www.mankier.com/ +- official_api_documentation: https://www.mankier.com/api +- use_official_api: true +- require_api_key: false +- results: JSON +- +- - name: odysee +- engine: odysee +- shortcut: od +- disabled: true +- +- - name: openairedatasets +- engine: json_engine +- paging: true +- search_url: https://api.openaire.eu/search/datasets?format=json&page={pageno}&size=10&title={query} +- results_query: response/results/result +- url_query: metadata/oaf:entity/oaf:result/children/instance/webresource/url/$ +- title_query: metadata/oaf:entity/oaf:result/title/$ +- content_query: metadata/oaf:entity/oaf:result/description/$ +- content_html_to_text: true +- categories: "science" +- shortcut: oad +- timeout: 5.0 +- about: +- website: https://www.openaire.eu/ +- wikidata_id: Q25106053 +- official_api_documentation: https://api.openaire.eu/ +- use_official_api: false +- require_api_key: false +- results: JSON +- +- - name: openairepublications +- engine: json_engine +- paging: true +- search_url: https://api.openaire.eu/search/publications?format=json&page={pageno}&size=10&title={query} +- results_query: response/results/result +- url_query: metadata/oaf:entity/oaf:result/children/instance/webresource/url/$ +- title_query: metadata/oaf:entity/oaf:result/title/$ +- content_query: metadata/oaf:entity/oaf:result/description/$ +- content_html_to_text: true +- categories: science +- shortcut: oap +- timeout: 5.0 +- about: +- website: https://www.openaire.eu/ +- wikidata_id: Q25106053 +- official_api_documentation: https://api.openaire.eu/ +- use_official_api: false +- require_api_key: false +- results: JSON +- +- # - name: opensemanticsearch +- # engine: opensemantic +- # shortcut: oss +- # base_url: 'http://localhost:8983/solr/opensemanticsearch/' +- +- - name: openstreetmap +- engine: openstreetmap +- shortcut: osm +- +- - name: openrepos +- engine: xpath +- paging: true +- search_url: https://openrepos.net/search/node/{query}?page={pageno} +- url_xpath: //li[@class="search-result"]//h3[@class="title"]/a/@href +- title_xpath: //li[@class="search-result"]//h3[@class="title"]/a +- content_xpath: //li[@class="search-result"]//div[@class="search-snippet-info"]//p[@class="search-snippet"] +- categories: files +- timeout: 4.0 +- disabled: true +- shortcut: or +- about: +- website: https://openrepos.net/ +- wikidata_id: +- official_api_documentation: +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: packagist +- engine: json_engine +- paging: true +- search_url: https://packagist.org/search.json?q={query}&page={pageno} +- results_query: results +- url_query: url +- title_query: name +- content_query: description +- categories: [it, packages] +- disabled: true +- timeout: 5.0 +- shortcut: pack +- about: +- website: https://packagist.org +- wikidata_id: Q108311377 +- official_api_documentation: https://packagist.org/apidoc +- use_official_api: true +- require_api_key: false +- results: JSON +- +- - name: pdbe +- engine: pdbe +- shortcut: pdb +- # Hide obsolete PDB entries. Default is not to hide obsolete structures +- # hide_obsolete: false +- +- - name: photon +- engine: photon +- shortcut: ph +- +- - name: piped +- engine: piped +- shortcut: ppd +- categories: videos +- piped_filter: videos +- timeout: 3.0 +- +- # URL to use as link and for embeds +- frontend_url: https://srv.piped.video +- # Instance will be selected randomly, for more see https://piped-instances.kavin.rocks/ +- backend_url: +- - https://pipedapi.kavin.rocks +- - https://pipedapi-libre.kavin.rocks +- - https://pipedapi.adminforge.de +- +- - name: piped.music +- engine: piped +- network: piped +- shortcut: ppdm +- categories: music +- piped_filter: music_songs +- timeout: 3.0 +- +- - name: piratebay +- engine: piratebay +- shortcut: tpb +- # You may need to change this URL to a proxy if piratebay is blocked in your +- # country +- url: https://thepiratebay.org/ +- timeout: 3.0 +- +- # Required dependency: psychopg2 +- # - name: postgresql +- # engine: postgresql +- # database: postgres +- # username: postgres +- # password: postgres +- # limit: 10 +- # query_str: 'SELECT * from my_table WHERE my_column = %(query)s' +- # shortcut : psql +- +- - name: pub.dev +- engine: xpath +- shortcut: pd +- search_url: https://pub.dev/packages?q={query}&page={pageno} +- paging: true +- results_xpath: //div[contains(@class,"packages-item")] +- url_xpath: ./div/h3/a/@href +- title_xpath: ./div/h3/a +- content_xpath: ./div/div/div[contains(@class,"packages-description")]/span +- categories: [packages, it] +- timeout: 3.0 +- disabled: true +- first_page_num: 1 +- about: +- website: https://pub.dev/ +- official_api_documentation: https://pub.dev/help/api +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: pubmed +- engine: pubmed +- shortcut: pub +- timeout: 3.0 +- +- - name: pypi +- shortcut: pypi +- engine: xpath +- paging: true +- search_url: https://pypi.org/search/?q={query}&page={pageno} +- results_xpath: /html/body/main/div/div/div/form/div/ul/li/a[@class="package-snippet"] +- url_xpath: ./@href +- title_xpath: ./h3/span[@class="package-snippet__name"] +- content_xpath: ./p +- suggestion_xpath: /html/body/main/div/div/div/form/div/div[@class="callout-block"]/p/span/a[@class="link"] +- first_page_num: 1 +- categories: [it, packages] +- about: +- website: https://pypi.org +- wikidata_id: Q2984686 +- official_api_documentation: https://warehouse.readthedocs.io/api-reference/index.html +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: qwant +- qwant_categ: web +- engine: qwant +- shortcut: qw +- categories: [general, web] +- additional_tests: +- rosebud: *test_rosebud +- +- - name: qwant news +- qwant_categ: news +- engine: qwant +- shortcut: qwn +- categories: news +- network: qwant +- +- - name: qwant images +- qwant_categ: images +- engine: qwant +- shortcut: qwi +- categories: [images, web] +- network: qwant +- +- - name: qwant videos +- qwant_categ: videos +- engine: qwant +- shortcut: qwv +- categories: [videos, web] +- network: qwant +- +- # - name: library +- # engine: recoll +- # shortcut: lib +- # base_url: 'https://recoll.example.org/' +- # search_dir: '' +- # mount_prefix: /export +- # dl_prefix: 'https://download.example.org' +- # timeout: 30.0 +- # categories: files +- # disabled: true +- +- # - name: recoll library reference +- # engine: recoll +- # base_url: 'https://recoll.example.org/' +- # search_dir: reference +- # mount_prefix: /export +- # dl_prefix: 'https://download.example.org' +- # shortcut: libr +- # timeout: 30.0 +- # categories: files +- # disabled: true +- +- - name: reddit +- engine: reddit +- shortcut: re +- page_size: 25 +- +- # Required dependency: redis +- # - name: myredis +- # shortcut : rds +- # engine: redis_server +- # exact_match_only: false +- # host: '127.0.0.1' +- # port: 6379 +- # enable_http: true +- # password: '' +- # db: 0 ++ - name: bing images ++ engine: bing_images ++ shortcut: bii + +- # tmp suspended: bad certificate +- # - name: scanr structures +- # shortcut: scs +- # engine: scanr_structures ++ # - name: 9gag ++ # engine: 9gag ++ # shortcut: 9g + # disabled: true +- +- - name: sepiasearch +- engine: sepiasearch +- shortcut: sep +- +- - name: soundcloud +- engine: soundcloud +- shortcut: sc +- +- - name: stackoverflow +- engine: stackexchange +- shortcut: st +- api_site: 'stackoverflow' +- categories: [it, q&a] +- +- - name: askubuntu +- engine: stackexchange +- shortcut: ubuntu +- api_site: 'askubuntu' +- categories: [it, q&a] +- +- - name: internetarchivescholar +- engine: internet_archive_scholar +- shortcut: ias +- timeout: 5.0 +- +- - name: superuser +- engine: stackexchange +- shortcut: su +- api_site: 'superuser' +- categories: [it, q&a] +- +- - name: searchcode code +- engine: searchcode_code +- shortcut: scc +- disabled: true +- +- - name: framalibre +- engine: framalibre +- shortcut: frl +- disabled: true +- +- # - name: searx +- # engine: searx_engine +- # shortcut: se +- # instance_urls : +- # - http://127.0.0.1:8888/ +- # - ... +- # disabled: true +- +- - name: semantic scholar +- engine: semantic_scholar +- disabled: true +- shortcut: se +- +- # Spotify needs API credentials +- # - name: spotify +- # engine: spotify +- # shortcut: stf +- # api_client_id: ******* +- # api_client_secret: ******* +- +- # - name: solr +- # engine: solr +- # shortcut: slr +- # base_url: http://localhost:8983 +- # collection: collection_name +- # sort: '' # sorting: asc or desc +- # field_list: '' # comma separated list of field names to display on the UI +- # default_fields: '' # default field to query +- # query_fields: '' # query fields +- # enable_http: true +- +- # - name: springer nature +- # engine: springer +- # # get your API key from: https://dev.springernature.com/signup +- # # working API key, for test & debug: "a69685087d07eca9f13db62f65b8f601" +- # api_key: 'unset' +- # shortcut: springer +- # timeout: 15.0 +- +- - name: startpage +- engine: startpage +- shortcut: sp +- timeout: 6.0 +- disabled: true +- additional_tests: +- rosebud: *test_rosebud +- +- - name: tokyotoshokan +- engine: tokyotoshokan +- shortcut: tt +- timeout: 6.0 +- disabled: true +- +- - name: solidtorrents +- engine: solidtorrents +- shortcut: solid +- timeout: 4.0 +- base_url: +- - https://solidtorrents.to +- - https://bitsearch.to +- +- # For this demo of the sqlite engine download: +- # https://liste.mediathekview.de/filmliste-v2.db.bz2 +- # and unpack into searx/data/filmliste-v2.db +- # Query to test: "!demo concert" +- # +- # - name: demo +- # engine: sqlite +- # shortcut: demo +- # categories: general +- # result_template: default.html +- # database: searx/data/filmliste-v2.db +- # query_str: >- +- # SELECT title || ' (' || time(duration, 'unixepoch') || ')' AS title, +- # COALESCE( NULLIF(url_video_hd,''), NULLIF(url_video_sd,''), url_video) AS url, +- # description AS content +- # FROM film +- # WHERE title LIKE :wildcard OR description LIKE :wildcard +- # ORDER BY duration DESC +- +- - name: tagesschau +- engine: tagesschau +- shortcut: ts +- disabled: true +- +- - name: tmdb +- engine: xpath +- paging: true +- search_url: https://www.themoviedb.org/search?page={pageno}&query={query} +- results_xpath: //div[contains(@class,"movie") or contains(@class,"tv")]//div[contains(@class,"card")] +- url_xpath: .//div[contains(@class,"poster")]/a/@href +- thumbnail_xpath: .//img/@src +- title_xpath: .//div[contains(@class,"title")]//h2 +- content_xpath: .//div[contains(@class,"overview")] +- shortcut: tm +- disabled: true +- +- # Requires Tor +- - name: torch +- engine: xpath +- paging: true +- search_url: +- http://xmh57jrknzkhv6y3ls3ubitzfqnkrwxhopf5aygthi7d6rplyvk3noyd.onion/cgi-bin/omega/omega?P={query}&DEFAULTOP=and +- results_xpath: //table//tr +- url_xpath: ./td[2]/a +- title_xpath: ./td[2]/b +- content_xpath: ./td[2]/small +- categories: onions +- enable_http: true +- shortcut: tch +- +- # torznab engine lets you query any torznab compatible indexer. Using this +- # engine in combination with Jackett opens the possibility to query a lot of +- # public and private indexers directly from SearXNG. More details at: +- # https://docs.searxng.org/dev/engines/online/torznab.html +- # +- # - name: Torznab EZTV +- # engine: torznab +- # shortcut: eztv +- # base_url: http://localhost:9117/api/v2.0/indexers/eztv/results/torznab +- # enable_http: true # if using localhost +- # api_key: xxxxxxxxxxxxxxx +- # show_magnet_links: true +- # show_torrent_files: false +- # # https://github.com/Jackett/Jackett/wiki/Jackett-Categories +- # torznab_categories: # optional +- # - 2000 +- # - 5000 +- +- - name: twitter +- shortcut: tw +- engine: twitter +- disabled: true +- +- # tmp suspended - too slow, too many errors +- # - name: urbandictionary +- # engine : xpath +- # search_url : https://www.urbandictionary.com/define.php?term={query} +- # url_xpath : //*[@class="word"]/@href +- # title_xpath : //*[@class="def-header"] +- # content_xpath: //*[@class="meaning"] +- # shortcut: ud +- +- - name: unsplash +- engine: unsplash +- shortcut: us +- +- - name: yahoo +- engine: yahoo +- shortcut: yh +- disabled: true +- +- - name: yahoo news +- engine: yahoo_news +- shortcut: yhn +- +- - name: youtube +- shortcut: yt +- # You can use the engine using the official stable API, but you need an API +- # key See: https://console.developers.google.com/project +- # +- # engine: youtube_api +- # api_key: 'apikey' # required! +- # +- # Or you can use the html non-stable engine, activated by default +- engine: youtube_noapi +- +- - name: dailymotion +- engine: dailymotion +- shortcut: dm +- +- - name: vimeo +- engine: vimeo +- shortcut: vm +- +- - name: wiby +- engine: json_engine +- paging: true +- search_url: https://wiby.me/json/?q={query}&p={pageno} +- url_query: URL +- title_query: Title +- content_query: Snippet +- categories: [general, web] +- shortcut: wib +- disabled: true +- about: +- website: https://wiby.me/ +- +- - name: alexandria +- engine: json_engine +- shortcut: alx +- categories: general +- paging: true +- search_url: https://api.alexandria.org/?a=1&q={query}&p={pageno} +- results_query: results +- title_query: title +- url_query: url +- content_query: snippet +- timeout: 1.5 +- disabled: true +- about: +- website: https://alexandria.org/ +- official_api_documentation: https://github.com/alexandria-org/alexandria-api/raw/master/README.md +- use_official_api: true +- require_api_key: false +- results: JSON +- +- - name: wikibooks +- engine: mediawiki +- weight: 0.5 +- shortcut: wb +- categories: [general, wikimedia] +- base_url: "https://{language}.wikibooks.org/" +- search_type: text +- disabled: true +- about: +- website: https://www.wikibooks.org/ +- wikidata_id: Q367 +- +- - name: wikinews +- engine: mediawiki +- shortcut: wn +- categories: [news, wikimedia] +- base_url: "https://{language}.wikinews.org/" +- search_type: text +- srsort: create_timestamp_desc +- about: +- website: https://www.wikinews.org/ +- wikidata_id: Q964 +- +- - name: wikiquote +- engine: mediawiki +- weight: 0.5 +- shortcut: wq +- categories: [general, wikimedia] +- base_url: "https://{language}.wikiquote.org/" +- search_type: text +- disabled: true +- additional_tests: +- rosebud: *test_rosebud +- about: +- website: https://www.wikiquote.org/ +- wikidata_id: Q369 +- +- - name: wikisource +- engine: mediawiki +- weight: 0.5 +- shortcut: ws +- categories: [general, wikimedia] +- base_url: "https://{language}.wikisource.org/" +- search_type: text +- disabled: true +- about: +- website: https://www.wikisource.org/ +- wikidata_id: Q263 +- +- - name: wikispecies +- engine: mediawiki +- shortcut: wsp +- categories: [general, science, wikimedia] +- base_url: "https://species.wikimedia.org/" +- search_type: text +- disabled: true +- about: +- website: https://species.wikimedia.org/ +- wikidata_id: Q13679 +- +- - name: wiktionary +- engine: mediawiki +- shortcut: wt +- categories: [dictionaries, wikimedia] +- base_url: "https://{language}.wiktionary.org/" +- search_type: text +- about: +- website: https://www.wiktionary.org/ +- wikidata_id: Q151 +- +- - name: wikiversity +- engine: mediawiki +- weight: 0.5 +- shortcut: wv +- categories: [general, wikimedia] +- base_url: "https://{language}.wikiversity.org/" +- search_type: text +- disabled: true +- about: +- website: https://www.wikiversity.org/ +- wikidata_id: Q370 +- +- - name: wikivoyage +- engine: mediawiki +- weight: 0.5 +- shortcut: wy +- categories: [general, wikimedia] +- base_url: "https://{language}.wikivoyage.org/" +- search_type: text +- disabled: true +- about: +- website: https://www.wikivoyage.org/ +- wikidata_id: Q373 +- +- - name: wikicommons.images +- engine: wikicommons +- shortcut: wc +- categories: images +- number_of_results: 10 +- +- - name: wolframalpha +- shortcut: wa +- # You can use the engine using the official stable API, but you need an API +- # key. See: https://products.wolframalpha.com/api/ +- # +- # engine: wolframalpha_api +- # api_key: '' +- # +- # Or you can use the html non-stable engine, activated by default +- engine: wolframalpha_noapi +- timeout: 6.0 +- categories: general +- disabled: true +- +- - name: dictzone +- engine: dictzone +- shortcut: dc +- +- - name: mymemory translated +- engine: translated +- shortcut: tl +- timeout: 5.0 +- # You can use without an API key, but you are limited to 1000 words/day +- # See: https://mymemory.translated.net/doc/usagelimits.php +- # api_key: '' +- +- # Required dependency: mysql-connector-python +- # - name: mysql +- # engine: mysql_server +- # database: mydatabase +- # username: user +- # password: pass +- # limit: 10 +- # query_str: 'SELECT * from mytable WHERE fieldname=%(query)s' +- # shortcut: mysql +- +- - name: 1337x +- engine: 1337x +- shortcut: 1337x +- disabled: true +- +- - name: duden +- engine: duden +- shortcut: du +- disabled: true +- +- - name: seznam +- shortcut: szn +- engine: seznam +- disabled: true +- +- # - name: deepl +- # engine: deepl +- # shortcut: dpl +- # # You can use the engine using the official stable API, but you need an API key +- # # See: https://www.deepl.com/pro-api?cta=header-pro-api +- # api_key: '' # required! +- # timeout: 5.0 +- # disabled: true +- +- - name: mojeek +- shortcut: mjk +- engine: xpath +- paging: true +- categories: [general, web] +- search_url: https://www.mojeek.com/search?q={query}&s={pageno}&lang={lang}&lb={lang} +- results_xpath: //ul[@class="results-standard"]/li/a[@class="ob"] +- url_xpath: ./@href +- title_xpath: ../h2/a +- content_xpath: ..//p[@class="s"] +- suggestion_xpath: //div[@class="top-info"]/p[@class="top-info spell"]/em/a +- first_page_num: 0 +- page_size: 10 +- disabled: true +- about: +- website: https://www.mojeek.com/ +- wikidata_id: Q60747299 +- official_api_documentation: https://www.mojeek.com/services/api.html/ +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: moviepilot +- engine: moviepilot +- shortcut: mp +- disabled: true +- +- - name: naver +- shortcut: nvr +- categories: [general, web] +- engine: xpath +- paging: true +- search_url: https://search.naver.com/search.naver?where=webkr&sm=osp_hty&ie=UTF-8&query={query}&start={pageno} +- url_xpath: //a[@class="link_tit"]/@href +- title_xpath: //a[@class="link_tit"] +- content_xpath: //a[@class="total_dsc"]/div +- first_page_num: 1 +- page_size: 10 +- disabled: true +- about: +- website: https://www.naver.com/ +- wikidata_id: Q485639 +- official_api_documentation: https://developers.naver.com/docs/nmt/examples/ +- use_official_api: false +- require_api_key: false +- results: HTML +- language: ko +- +- - name: rubygems +- shortcut: rbg +- engine: xpath +- paging: true +- search_url: https://rubygems.org/search?page={pageno}&query={query} +- results_xpath: /html/body/main/div/a[@class="gems__gem"] +- url_xpath: ./@href +- title_xpath: ./span/h2 +- content_xpath: ./span/p +- suggestion_xpath: /html/body/main/div/div[@class="search__suggestions"]/p/a +- first_page_num: 1 +- categories: [it, packages] +- disabled: true +- about: +- website: https://rubygems.org/ +- wikidata_id: Q1853420 +- official_api_documentation: https://guides.rubygems.org/rubygems-org-api/ +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: peertube +- engine: peertube +- shortcut: ptb +- paging: true +- # alternatives see: https://instances.joinpeertube.org/instances +- # base_url: https://tube.4aem.com +- categories: videos +- disabled: true +- timeout: 6.0 +- +- - name: mediathekviewweb +- engine: mediathekviewweb +- shortcut: mvw +- disabled: true +- +- # - name: yacy +- # engine: yacy +- # shortcut: ya +- # base_url: http://localhost:8090 +- # # required if you aren't using HTTPS for your local yacy instance' +- # enable_http: true +- # timeout: 3.0 +- # # Yacy search mode. 'global' or 'local'. +- # search_mode: 'global' +- +- - name: rumble +- engine: rumble +- shortcut: ru +- base_url: https://rumble.com/ +- paging: true +- categories: videos +- disabled: true +- +- - name: wordnik +- engine: wordnik +- shortcut: def +- base_url: https://www.wordnik.com/ +- categories: [dictionaries] +- timeout: 5.0 +- +- - name: woxikon.de synonyme +- engine: xpath +- shortcut: woxi +- categories: [dictionaries] +- timeout: 5.0 +- disabled: true +- search_url: https://synonyme.woxikon.de/synonyme/{query}.php +- url_xpath: //div[@class="upper-synonyms"]/a/@href +- content_xpath: //div[@class="synonyms-list-group"] +- title_xpath: //div[@class="upper-synonyms"]/a +- no_result_for_http_status: [404] +- about: +- website: https://www.woxikon.de/ +- wikidata_id: # No Wikidata ID +- use_official_api: false +- require_api_key: false +- results: HTML +- language: de +- +- - name: seekr news +- engine: seekr +- shortcut: senews +- categories: news +- seekr_category: news +- disabled: true +- +- - name: seekr images +- engine: seekr +- network: seekr news +- shortcut: seimg +- categories: images +- seekr_category: images +- disabled: true +- +- - name: seekr videos +- engine: seekr +- network: seekr news +- shortcut: sevid +- categories: videos +- seekr_category: videos +- disabled: true +- +- - name: sjp.pwn +- engine: sjp +- shortcut: sjp +- base_url: https://sjp.pwn.pl/ +- timeout: 5.0 +- disabled: true +- +- - name: svgrepo +- engine: svgrepo +- shortcut: svg +- timeout: 10.0 +- disabled: true +- +- - name: wallhaven +- engine: wallhaven +- # api_key: abcdefghijklmnopqrstuvwxyz +- shortcut: wh +- +- # wikimini: online encyclopedia for children +- # The fulltext and title parameter is necessary for Wikimini because +- # sometimes it will not show the results and redirect instead +- - name: wikimini +- engine: xpath +- shortcut: wkmn +- search_url: https://fr.wikimini.org/w/index.php?search={query}&title=Sp%C3%A9cial%3ASearch&fulltext=Search +- url_xpath: //li/div[@class="mw-search-result-heading"]/a/@href +- title_xpath: //li//div[@class="mw-search-result-heading"]/a +- content_xpath: //li/div[@class="searchresult"] +- categories: general +- disabled: true +- about: +- website: https://wikimini.org/ +- wikidata_id: Q3568032 +- use_official_api: false +- require_api_key: false +- results: HTML +- language: fr +- +- - name: wttr.in +- engine: wttr +- shortcut: wttr +- timeout: 9.0 +- +- - name: yummly +- engine: yummly +- shortcut: yum +- disabled: true +- +- - name: brave +- engine: brave +- shortcut: br +- time_range_support: true +- paging: true +- categories: [general, web] +- brave_category: search +- # brave_spellcheck: true +- +- - name: brave.images +- engine: brave +- network: brave +- shortcut: brimg +- categories: [images, web] +- brave_category: images +- +- - name: brave.videos +- engine: brave +- network: brave +- shortcut: brvid +- categories: [videos, web] +- brave_category: videos +- +- - name: brave.news +- engine: brave +- network: brave +- shortcut: brnews +- categories: news +- brave_category: news +- +- - name: lib.rs +- shortcut: lrs +- engine: xpath +- search_url: https://lib.rs/search?q={query} +- results_xpath: /html/body/main/div/ol/li/a +- url_xpath: ./@href +- title_xpath: ./div[@class="h"]/h4 +- content_xpath: ./div[@class="h"]/p +- categories: [it, packages] +- disabled: true +- about: +- website: https://lib.rs +- wikidata_id: Q113486010 +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: sourcehut +- shortcut: srht +- engine: xpath +- paging: true +- search_url: https://sr.ht/projects?page={pageno}&search={query} +- results_xpath: (//div[@class="event-list"])[1]/div[@class="event"] +- url_xpath: ./h4/a[2]/@href +- title_xpath: ./h4/a[2] +- content_xpath: ./p +- first_page_num: 1 +- categories: [it, repos] +- disabled: true +- about: +- website: https://sr.ht +- wikidata_id: Q78514485 +- official_api_documentation: https://man.sr.ht/ +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: goo +- shortcut: goo +- engine: xpath +- paging: true +- search_url: https://search.goo.ne.jp/web.jsp?MT={query}&FR={pageno}0 +- url_xpath: //div[@class="result"]/p[@class='title fsL1']/a/@href +- title_xpath: //div[@class="result"]/p[@class='title fsL1']/a +- content_xpath: //p[contains(@class,'url fsM')]/following-sibling::p +- first_page_num: 0 +- categories: [general, web] +- disabled: true +- timeout: 4.0 +- about: +- website: https://search.goo.ne.jp +- wikidata_id: Q249044 +- use_official_api: false +- require_api_key: false +- results: HTML +- language: ja +- +- - name: bt4g +- engine: bt4g +- shortcut: bt4g +- +- - name: pkg.go.dev +- engine: xpath +- shortcut: pgo +- search_url: https://pkg.go.dev/search?limit=100&m=package&q={query} +- results_xpath: /html/body/main/div[contains(@class,"SearchResults")]/div[not(@class)]/div[@class="SearchSnippet"] +- url_xpath: ./div[@class="SearchSnippet-headerContainer"]/h2/a/@href +- title_xpath: ./div[@class="SearchSnippet-headerContainer"]/h2/a +- content_xpath: ./p[@class="SearchSnippet-synopsis"] +- categories: [packages, it] +- timeout: 3.0 +- disabled: true +- about: +- website: https://pkg.go.dev/ +- use_official_api: false +- require_api_key: false +- results: HTML +- +-# Doku engine lets you access to any Doku wiki instance: +-# A public one or a privete/corporate one. +-# - name: ubuntuwiki +-# engine: doku +-# shortcut: uw +-# base_url: 'https://doc.ubuntu-fr.org' +- +-# Be careful when enabling this engine if you are +-# running a public instance. Do not expose any sensitive +-# information. You can restrict access by configuring a list +-# of access tokens under tokens. +-# - name: git grep +-# engine: command +-# command: ['git', 'grep', '{{QUERY}}'] +-# shortcut: gg +-# tokens: [] +-# disabled: true +-# delimiter: +-# chars: ':' +-# keys: ['filepath', 'code'] +- +-# Be careful when enabling this engine if you are +-# running a public instance. Do not expose any sensitive +-# information. You can restrict access by configuring a list +-# of access tokens under tokens. +-# - name: locate +-# engine: command +-# command: ['locate', '{{QUERY}}'] +-# shortcut: loc +-# tokens: [] +-# disabled: true +-# delimiter: +-# chars: ' ' +-# keys: ['line'] +- +-# Be careful when enabling this engine if you are +-# running a public instance. Do not expose any sensitive +-# information. You can restrict access by configuring a list +-# of access tokens under tokens. +-# - name: find +-# engine: command +-# command: ['find', '.', '-name', '{{QUERY}}'] +-# query_type: path +-# shortcut: fnd +-# tokens: [] +-# disabled: true +-# delimiter: +-# chars: ' ' +-# keys: ['line'] +- +-# Be careful when enabling this engine if you are +-# running a public instance. Do not expose any sensitive +-# information. You can restrict access by configuring a list +-# of access tokens under tokens. +-# - name: pattern search in files +-# engine: command +-# command: ['fgrep', '{{QUERY}}'] +-# shortcut: fgr +-# tokens: [] +-# disabled: true +-# delimiter: +-# chars: ' ' +-# keys: ['line'] +- +-# Be careful when enabling this engine if you are +-# running a public instance. Do not expose any sensitive +-# information. You can restrict access by configuring a list +-# of access tokens under tokens. +-# - name: regex search in files +-# engine: command +-# command: ['grep', '{{QUERY}}'] +-# shortcut: gr +-# tokens: [] +-# disabled: true +-# delimiter: +-# chars: ' ' +-# keys: ['line'] ++ # ++ # - name: annas archive ++ # engine: annas_archive ++ # disabled: true ++ # shortcut: aa ++ # ++ # # - name: annas articles ++ # # engine: annas_archive ++ # # shortcut: aaa ++ # # # https://docs.searxng.org/dev/engines/online/annas_archive.html ++ # # aa_content: 'journal_article' # book_any .. magazine, standards_document ++ # # aa_ext: 'pdf' # pdf, epub, .. ++ # # aa_sort: 'newest' # newest, oldest, largest, smallest ++ # ++ # - name: apk mirror ++ # engine: apkmirror ++ # timeout: 4.0 ++ # shortcut: apkm ++ # disabled: true ++ # ++ # - name: apple app store ++ # engine: apple_app_store ++ # shortcut: aps ++ # disabled: true ++ # ++ # # Requires Tor ++ # - name: ahmia ++ # engine: ahmia ++ # categories: onions ++ # enable_http: true ++ # shortcut: ah ++ # ++ # - name: anaconda ++ # engine: xpath ++ # paging: true ++ # first_page_num: 0 ++ # search_url: https://anaconda.org/search?q={query}&page={pageno} ++ # results_xpath: //tbody/tr ++ # url_xpath: ./td/h5/a[last()]/@href ++ # title_xpath: ./td/h5 ++ # content_xpath: ./td[h5]/text() ++ # categories: it ++ # timeout: 6.0 ++ # shortcut: conda ++ # disabled: true ++ # ++ # - name: arch linux wiki ++ # engine: archlinux ++ # shortcut: al ++ # ++ # - name: artic ++ # engine: artic ++ # shortcut: arc ++ # timeout: 4.0 ++ # ++ # - name: arxiv ++ # engine: arxiv ++ # shortcut: arx ++ # timeout: 4.0 ++ # ++ # # tmp suspended: dh key too small ++ # # - name: base ++ # # engine: base ++ # # shortcut: bs ++ # ++ # - name: bandcamp ++ # engine: bandcamp ++ # shortcut: bc ++ # categories: music ++ # ++ # - name: wikipedia ++ # engine: wikipedia ++ # shortcut: wp ++ # # add "list" to the array to get results in the results list ++ # display_type: ["infobox"] ++ # base_url: 'https://{language}.wikipedia.org/' ++ # categories: [general] ++ # ++ # - name: bilibili ++ # engine: bilibili ++ # shortcut: bil ++ # disabled: true ++ # ++ # - name: bing ++ # engine: bing ++ # shortcut: bi ++ # disabled: true ++ # ++ # ++ # - name: bing news ++ # engine: bing_news ++ # shortcut: bin ++ # ++ # - name: bing videos ++ # engine: bing_videos ++ # shortcut: biv ++ # ++ # - name: bitbucket ++ # engine: xpath ++ # paging: true ++ # search_url: https://bitbucket.org/repo/all/{pageno}?name={query} ++ # url_xpath: //article[@class="repo-summary"]//a[@class="repo-link"]/@href ++ # title_xpath: //article[@class="repo-summary"]//a[@class="repo-link"] ++ # content_xpath: //article[@class="repo-summary"]/p ++ # categories: [it, repos] ++ # timeout: 4.0 ++ # disabled: true ++ # shortcut: bb ++ # about: ++ # website: https://bitbucket.org/ ++ # wikidata_id: Q2493781 ++ # official_api_documentation: https://developer.atlassian.com/bitbucket ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: btdigg ++ # engine: btdigg ++ # shortcut: bt ++ # disabled: true ++ # ++ # - name: ccc-tv ++ # engine: xpath ++ # paging: false ++ # search_url: https://media.ccc.de/search/?q={query} ++ # url_xpath: //div[@class="caption"]/h3/a/@href ++ # title_xpath: //div[@class="caption"]/h3/a/text() ++ # content_xpath: //div[@class="caption"]/h4/@title ++ # categories: videos ++ # disabled: true ++ # shortcut: c3tv ++ # about: ++ # website: https://media.ccc.de/ ++ # wikidata_id: Q80729951 ++ # official_api_documentation: https://github.com/voc/voctoweb ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # # We don't set language: de here because media.ccc.de is not just ++ # # for a German audience. It contains many English videos and many ++ # # German videos have English subtitles. ++ # ++ # - name: openverse ++ # engine: openverse ++ # categories: images ++ # shortcut: opv ++ # ++ # - name: chefkoch ++ # engine: chefkoch ++ # shortcut: chef ++ # # to show premium or plus results too: ++ # # skip_premium: false ++ # ++ # # - name: core.ac.uk ++ # # engine: core ++ # # categories: science ++ # # shortcut: cor ++ # # # get your API key from: https://core.ac.uk/api-keys/register/ ++ # # api_key: 'unset' ++ # ++ # - name: crossref ++ # engine: crossref ++ # shortcut: cr ++ # timeout: 30 ++ # disabled: true ++ # ++ # - name: crowdview ++ # engine: json_engine ++ # shortcut: cv ++ # categories: general ++ # paging: false ++ # search_url: https://crowdview-next-js.onrender.com/api/search-v3?query={query} ++ # results_query: results ++ # url_query: link ++ # title_query: title ++ # content_query: snippet ++ # disabled: true ++ # about: ++ # website: https://crowdview.ai/ ++ # ++ # - name: yep ++ # engine: json_engine ++ # shortcut: yep ++ # categories: general ++ # disabled: true ++ # paging: false ++ # content_html_to_text: true ++ # title_html_to_text: true ++ # search_url: https://api.yep.com/fs/1/?type=web&q={query}&no_correct=false&limit=100 ++ # results_query: 1/results ++ # title_query: title ++ # url_query: url ++ # content_query: snippet ++ # about: ++ # website: https://yep.com ++ # use_official_api: false ++ # require_api_key: false ++ # results: JSON ++ # ++ # - name: curlie ++ # engine: xpath ++ # shortcut: cl ++ # categories: general ++ # disabled: true ++ # paging: true ++ # lang_all: '' ++ # search_url: https://curlie.org/search?q={query}&lang={lang}&start={pageno}&stime=92452189 ++ # page_size: 20 ++ # results_xpath: //div[@id="site-list-content"]/div[@class="site-item"] ++ # url_xpath: ./div[@class="title-and-desc"]/a/@href ++ # title_xpath: ./div[@class="title-and-desc"]/a/div ++ # content_xpath: ./div[@class="title-and-desc"]/div[@class="site-descr"] ++ # about: ++ # website: https://curlie.org/ ++ # wikidata_id: Q60715723 ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: currency ++ # engine: currency_convert ++ # categories: general ++ # shortcut: cc ++ # ++ # - name: deezer ++ # engine: deezer ++ # shortcut: dz ++ # disabled: true ++ # ++ # - name: deviantart ++ # engine: deviantart ++ # shortcut: da ++ # timeout: 3.0 ++ # ++ # - name: ddg definitions ++ # engine: duckduckgo_definitions ++ # shortcut: ddd ++ # weight: 2 ++ # disabled: true ++ # tests: *tests_infobox ++ # ++ # # cloudflare protected ++ # # - name: digbt ++ # # engine: digbt ++ # # shortcut: dbt ++ # # timeout: 6.0 ++ # # disabled: true ++ # ++ # - name: docker hub ++ # engine: docker_hub ++ # shortcut: dh ++ # categories: [it, packages] ++ # ++ # - name: erowid ++ # engine: xpath ++ # paging: true ++ # first_page_num: 0 ++ # page_size: 30 ++ # search_url: https://www.erowid.org/search.php?q={query}&s={pageno} ++ # url_xpath: //dl[@class="results-list"]/dt[@class="result-title"]/a/@href ++ # title_xpath: //dl[@class="results-list"]/dt[@class="result-title"]/a/text() ++ # content_xpath: //dl[@class="results-list"]/dd[@class="result-details"] ++ # categories: [] ++ # shortcut: ew ++ # disabled: true ++ # about: ++ # website: https://www.erowid.org/ ++ # wikidata_id: Q1430691 ++ # official_api_documentation: ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # # - name: elasticsearch ++ # # shortcut: es ++ # # engine: elasticsearch ++ # # base_url: http://localhost:9200 ++ # # username: elastic ++ # # password: changeme ++ # # index: my-index ++ # # # available options: match, simple_query_string, term, terms, custom ++ # # query_type: match ++ # # # if query_type is set to custom, provide your query here ++ # # #custom_query_json: {"query":{"match_all": {}}} ++ # # #show_metadata: false ++ # # disabled: true ++ # ++ # - name: wikidata ++ # engine: wikidata ++ # shortcut: wd ++ # timeout: 3.0 ++ # weight: 2 ++ # # add "list" to the array to get results in the results list ++ # display_type: ["infobox"] ++ # tests: *tests_infobox ++ # categories: [general] ++ # ++ # - name: duckduckgo ++ # engine: duckduckgo ++ # shortcut: ddg ++ # ++ # - name: duckduckgo images ++ # engine: duckduckgo_images ++ # shortcut: ddi ++ # timeout: 3.0 ++ # disabled: true ++ # ++ # - name: duckduckgo weather ++ # engine: duckduckgo_weather ++ # shortcut: ddw ++ # disabled: true ++ # ++ # - name: apple maps ++ # engine: apple_maps ++ # shortcut: apm ++ # disabled: true ++ # timeout: 5.0 ++ # ++ # - name: emojipedia ++ # engine: emojipedia ++ # timeout: 4.0 ++ # shortcut: em ++ # disabled: true ++ # ++ # - name: tineye ++ # engine: tineye ++ # shortcut: tin ++ # timeout: 9.0 ++ # disabled: true ++ # ++ # - name: etymonline ++ # engine: xpath ++ # paging: true ++ # search_url: https://etymonline.com/search?page={pageno}&q={query} ++ # url_xpath: //a[contains(@class, "word__name--")]/@href ++ # title_xpath: //a[contains(@class, "word__name--")] ++ # content_xpath: //section[contains(@class, "word__defination")] ++ # first_page_num: 1 ++ # shortcut: et ++ # categories: [dictionaries] ++ # about: ++ # website: https://www.etymonline.com/ ++ # wikidata_id: Q1188617 ++ # official_api_documentation: ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # # - name: ebay ++ # # engine: ebay ++ # # shortcut: eb ++ # # base_url: 'https://www.ebay.com' ++ # # disabled: true ++ # # timeout: 5 ++ # ++ # - name: 1x ++ # engine: www1x ++ # shortcut: 1x ++ # timeout: 3.0 ++ # disabled: true ++ # ++ # - name: fdroid ++ # engine: fdroid ++ # shortcut: fd ++ # disabled: true ++ # ++ # - name: flickr ++ # categories: images ++ # shortcut: fl ++ # # You can use the engine using the official stable API, but you need an API ++ # # key, see: https://www.flickr.com/services/apps/create/ ++ # # engine: flickr ++ # # api_key: 'apikey' # required! ++ # # Or you can use the html non-stable engine, activated by default ++ # engine: flickr_noapi ++ # ++ # - name: free software directory ++ # engine: mediawiki ++ # shortcut: fsd ++ # categories: [it, software wikis] ++ # base_url: https://directory.fsf.org/ ++ # search_type: title ++ # timeout: 5.0 ++ # disabled: true ++ # about: ++ # website: https://directory.fsf.org/ ++ # wikidata_id: Q2470288 ++ # ++ # # - name: freesound ++ # # engine: freesound ++ # # shortcut: fnd ++ # # disabled: true ++ # # timeout: 15.0 ++ # # API key required, see: https://freesound.org/docs/api/overview.html ++ # # api_key: MyAPIkey ++ # ++ # - name: frinkiac ++ # engine: frinkiac ++ # shortcut: frk ++ # disabled: true ++ # ++ # - name: genius ++ # engine: genius ++ # shortcut: gen ++ # ++ # - name: gentoo ++ # engine: gentoo ++ # shortcut: ge ++ # timeout: 10.0 ++ # ++ # - name: gitlab ++ # engine: json_engine ++ # paging: true ++ # search_url: https://gitlab.com/api/v4/projects?search={query}&page={pageno} ++ # url_query: web_url ++ # title_query: name_with_namespace ++ # content_query: description ++ # page_size: 20 ++ # categories: [it, repos] ++ # shortcut: gl ++ # timeout: 10.0 ++ # disabled: true ++ # about: ++ # website: https://about.gitlab.com/ ++ # wikidata_id: Q16639197 ++ # official_api_documentation: https://docs.gitlab.com/ee/api/ ++ # use_official_api: false ++ # require_api_key: false ++ # results: JSON ++ # ++ # - name: github ++ # engine: github ++ # shortcut: gh ++ # ++ # # This a Gitea service. If you would like to use a different instance, ++ # # change codeberg.org to URL of the desired Gitea host. Or you can create a ++ # # new engine by copying this and changing the name, shortcut and search_url. ++ # ++ # - name: codeberg ++ # engine: json_engine ++ # search_url: https://codeberg.org/api/v1/repos/search?q={query}&limit=10 ++ # url_query: html_url ++ # title_query: name ++ # content_query: description ++ # categories: [it, repos] ++ # shortcut: cb ++ # disabled: true ++ # about: ++ # website: https://codeberg.org/ ++ # wikidata_id: ++ # official_api_documentation: https://try.gitea.io/api/swagger ++ # use_official_api: false ++ # require_api_key: false ++ # results: JSON ++ # ++ # ++ # ++ # - name: google news ++ # engine: google_news ++ # shortcut: gon ++ # # additional_tests: ++ # # android: *test_android ++ # ++ # - name: google videos ++ # engine: google_videos ++ # shortcut: gov ++ # # additional_tests: ++ # # android: *test_android ++ # ++ # - name: google scholar ++ # engine: google_scholar ++ # shortcut: gos ++ # ++ # - name: google play apps ++ # engine: google_play ++ # categories: [files, apps] ++ # shortcut: gpa ++ # play_categ: apps ++ # disabled: true ++ # ++ # - name: google play movies ++ # engine: google_play ++ # categories: videos ++ # shortcut: gpm ++ # play_categ: movies ++ # disabled: true ++ # ++ # - name: material icons ++ # engine: material_icons ++ # categories: images ++ # shortcut: mi ++ # disabled: true ++ # ++ # - name: gpodder ++ # engine: json_engine ++ # shortcut: gpod ++ # timeout: 4.0 ++ # paging: false ++ # search_url: https://gpodder.net/search.json?q={query} ++ # url_query: url ++ # title_query: title ++ # content_query: description ++ # page_size: 19 ++ # categories: music ++ # disabled: true ++ # about: ++ # website: https://gpodder.net ++ # wikidata_id: Q3093354 ++ # official_api_documentation: https://gpoddernet.readthedocs.io/en/latest/api/ ++ # use_official_api: false ++ # requires_api_key: false ++ # results: JSON ++ # ++ # - name: habrahabr ++ # engine: xpath ++ # paging: true ++ # search_url: https://habr.com/en/search/page{pageno}/?q={query} ++ # results_xpath: //article[contains(@class, "tm-articles-list__item")] ++ # url_xpath: .//a[@class="tm-title__link"]/@href ++ # title_xpath: .//a[@class="tm-title__link"] ++ # content_xpath: .//div[contains(@class, "article-formatted-body")] ++ # categories: it ++ # timeout: 4.0 ++ # disabled: true ++ # shortcut: habr ++ # about: ++ # website: https://habr.com/ ++ # wikidata_id: Q4494434 ++ # official_api_documentation: https://habr.com/en/docs/help/api/ ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: hoogle ++ # engine: xpath ++ # paging: true ++ # search_url: https://hoogle.haskell.org/?hoogle={query}&start={pageno} ++ # results_xpath: '//div[@class="result"]' ++ # title_xpath: './/div[@class="ans"]//a' ++ # url_xpath: './/div[@class="ans"]//a/@href' ++ # content_xpath: './/div[@class="from"]' ++ # page_size: 20 ++ # categories: [it, packages] ++ # shortcut: ho ++ # about: ++ # website: https://hoogle.haskell.org/ ++ # wikidata_id: Q34010 ++ # official_api_documentation: https://hackage.haskell.org/api ++ # use_official_api: false ++ # require_api_key: false ++ # results: JSON ++ # ++ # - name: imdb ++ # engine: imdb ++ # shortcut: imdb ++ # timeout: 6.0 ++ # disabled: true ++ # ++ # - name: ina ++ # engine: ina ++ # shortcut: in ++ # timeout: 6.0 ++ # disabled: true ++ # ++ # - name: invidious ++ # engine: invidious ++ # # Instanes will be selected randomly, see https://api.invidious.io/ for ++ # # instances that are stable (good uptime) and close to you. ++ # base_url: ++ # - https://invidious.io.lol ++ # - https://invidious.fdn.fr ++ # - https://yt.artemislena.eu ++ # - https://invidious.tiekoetter.com ++ # - https://invidious.flokinet.to ++ # - https://vid.puffyan.us ++ # - https://invidious.privacydev.net ++ # - https://inv.tux.pizza ++ # shortcut: iv ++ # timeout: 3.0 ++ # disabled: true ++ # ++ # - name: jisho ++ # engine: jisho ++ # shortcut: js ++ # timeout: 3.0 ++ # disabled: true ++ # ++ # - name: kickass ++ # engine: kickass ++ # shortcut: kc ++ # timeout: 4.0 ++ # disabled: true ++ # ++ # - name: lemmy communities ++ # engine: lemmy ++ # lemmy_type: Communities ++ # shortcut: leco ++ # ++ # - name: lemmy users ++ # engine: lemmy ++ # network: lemmy communities ++ # lemmy_type: Users ++ # shortcut: leus ++ # ++ # - name: lemmy posts ++ # engine: lemmy ++ # network: lemmy communities ++ # lemmy_type: Posts ++ # shortcut: lepo ++ # ++ # - name: lemmy comments ++ # engine: lemmy ++ # network: lemmy communities ++ # lemmy_type: Comments ++ # shortcut: lecom ++ # ++ # - name: library genesis ++ # engine: xpath ++ # search_url: https://libgen.fun/search.php?req={query} ++ # url_xpath: //a[contains(@href,"get.php?md5")]/@href ++ # title_xpath: //a[contains(@href,"book/")]/text()[1] ++ # content_xpath: //td/a[1][contains(@href,"=author")]/text() ++ # categories: files ++ # timeout: 7.0 ++ # disabled: true ++ # shortcut: lg ++ # about: ++ # website: https://libgen.fun/ ++ # wikidata_id: Q22017206 ++ # official_api_documentation: ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: z-library ++ # engine: zlibrary ++ # shortcut: zlib ++ # categories: files ++ # timeout: 7.0 ++ # ++ # - name: library of congress ++ # engine: loc ++ # shortcut: loc ++ # categories: images ++ # ++ # - name: lingva ++ # engine: lingva ++ # shortcut: lv ++ # # set lingva instance in url, by default it will use the official instance ++ # # url: https://lingva.ml ++ # ++ # - name: lobste.rs ++ # engine: xpath ++ # search_url: https://lobste.rs/search?utf8=%E2%9C%93&q={query}&what=stories&order=relevance ++ # results_xpath: //li[contains(@class, "story")] ++ # url_xpath: .//a[@class="u-url"]/@href ++ # title_xpath: .//a[@class="u-url"] ++ # content_xpath: .//a[@class="domain"] ++ # categories: it ++ # shortcut: lo ++ # timeout: 5.0 ++ # disabled: true ++ # about: ++ # website: https://lobste.rs/ ++ # wikidata_id: Q60762874 ++ # official_api_documentation: ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: azlyrics ++ # shortcut: lyrics ++ # engine: xpath ++ # timeout: 4.0 ++ # disabled: true ++ # categories: [music, lyrics] ++ # paging: true ++ # search_url: https://search.azlyrics.com/search.php?q={query}&w=lyrics&p={pageno} ++ # url_xpath: //td[@class="text-left visitedlyr"]/a/@href ++ # title_xpath: //span/b/text() ++ # content_xpath: //td[@class="text-left visitedlyr"]/a/small ++ # about: ++ # website: https://azlyrics.com ++ # wikidata_id: Q66372542 ++ # official_api_documentation: ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: metacpan ++ # engine: metacpan ++ # shortcut: cpan ++ # disabled: true ++ # number_of_results: 20 ++ # ++ # # - name: meilisearch ++ # # engine: meilisearch ++ # # shortcut: mes ++ # # enable_http: true ++ # # base_url: http://localhost:7700 ++ # # index: my-index ++ # ++ # - name: mixcloud ++ # engine: mixcloud ++ # shortcut: mc ++ # ++ # # MongoDB engine ++ # # Required dependency: pymongo ++ # # - name: mymongo ++ # # engine: mongodb ++ # # shortcut: md ++ # # exact_match_only: false ++ # # host: '127.0.0.1' ++ # # port: 27017 ++ # # enable_http: true ++ # # results_per_page: 20 ++ # # database: 'business' ++ # # collection: 'reviews' # name of the db collection ++ # # key: 'name' # key in the collection to search for ++ # ++ # - name: mwmbl ++ # engine: mwmbl ++ # # api_url: https://api.mwmbl.org ++ # shortcut: mwm ++ # disabled: true ++ # ++ # - name: npm ++ # engine: json_engine ++ # paging: true ++ # first_page_num: 0 ++ # search_url: https://api.npms.io/v2/search?q={query}&size=25&from={pageno} ++ # results_query: results ++ # url_query: package/links/npm ++ # title_query: package/name ++ # content_query: package/description ++ # page_size: 25 ++ # categories: [it, packages] ++ # disabled: true ++ # timeout: 5.0 ++ # shortcut: npm ++ # about: ++ # website: https://npms.io/ ++ # wikidata_id: Q7067518 ++ # official_api_documentation: https://api-docs.npms.io/ ++ # use_official_api: false ++ # require_api_key: false ++ # results: JSON ++ # ++ # - name: nyaa ++ # engine: nyaa ++ # shortcut: nt ++ # disabled: true ++ # ++ # - name: mankier ++ # engine: json_engine ++ # search_url: https://www.mankier.com/api/v2/mans/?q={query} ++ # results_query: results ++ # url_query: url ++ # title_query: name ++ # content_query: description ++ # categories: it ++ # shortcut: man ++ # about: ++ # website: https://www.mankier.com/ ++ # official_api_documentation: https://www.mankier.com/api ++ # use_official_api: true ++ # require_api_key: false ++ # results: JSON ++ # ++ # - name: odysee ++ # engine: odysee ++ # shortcut: od ++ # disabled: true ++ # ++ # - name: openairedatasets ++ # engine: json_engine ++ # paging: true ++ # search_url: https://api.openaire.eu/search/datasets?format=json&page={pageno}&size=10&title={query} ++ # results_query: response/results/result ++ # url_query: metadata/oaf:entity/oaf:result/children/instance/webresource/url/$ ++ # title_query: metadata/oaf:entity/oaf:result/title/$ ++ # content_query: metadata/oaf:entity/oaf:result/description/$ ++ # content_html_to_text: true ++ # categories: "science" ++ # shortcut: oad ++ # timeout: 5.0 ++ # about: ++ # website: https://www.openaire.eu/ ++ # wikidata_id: Q25106053 ++ # official_api_documentation: https://api.openaire.eu/ ++ # use_official_api: false ++ # require_api_key: false ++ # results: JSON ++ # ++ # - name: openairepublications ++ # engine: json_engine ++ # paging: true ++ # search_url: https://api.openaire.eu/search/publications?format=json&page={pageno}&size=10&title={query} ++ # results_query: response/results/result ++ # url_query: metadata/oaf:entity/oaf:result/children/instance/webresource/url/$ ++ # title_query: metadata/oaf:entity/oaf:result/title/$ ++ # content_query: metadata/oaf:entity/oaf:result/description/$ ++ # content_html_to_text: true ++ # categories: science ++ # shortcut: oap ++ # timeout: 5.0 ++ # about: ++ # website: https://www.openaire.eu/ ++ # wikidata_id: Q25106053 ++ # official_api_documentation: https://api.openaire.eu/ ++ # use_official_api: false ++ # require_api_key: false ++ # results: JSON ++ # ++ # # - name: opensemanticsearch ++ # # engine: opensemantic ++ # # shortcut: oss ++ # # base_url: 'http://localhost:8983/solr/opensemanticsearch/' ++ # ++ # - name: openstreetmap ++ # engine: openstreetmap ++ # shortcut: osm ++ # ++ # - name: openrepos ++ # engine: xpath ++ # paging: true ++ # search_url: https://openrepos.net/search/node/{query}?page={pageno} ++ # url_xpath: //li[@class="search-result"]//h3[@class="title"]/a/@href ++ # title_xpath: //li[@class="search-result"]//h3[@class="title"]/a ++ # content_xpath: //li[@class="search-result"]//div[@class="search-snippet-info"]//p[@class="search-snippet"] ++ # categories: files ++ # timeout: 4.0 ++ # disabled: true ++ # shortcut: or ++ # about: ++ # website: https://openrepos.net/ ++ # wikidata_id: ++ # official_api_documentation: ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: packagist ++ # engine: json_engine ++ # paging: true ++ # search_url: https://packagist.org/search.json?q={query}&page={pageno} ++ # results_query: results ++ # url_query: url ++ # title_query: name ++ # content_query: description ++ # categories: [it, packages] ++ # disabled: true ++ # timeout: 5.0 ++ # shortcut: pack ++ # about: ++ # website: https://packagist.org ++ # wikidata_id: Q108311377 ++ # official_api_documentation: https://packagist.org/apidoc ++ # use_official_api: true ++ # require_api_key: false ++ # results: JSON ++ # ++ # - name: pdbe ++ # engine: pdbe ++ # shortcut: pdb ++ # # Hide obsolete PDB entries. Default is not to hide obsolete structures ++ # # hide_obsolete: false ++ # ++ # - name: photon ++ # engine: photon ++ # shortcut: ph ++ # ++ # - name: piped ++ # engine: piped ++ # shortcut: ppd ++ # categories: videos ++ # piped_filter: videos ++ # timeout: 3.0 ++ # ++ # # URL to use as link and for embeds ++ # frontend_url: https://srv.piped.video ++ # # Instance will be selected randomly, for more see https://piped-instances.kavin.rocks/ ++ # backend_url: ++ # - https://pipedapi.kavin.rocks ++ # - https://pipedapi-libre.kavin.rocks ++ # - https://pipedapi.adminforge.de ++ # ++ # - name: piped.music ++ # engine: piped ++ # network: piped ++ # shortcut: ppdm ++ # categories: music ++ # piped_filter: music_songs ++ # timeout: 3.0 ++ # ++ # - name: piratebay ++ # engine: piratebay ++ # shortcut: tpb ++ # # You may need to change this URL to a proxy if piratebay is blocked in your ++ # # country ++ # url: https://thepiratebay.org/ ++ # timeout: 3.0 ++ # ++ # # Required dependency: psychopg2 ++ # # - name: postgresql ++ # # engine: postgresql ++ # # database: postgres ++ # # username: postgres ++ # # password: postgres ++ # # limit: 10 ++ # # query_str: 'SELECT * from my_table WHERE my_column = %(query)s' ++ # # shortcut : psql ++ # ++ # - name: pub.dev ++ # engine: xpath ++ # shortcut: pd ++ # search_url: https://pub.dev/packages?q={query}&page={pageno} ++ # paging: true ++ # results_xpath: //div[contains(@class,"packages-item")] ++ # url_xpath: ./div/h3/a/@href ++ # title_xpath: ./div/h3/a ++ # content_xpath: ./div/div/div[contains(@class,"packages-description")]/span ++ # categories: [packages, it] ++ # timeout: 3.0 ++ # disabled: true ++ # first_page_num: 1 ++ # about: ++ # website: https://pub.dev/ ++ # official_api_documentation: https://pub.dev/help/api ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: pubmed ++ # engine: pubmed ++ # shortcut: pub ++ # timeout: 3.0 ++ # ++ # - name: pypi ++ # shortcut: pypi ++ # engine: xpath ++ # paging: true ++ # search_url: https://pypi.org/search/?q={query}&page={pageno} ++ # results_xpath: /html/body/main/div/div/div/form/div/ul/li/a[@class="package-snippet"] ++ # url_xpath: ./@href ++ # title_xpath: ./h3/span[@class="package-snippet__name"] ++ # content_xpath: ./p ++ # suggestion_xpath: /html/body/main/div/div/div/form/div/div[@class="callout-block"]/p/span/a[@class="link"] ++ # first_page_num: 1 ++ # categories: [it, packages] ++ # about: ++ # website: https://pypi.org ++ # wikidata_id: Q2984686 ++ # official_api_documentation: https://warehouse.readthedocs.io/api-reference/index.html ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: qwant ++ # qwant_categ: web ++ # engine: qwant ++ # shortcut: qw ++ # categories: [general, web] ++ # additional_tests: ++ # rosebud: *test_rosebud ++ # ++ # - name: qwant news ++ # qwant_categ: news ++ # engine: qwant ++ # shortcut: qwn ++ # categories: news ++ # network: qwant ++ # ++ # - name: qwant images ++ # qwant_categ: images ++ # engine: qwant ++ # shortcut: qwi ++ # categories: [images, web] ++ # network: qwant ++ # ++ # - name: qwant videos ++ # qwant_categ: videos ++ # engine: qwant ++ # shortcut: qwv ++ # categories: [videos, web] ++ # network: qwant ++ # ++ # # - name: library ++ # # engine: recoll ++ # # shortcut: lib ++ # # base_url: 'https://recoll.example.org/' ++ # # search_dir: '' ++ # # mount_prefix: /export ++ # # dl_prefix: 'https://download.example.org' ++ # # timeout: 30.0 ++ # # categories: files ++ # # disabled: true ++ # ++ # # - name: recoll library reference ++ # # engine: recoll ++ # # base_url: 'https://recoll.example.org/' ++ # # search_dir: reference ++ # # mount_prefix: /export ++ # # dl_prefix: 'https://download.example.org' ++ # # shortcut: libr ++ # # timeout: 30.0 ++ # # categories: files ++ # # disabled: true ++ # ++ # - name: reddit ++ # engine: reddit ++ # shortcut: re ++ # page_size: 25 ++ # ++ # # Required dependency: redis ++ # # - name: myredis ++ # # shortcut : rds ++ # # engine: redis_server ++ # # exact_match_only: false ++ # # host: '127.0.0.1' ++ # # port: 6379 ++ # # enable_http: true ++ # # password: '' ++ # # db: 0 ++ # ++ # # tmp suspended: bad certificate ++ # # - name: scanr structures ++ # # shortcut: scs ++ # # engine: scanr_structures ++ # # disabled: true ++ # ++ # - name: sepiasearch ++ # engine: sepiasearch ++ # shortcut: sep ++ # ++ # - name: soundcloud ++ # engine: soundcloud ++ # shortcut: sc ++ # ++ # - name: stackoverflow ++ # engine: stackexchange ++ # shortcut: st ++ # api_site: 'stackoverflow' ++ # categories: [it, q&a] ++ # ++ # - name: askubuntu ++ # engine: stackexchange ++ # shortcut: ubuntu ++ # api_site: 'askubuntu' ++ # categories: [it, q&a] ++ # ++ # - name: internetarchivescholar ++ # engine: internet_archive_scholar ++ # shortcut: ias ++ # timeout: 5.0 ++ # ++ # - name: superuser ++ # engine: stackexchange ++ # shortcut: su ++ # api_site: 'superuser' ++ # categories: [it, q&a] ++ # ++ # - name: searchcode code ++ # engine: searchcode_code ++ # shortcut: scc ++ # disabled: true ++ # ++ # - name: framalibre ++ # engine: framalibre ++ # shortcut: frl ++ # disabled: true ++ # ++ # # - name: searx ++ # # engine: searx_engine ++ # # shortcut: se ++ # # instance_urls : ++ # # - http://127.0.0.1:8888/ ++ # # - ... ++ # # disabled: true ++ # ++ # - name: semantic scholar ++ # engine: semantic_scholar ++ # disabled: true ++ # shortcut: se ++ # ++ # # Spotify needs API credentials ++ # # - name: spotify ++ # # engine: spotify ++ # # shortcut: stf ++ # # api_client_id: ******* ++ # # api_client_secret: ******* ++ # ++ # # - name: solr ++ # # engine: solr ++ # # shortcut: slr ++ # # base_url: http://localhost:8983 ++ # # collection: collection_name ++ # # sort: '' # sorting: asc or desc ++ # # field_list: '' # comma separated list of field names to display on the UI ++ # # default_fields: '' # default field to query ++ # # query_fields: '' # query fields ++ # # enable_http: true ++ # ++ # # - name: springer nature ++ # # engine: springer ++ # # # get your API key from: https://dev.springernature.com/signup ++ # # # working API key, for test & debug: "a69685087d07eca9f13db62f65b8f601" ++ # # api_key: 'unset' ++ # # shortcut: springer ++ # # timeout: 15.0 ++ # ++ # - name: startpage ++ # engine: startpage ++ # shortcut: sp ++ # timeout: 6.0 ++ # disabled: true ++ # additional_tests: ++ # rosebud: *test_rosebud ++ # ++ # - name: tokyotoshokan ++ # engine: tokyotoshokan ++ # shortcut: tt ++ # timeout: 6.0 ++ # disabled: true ++ # ++ # - name: solidtorrents ++ # engine: solidtorrents ++ # shortcut: solid ++ # timeout: 4.0 ++ # base_url: ++ # - https://solidtorrents.to ++ # - https://bitsearch.to ++ # ++ # # For this demo of the sqlite engine download: ++ # # https://liste.mediathekview.de/filmliste-v2.db.bz2 ++ # # and unpack into searx/data/filmliste-v2.db ++ # # Query to test: "!demo concert" ++ # # ++ # # - name: demo ++ # # engine: sqlite ++ # # shortcut: demo ++ # # categories: general ++ # # result_template: default.html ++ # # database: searx/data/filmliste-v2.db ++ # # query_str: >- ++ # # SELECT title || ' (' || time(duration, 'unixepoch') || ')' AS title, ++ # # COALESCE( NULLIF(url_video_hd,''), NULLIF(url_video_sd,''), url_video) AS url, ++ # # description AS content ++ # # FROM film ++ # # WHERE title LIKE :wildcard OR description LIKE :wildcard ++ # # ORDER BY duration DESC ++ # ++ # - name: tagesschau ++ # engine: tagesschau ++ # shortcut: ts ++ # disabled: true ++ # ++ # - name: tmdb ++ # engine: xpath ++ # paging: true ++ # search_url: https://www.themoviedb.org/search?page={pageno}&query={query} ++ # results_xpath: //div[contains(@class,"movie") or contains(@class,"tv")]//div[contains(@class,"card")] ++ # url_xpath: .//div[contains(@class,"poster")]/a/@href ++ # thumbnail_xpath: .//img/@src ++ # title_xpath: .//div[contains(@class,"title")]//h2 ++ # content_xpath: .//div[contains(@class,"overview")] ++ # shortcut: tm ++ # disabled: true ++ # ++ # # Requires Tor ++ # - name: torch ++ # engine: xpath ++ # paging: true ++ # search_url: ++ # http://xmh57jrknzkhv6y3ls3ubitzfqnkrwxhopf5aygthi7d6rplyvk3noyd.onion/cgi-bin/omega/omega?P={query}&DEFAULTOP=and ++ # results_xpath: //table//tr ++ # url_xpath: ./td[2]/a ++ # title_xpath: ./td[2]/b ++ # content_xpath: ./td[2]/small ++ # categories: onions ++ # enable_http: true ++ # shortcut: tch ++ # ++ # # torznab engine lets you query any torznab compatible indexer. Using this ++ # # engine in combination with Jackett opens the possibility to query a lot of ++ # # public and private indexers directly from SearXNG. More details at: ++ # # https://docs.searxng.org/dev/engines/online/torznab.html ++ # # ++ # # - name: Torznab EZTV ++ # # engine: torznab ++ # # shortcut: eztv ++ # # base_url: http://localhost:9117/api/v2.0/indexers/eztv/results/torznab ++ # # enable_http: true # if using localhost ++ # # api_key: xxxxxxxxxxxxxxx ++ # # show_magnet_links: true ++ # # show_torrent_files: false ++ # # # https://github.com/Jackett/Jackett/wiki/Jackett-Categories ++ # # torznab_categories: # optional ++ # # - 2000 ++ # # - 5000 ++ # ++ # - name: twitter ++ # shortcut: tw ++ # engine: twitter ++ # disabled: true ++ # ++ # # tmp suspended - too slow, too many errors ++ # # - name: urbandictionary ++ # # engine : xpath ++ # # search_url : https://www.urbandictionary.com/define.php?term={query} ++ # # url_xpath : //*[@class="word"]/@href ++ # # title_xpath : //*[@class="def-header"] ++ # # content_xpath: //*[@class="meaning"] ++ # # shortcut: ud ++ # ++ # - name: unsplash ++ # engine: unsplash ++ # shortcut: us ++ # ++ # - name: yahoo ++ # engine: yahoo ++ # shortcut: yh ++ # disabled: true ++ # ++ # - name: yahoo news ++ # engine: yahoo_news ++ # shortcut: yhn ++ # ++ # - name: youtube ++ # shortcut: yt ++ # # You can use the engine using the official stable API, but you need an API ++ # # key See: https://console.developers.google.com/project ++ # # ++ # # engine: youtube_api ++ # # api_key: 'apikey' # required! ++ # # ++ # # Or you can use the html non-stable engine, activated by default ++ # engine: youtube_noapi ++ # ++ # - name: dailymotion ++ # engine: dailymotion ++ # shortcut: dm ++ # ++ # - name: vimeo ++ # engine: vimeo ++ # shortcut: vm ++ # ++ # - name: wiby ++ # engine: json_engine ++ # paging: true ++ # search_url: https://wiby.me/json/?q={query}&p={pageno} ++ # url_query: URL ++ # title_query: Title ++ # content_query: Snippet ++ # categories: [general, web] ++ # shortcut: wib ++ # disabled: true ++ # about: ++ # website: https://wiby.me/ ++ # ++ # - name: alexandria ++ # engine: json_engine ++ # shortcut: alx ++ # categories: general ++ # paging: true ++ # search_url: https://api.alexandria.org/?a=1&q={query}&p={pageno} ++ # results_query: results ++ # title_query: title ++ # url_query: url ++ # content_query: snippet ++ # timeout: 1.5 ++ # disabled: true ++ # about: ++ # website: https://alexandria.org/ ++ # official_api_documentation: https://github.com/alexandria-org/alexandria-api/raw/master/README.md ++ # use_official_api: true ++ # require_api_key: false ++ # results: JSON ++ # ++ # - name: wikibooks ++ # engine: mediawiki ++ # weight: 0.5 ++ # shortcut: wb ++ # categories: [general, wikimedia] ++ # base_url: "https://{language}.wikibooks.org/" ++ # search_type: text ++ # disabled: true ++ # about: ++ # website: https://www.wikibooks.org/ ++ # wikidata_id: Q367 ++ # ++ # - name: wikinews ++ # engine: mediawiki ++ # shortcut: wn ++ # categories: [news, wikimedia] ++ # base_url: "https://{language}.wikinews.org/" ++ # search_type: text ++ # srsort: create_timestamp_desc ++ # about: ++ # website: https://www.wikinews.org/ ++ # wikidata_id: Q964 ++ # ++ # - name: wikiquote ++ # engine: mediawiki ++ # weight: 0.5 ++ # shortcut: wq ++ # categories: [general, wikimedia] ++ # base_url: "https://{language}.wikiquote.org/" ++ # search_type: text ++ # disabled: true ++ # additional_tests: ++ # rosebud: *test_rosebud ++ # about: ++ # website: https://www.wikiquote.org/ ++ # wikidata_id: Q369 ++ # ++ # - name: wikisource ++ # engine: mediawiki ++ # weight: 0.5 ++ # shortcut: ws ++ # categories: [general, wikimedia] ++ # base_url: "https://{language}.wikisource.org/" ++ # search_type: text ++ # disabled: true ++ # about: ++ # website: https://www.wikisource.org/ ++ # wikidata_id: Q263 ++ # ++ # - name: wikispecies ++ # engine: mediawiki ++ # shortcut: wsp ++ # categories: [general, science, wikimedia] ++ # base_url: "https://species.wikimedia.org/" ++ # search_type: text ++ # disabled: true ++ # about: ++ # website: https://species.wikimedia.org/ ++ # wikidata_id: Q13679 ++ # ++ # - name: wiktionary ++ # engine: mediawiki ++ # shortcut: wt ++ # categories: [dictionaries, wikimedia] ++ # base_url: "https://{language}.wiktionary.org/" ++ # search_type: text ++ # about: ++ # website: https://www.wiktionary.org/ ++ # wikidata_id: Q151 ++ # ++ # - name: wikiversity ++ # engine: mediawiki ++ # weight: 0.5 ++ # shortcut: wv ++ # categories: [general, wikimedia] ++ # base_url: "https://{language}.wikiversity.org/" ++ # search_type: text ++ # disabled: true ++ # about: ++ # website: https://www.wikiversity.org/ ++ # wikidata_id: Q370 ++ # ++ # - name: wikivoyage ++ # engine: mediawiki ++ # weight: 0.5 ++ # shortcut: wy ++ # categories: [general, wikimedia] ++ # base_url: "https://{language}.wikivoyage.org/" ++ # search_type: text ++ # disabled: true ++ # about: ++ # website: https://www.wikivoyage.org/ ++ # wikidata_id: Q373 ++ # ++ # - name: wikicommons.images ++ # engine: wikicommons ++ # shortcut: wc ++ # categories: images ++ # number_of_results: 10 ++ # ++ # - name: wolframalpha ++ # shortcut: wa ++ # # You can use the engine using the official stable API, but you need an API ++ # # key. See: https://products.wolframalpha.com/api/ ++ # # ++ # # engine: wolframalpha_api ++ # # api_key: '' ++ # # ++ # # Or you can use the html non-stable engine, activated by default ++ # engine: wolframalpha_noapi ++ # timeout: 6.0 ++ # categories: general ++ # disabled: true ++ # ++ # - name: dictzone ++ # engine: dictzone ++ # shortcut: dc ++ # ++ # - name: mymemory translated ++ # engine: translated ++ # shortcut: tl ++ # timeout: 5.0 ++ # # You can use without an API key, but you are limited to 1000 words/day ++ # # See: https://mymemory.translated.net/doc/usagelimits.php ++ # # api_key: '' ++ # ++ # # Required dependency: mysql-connector-python ++ # # - name: mysql ++ # # engine: mysql_server ++ # # database: mydatabase ++ # # username: user ++ # # password: pass ++ # # limit: 10 ++ # # query_str: 'SELECT * from mytable WHERE fieldname=%(query)s' ++ # # shortcut: mysql ++ # ++ # - name: 1337x ++ # engine: 1337x ++ # shortcut: 1337x ++ # disabled: true ++ # ++ # - name: duden ++ # engine: duden ++ # shortcut: du ++ # disabled: true ++ # ++ # - name: seznam ++ # shortcut: szn ++ # engine: seznam ++ # disabled: true ++ # ++ # # - name: deepl ++ # # engine: deepl ++ # # shortcut: dpl ++ # # # You can use the engine using the official stable API, but you need an API key ++ # # # See: https://www.deepl.com/pro-api?cta=header-pro-api ++ # # api_key: '' # required! ++ # # timeout: 5.0 ++ # # disabled: true ++ # ++ # - name: mojeek ++ # shortcut: mjk ++ # engine: xpath ++ # paging: true ++ # categories: [general, web] ++ # search_url: https://www.mojeek.com/search?q={query}&s={pageno}&lang={lang}&lb={lang} ++ # results_xpath: //ul[@class="results-standard"]/li/a[@class="ob"] ++ # url_xpath: ./@href ++ # title_xpath: ../h2/a ++ # content_xpath: ..//p[@class="s"] ++ # suggestion_xpath: //div[@class="top-info"]/p[@class="top-info spell"]/em/a ++ # first_page_num: 0 ++ # page_size: 10 ++ # disabled: true ++ # about: ++ # website: https://www.mojeek.com/ ++ # wikidata_id: Q60747299 ++ # official_api_documentation: https://www.mojeek.com/services/api.html/ ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: moviepilot ++ # engine: moviepilot ++ # shortcut: mp ++ # disabled: true ++ # ++ # - name: naver ++ # shortcut: nvr ++ # categories: [general, web] ++ # engine: xpath ++ # paging: true ++ # search_url: https://search.naver.com/search.naver?where=webkr&sm=osp_hty&ie=UTF-8&query={query}&start={pageno} ++ # url_xpath: //a[@class="link_tit"]/@href ++ # title_xpath: //a[@class="link_tit"] ++ # content_xpath: //a[@class="total_dsc"]/div ++ # first_page_num: 1 ++ # page_size: 10 ++ # disabled: true ++ # about: ++ # website: https://www.naver.com/ ++ # wikidata_id: Q485639 ++ # official_api_documentation: https://developers.naver.com/docs/nmt/examples/ ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # language: ko ++ # ++ # - name: rubygems ++ # shortcut: rbg ++ # engine: xpath ++ # paging: true ++ # search_url: https://rubygems.org/search?page={pageno}&query={query} ++ # results_xpath: /html/body/main/div/a[@class="gems__gem"] ++ # url_xpath: ./@href ++ # title_xpath: ./span/h2 ++ # content_xpath: ./span/p ++ # suggestion_xpath: /html/body/main/div/div[@class="search__suggestions"]/p/a ++ # first_page_num: 1 ++ # categories: [it, packages] ++ # disabled: true ++ # about: ++ # website: https://rubygems.org/ ++ # wikidata_id: Q1853420 ++ # official_api_documentation: https://guides.rubygems.org/rubygems-org-api/ ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: peertube ++ # engine: peertube ++ # shortcut: ptb ++ # paging: true ++ # # alternatives see: https://instances.joinpeertube.org/instances ++ # # base_url: https://tube.4aem.com ++ # categories: videos ++ # disabled: true ++ # timeout: 6.0 ++ # ++ # - name: mediathekviewweb ++ # engine: mediathekviewweb ++ # shortcut: mvw ++ # disabled: true ++ # ++ # # - name: yacy ++ # # engine: yacy ++ # # shortcut: ya ++ # # base_url: http://localhost:8090 ++ # # # required if you aren't using HTTPS for your local yacy instance' ++ # # enable_http: true ++ # # timeout: 3.0 ++ # # # Yacy search mode. 'global' or 'local'. ++ # # search_mode: 'global' ++ # ++ # - name: rumble ++ # engine: rumble ++ # shortcut: ru ++ # base_url: https://rumble.com/ ++ # paging: true ++ # categories: videos ++ # disabled: true ++ # ++ # - name: wordnik ++ # engine: wordnik ++ # shortcut: def ++ # base_url: https://www.wordnik.com/ ++ # categories: [dictionaries] ++ # timeout: 5.0 ++ # ++ # - name: woxikon.de synonyme ++ # engine: xpath ++ # shortcut: woxi ++ # categories: [dictionaries] ++ # timeout: 5.0 ++ # disabled: true ++ # search_url: https://synonyme.woxikon.de/synonyme/{query}.php ++ # url_xpath: //div[@class="upper-synonyms"]/a/@href ++ # content_xpath: //div[@class="synonyms-list-group"] ++ # title_xpath: //div[@class="upper-synonyms"]/a ++ # no_result_for_http_status: [404] ++ # about: ++ # website: https://www.woxikon.de/ ++ # wikidata_id: # No Wikidata ID ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # language: de ++ # ++ # - name: seekr news ++ # engine: seekr ++ # shortcut: senews ++ # categories: news ++ # seekr_category: news ++ # disabled: true ++ # ++ # - name: seekr images ++ # engine: seekr ++ # network: seekr news ++ # shortcut: seimg ++ # categories: images ++ # seekr_category: images ++ # disabled: true ++ # ++ # - name: seekr videos ++ # engine: seekr ++ # network: seekr news ++ # shortcut: sevid ++ # categories: videos ++ # seekr_category: videos ++ # disabled: true ++ # ++ # - name: sjp.pwn ++ # engine: sjp ++ # shortcut: sjp ++ # base_url: https://sjp.pwn.pl/ ++ # timeout: 5.0 ++ # disabled: true ++ # ++ # - name: svgrepo ++ # engine: svgrepo ++ # shortcut: svg ++ # timeout: 10.0 ++ # disabled: true ++ # ++ # - name: wallhaven ++ # engine: wallhaven ++ # # api_key: abcdefghijklmnopqrstuvwxyz ++ # shortcut: wh ++ # ++ # # wikimini: online encyclopedia for children ++ # # The fulltext and title parameter is necessary for Wikimini because ++ # # sometimes it will not show the results and redirect instead ++ # - name: wikimini ++ # engine: xpath ++ # shortcut: wkmn ++ # search_url: https://fr.wikimini.org/w/index.php?search={query}&title=Sp%C3%A9cial%3ASearch&fulltext=Search ++ # url_xpath: //li/div[@class="mw-search-result-heading"]/a/@href ++ # title_xpath: //li//div[@class="mw-search-result-heading"]/a ++ # content_xpath: //li/div[@class="searchresult"] ++ # categories: general ++ # disabled: true ++ # about: ++ # website: https://wikimini.org/ ++ # wikidata_id: Q3568032 ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # language: fr ++ # ++ # - name: wttr.in ++ # engine: wttr ++ # shortcut: wttr ++ # timeout: 9.0 ++ # ++ # - name: yummly ++ # engine: yummly ++ # shortcut: yum ++ # disabled: true ++ # ++ # - name: brave ++ # engine: brave ++ # shortcut: br ++ # time_range_support: true ++ # paging: true ++ # categories: [general, web] ++ # brave_category: search ++ # # brave_spellcheck: true ++ # ++ # - name: brave.images ++ # engine: brave ++ # network: brave ++ # shortcut: brimg ++ # categories: [images, web] ++ # brave_category: images ++ # ++ # - name: brave.videos ++ # engine: brave ++ # network: brave ++ # shortcut: brvid ++ # categories: [videos, web] ++ # brave_category: videos ++ # ++ # - name: brave.news ++ # engine: brave ++ # network: brave ++ # shortcut: brnews ++ # categories: news ++ # brave_category: news ++ # ++ # - name: lib.rs ++ # shortcut: lrs ++ # engine: xpath ++ # search_url: https://lib.rs/search?q={query} ++ # results_xpath: /html/body/main/div/ol/li/a ++ # url_xpath: ./@href ++ # title_xpath: ./div[@class="h"]/h4 ++ # content_xpath: ./div[@class="h"]/p ++ # categories: [it, packages] ++ # disabled: true ++ # about: ++ # website: https://lib.rs ++ # wikidata_id: Q113486010 ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: sourcehut ++ # shortcut: srht ++ # engine: xpath ++ # paging: true ++ # search_url: https://sr.ht/projects?page={pageno}&search={query} ++ # results_xpath: (//div[@class="event-list"])[1]/div[@class="event"] ++ # url_xpath: ./h4/a[2]/@href ++ # title_xpath: ./h4/a[2] ++ # content_xpath: ./p ++ # first_page_num: 1 ++ # categories: [it, repos] ++ # disabled: true ++ # about: ++ # website: https://sr.ht ++ # wikidata_id: Q78514485 ++ # official_api_documentation: https://man.sr.ht/ ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: goo ++ # shortcut: goo ++ # engine: xpath ++ # paging: true ++ # search_url: https://search.goo.ne.jp/web.jsp?MT={query}&FR={pageno}0 ++ # url_xpath: //div[@class="result"]/p[@class='title fsL1']/a/@href ++ # title_xpath: //div[@class="result"]/p[@class='title fsL1']/a ++ # content_xpath: //p[contains(@class,'url fsM')]/following-sibling::p ++ # first_page_num: 0 ++ # categories: [general, web] ++ # disabled: true ++ # timeout: 4.0 ++ # about: ++ # website: https://search.goo.ne.jp ++ # wikidata_id: Q249044 ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # language: ja ++ # ++ # - name: bt4g ++ # engine: bt4g ++ # shortcut: bt4g ++ # ++ # - name: pkg.go.dev ++ # engine: xpath ++ # shortcut: pgo ++ # search_url: https://pkg.go.dev/search?limit=100&m=package&q={query} ++ # results_xpath: /html/body/main/div[contains(@class,"SearchResults")]/div[not(@class)]/div[@class="SearchSnippet"] ++ # url_xpath: ./div[@class="SearchSnippet-headerContainer"]/h2/a/@href ++ # title_xpath: ./div[@class="SearchSnippet-headerContainer"]/h2/a ++ # content_xpath: ./p[@class="SearchSnippet-synopsis"] ++ # categories: [packages, it] ++ # timeout: 3.0 ++ # disabled: true ++ # about: ++ # website: https://pkg.go.dev/ ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ ## Doku engine lets you access to any Doku wiki instance: ++ ## A public one or a privete/corporate one. ++ ## - name: ubuntuwiki ++ ## engine: doku ++ ## shortcut: uw ++ ## base_url: 'https://doc.ubuntu-fr.org' ++ # ++ ## Be careful when enabling this engine if you are ++ ## running a public instance. Do not expose any sensitive ++ ## information. You can restrict access by configuring a list ++ ## of access tokens under tokens. ++ ## - name: git grep ++ ## engine: command ++ ## command: ['git', 'grep', '{{QUERY}}'] ++ ## shortcut: gg ++ ## tokens: [] ++ ## disabled: true ++ ## delimiter: ++ ## chars: ':' ++ ## keys: ['filepath', 'code'] ++ # ++ ## Be careful when enabling this engine if you are ++ ## running a public instance. Do not expose any sensitive ++ ## information. You can restrict access by configuring a list ++ ## of access tokens under tokens. ++ ## - name: locate ++ ## engine: command ++ ## command: ['locate', '{{QUERY}}'] ++ ## shortcut: loc ++ ## tokens: [] ++ ## disabled: true ++ ## delimiter: ++ ## chars: ' ' ++ ## keys: ['line'] ++ # ++ ## Be careful when enabling this engine if you are ++ ## running a public instance. Do not expose any sensitive ++ ## information. You can restrict access by configuring a list ++ ## of access tokens under tokens. ++ ## - name: find ++ ## engine: command ++ ## command: ['find', '.', '-name', '{{QUERY}}'] ++ ## query_type: path ++ ## shortcut: fnd ++ ## tokens: [] ++ ## disabled: true ++ ## delimiter: ++ ## chars: ' ' ++ ## keys: ['line'] ++ # ++ ## Be careful when enabling this engine if you are ++ ## running a public instance. Do not expose any sensitive ++ ## information. You can restrict access by configuring a list ++ ## of access tokens under tokens. ++ ## - name: pattern search in files ++ ## engine: command ++ ## command: ['fgrep', '{{QUERY}}'] ++ ## shortcut: fgr ++ ## tokens: [] ++ ## disabled: true ++ ## delimiter: ++ ## chars: ' ' ++ ## keys: ['line'] ++ # ++ ## Be careful when enabling this engine if you are ++ ## running a public instance. Do not expose any sensitive ++ ## information. You can restrict access by configuring a list ++ ## of access tokens under tokens. ++ ## - name: regex search in files ++ ## engine: command ++ ## command: ['grep', '{{QUERY}}'] ++ ## shortcut: gr ++ ## tokens: [] ++ ## disabled: true ++ ## delimiter: ++ ## chars: ' ' ++ ## keys: ['line'] + + doi_resolvers: + oadoi.org: 'https://oadoi.org/' diff --git a/src/portal/patches/snscrape_1036.diff b/src/portal/patches/snscrape_1036.diff new file mode 100644 index 0000000..0512f56 --- /dev/null +++ b/src/portal/patches/snscrape_1036.diff @@ -0,0 +1,28 @@ +diff --git a/.editorconfig b/.editorconfig +new file mode 100644 +index 0000000..e7c6935 +--- /dev/null ++++ b/.editorconfig +@@ -0,0 +1,3 @@ ++[*.py] ++indent_style = tab ++tab_width = 8 +diff --git a/snscrape/modules/__init__.py b/snscrape/modules/__init__.py +index e15a7b9..8963b93 100644 +--- a/snscrape/modules/__init__.py ++++ b/snscrape/modules/__init__.py +@@ -1,4 +1,5 @@ + import pkgutil ++import importlib + + + __all__ = [] +@@ -10,7 +11,7 @@ def _import_modules(): + assert not isPkg + moduleNameWithoutPrefix = moduleName[prefixLen:] + __all__.append(moduleNameWithoutPrefix) +- module = importer.find_module(moduleName).load_module(moduleName) ++ module = importlib.import_module(moduleName) + globals()[moduleNameWithoutPrefix] = module + + diff --git a/src/portal/patches/snscrape_auth.diff b/src/portal/patches/snscrape_auth.diff new file mode 100644 index 0000000..67962a9 --- /dev/null +++ b/src/portal/patches/snscrape_auth.diff @@ -0,0 +1,216 @@ +diff --git a/snscrape/modules/twitter.py b/snscrape/modules/twitter.py +index d1719a8..43d9076 100644 +--- a/snscrape/modules/twitter.py ++++ b/snscrape/modules/twitter.py +@@ -92,6 +92,7 @@ import typing + import urllib.parse + import urllib3.util.ssl_ + import warnings ++import hashlib + + + _logger = logging.getLogger(__name__) +@@ -99,10 +100,12 @@ _API_AUTHORIZATION_HEADER = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOu + _globalGuestTokenManager = None + _GUEST_TOKEN_VALIDITY = 10800 + _CIPHERS_CHROME = 'TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA:AES256-SHA' ++_AGE_RESTRICTED_MSG = 'Age-restricted adult content. This content might not be appropriate for people under 18 years old. To view this media, you’ll need to log in to Twitter. Learn more' + + + @dataclasses.dataclass + class Tweet(snscrape.base.Item): ++ rawResponses: typing.Dict[str, str] + url: str + date: datetime.datetime + rawContent: str +@@ -793,17 +796,23 @@ class _TwitterAPIType(enum.Enum): + V2 = 0 # Introduced with the redesign + GRAPHQL = 1 + ++class _TwitterAPIAuthType(enum.Enum): ++ GUEST_TOKEN = 0 ++ OAUTH2 = 1 + + class _TwitterAPIScraper(snscrape.base.Scraper): +- def __init__(self, baseUrl, *, guestTokenManager = None, maxEmptyPages = 0, **kwargs): ++ def __init__(self, baseUrl, *, guestTokenManager = None, maxEmptyPages = 0, cookies = None, **kwargs): + super().__init__(**kwargs) + self._baseUrl = baseUrl +- if guestTokenManager is None: ++ self._syndicationUrl = 'https://cdn.syndication.twimg.com/tweet?id=' ++ self._authType = _TwitterAPIAuthType.OAUTH2 ++ if self._authType == _TwitterAPIAuthType.GUEST_TOKEN and guestTokenManager is None: + global _globalGuestTokenManager + if _globalGuestTokenManager is None: + _globalGuestTokenManager = GuestTokenManager() + guestTokenManager = _globalGuestTokenManager + self._guestTokenManager = guestTokenManager ++ self._cookies = cookies + self._maxEmptyPages = maxEmptyPages + self._apiHeaders = { + 'Authorization': _API_AUTHORIZATION_HEADER, +@@ -813,6 +822,11 @@ class _TwitterAPIScraper(snscrape.base.Scraper): + adapter = _TwitterTLSAdapter() + self._session.mount('https://twitter.com', adapter) + self._session.mount('https://api.twitter.com', adapter) ++ if self._authType == _TwitterAPIAuthType.OAUTH2: ++ self._ensure_oauth() ++ self._lastResponse = None ++ self._lastResponseHash = None ++ self._lastSentResponseHash = None + + def _check_guest_token_response(self, r): + if r.status_code != 200: +@@ -820,6 +834,8 @@ class _TwitterAPIScraper(snscrape.base.Scraper): + return True, None + + def _ensure_guest_token(self, url = None): ++ if not self._guestTokenManager: ++ return + if self._guestTokenManager.token is None: + _logger.info('Retrieving guest token') + r = self._get(self._baseUrl if url is None else url, responseOkCallback = self._check_guest_token_response) +@@ -843,18 +859,42 @@ class _TwitterAPIScraper(snscrape.base.Scraper): + self._apiHeaders['x-guest-token'] = self._guestTokenManager.token + + def _unset_guest_token(self, blockUntil): ++ if not self._guestTokenManager: ++ return + self._guestTokenManager.reset(blockUntil = blockUntil) + del self._session.cookies['gt'] + del self._apiHeaders['x-guest-token'] + ++ def _ensure_oauth(self): ++ if not self._cookies: ++ return ++ for k, v in self._cookies.items(): ++ self._session.cookies.set(k, v, domain = '.twitter.com', path = '/', secure = True) ++ self._apiHeaders['x-csrf-token'] = self._session.cookies.get('ct0') ++ self._apiHeaders['x-twitter-active-user'] = 'yes' ++ self._apiHeaders['x-twitter-auth-type'] = 'OAuth2Session' ++ self._apiHeaders['x-twitter-client-language'] = 'en' ++ ++ def _ensure_auth_updated(self, url = None): ++ if self._authType == _TwitterAPIAuthType.OAUTH2: ++ self._ensure_oauth() ++ else: ++ self._ensure_guest_token(url) ++ + def _check_api_response(self, r, apiType, instructionsPath): + if r.status_code in (403, 404, 429): ++ currentTime = time.time() + if r.status_code == 429 and r.headers.get('x-rate-limit-remaining', '') == '0' and 'x-rate-limit-reset' in r.headers: +- blockUntil = min(int(r.headers['x-rate-limit-reset']), int(time.time()) + 900) ++ blockUntil = min(int(r.headers['x-rate-limit-reset']), int(currentTime) + 900) ++ else: ++ blockUntil = int(currentTime) + 300 ++ if self._authType == _TwitterAPIAuthType.GUEST_TOKEN: ++ self._unset_guest_token(blockUntil) ++ self._ensure_guest_token() + else: +- blockUntil = int(time.time()) + 300 +- self._unset_guest_token(blockUntil) +- self._ensure_guest_token() ++ blockFor = (blockUntil - currentTime) * (int(r.headers.get('x-rate-limit-remaining', '0')) + 1) ++ _logger.warn(f'Got too many requests, blocking for {blockFor} seconds ({r.headers['x-rate-limit-reset']}) ({r.headers.get('x-rate-limit-remaining', '0')})') ++ time.sleep(blockFor) + return False, f'blocked ({r.status_code})' + if r.headers.get('content-type', '').replace(' ', '') != 'application/json;charset=utf-8': + return False, 'content type is not JSON' +@@ -868,6 +908,11 @@ class _TwitterAPIScraper(snscrape.base.Scraper): + r._snscrapeObj = obj + if apiType is _TwitterAPIType.GRAPHQL and 'errors' in obj: + msg = 'Twitter responded with an error: ' + ', '.join(f'{e["name"]}: {e["message"]}' for e in obj['errors']) ++ blockFor = 30.0 ++ _logger.warn(f'Got errors waiting for {blockFor} seconds') ++ time.sleep(blockFor) ++ return False, msg ++ ''' + instructions = obj + for k in instructionsPath: + instructions = instructions.get(k, {}) +@@ -877,13 +922,23 @@ class _TwitterAPIScraper(snscrape.base.Scraper): + return True, None + else: + return False, msg ++ ''' + return True, None + + def _get_api_data(self, endpoint, apiType, params, instructionsPath = None): +- self._ensure_guest_token() ++ self._ensure_auth_updated() + if apiType is _TwitterAPIType.GRAPHQL: + params = urllib.parse.urlencode({k: json.dumps(v, separators = (',', ':')) for k, v in params.items()}, quote_via = urllib.parse.quote) + r = self._get(endpoint, params = params, headers = self._apiHeaders, responseOkCallback = functools.partial(self._check_api_response, apiType = apiType, instructionsPath = instructionsPath)) ++ if self._authType == _TwitterAPIAuthType.OAUTH2: ++ csrf = r.cookies.get('ct0') ++ if csrf: ++ _logger.warn('Response contained updated csrf token') ++ self._apiHeaders['x-csrf-token'] = csrf ++ self._lastResponse = r._snscrapeObj ++ hasher = hashlib.new('sha256') ++ hasher.update(json.dumps(self._lastResponse).encode('utf-8')) ++ self._lastResponseHash = hasher.hexdigest() + return r._snscrapeObj + + def _iter_api_data(self, endpoint, apiType, params, paginationParams = None, cursor = None, direction = _ScrollDirection.BOTTOM, instructionsPath = None): +@@ -988,8 +1043,50 @@ class _TwitterAPIScraper(snscrape.base.Scraper): + def _get_tweet_id(self, tweet): + return tweet['id'] if 'id' in tweet else int(tweet['id_str']) + ++ def _entry_is_age_restricted(self, entry): ++ if entry['entryId'].startswith('tombstone-') and entry['content']['entryType'] == 'TimelineTimelineItem': ++ if entry['content']['itemContent']['tombstoneInfo']['richText']['text'] == _AGE_RESTRICTED_MSG: ++ return True ++ return False ++ ++ # Can we parse out more data here? ++ def _syndication_make_tweet(self, tweet): ++ tweetId = self._get_tweet_id(tweet) ++ kwargs = {} ++ kwargs['rawResponses'] = { ++ 'detached_api' : json.dumps(tweet) ++ } ++ kwargs['id'] = tweetId ++ kwargs['rawContent'] = tweet['text'] ++ kwargs['renderedContent'] = '' ++ username = tweet['user']['screen_name'] ++ kwargs['user'] = User(id = int(tweet['user']['id_str']), username = username, displayname = tweet['user']['name']) ++ kwargs['date'] = datetime.datetime.strptime(tweet['created_at'], '%Y-%m-%dT%H:%M:%S.%fZ') ++ kwargs['replyCount'] = tweet['conversation_count'] ++ kwargs['retweetCount'] = 0 ++ kwargs['quoteCount'] = 0 ++ kwargs['conversationId'] = 0 ++ kwargs['lang'] = tweet['lang'] ++ kwargs['source'] = '' ++ kwargs['url'] = f'https://twitter.com/{username}/status/{tweetId}' ++ kwargs['likeCount'] = tweet['favorite_count'] ++ media = [] ++ for p in tweet['photos']: ++ media.append(Photo(previewUrl = '', fullUrl = p['url'])) ++ kwargs['media'] = media ++ return Tweet(**kwargs) ++ ++ + def _make_tweet(self, tweet, user, retweetedTweet = None, quotedTweet = None, card = None, noteTweet = None, **kwargs): + tweetId = self._get_tweet_id(tweet) ++ kwargs['rawResponses'] = { ++ 'hash': self._lastResponseHash, ++ 'tweet': json.dumps(tweet), ++ 'user': user.json() ++ } ++ if not self._lastSentResponseHash or self._lastResponseHash != self._lastSentResponseHash: ++ kwargs['rawResponses']['detached_api'] = json.dumps(self._lastResponse) ++ self._lastSentResponseHash = self._lastResponseHash + kwargs['id'] = tweetId + if noteTweet and 'text' in noteTweet: + kwargs['rawContent'] = noteTweet['text'] +@@ -1789,7 +1886,7 @@ class TwitterUserScraper(TwitterSearchScraper): + self._baseUrl = f'https://twitter.com/{self._user}' if not self._isUserId else f'https://twitter.com/i/user/{self._user}' + + def _get_entity(self): +- self._ensure_guest_token() ++ self._ensure_auth_updated() + if not self._isUserId: + fieldName = 'screen_name' + endpoint = 'https://twitter.com/i/api/graphql/pVrmNaXcxPjisIvKtLDMEA/UserByScreenName' diff --git a/src/portal/py/__init__.py b/src/portal/py/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/src/portal/py/__init__.py diff --git a/src/portal/py/base.py b/src/portal/py/base.py new file mode 100644 index 0000000..c0a0b4f --- /dev/null +++ b/src/portal/py/base.py @@ -0,0 +1,116 @@ +import log +import time +import httpx +import threading +from enum import Enum +from collections import OrderedDict +from typing import Optional, Self, Any +from post import Post, User + +class Method(Enum): + HEAD = "HEAD" + GET = "GET" + POST = "POST" + +ParsedJson = Any + +class Search(): + def __init__(self): + self.pages: dict[int, list[str]] = {} + self.completed: bool = False + self.errored: bool = False + self.iter_page: int = 0 + self.iter_index: int = 0 + + def __iter__(self) -> Self: + self.iter_page = 0 + self.iter_index = 0 + return self + + def __next__(self) -> str: + page = self.get_page(self.iter_page) + if not page: + raise StopIteration + unique_id = page[self.iter_index] + self.iter_index += 1 + if self.iter_index >= len(page): + self.iter_page += 1 + self.iter_index = 0 + return unique_id + + def request_page(self, num: int) -> bool: + return False + + def get_page(self, num: int) -> Optional[list[str]]: + if self.errored: + return None + if num not in self.pages: + if self.completed: + return None + if not self.request_page(num): + if not self.completed: + self.errored = True + return None + return self.pages[num] + +class Module(): + def __init__(self): + self.headers: dict[str, str] = {} + self.cookies: dict[str, str] = {} + self.unique_id_map: OrderedDict = OrderedDict() + self.map_mutex: threading.Lock = threading.Lock() + self.session: httpx.Client = httpx.Client(follow_redirects=True, timeout=20.0, http2=True) + + def init(self) -> bool: + return True + + def do_request(self, method: Method, url: str, params: Optional[dict[str, str]] = None, retries: int = 2) -> Optional[httpx.Response]: + log.debug(f'HTTP {method.value} request {url} {params}.') + response: Optional[httpx.Response] = None + for _ in range(0, retries + 1): + try: + response = self.session.request(method.value, url, headers=self.headers, cookies=self.cookies, params=params) + except Exception as e: + log.info(f'Connection exception ({e}), retrying in 20 seconds...') + time.sleep(20) + continue + if not response: + continue + if response.status_code == 429: + log.info('Got too many requests, waiting for 3 minutes...') + time.sleep(60 * 3) + response = None + continue + if response.status_code == 404 or response.status_code == 403: # Shortcut 404, 403 + return None + if int(response.status_code / 100) != 2: # Non-200 response + log.error(f'Non-200 status code ({response.status_code}), retrying in 5 seconds...') + time.sleep(5) + response = None + continue + try: + response.read() + except Exception as e: + log.info(f'Read exception ({e}), retrying in 20 seconds...') + time.sleep(20) + response = None + continue + else: + break + return response + + def add_to_map(self, unique_id: str, item: Post | User): + with self.map_mutex: + self.unique_id_map[unique_id] = item + if len(self.unique_id_map) > 10240: + self.unique_id_map.popitem(last=False) + + def search(self, query: str, *extra_args: Any) -> Optional[Search]: + return None + + def get_item(self, unique_id: str) -> Optional[Post | User]: + with self.map_mutex: + return self.unique_id_map.get(unique_id, None) + + def get_download(self, unique_id: str, key: str) -> Optional[dict[str, Any]]: + return None diff --git a/src/portal/py/config.def.py b/src/portal/py/config.def.py new file mode 100644 index 0000000..c4e11c6 --- /dev/null +++ b/src/portal/py/config.def.py @@ -0,0 +1,20 @@ +from pathlib import Path + +BASE_DIR = '' + +TWITTER_ACCESS_TOKEN = '' +TWITTER_ACCESS_TOKEN_SECRET = '' +TWITTER_CONSUMER_TOKEN = '' +TWITTER_CONSUMER_TOKEN_SECRET = '' +TWITTER_COOKIES_PATH = '{}/cookies_twitter.txt'.format(BASE_DIR) +TWITTER_TIMEOUT = 5 # seconds + +FANBOX_SESSID = '' + +PIXIV_REFRESH_TOKEN = '' + +INSTAGRAM_USERNAME = '' +INSTAGRAM_PASSWORD = '' +INSTAGRAM_SESSION_ID = '' +INSTAGRAM_SETTINGS_PATH = Path('{}/ig_settings.json'.format(BASE_DIR)) +INSTAGRAM_USER_AGENT = '' diff --git a/src/portal/py/log.py b/src/portal/py/log.py new file mode 100644 index 0000000..b693a4c --- /dev/null +++ b/src/portal/py/log.py @@ -0,0 +1,34 @@ +class DefaultLogger(): + def debug(self, msg): + print(msg) + + def info(self, msg): + print(msg) + + def warn(self, msg): + print(msg) + + def error(self, msg): + print(msg) + +LOGGER = DefaultLogger() + +def set_logger(logger): + global LOGGER + LOGGER = logger + +def debug(msg): + global LOGGER + LOGGER.debug(msg) + +def info(msg): + global LOGGER + LOGGER.info(msg) + +def warn(msg): + global LOGGER + LOGGER.warn(msg) + +def error(msg): + global LOGGER + LOGGER.error(msg) diff --git a/src/portal/py/modules/__init__.py b/src/portal/py/modules/__init__.py new file mode 100644 index 0000000..480660c --- /dev/null +++ b/src/portal/py/modules/__init__.py @@ -0,0 +1,22 @@ +import config + +from modules.youtube import YoutubeModule +from modules.twitter import TwitterScrapeModule +from modules.pixiv_app import PixivAppModule +from modules.pixiv_web import PixivWebModule +from modules.fanbox import FanboxModule +from modules.instagram import InstagramModule + +ALL_MODULES = { + 'youtube': (YoutubeModule(), []), +# 'twitter': (TwitterModule( +# config.TWITTER_ACCESS_TOKEN, config.TWITTER_ACCESS_TOKEN_SECRET, +# config.TWITTER_CONSUMER_TOKEN, config.TWITTER_CONSUMER_TOKEN_SECRET), []), +# 'twitter': (TwitterScrapeModule(config.TWITTER_COOKIES_PATH), []), +# 'pixiv_web': (PixivWebModule(config.PIXIV_SESSID), []), +# 'pixiv_app': (PixivAppModule(config.PIXIV_REFRESH_TOKEN), []), +# 'fanbox': (FanboxModule(config.FANBOX_SESSID), []), +# 'instagram': (InstagramModule( +# config.INSTAGRAM_USER_AGENT, config.INSTAGRAM_SETTINGS_PATH, +# config.INSTAGRAM_SESSION_ID), []) +} diff --git a/src/portal/py/modules/common.py b/src/portal/py/modules/common.py new file mode 100644 index 0000000..62c3d5d --- /dev/null +++ b/src/portal/py/modules/common.py @@ -0,0 +1,9 @@ +from datetime import datetime, timezone + +def parse_pixiv_date(date_str: str) -> datetime: + rindex = date_str.rfind(':') + date_str = date_str[:rindex] + date_str[rindex + 1:] + return datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S%z") + +def get_current_utc_time() -> datetime: + return datetime.now(timezone.utc) diff --git a/src/portal/py/modules/fanbox.py b/src/portal/py/modules/fanbox.py new file mode 100644 index 0000000..01e66c3 --- /dev/null +++ b/src/portal/py/modules/fanbox.py @@ -0,0 +1,192 @@ +import json +import httpx +from typing import Optional, Any +from json.decoder import JSONDecodeError +from base import Search, Module, Method, ParsedJson +from post import MediaUrl, PostType, DateType, Post, User, Image, File, Tag +from query_parser import QueryParser +from modules.common import parse_pixiv_date, get_current_utc_time + +BASE_URL = "https://api.fanbox.cc" + +class FanboxBase(Search): + def __init__(self, userdata: Any): + super().__init__() + self.module: FanboxModule = userdata + + def check_api_response(self, response: Optional[httpx.Response]) -> Optional[ParsedJson]: + if not response: + return None + try: + obj = response.json() + except JSONDecodeError: + return None + return obj['body'] + + def create_media_url(self, url: str) -> MediaUrl: + return MediaUrl(url, url.split('.')[-1]) + + def create_post(self, data: ParsedJson) -> Optional[Post]: + kwargs = {} + post_id = data['id'] + unique_id = f'fanbox:p:{post_id}' + kwargs['unique_id'] = unique_id + kwargs['raw_responses'] = {} + kwargs['raw_responses']['api'] = json.dumps(data) + publish_date = parse_pixiv_date(data['publishedDatetime']).timestamp() + kwargs['dates'] = { + DateType.CREATED: publish_date, + DateType.RETRIEVED: get_current_utc_time().timestamp() + } + update_date = parse_pixiv_date(data['updatedDatetime']).timestamp() + if update_date != publish_date: + kwargs['dates'][DateType.EDITED] = update_date + user = User(unique_id=f'fanbox:u:{data['user']['userId']}', username=data['user']['name']) + user.profile_picture_url = self.create_media_url(data['user']['iconUrl']) + kwargs['author'] = user + kwargs['title'] = data['title'] + kwargs['likes'] = data['likeCount'] + kwargs['comments'] = data['commentCount'] + kwargs['tags'] = [Tag(name=tag) for tag in data['tags']] + if data['isRestricted']: + kwargs['type'] = PostType.PREVIEW + else: + kwargs['type'] = PostType.POST + text = '' + media = {} + if data['type'] == 'image': + text = data['body']['text'] + for i, value in enumerate(data['body']['images']): + media[str(i)] = Image(url=MediaUrl(value['originalUrl'], value['extension']), thumbnail_url=self.create_media_url(value['thumbnailUrl'])) + elif data['type'] == 'file': + text = data['body']['text'] + for i, value in enumerate(data['body']['files']): + media[str(i)] = File(url=MediaUrl(value['url'], value['extension']), name=value['name']) + elif data['type'] == 'article': + for block in data['body']['blocks']: + if block['type'] == 'p': + text += block['text'] + '\n' + elif block['type'] == 'image': + text += f'image::{block['imageId']}[]' + '\n' + elif block['type'] == 'url_embed': + text += f'embed::{block['urlEmbedId']}[]' + '\n' + for key, value in data['body']['imageMap'].items(): + media[key] = Image(url=MediaUrl(value['originalUrl'], value['extension']), thumbnail_url=self.create_media_url(value['thumbnailUrl'])) + for key, value in data['body']['fileMap'].items(): + media[key] = File(url=MediaUrl(value['url'], value['extension']), name=value['name']) + kwargs['text'] = text + kwargs['media'] = media + post = Post(**kwargs) + self.module.add_to_map(unique_id, post) + return post + + def request_post(self, post_id: int) -> Optional[Post]: + url = f'{BASE_URL}/post.info?postId={post_id}' + obj = self.check_api_response(self.module.do_request(Method.GET, url)) + if not obj: + return None + return self.create_post(obj) + + def create_user(self, data: ParsedJson) -> User: + unique_id = f'fanbox:u:{data['id']}' + user = User(unique_id=unique_id, username=data['creatorId'], display_name=data['user']['name']) + user.profile_picture_url = self.create_media_url(data['user']['iconUrl']) + self.module.add_to_map(unique_id, user) + return user + +class FanboxPost(FanboxBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + try: + self.id: int = int(arg) + except ValueError: + self.errored = True + + def request_page(self, num: int) -> bool: + post = self.request_post(self.id) + if not post: + self.errored = True + return False + self.pages[num] = [post.unique_id] + self.completed = True + return True + +class FanboxUser(FanboxBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.id = arg + self.pagination = [] + + def request_pagination(self) -> bool: + url = f'{BASE_URL}/post.paginateCreator?creatorId={self.id}' + obj = self.check_api_response(self.module.do_request(Method.GET, url)) + if not obj: + return False + for link in obj: + self.pagination.append(link) + return True + + def request_page(self, num: int) -> bool: + if len(self.pagination) == 0: + if not self.request_pagination(): + self.errored = True + return False + if num >= len(self.pagination): + return False + obj = self.check_api_response(self.module.do_request(Method.GET, self.pagination[num])) + if not obj: + self.errored = True + return False + self.pages[num] = [] + for item in obj['items']: + post = self.request_post(item['id']) + if post: + self.pages[num].append(post.unique_id) + if len(self.pages) == len(self.pagination): + self.completed = True + return True + +class FanboxSupporting(FanboxBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + + def request_page(self, num: int) -> bool: + url = f'{BASE_URL}/plan.listSupporting' + obj = self.check_api_response(self.module.do_request(Method.GET, url)) + if not obj: + self.errored = True + return False + self.pages[num] = [] + for user in obj: + self.pages[num].append(self.create_user(user).unique_id) + self.completed = True + return True + +class FanboxModule(Module): + def __init__(self, sessid: str): + super().__init__() + self.parser: QueryParser = QueryParser(self, 'post') + self.parser.add_command('post', FanboxPost) + self.parser.add_command('user', FanboxUser) + self.parser.add_command('supporting', FanboxSupporting) + self.headers['Accept-Encoding'] = 'gzip, deflate, br' + self.headers['Accept-Language'] = 'en-US,en;q=0.5' + self.headers['Origin'] = 'https://www.fanbox.cc' + self.headers['Referer'] = 'https://www.fanbox.cc/' + self.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0' + self.cookies['FANBOXSESSID'] = sessid + + def search(self, query: str, *extra_args: Any) -> Optional[Search]: + return self.parser.parse_query(query) + + def get_download(self, unique_id: str, key: str) -> Optional[dict[str, Any]]: + if unique_id not in self.unique_id_map: + return None + post = self.unique_id_map[unique_id] + if key not in post.media: + return None + return { + 'urls': [post.media[key].url], + 'headers': {}, + 'cookies': {} + } diff --git a/src/portal/py/modules/instagram.py b/src/portal/py/modules/instagram.py new file mode 100644 index 0000000..325caf6 --- /dev/null +++ b/src/portal/py/modules/instagram.py @@ -0,0 +1,154 @@ +import os +import log +import email.utils +from typing import Optional, Any +from pathlib import Path +from base import Module, Search +from post import MediaUrl, PostType, DateType, Post, User, Image, Video +from modules.common import get_current_utc_time +from instagrapi import Client +from instagrapi.exceptions import UserNotFound, LoginRequired, ChallengeRequired + +class InstagramSearch(Search): + REACH_LIMIT = 3 + + def __init__(self, module: Any, pk: str): + super().__init__() + self.module: InstagramModule = module + self.pk: str = pk + self.page: int = 0 + self.cursor: Any = None + + def create_media_url(self, url: str) -> MediaUrl: + question = url.find('?') + if question >= 0: + ext = url[:question].split('.')[-1] + else: + ext = url.split('.')[-1] + return MediaUrl(url, ext) + + def request_page(self, num: int) -> bool: + if num >= self.page + self.REACH_LIMIT: + return False + request_satisfied = False + for i in range(self.page, num + 1): + try: + media, self.cursor = self.module.cl.user_medias_paginated_v1(self.pk, 0, end_cursor=self.cursor) + except LoginRequired: + self.errored = True + return False + except ChallengeRequired: + self.errored = True + return False + if not self.cursor: + self.completed = True + self.pages[i] = [] + for m in media: + kwargs = {} + kwargs['type'] = PostType.POST + unique_id = 'instagram:p:{}'.format(m.id) + kwargs['unique_id'] = unique_id + kwargs['raw_responses'] = {} + kwargs['raw_responses']['tile'] = m.json() + kwargs['url'] = 'https://www.instagram.com/p/{}'.format(m.code) + kwargs['dates'] = { + DateType.CREATED: m.taken_at.timestamp(), + DateType.RETRIEVED: get_current_utc_time().timestamp() + } + user = User(unique_id='instagram:u:{}'.format(m.user.pk)) + if m.user.username: + user.username = m.user.username + if m.user.full_name: + user.display_name = m.user.full_name + if m.user.profile_pic_url: + user.profile_picture_url = self.create_media_url(str(m.user.profile_pic_url)) + kwargs['author'] = user + kwargs['title'] = m.title + kwargs['text'] = m.caption_text + kwargs['likes'] = m.like_count + kwargs['comments'] = m.comment_count + media = {} + if m.media_type == 8: # Album + for k, r in enumerate(m.resources): + if r.media_type == 1: # Photo + media[str(k)] = Image(url=self.create_media_url(str(r.thumbnail_url))) + elif r.media_type == 2: # Video + media[str(k)] = Video(url=self.create_media_url(str(r.video_url)), thumbnail_url=self.create_media_url(str(r.thumbnail_url))) + elif m.media_type == 2: # Video + kwargs['views'] = m.play_count + media[str(0)] = Video(url=self.create_media_url(str(m.video_url)), thumbnail_url=self.create_media_url(str(m.thumbnail_url))) + elif m.media_type == 1: # Photo + candidates = sorted(m.image_versions2['candidates'], key=lambda x: x['width'], reverse=True) + media[str(0)] = Image(url=self.create_media_url(candidates[0]['url']), thumbnail_url=self.create_media_url(candidates[1]['url'])) + kwargs['media'] = media + post = Post(**kwargs) + self.module.add_to_map(unique_id, post) + self.pages[i].append(unique_id) + self.page += 1 + if i == num: + request_satisfied = True + return request_satisfied + +class InstagramModule(Module): + def __init__(self, user_agent: str, settings: Path, session_id: str): + super().__init__() + self.cl: Client = Client() + # https://specdevice.com/showspec.php?id=c318-0c39-0033-c5870033c587 + device_set = { + 'app_version': '324.0.0.0.16', + 'android_version': 33, + 'android_release': '13.0.0', + 'dpi': '476dpi', + 'resolution': '1440x3120', + 'manufacturer': 'Google', + 'device': 'cheetah', + 'model': 'Pixel 7 Pro', + 'cpu': 'cheetah', + 'version_code': '9981770' + } + user_agent = f'Instagram {device_set['app_version']} Android ({device_set['android_version']}/{device_set['android_release']}; {device_set['dpi']}; {device_set['resolution']}; {device_set['manufacturer']}; {device_set['device']}; {device_set['model']}; {device_set['cpu']}; en_US; {device_set['version_code']})' + self.cl.set_country('US') + self.cl.set_country_code(1) # Phone code + self.cl.set_locale('en_US') + self.cl.set_timezone_offset(-14400) # New_York GMT-4 + self.cl.set_user_agent(user_agent) + self.cl.set_device(device = device_set) + self.settings: Path = settings + self.session_id: str = session_id + + def init(self) -> bool: + if os.path.exists(self.settings): + self.cl.load_settings(self.settings) + if not self.cl.login_by_sessionid(self.session_id): + return False + #if not self.cl.login(self.username, self.password): + # return False + self.cl.dump_settings(self.settings) + return True + + def search(self, query: str, *extra_args: Any) -> Optional[Search]: + try: + user = self.cl.user_info_by_username_v1(query) + except UserNotFound: + log.warn('User not found.') + return None + except LoginRequired: + log.error('Fetching user failed: Login required.') + return None + except ChallengeRequired: + log.error('Fetching user failed: Challenge required.') + return None + return InstagramSearch(self, user.pk) + + def get_download(self, unique_id: str, key: str) -> Optional[dict[str, Any]]: + if unique_id not in self.unique_id_map: + return None + post = self.unique_id_map[unique_id] + if key not in post.media: + return None + return { + 'urls': [post.media[key].url], + 'headers': { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0' + } + } diff --git a/src/portal/py/modules/pixiv_app.py b/src/portal/py/modules/pixiv_app.py new file mode 100644 index 0000000..8be3f9c --- /dev/null +++ b/src/portal/py/modules/pixiv_app.py @@ -0,0 +1,201 @@ +import json +from typing import Optional, Any +from base import Search, Module, ParsedJson +from post import MediaUrl, PostType, DateType, User, Post, Image, Tag, TagType +from query_parser import QueryParser +from pixivpy3 import * +from modules.common import get_current_utc_time, parse_pixiv_date + +class PixivAppBase(Search): + def __init__(self, userdata: Any): + super().__init__() + self.module: PixivAppModule = userdata + self.page: int = 0 + self.next_qs: Optional[dict[str, Any]] = None + + def api_request_page(self, num: int) -> tuple[bool, Optional[dict[str, Any]]]: + return False, None + + def request_page(self, num: int) -> bool: + request_satisfied = False + for i in range(self.page, num + 1): + if not self.next_qs: + break + if i in self.pages: + continue + self.pages[i] = [] + success, self.next_qs = self.api_request_page(i) + if not success: + self.errored = True + break + if i == num: + request_satisfied = True + self.page += 1 + return request_satisfied + + def create_media_url(self, url: str) -> MediaUrl: + return MediaUrl(url, url.split('.')[-1]) + + def create_tag(self, data: ParsedJson) -> Tag: + tag = Tag(type=TagType.GENERAL, name=data['name']) + if data['translated_name']: + tag.alts['en'] = data['translated_name'] + return tag + + def create_post(self, data: ParsedJson) -> Post: + if len(data['meta_pages']) == 0: + data['meta_pages'].append({ + 'image_urls': { + 'medium': data['image_urls']['medium'], + 'original': data['meta_single_page']['original_image_url'] + } + }) + kwargs = {} + kwargs['type'] = PostType.POST + unique_id = 'pixiv:i:{}'.format(data['id']) + kwargs['unique_id'] = unique_id + kwargs['raw_responses'] = {} + kwargs['raw_responses']['api'] = json.dumps(data) + kwargs['url'] = 'https://www.pixiv.net/en/artworks/{}'.format(data['id']) + kwargs['dates'] = { + DateType.CREATED: parse_pixiv_date(data['create_date']).timestamp(), + DateType.RETRIEVED: get_current_utc_time().timestamp() + } + user = User(unique_id=f'pixiv:u:{data['user']['id']}', username=data['user']['account'], display_name=data['user']['name']) + user.profile_picture_url = data['user']['profile_image_urls']['medium'] + kwargs['author'] = user + kwargs['title'] = data['title'] + kwargs['text'] = data['caption'] + kwargs['likes'] = data['total_bookmarks'] + if 'total_comments' in data: + kwargs['comments'] = data['total_comments'] + kwargs['views'] = data['total_view'] + kwargs['tags'] = [self.create_tag(tag) for tag in data['tags']] + media = {} + for i, page in enumerate(data['meta_pages']): + media[str(i)] = Image(url=self.create_media_url(page['image_urls']['original']), thumbnail_url=self.create_media_url(page['image_urls']['medium'])) + kwargs['media'] = media + post = Post(**kwargs) + self.module.add_to_map(unique_id, post) + return post + + def create_user(self, data: ParsedJson) -> User: + unique_id = f'pixiv:u:{data['id']}' + user = User(unique_id=unique_id, username=data['account'], display_name=data['name']) + self.module.add_to_map(unique_id, user) + return user + +class PixivAppSearch(PixivAppBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.next_qs: Optional[dict[str, Any]] = {} + self.next_qs['word'] = arg + self.next_qs['sort'] = 'popular_desc' + self.next_qs['search_ai_type'] = 1 + self.next_qs['start_date'] = '2019-01-01' + self.next_qs['end_date'] = '2019-12-31' + + def api_request_page(self, num: int) -> tuple[bool, Optional[dict[str, Any]]]: + if not self.next_qs: + return False, None + obj = self.module.api.search_illust(**self.next_qs) + if 'illusts' not in obj: + return False, None + for post in obj['illusts']: + self.pages[num].append(self.create_post(post).unique_id) + return True, self.module.api.parse_qs(obj['next_url']) + +class PixivAppIllust(PixivAppBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.next_qs: Optional[dict[str, Any]] = {} + self.next_qs['illust_id'] = arg + + def api_request_page(self, num: int) -> tuple[bool, Optional[dict[str, Any]]]: + if not self.next_qs: + return False, None + obj = self.module.api.illust_detail(**self.next_qs) + if 'illust' not in obj: + return False, None + self.pages[num].append(self.create_post(obj['illust']).unique_id) + return True, None + +class PixivAppUser(PixivAppBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.next_qs: Optional[dict[str, Any]] = {} + self.next_qs['user_id'] = arg + self.next_qs['type'] = 'illust' + + def api_request_page(self, num: int) -> tuple[bool, Optional[dict[str, Any]]]: + if not self.next_qs: + return False, None + obj = self.module.api.user_illusts(**self.next_qs) + if 'illusts' not in obj: + return False, None + for post in obj['illusts']: + self.pages[num].append(self.create_post(post).unique_id) + return True, self.module.api.parse_qs(obj['next_url']) + +class PixivAppBookmarks(PixivAppBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.next_qs: Optional[dict[str, Any]] = {} + self.next_qs['user_id'] = arg + self.next_qs['restrict'] = 'public' + + def api_request_page(self, num: int) -> tuple[bool, Optional[dict[str, Any]]]: + if not self.next_qs: + return False, None + obj = self.module.api.user_bookmarks_illust(**self.next_qs) + if 'illusts' not in obj: + return False, None + for post in obj['illusts']: + self.pages[num].append(self.create_post(post).unique_id) + return True, self.module.api.parse_qs(obj['next_url']) + +class PixivAppFollowing(PixivAppBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.next_qs: Optional[dict[str, Any]] = {} + self.next_qs['user_id'] = arg + self.next_qs['restrict'] = 'public' + + def api_request_page(self, num: int) -> tuple[bool, Optional[dict[str, Any]]]: + if not self.next_qs: + return False, None + obj = self.module.api.user_following(**self.next_qs) + if 'user_previews' not in obj: + return False, None + for user in obj['user_previews']: + self.pages[num].append(self.create_user(user['user']).unique_id) + return True, self.module.api.parse_qs(obj['next_url']) + +class PixivAppModule(Module): + def __init__(self, refresh_token: str): + super().__init__() + self.parser: QueryParser = QueryParser(self, 'search') + self.parser.add_command('search', PixivAppSearch) + self.parser.add_command('illust', PixivAppIllust) + self.parser.add_command('user', PixivAppUser) + self.parser.add_command('bookmarks', PixivAppBookmarks) + self.parser.add_command('following', PixivAppFollowing) + self.api: AppPixivAPI = AppPixivAPI() + self.api.auth(refresh_token=refresh_token) + + def search(self, query: str, *extra_args: Any) -> Optional[Search]: + return self.parser.parse_query(query) + + def get_download(self, unique_id: str, key: str) -> Optional[dict[str, Any]]: + if unique_id not in self.unique_id_map: + return None + post = self.unique_id_map[unique_id] + if key not in post.media: + return None + return { + 'urls': [post.media[key].url], + 'headers': { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0', + 'Referer': 'https://www.pixiv.net/' + } + } diff --git a/src/portal/py/modules/pixiv_web.py b/src/portal/py/modules/pixiv_web.py new file mode 100644 index 0000000..7bc043b --- /dev/null +++ b/src/portal/py/modules/pixiv_web.py @@ -0,0 +1,263 @@ +import log +import json +import httpx +import urllib.parse +from typing import Optional, Any +from json.decoder import JSONDecodeError +from base import Search, Module, Method, ParsedJson +from post import MediaUrl, PostType, DateType, Post, User, Media, Image, Animation, Tag, TagType +from query_parser import QueryParser +from modules.common import parse_pixiv_date, get_current_utc_time + +BASE_URL = 'https://www.pixiv.net/ajax' +LANG = 'en' +VERSION = '5f53980f6b59c9376220b6d86e8feb5ea9e22e10' + +class PixivWebBase(Search): + def __init__(self, userdata: Any): + super().__init__() + self.module: PixivWebModule = userdata + + def check_api_response(self, response: Optional[httpx.Response]) -> Optional[ParsedJson]: + if not response: + return None + try: + obj = response.json() + except JSONDecodeError: + return None + if obj['error']: + log.error(obj['message']) + return None + return obj['body'] + + def create_media_url(self, url: str) -> MediaUrl: + return MediaUrl(url, url.split('.')[-1]) + + def parse_pages(self, data: ParsedJson) -> dict[str, Media]: + media = {} + for i, m in enumerate(data): + media[str(i)] = Image(url=self.create_media_url(m['urls']['original']), thumbnail_url=self.create_media_url(m['urls']['small'])) + return media + + def parse_ugoira(self, data: ParsedJson, thumbnail_url: str) -> Animation: + animation = Animation(url=self.create_media_url(data['originalSrc']), thumbnail_url=self.create_media_url(thumbnail_url)) + for frame in data['frames']: + animation.frames.append((frame['file'], frame['delay'])) + return animation + + def seek_profile_picture_url(self, user_illusts: ParsedJson) -> Optional[MediaUrl]: + for illust in user_illusts.values(): + if illust is None: + continue + if 'profileImageUrl' in illust: + return self.create_media_url(illust['profileImageUrl']) + return None + + def create_tag(self, data: ParsedJson) -> Tag: + tag = Tag(type=TagType.GENERAL, name=data['tag']) + if 'romaji' in data: + tag.alts['romaji'] = data['romaji'] + if 'translation' in data: + if 'en' in data['translation']: + tag.alts['en'] = data['translation']['en'] + return tag + + def create_post(self, data: ParsedJson, pages: Optional[ParsedJson]=None, ugoira: Optional[ParsedJson]=None) -> Optional[Post]: + kwargs = {} + kwargs['type'] = PostType.POST + illust_id = data['illustId'] + unique_id = f'pixiv:i:{illust_id}' + kwargs['unique_id'] = unique_id + kwargs['raw_responses'] = {} + kwargs['raw_responses']['api'] = json.dumps(data) + kwargs['url'] = f'https://www.pixiv.net/{LANG}/artworks/{data['illustId']}' + create_date = parse_pixiv_date(data['createDate']).timestamp() + kwargs['dates'] = { + DateType.CREATED: create_date, + DateType.RETRIEVED: get_current_utc_time().timestamp() + } + upload_date = parse_pixiv_date(data['uploadDate']).timestamp() + if upload_date != create_date: + kwargs['dates'][DateType.EDITED] = upload_date + user = User(unique_id=f'pixiv:u:{data['userId']}', username=data['userAccount'], display_name=data['userName']) + profile_picture_url = self.seek_profile_picture_url(data['userIllusts']) + if profile_picture_url: + user.profile_picture_url = profile_picture_url + kwargs['author'] = user + kwargs['title'] = data['illustTitle'] + kwargs['text'] = data['illustComment'] + kwargs['likes'] = data['likeCount'] + kwargs['bookmarks'] = data['bookmarkCount'] + kwargs['comments'] = data['commentCount'] + kwargs['views'] = data['viewCount'] + kwargs['tags'] = [self.create_tag(tag) for tag in data['tags']['tags']] + if data['illustType'] == 2: # Ugoira + if not ugoira: + url = f'{BASE_URL}/illust/{illust_id}/ugoira_meta?lang={LANG}&version={VERSION}' + ugoira = self.check_api_response(self.module.do_request(Method.GET, url)) + if not ugoira: + return None + kwargs['raw_responses']['ugoira'] = json.dumps(ugoira) + kwargs['media'] = { '0': self.parse_ugoira(ugoira, data['urls']['original']) } + elif data['pageCount'] > 1: + if not pages: + url = f'{BASE_URL}/illust/{illust_id}/pages?lang={LANG}&version={VERSION}' + pages = self.check_api_response(self.module.do_request(Method.GET, url)) + if not pages: + return None + kwargs['raw_responses']['pages'] = json.dumps(pages) + kwargs['media'] = self.parse_pages(pages) + else: + kwargs['media'] = self.parse_pages([{ 'urls': data['urls'] }]) + post = Post(**kwargs) + self.module.add_to_map(unique_id, post) + return post + + def remake_post(self, raw_responses: ParsedJson, original_date: float) -> Optional[Post]: + api = json.loads(raw_responses['api']) + pages = raw_responses.get('pages', None) + if pages: + pages = json.loads(pages) + ugoira = raw_responses.get('ugoira', None) + if ugoira: + ugoira = json.loads(ugoira) + post = self.create_post(api, pages, ugoira) + if post: + post.dates[DateType.RETRIEVED] = original_date + return post + + def request_illust(self, illust_id: int) -> Optional[Post]: + url = f'{BASE_URL}/illust/{illust_id}?lang={LANG}&version={VERSION}' + obj = self.check_api_response(self.module.do_request(Method.GET, url)) + if not obj: + return None + return self.create_post(obj) + + def create_user(self, data: ParsedJson) -> User: + unique_id=f'pixiv:u:{data['userId']}' + user = User(unique_id=unique_id, display_name=data['userName']) + user.profile_picture_url = self.create_media_url(data['profileImageUrl']) + self.module.add_to_map(unique_id, user) + return user + +class PixivWebSearch(PixivWebBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.query: str = arg + self.last_page: Optional[int] = None + + def request_page(self, num: int) -> bool: + if self.last_page and num >= self.last_page: + return False + start = '2019-01-01' + end = '2019-12-31' + order = 'popular_d' + url = f'{BASE_URL}/search/artworks/{urllib.parse.quote(self.query)}?word={self.query}&order={order}&mode=all&scd={start}&ecd={end}&p={num + 1}&csw=0&s_mode=s_tag&type=all&lang={LANG}&version={VERSION}' + obj = self.check_api_response(self.module.do_request(Method.GET, url)) + if not obj: + self.errored = True + return False + self.last_page = obj['illustManga']['lastPage'] + self.pages[num] = [] + for partial_post in obj['illustManga']['data']: + post = self.request_illust(int(partial_post['id'])) + if not post: + self.errored = True + return False + self.pages[num].append(post.unique_id) + return True + +class PixivWebIllust(PixivWebBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + try: + self.id: int = int(arg) + except ValueError: + self.errored = True + + def request_page(self, num: int) -> bool: + post = self.request_illust(self.id) + if not post: + self.errored = True + return False + self.pages[num] = [post.unique_id] + self.completed = True + return True + +class PixivWebUser(PixivWebBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + try: + self.id: int = int(arg) + except ValueError: + self.errored = True + + def request_page(self, num: int) -> bool: + url = f'{BASE_URL}/user/{self.id}/profile/all?lang={LANG}&version={VERSION}' + obj = self.check_api_response(self.module.do_request(Method.GET, url)) + if not obj: + self.errored = True + return False + self.pages[num] = [] + if obj['manga']: + for illust in obj['manga'].keys(): + post = self.request_illust(int(illust)) + if not post: + self.errored = True + return False + self.pages[num].append(post.unique_id) + self.completed = True + return True + +class PixivWebFollowing(PixivWebBase): + LIMIT = 24 + + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + try: + self.id: int = int(arg) + except ValueError: + self.errored = True + + def request_page(self, num: int) -> bool: + url = f'{BASE_URL}/user/{self.id}/following?offset={self.LIMIT * num}&limit={self.LIMIT}&rest=show&tag=&acceptingRequests=0&lang={LANG}&version={VERSION}' + obj = self.check_api_response(self.module.do_request(Method.GET, url)) + if not obj: + self.errored = True + return False + self.pages[num] = [] + for user in obj['users']: + self.pages[num].append(self.create_user(user).unique_id) + return True + +class PixivWebModule(Module): + def __init__(self, sessid: str): + super().__init__() + self.parser: QueryParser = QueryParser(self, 'search') + self.parser.add_command('search', PixivWebSearch) + self.parser.add_command('illust', PixivWebIllust) + self.parser.add_command('user', PixivWebUser) + self.parser.add_command('following', PixivWebFollowing) + self.headers['Accept-Encoding'] = 'gzip, deflate, br' + self.headers['Accept-Language'] = 'en-US,en;q=0.5' + self.headers['Referer'] = 'https://www.pixiv.net/' + self.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0' + self.headers['x-user-id'] = '22781328' + self.cookies['PHPSESSID'] = sessid + + def search(self, query: str, *extra_args: Any) -> Optional[Search]: + return self.parser.parse_query(query) + + def get_download(self, unique_id: str, key: str) -> Optional[dict[str, Any]]: + if unique_id not in self.unique_id_map: + return None + post = self.unique_id_map[unique_id] + if key not in post.media: + return None + return { + 'urls': [post.media[key].url], + 'headers': { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0', + 'Referer': 'https://www.pixiv.net/' + } + } diff --git a/src/portal/py/modules/searx.py b/src/portal/py/modules/searx.py new file mode 100644 index 0000000..4cc0b04 --- /dev/null +++ b/src/portal/py/modules/searx.py @@ -0,0 +1,95 @@ +import sys +import json + +from base import Module, Search +from post import Post, Image + +sys.path.append('../../vendor/searxng') +from searx import settings +from searx.utils import gen_useragent +from searx.engines import load_engine, register_engine + +# TODO: Put these on the Provider object. +ENGINES = {} + +ENGINE_INITIAL_INDEX = { + 'google images': 0, + 'google': 1, + 'bing images': 0 +} + +class SearxSearch(Search): + def __init__(self, plugin, query, engine): + super().__init__() + global ENGINES + self.query = query + self.plugin = plugin + self.key = engine.replace(' ', '_') + self.engine = ENGINES[engine] + self.index = ENGINE_INITIAL_INDEX[engine] + + def load_page(self, index): + self.pages[index] = [] + res = self.plugin.send_http_request(self.engine.request(self.query, { + 'language': 'en-US', + 'safesearch': 0, + 'time_range': None, + 'pageno': self.index + index, + 'cookies': self.plugin.cookies, + 'headers': self.plugin.headers, + 'data': None + })) + if not res: + return False + try: + ret = self.engine.response(res) + except: + return False + if len(ret) == 0: + # TODO: Test. + self.completed = True + return False + for i, p in enumerate(ret): + if 'url' not in p: + continue + # Don't add this to the global map because it's not actually unique. + unique_id = '{}:{}_{}:{}'.format(self.key, self.query, i, index) + media = [] + if 'img_src' in p: + media.append(Image(url=p['img_src'], thumbnail_url='')) + elif 'image_url' in p: + media.append(Image(url=p['image_url'], thumbnail_url='')) + kwargs = {} + kwargs['unique_id'] = unique_id + kwargs['raw_responses'] = {} + kwargs['raw_responses']['result'] = json.dumps(p) + kwargs['url'] = p['url'] + kwargs['title'] = p['title'] + kwargs['text'] = p['content'] + kwargs['media'] = media + self.pages[index].append(Post(**kwargs)) + return True + +class SearxModule(Module): + def __init__(self): + super().__init__() + global ENGINES + self.headers = { + 'User-Agent': gen_useragent(), + 'Accept-Language': 'en-US,en;q=0.5', + } + for engine_data in settings['engines']: + engine = load_engine(engine_data) + if engine: + register_engine(engine) + ENGINES[engine.name] = engine + + def send_http_request(self, params): + self.cookies.update(params['cookies']) + self.headers.update(params['headers']) + return self.get(params['url'], params=params['data']) + + def search(self, query, *extra_args): + if not extra_args: + return None + return SearxSearch(self, query, extra_args[0]) diff --git a/src/portal/py/modules/twitter.py b/src/portal/py/modules/twitter.py new file mode 100644 index 0000000..7e77253 --- /dev/null +++ b/src/portal/py/modules/twitter.py @@ -0,0 +1,190 @@ +import log +import http.cookiejar +from typing import Optional, Any, Iterator +from datetime import timezone +from base import Search, Module +from post import MediaUrl, PostType, DateType, PostRef, Post, User, Media, Image, Video +from query_parser import QueryParser +from modules.common import get_current_utc_time + +# GraphQL API +from snscrape.modules import twitter +from snscrape.base import ScraperException +from snscrape.modules.twitter import (Tweet, TweetRef, Tombstone, UserRef, + TwitterSearchScraper, TwitterProfileScraper, TwitterUserScraper, TwitterTweetScraper) + +class TwitterScrapeBase(Search): + # Posts per emulated page because snscrape returns an iterator. + POSTS_PER_PAGE = 8 + + # Don't allow requests for a page more than this amount past the current page. + REACH_LIMIT = 4 + + def __init__(self, userdata: Any): + super().__init__() + self.module: TwitterScrapeModule = userdata + self.iterator: Iterator[Tweet | TweetRef | Tombstone] + self.page: int = 0 + + def create_media_url(self, url: str) -> MediaUrl: + format = url.find('format=') + if format >= 0: + ext = url[format + 7:format + 10] + else: + question = url.rfind('?') + if question >= 0: + url = url[0:question] + ext = url.split('.')[-1] + return MediaUrl(url, ext) + + def parse_media(self, data: list[twitter.Medium]) -> dict[str, Media]: + media = {} + for i, m in enumerate(data): + if isinstance(m, twitter.Photo): + media[str(i)] = Image(url=self.create_media_url(m.fullUrl), thumbnail_url=self.create_media_url(m.previewUrl)) + elif isinstance(m, twitter.Video) or isinstance(m, twitter.Gif): + videos = sorted(m.variants, key=lambda x: x.bitrate if x.bitrate else 0, reverse=True) + media[str(i)] = Video(url=self.create_media_url(videos[0].url), thumbnail_url=self.create_media_url(m.thumbnailUrl)) + return media + + def create_post(self, tweet: Tweet | TweetRef | Tombstone) -> Post: + kwargs = {} + kwargs['unique_id'] = f'twitter:t:{tweet.id}' + if isinstance(tweet, TweetRef) or isinstance(tweet, Tombstone): + kwargs['type'] = PostType.TOMBSTONE + return Post(**kwargs) + kwargs['raw_responses'] = tweet.rawResponses + kwargs['url'] = tweet.url + # TODO: replace correct? + kwargs['dates'] = { + DateType.CREATED: tweet.date.replace(tzinfo=timezone.utc).timestamp(), + DateType.RETRIEVED: get_current_utc_time().timestamp() + } + author = User(unique_id=f'twitter:u:{tweet.user.id}') + if not isinstance(tweet.user, UserRef): + author.username = tweet.user.username + if tweet.user.displayname: + author.display_name = tweet.user.displayname + if tweet.user.profileImageUrl: + author.profile_picture_url = self.create_media_url(tweet.user.profileImageUrl) + kwargs['author'] = author + if tweet.retweetedTweet: + kwargs['type'] = PostType.REPOST + kwargs['post'] = PostRef(f'twitter:t:{tweet.retweetedTweet.id}') + return Post(**kwargs) + kwargs['type'] = PostType.POST + kwargs['text'] = tweet.rawContent + kwargs['likes'] = tweet.likeCount + kwargs['reposts'] = tweet.retweetCount + kwargs['quotes'] = tweet.quoteCount + kwargs['comments'] = tweet.replyCount + kwargs['views'] = tweet.viewCount + if tweet.media: + kwargs['media'] = self.parse_media(tweet.media) + if tweet.inReplyToTweetId: + kwargs['in_reply_to'] = PostRef(f'twitter:t:{tweet.inReplyToTweetId}') + if tweet.quotedTweet: + kwargs['quoted'] = PostRef(f'twitter:t:{tweet.quotedTweet.id}') + return Post(**kwargs) + + def add_post_to_page(self, num: int, item: Post | User) -> None: + self.module.add_to_map(item.unique_id, item) + self.pages[num].append(item.unique_id) + + def step_iterator_for_page(self, num: int) -> bool: + for _ in range(0, self.POSTS_PER_PAGE): + try: + tweet = next(self.iterator) + except StopIteration: + self.completed = True + break + except ScraperException as e: + log.error(repr(e)) + continue + self.add_post_to_page(num, self.create_post(tweet)) + if isinstance(tweet, TweetRef) or isinstance(tweet, Tombstone): + continue + if tweet.retweetedTweet: + retweet = self.create_post(tweet.retweetedTweet) + self.module.add_to_map(retweet.unique_id, retweet) + if tweet.quotedTweet: + quote = self.create_post(tweet.quotedTweet) + self.module.add_to_map(quote.unique_id, quote) + return len(self.pages[num]) > 0 + + def request_page(self, num: int) -> bool: + if num - self.page >= self.REACH_LIMIT: + return False + request_satisfied = False + for i in range(self.page, num + 1): + if self.completed: + break + if i in self.pages: + continue + self.pages[i] = [] + if not self.step_iterator_for_page(i): + break + self.page = i + if i == num: + request_satisfied = True + return request_satisfied + +class TwitterScrapeSearch(TwitterScrapeBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.iterator = TwitterSearchScraper(arg, top=True, cookies=self.module.cookies).get_items() + +class TwitterScrapeUser(TwitterScrapeBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + if arg.startswith('@'): + arg = arg[1:] + self.iterator = TwitterUserScraper(arg, cookies=self.module.cookies).get_items() + +class TwitterScrapeProfile(TwitterScrapeBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + if arg.startswith('@'): + arg = arg[1:] + self.iterator = TwitterProfileScraper(arg, cookies=self.module.cookies).get_items() + +class TwitterScrapeTweet(TwitterScrapeBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + if arg.startswith('https://'): + arg = arg[arg.rfind('/') + 1:] + question = arg.find('?') + if question >= 0: + arg = arg[:question] + self.iterator = TwitterTweetScraper(arg, cookies=self.module.cookies).get_items() + +class TwitterScrapeModule(Module): + def __init__(self, cookies_path: str): + super().__init__() + self.parser: QueryParser = QueryParser(self, 'search') + self.parser.add_command('search', TwitterScrapeSearch) + self.parser.add_command('user', TwitterScrapeUser) + self.parser.add_command('profile', TwitterScrapeProfile) + self.parser.add_command('tweet', TwitterScrapeTweet) + if cookies_path: + cookie_jar = http.cookiejar.MozillaCookieJar() + cookie_jar.load(filename=cookies_path, ignore_expires=True) + for c in cookie_jar: + if c.value: + self.cookies[c.name] = c.value + + def search(self, query: str, *extra_args: Any) -> Optional[Search]: + return self.parser.parse_query(query) + + def get_download(self, unique_id: str, key: str) -> Optional[dict[str, Any]]: + if unique_id not in self.unique_id_map: + return None + post = self.unique_id_map[unique_id] + if key not in post.media: + return None + return { + 'urls': [post.media[key].url], + 'headers': { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0' + } + } diff --git a/src/portal/py/modules/twitter_api.py b/src/portal/py/modules/twitter_api.py new file mode 100644 index 0000000..6d04855 --- /dev/null +++ b/src/portal/py/modules/twitter_api.py @@ -0,0 +1,269 @@ +from base import Search, Module +from query_parser import QueryParser + +# Official Twitter API +from twitter import Twitter2, TwitterError, OAuth + +# Quite incomplete API based backend. Should *not* be used for archiving. +# HOLD: I can't test this without paying $100 for the X API WTFFF. +''' +class TwitterBase(Search): + ALL_PARAMS = { + 'tweet.fields': 'attachments,author_id,context_annotations,conversation_id,created_at,entities,geo,id,in_reply_to_user_id,lang,public_metrics,possibly_sensitive,referenced_tweets,reply_settings,source,text,withheld', + 'user.fields': 'created_at,description,entities,id,location,name,pinned_tweet_id,profile_image_url,protected,public_metrics,url,username,verified,withheld', + 'media.fields': 'duration_ms,height,media_key,preview_image_url,type,url,width,public_metrics', + 'place.fields': 'contained_within,country,country_code,full_name,geo,id,name,place_type', + 'poll.fields': 'duration_minutes,end_datetime,id,options,voting_status', + } + ALL_EXPRESSIONS = 'author_id,referenced_tweets.id,referenced_tweets.id.author_id,entities.mentions.username,attachments.poll_ids,attachments.media_keys,in_reply_to_user_id,geo.place_id' + + SOME_PARAMS = { + 'tweet.fields': 'attachments,author_id,text,entities,referenced_tweets', + 'user.fields': 'id,name,profile_image_url,url,username', + 'media.fields': 'duration_ms,height,media_key,preview_image_url,type,url,width,public_metrics', + 'place.fields': '', + 'poll.fields': '', + } + SOME_EXPRESSIONS = 'author_id,referenced_tweets.id,referenced_tweets.id.author_id,entities.mentions.username,attachments.media_keys,in_reply_to_user_id' + + def __init__(self, userdata, arg): + super().__init__() + self.provider = userdata + self.params = self.SOME_PARAMS.copy() + self.media_map = {} + self.user_id = None + self.pagination_page = 0 + self.pagination_token = None + self.arg = arg + + def add_attachemnts(self, l, r, data): + if 'media_keys' not in data: + return + for m in data['media_keys']: + if m in r: + continue + r.append(m) + if m in self.media_map: + l.append(Image(url=self.media_map[m], thumbnail_url='')) + + def add_entity_urls(self, data): + if 'urls' in data: + for u in data['urls']: + if 'media_key' in u and u['media_key'] not in self.media_map: + self.media_map[u['media_key']] = u['expanded_url'] + + def make_post(self, tweet): + print(json.dumps(tweet, indent=4)) + media = [] + repeats = [] + if 'referenced_tweets' in t: + print(len(referenced_tweets)) + #for rt in t['referenced_tweets']: + # if rt['id'] in self.tweet_map: + # rrt = self.tweet_map[rt['id']] + # if 'entities' in rrt: + # self.add_entity_urls(rrt['entities']) + # if 'attachments' in rrt: + # self.add_attachemnts(media, repeats, rrt['attachments']) + if 'attachments' in t: + self.add_attachemnts(media, repeats, t['attachments']) + unique_id = 'twitter:t:{}'.format(t['id']) + url = 'https://twitter.com/{}/status/{}'.format(t['author_id'], t['id']) + kwargs = {} + kwargs['unique_id'] = unique_id + kwargs['raw_responses'] = { 'tweet' : json.dumps(t) } + kwargs['url'] = url + kwargs['author'] = User(unique_id='twitter:u:{}'.format(t['author_id'])) + kwargs['title'] = '' + kwargs['text'] = t['text'] + kwargs['media'] = media + return Post(**kwargs) + + def add_items_from_search(self, data): + if 'includes' in data: + includes = data['includes'] + if 'media' in includes: + for i in includes['media']: + if 'url' in i: + self.media_map[i['media_key']] = i['url'] + #if 'tweets' in includes: + # for t in includes['tweets']: + # self.tweet_map[t['id']] = t + if 'data' not in data or not data['data']: + return False + l = data['data'] if type(data['data']) == list else [data['data']] + for t in l: + if 'entities' in t: + self.add_entity_urls(t['entities']) + post = self.make_post(t) + self.pages[self.pagination_page].append(post) + return True + + def add_user_ids_from_search(self, data): + if not data['data']: + return False + for u in data['data']: + self.pages[self.pagination_page].append(User( + 'twitter:u:{}'.format(u['id']), username=u['username'], display_name=u['name'])) + return True + + def should_process_index(self, index): + if index - self.pagination_page >= REACH_LIMIT: + return False + if self.pagination_page > 0 and not self.pagination_token: + self.completed = True + return False + if self.pagination_token: + self.params['pagination_token'] = self.pagination_token + self.pages[self.pagination_page] = [] + return True + + def handle_data(self, data, user_ids=False): + if 'next_token' not in data['meta'].keys(): + self.pagination_token = None + else: + self.pagination_token = data['meta']['next_token'] + if user_ids: + if not self.add_user_ids_from_search(data): + return False + else: + if not self.add_items_from_search(data): + return False + self.pagination_page += 1 + return True + + def get_user_id(self, username): + if self.user_id: + return True + try: + data = self.provider.t.users.by.username._username( + _username=username, _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + self.user_id = data['data']['id'] + return True + +class TwitterSearch(TwitterBase): + def load_page(self, index): + for _ in range(self.pagination_page, index + 1): + if not self.should_process_index(index): + return False + try: + data = self.provider.t.tweets.search.recent( + query=self.arg, expansions=self.SOME_EXPRESSIONS, params=self.params, + sort_order='relevancy', max_results=25, _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + return self.handle_data(data) + +class TwitterUser(TwitterBase): + def __init__(self, userdata, arg): + if arg.startswith('@'): + arg = arg[1:] + super().__init__(userdata, arg) + + def load_page(self, index): + for _ in range(self.pagination_page, index + 1): + if not self.should_process_index(index): + return False + if not self.get_user_id(self.arg): + return False + try: + data = self.provider.t.users._id.tweets( + _id=self.user_id, expansions=self.SOME_EXPRESSIONS, params=self.params, + max_results=25, _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + return self.handle_data(data) + +class TwitterTimeline(TwitterBase): + def __init__(self, userdata, arg): + if not arg: + arg = 'pizzabelly' + super().__init__(userdata, arg) + self.params['exclude'] = 'replies' + + def load_page(self, index): + for _ in range(self.pagination_page, index + 1): + if not self.should_process_index(index): + return False + if not self.get_user_id(self.arg): + return False + try: + data = self.provider.t.users._id.timelines.reverse_chronological( + _id=self.user_id, expansions=self.SOME_EXPRESSIONS, params=self.params, + max_results=25, _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + return self.handle_data(data) + +class TwitterLikes(TwitterBase): + def __init__(self, userdata, arg): + if arg.startswith('@'): + arg = arg[1:] + super().__init__(userdata, arg) + + def load_page(self, index): + for _ in range(self.pagination_page, index + 1): + if not self.should_process_index(index): + return False + if not self.get_user_id(self.arg): + return False + try: + data = self.provider.t.users._id.liked_tweets( + _id=self.user_id, expansions=self.SOME_EXPRESSIONS, params=self.params, + max_results=25, _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + return self.handle_data(data) + +class TwitterTweet(TwitterBase): + def load_page(self, index): + for _ in range(self.pagination_page, index + 1): + if not self.should_process_index(index): + return False + try: + data = self.provider.t.tweets( + ids=self.arg, expansions=self.SOME_EXPRESSIONS, params=self.params, + _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + return self.handle_data(data) + +class TwitterFollowing(TwitterBase): + def __init__(self, userdata, arg): + if arg.startswith('@'): + arg = arg[1:] + super().__init__(userdata, arg) + del self.params['media.fields'] + del self.params['place.fields'] + del self.params['poll.fields'] + + def load_page(self, index): + for _ in range(self.pagination_page, index + 1): + if not self.should_process_index(index): + return False + if not self.get_user_id(self.arg): + return False + try: + data = self.provider.t.users._id.following( + _id=self.user_id, params=self.params, max_results=25, _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + return self.handle_data(data, True) +''' + +class TwitterApiModule(Module): + def __init__(self, access_key: str, access_secret: str, consumer_key: str, consumer_secret: str): + self.t: Twitter2 = Twitter2(auth=OAuth(access_key, access_secret, consumer_key, consumer_secret), retry=True) + self.parser: QueryParser = QueryParser(self, 'timeline') + #self.parser.add_command('timeline', TwitterTimeline) + #self.parser.add_command('likes', TwitterLikes) + #self.parser.add_command('following', TwitterFollowing) diff --git a/src/portal/py/modules/youtube.py b/src/portal/py/modules/youtube.py new file mode 100644 index 0000000..d14dfe8 --- /dev/null +++ b/src/portal/py/modules/youtube.py @@ -0,0 +1,133 @@ +import log +from typing import Optional, Any +from base import Search, Module, ParsedJson +from post import MediaUrl, Post, PostType, Media, Video +from query_parser import QueryParser +from yt_dlp import YoutubeDL + +class YDLLogger(): + def debug(self, msg): + if msg.startswith('[debug] '): + log.debug(msg) + else: + log.info(msg) + + def info(self, msg): + log.info(msg) + + def warning(self, msg): + log.warn(msg) + + def error(self, msg): + log.error(msg) + +ydl_opts = { + 'quiet': False, + 'logger': YDLLogger(), + 'cachedir': False, +# 'cookiefile': '', +} + +ydl = YoutubeDL(ydl_opts) + +def get_playback_url(data, video=True): + if 'entries' in data: + if len(data['entries']) == 0: + return None + data = data['entries'][0] + + if 'formats' not in data: + if 'url' in data: + return data['url'] + return None + + # Filter out hls temporarily. + data['formats'] = list(filter(lambda f: not f['protocol'].startswith('m3u8'), data['formats'])) + + if len(data['formats']) == 0: + return None + + url = data['formats'][0]['url'] + + # audio_ext? + has_audio = list(filter(lambda f: 'acodec' not in f or f['acodec'] != 'none', data['formats'])) + + if video: + if len(has_audio) > 0: + data['formats'] = has_audio + try: + data['formats'] = list(filter(lambda f: 'quality' in f, data['formats'])) + url = max(data['formats'], key=lambda f: f['quality'])['url'] + except: + pass + else: + if len(has_audio) == 0: + return None + data['formats'] = has_audio + try: + audio_only = list(filter(lambda f: f['vcodec'] == 'none', data['formats'])) + if len(audio_only) > 0: + data['formats'] = audio_only + else: + data['formats'] = list(filter(lambda f: f['ext'] in ['mp4'], data['formats'])) + except: + pass + try: + data['formats'] = list(filter(lambda f: 'abr' in f, data['formats'])) + url = max(data['formats'], key=lambda f: f['abr'])['url'] + except: + pass + + return url + +class YoutubeBase(Search): + def __init__(self, userdata: Any): + super().__init__() + self.module: YoutubeModule = userdata + + def get_info(self) -> Optional[ParsedJson]: + return None + + def request_page(self, num: int) -> bool: + info = self.get_info() + if not info: + return False + url = get_playback_url(info) + if not url: + return False + media: dict[str, Media] = { '0': Video(url=MediaUrl(url=url)) } + unique_id = f'youtube:v:{info['id']}' + self.module.add_to_map(unique_id, Post(type=PostType.POST, unique_id=unique_id, url='', title=info['title'], text='', media=media)) + self.pages[num] = [unique_id] + return True + +class YoutubeSearch(YoutubeBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.query: str = arg + + def get_info(self) -> Optional[ParsedJson]: + return ydl.extract_info(f'ytsearch1:{self.query}', download=False) + +class YoutubeLink(YoutubeBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.link: str = arg + + def get_info(self) -> Optional[ParsedJson]: + return ydl.extract_info(self.link, download=False) + +class YoutubeModule(Module): + def __init__(self): + super().__init__() + self.parser: QueryParser = QueryParser(self, 'search') + self.parser.add_command('search', YoutubeSearch) + self.parser.add_command('link', YoutubeLink) + + def search(self, query: str, *extra_args: Any) -> Optional[Search]: + return self.parser.parse_query(query) + + def get_download(self, unique_id: str, key: str) -> Optional[dict[str, Any]]: + if unique_id not in self.unique_id_map: + return None + return None diff --git a/src/portal/py/post.py b/src/portal/py/post.py new file mode 100644 index 0000000..b064f97 --- /dev/null +++ b/src/portal/py/post.py @@ -0,0 +1,132 @@ +import dataclasses +from dataclasses import field +from typing import Optional +from enum import Enum +from json import JSONEncoder + +class PostType(int, Enum): + UNKNOWN = 0 + POST = 1 + REPOST = 2 + PREVIEW = 3 + TOMBSTONE = 4 + +class DateType(int, Enum): + CREATED = 0 + EDITED = 1 + RETRIEVED = 2 + +class MediaType(int, Enum): + UNKNOWN = 0 + FILE = 1 + AUDIO = 2 + IMAGE = 3 + VIDEO = 4 + VIDEO_SPLIT = 5 + ANIMATION = 6 + +class TagType(int, Enum): + UNKNOWN = 0 + GENERAL = 1 + ARTIST = 2 + CHARACTER = 3 + COPYRIGHT = 4 + META = 5 + DEPRECATED = 6 + +@dataclasses.dataclass +class MediaUrl(): + url: str = '' + ext: str = 'unknown' + +@dataclasses.dataclass +class Media(): + type: MediaType = MediaType.UNKNOWN + url: MediaUrl = field(default_factory=lambda: MediaUrl()) + +@dataclasses.dataclass +class File(Media): + type: MediaType = MediaType.FILE + name: str = '' + +@dataclasses.dataclass +class Audio(Media): + type: MediaType = MediaType.AUDIO + +@dataclasses.dataclass +class Image(Media): + type: MediaType = MediaType.IMAGE + thumbnail_url: MediaUrl = field(default_factory=lambda: MediaUrl()) + +@dataclasses.dataclass +class Video(Media): + type: MediaType = MediaType.VIDEO + thumbnail_url: MediaUrl = field(default_factory=lambda: MediaUrl()) + +@dataclasses.dataclass +class VideoSplit(Media): + type: MediaType = MediaType.VIDEO_SPLIT + audio_url: MediaUrl = field(default_factory=lambda: MediaUrl()) + subtitle_url: MediaUrl = field(default_factory=lambda: MediaUrl()) + thumbnail_url: MediaUrl = field(default_factory=lambda: MediaUrl()) + +@dataclasses.dataclass +class Animation(Media): + type: MediaType = MediaType.ANIMATION + thumbnail_url: MediaUrl = field(default_factory=lambda: MediaUrl()) + frames: list[tuple[str, int]] = field(default_factory=lambda: []) + +@dataclasses.dataclass +class Tag(): + type: TagType = TagType.UNKNOWN + name: str = '' + alts: dict[str, str] = field(default_factory=lambda: {}) + +@dataclasses.dataclass +class User(): + unique_id: str = '' + username: str = '' + display_name: str = '' + profile_picture_url: MediaUrl = field(default_factory=lambda: MediaUrl()) + +@dataclasses.dataclass +class PostRef(): + unique_id: str = '' + +# V4 Ideas: +# - Date estimate and range +# - Edited but unknown when +# - Generally an estimate/guess +# - Formated text/body + +@dataclasses.dataclass +class Post(): + version: int = 3 + type: PostType = PostType.UNKNOWN + unique_id: str = '' + raw_responses: dict[str, str] = field(default_factory=lambda: {}) + url: str = '' + dates: dict[DateType, float] = field(default_factory=lambda: {}) + author: User = field(default_factory=lambda: User()) + title: str = '' + text: str = '' + likes: Optional[int] = None + bookmarks: Optional[int] = None + reposts: Optional[int] = None + quotes: Optional[int] = None + comments: Optional[int] = None + views: Optional[int] = None + tags: list[Tag] = field(default_factory=lambda: []) + links: list[str] = field(default_factory=lambda: []) + media: dict[str, Media] = field(default_factory=lambda: {}) + post: PostRef = field(default_factory=lambda: PostRef()) + quoted: PostRef = field(default_factory=lambda: PostRef()) + in_reply_to: PostRef = field(default_factory=lambda: PostRef()) + +class PostEncoder(JSONEncoder): + def default(self, o): + if isinstance(o, Enum): + return o.value + if dataclasses.is_dataclass(o): + return dataclasses.asdict(o) + return o.__dict__ diff --git a/src/portal/py/query_parser.py b/src/portal/py/query_parser.py new file mode 100644 index 0000000..c16ded2 --- /dev/null +++ b/src/portal/py/query_parser.py @@ -0,0 +1,45 @@ +from typing import Optional, Any, Callable +from base import Search + +type QueryCommand = Callable[[Any, Optional[str]], Search] + +class QueryParser(): + def __init__(self, userdata: Any, default_command: str): + self.commands: dict[str, QueryCommand] = {} + self.escaped_commands: dict[str, str] = {} + self.userdata: Any = userdata + self.default_command: str = default_command + + def add_command(self, name, search) -> None: + self.commands[name] = search + self.escaped_commands['\\' + name] = name + + def parse_query(self, query: str) -> Search: + arg = query + search = self.commands[self.default_command] + + sp = query.split(' ') + + for i, word in enumerate(sp): + col = word.find(':') + if col == -1: + continue + command = word[0:col] + if command in self.escaped_commands: + query = query.replace(command, self.escaped_commands[command], 1) + continue + if command in self.commands: + if col + 1 < len(word): + arg = word[col+1:] + else: + arg = None + search = self.commands[command] + if i == 0: + if len(sp) > 1: + query = query.replace(word + ' ', '', 1) + else: + query = query.replace(word, '', 1) + else: + query = query.replace(' ' + word + ' ', '', 1) + + return search(self.userdata, arg) diff --git a/src/portal/requirements.txt b/src/portal/requirements.txt new file mode 100644 index 0000000..d241387 --- /dev/null +++ b/src/portal/requirements.txt @@ -0,0 +1,2 @@ +httpx[http2,brotli] +pillow diff --git a/src/portal/scripts/create_vendor.sh b/src/portal/scripts/create_vendor.sh new file mode 100755 index 0000000..f4d551e --- /dev/null +++ b/src/portal/scripts/create_vendor.sh @@ -0,0 +1,28 @@ +#! /usr/bin/env sh + +python3 -m venv ./venv +source ./venv/bin/activate + +cd vendor/ +rm -r vendor/ + +pip3 install -r ../requirements.txt --upgrade --target=./vendor + +cd yt-dlp/ +pip3 install . --upgrade --target=../vendor +cd ../ + +cd snscrape/ +pip3 install . --upgrade --target=../vendor +cd ../ + +cd pixivpy/ +pip3 install . --upgrade --target=../vendor +cd ../ + +cd instagrapi/ +pip3 install . --upgrade --target=../vendor +cd ../ + +cd ../ +rm -r venv diff --git a/src/portal/scripts/run_py.sh b/src/portal/scripts/run_py.sh new file mode 100755 index 0000000..ddbe869 --- /dev/null +++ b/src/portal/scripts/run_py.sh @@ -0,0 +1,4 @@ +#! /usr/bin/env sh +export PYTHONDONTWRITEBYTECODE=1 +export PYTHONPATH=$HOME/c/camu/src/portal/vendor/vendor +python3 $@ diff --git a/src/portal/src/packet_ext.c b/src/portal/src/packet_ext.c new file mode 100644 index 0000000..580c307 --- /dev/null +++ b/src/portal/src/packet_ext.c @@ -0,0 +1,98 @@ +#include "packet_ext.h" + +static void aki_packet_write_optional_int(struct aki_packet *packet, optional_int *o) +{ + AKI_PACKET_WRITE_TYPE(packet, s64, o->i); + AKI_PACKET_WRITE_TYPE(packet, bool, o->set); +} + +void aki_packet_write_camu_post(struct aki_packet *packet, struct camu_post *post) +{ + AKI_PACKET_WRITE_TYPE(packet, u16, post->version); + AKI_PACKET_WRITE_TYPE(packet, u8, post->type); + aki_packet_write_str(packet, &post->unique_id); + aki_packet_write_str(packet, &post->url); + AKI_PACKET_WRITE_TYPE(packet, u32, post->dates.size); + struct camu_post_date *date; + al_array_foreach_ptr(post->dates, i, date) { + AKI_PACKET_WRITE_TYPE(packet, u8, date->type); + AKI_PACKET_WRITE_TYPE(packet, f32, date->timestamp); + } + aki_packet_write_str(packet, &post->author.unique_id); + aki_packet_write_wstr(packet, &post->author.username); + aki_packet_write_wstr(packet, &post->author.display_name); + aki_packet_write_str(packet, &post->author.profile_picture_url); + aki_packet_write_wstr(packet, &post->title); + aki_packet_write_wstr(packet, &post->text); + aki_packet_write_optional_int(packet, &post->likes); + aki_packet_write_optional_int(packet, &post->bookmarks); + aki_packet_write_optional_int(packet, &post->reposts); + aki_packet_write_optional_int(packet, &post->quotes); + aki_packet_write_optional_int(packet, &post->comments); + aki_packet_write_optional_int(packet, &post->views); + AKI_PACKET_WRITE_TYPE(packet, u32, post->media.size); + struct camu_post_media *media; + al_array_foreach_ptr(post->media, i, media) { + AKI_PACKET_WRITE_TYPE(packet, u8, media->type); + aki_packet_write_str(packet, &media->key); + aki_packet_write_str(packet, &media->url); + aki_packet_write_str(packet, &media->ext); + aki_packet_write_str(packet, &media->thumbnail_url); + aki_packet_write_str(packet, &media->thumbnail_ext); + } + aki_packet_write_str(packet, &post->post.unique_id); + aki_packet_write_str(packet, &post->quoted.unique_id); + aki_packet_write_str(packet, &post->in_reply_to.unique_id); +} + +static void aki_packet_read_optional_int(struct aki_packet *packet, optional_int *o) +{ + AKI_PACKET_READ_TYPE(packet, s64, o->i); + AKI_PACKET_READ_TYPE(packet, bool, o->set); +} + +void aki_packet_read_camu_post(struct aki_packet *packet, struct camu_post *post) +{ + camu_post_reset(post); + AKI_PACKET_READ_TYPE(packet, u16, post->version); + AKI_PACKET_READ_TYPE(packet, u8, post->type); + aki_packet_read_str(packet, &post->unique_id); + aki_packet_read_str(packet, &post->url); + u32 dates_size; + AKI_PACKET_READ_TYPE(packet, u32, dates_size); + al_array_reserve(post->dates, dates_size); + for (u32 i = 0; i < dates_size; i++) { + struct camu_post_date date; + AKI_PACKET_READ_TYPE(packet, u8, date.type); + AKI_PACKET_READ_TYPE(packet, f32, date.timestamp); + al_array_push(post->dates, date); + } + aki_packet_read_str(packet, &post->author.unique_id); + aki_packet_read_wstr(packet, &post->author.username); + aki_packet_read_wstr(packet, &post->author.display_name); + aki_packet_read_str(packet, &post->author.profile_picture_url); + aki_packet_read_wstr(packet, &post->title); + aki_packet_read_wstr(packet, &post->text); + aki_packet_read_optional_int(packet, &post->likes); + aki_packet_read_optional_int(packet, &post->bookmarks); + aki_packet_read_optional_int(packet, &post->reposts); + aki_packet_read_optional_int(packet, &post->quotes); + aki_packet_read_optional_int(packet, &post->comments); + aki_packet_read_optional_int(packet, &post->views); + u32 media_size; + AKI_PACKET_READ_TYPE(packet, u32, media_size); + al_array_reserve(post->media, media_size); + for (u32 i = 0; i < media_size; i++) { + struct camu_post_media media; + AKI_PACKET_READ_TYPE(packet, u8, media.type); + aki_packet_read_str(packet, &media.key); + aki_packet_read_str(packet, &media.url); + aki_packet_read_str(packet, &media.ext); + aki_packet_read_str(packet, &media.thumbnail_url); + aki_packet_read_str(packet, &media.thumbnail_ext); + al_array_push(post->media, media); + } + aki_packet_read_str(packet, &post->post.unique_id); + aki_packet_read_str(packet, &post->quoted.unique_id); + aki_packet_read_str(packet, &post->in_reply_to.unique_id); +} diff --git a/src/portal/src/packet_ext.h b/src/portal/src/packet_ext.h new file mode 100644 index 0000000..eae1e07 --- /dev/null +++ b/src/portal/src/packet_ext.h @@ -0,0 +1,10 @@ +#pragma once + +#include <al/types.h> +#include <aki/common.h> +#include <aki/event_loop.h> + +#include "post.h" + +void aki_packet_write_post(struct aki_packet *packet, struct camu_post *post); +void aki_packet_read_post(struct aki_packet *packet, struct camu_post *post); diff --git a/src/portal/src/post.c b/src/portal/src/post.c new file mode 100644 index 0000000..c5eb4b5 --- /dev/null +++ b/src/portal/src/post.c @@ -0,0 +1,64 @@ +#include "post.h" + +void camu_post_reset(struct camu_post *post) +{ + al_bzero(post, sizeof(struct camu_post)); + al_array_init(post->dates); + al_array_init(post->media); +} + +void camu_post_clone(struct camu_post *dest, struct camu_post *src) +{ + camu_post_reset(dest); + dest->version = src->version; + dest->type = src->type; + al_str_clone(&dest->unique_id, &src->unique_id); + al_str_clone(&dest->url, &src->url); + al_array_clone(dest->dates, src->dates); + al_str_clone(&dest->author.unique_id, &src->author.unique_id); + al_wstr_clone(&dest->author.username, &src->author.username); + al_wstr_clone(&dest->author.display_name, &src->author.display_name); + al_str_clone(&dest->author.profile_picture_url, &src->author.profile_picture_url); + al_wstr_clone(&dest->title, &src->title); + al_wstr_clone(&dest->text, &src->text); + dest->likes = src->likes; + dest->bookmarks = src->bookmarks; + dest->reposts = src->reposts; + dest->quotes = src->quotes; + dest->comments = src->comments; + dest->views = src->views; + struct camu_post_media *media; + struct camu_post_media media_copy; + al_array_foreach_ptr(src->media, i, media) { + al_bzero(&media_copy, sizeof(struct camu_post_media)); + media_copy.type = media->type; + al_str_clone(&media_copy.key, &media->key); + al_str_clone(&media_copy.url, &media->url); + al_str_clone(&media_copy.ext, &media->ext); + al_str_clone(&media_copy.thumbnail_url, &media->thumbnail_url); + al_str_clone(&media_copy.thumbnail_ext, &media->thumbnail_ext); + al_array_push(dest->media, media_copy); + } + al_str_clone(&dest->post.unique_id, &src->post.unique_id); + al_str_clone(&dest->quoted.unique_id, &src->quoted.unique_id); + al_str_clone(&dest->in_reply_to.unique_id, &src->in_reply_to.unique_id); +} + +void camu_post_add_media(struct camu_post *post, u8 type, str *key, str *url, str *ext, str *thumbnail_url, str *thumbnail_ext) +{ + al_array_push(post->media, ((struct camu_post_media){ + .type = type, + .key = *key, + .url = *url, + .ext = *ext, + .thumbnail_url = *thumbnail_url, + .thumbnail_ext = *thumbnail_ext + })); +} + +void camu_post_add_date(struct camu_post *post, u8 type, f32 timestamp) +{ + al_array_push(post->dates, ((struct camu_post_date){ + .type = type, .timestamp = timestamp + })); +} diff --git a/src/portal/src/post.h b/src/portal/src/post.h new file mode 100644 index 0000000..11fa9b6 --- /dev/null +++ b/src/portal/src/post.h @@ -0,0 +1,102 @@ +#pragma once + +#include <al/types.h> +#include <al/str.h> +#include <al/wstr.h> +#include <al/array.h> + +enum { + CAMU_POST_UNKNOWN = 0, + CAMU_POST_POST, + CAMU_POST_REPOST, + CAMU_POST_PREVIEW, + CAMU_POST_TOMBSTONE +}; + +enum { + CAMU_DATE_CREATED = 0, + CAMU_DATE_EDITED, + CAMU_DATE_RETRIEVED +}; + +enum { + CAMU_MEDIA_UNKNOWN = 0, + CAMU_MEDIA_FILE, + CAMU_MEDIA_AUDIO, + CAMU_MEDIA_IMAGE, + CAMU_MEDIA_VIDEO, + CAMU_MEDIA_VIDEO_SPLIT, + CAMU_MEDIA_ANIMATION +}; + +enum { + CAMU_TAG_UNKNOWN = 0, + CAMU_TAG_GENERAL, + CAMU_TAG_ARTIST, + CAMU_TAG_CHARACTER, + CAMU_TAG_COPYRIGHT, + CAMU_TAG_META, + CAMU_TAG_DEPRECATED +}; + +typedef struct { + s64 i; + bool set; +} optional_int; + +struct camu_post_date { + u8 type; + f32 timestamp; +}; + +typedef array(struct camu_post_date) camu_dates_array; + +struct camu_post_media { + u8 type; + str key; + str url; + str ext; + str thumbnail_url; + str thumbnail_ext; +}; + +typedef array(struct camu_post_media) camu_media_array; + +struct camu_post_user { + str unique_id; + wstr username; + wstr display_name; + str profile_picture_url; +}; + +struct camu_post_ref { + str unique_id; +}; + +struct camu_post { + u16 version; + u8 type; + str unique_id; + str url; + camu_dates_array dates; + struct camu_post_user author; + wstr title; + wstr text; + optional_int likes; + optional_int bookmarks; + optional_int reposts; + optional_int quotes; + optional_int comments; + optional_int views; + camu_media_array media; + struct camu_post_ref post; // reposted post. + struct camu_post_ref quoted; + struct camu_post_ref in_reply_to; +}; + +void camu_post_reset(struct camu_post *post); +void camu_post_clone(struct camu_post *dest, struct camu_post *src); + +// Python internal. +void camu_post_add_date(struct camu_post *post, u8 type, f32 timestamp); +void camu_post_add_media(struct camu_post *post, u8 type, str *key, str *url, str *ext, str *thumbnail_url, str *thumbnail_ext); diff --git a/src/portal/src/post_cache.c b/src/portal/src/post_cache.c new file mode 100644 index 0000000..ad50404 --- /dev/null +++ b/src/portal/src/post_cache.c @@ -0,0 +1,30 @@ +#include "post_cache.h" + +void camu_post_cache_init(struct camu_post_cache *cache) +{ + al_array_init(cache->cache); +} + +void camu_post_cache_push(struct camu_post_cache *cache, struct camu_post *post) +{ + struct camu_post *check = camu_post_cache_get(cache, &post->unique_id); + if (!check) { + struct camu_post *npost = al_alloc_object(struct camu_post); + camu_post_clone(npost, post); + al_array_push(cache->cache, npost); + } +} + +struct camu_post *camu_post_cache_get(struct camu_post_cache *cache, str *unique_id) +{ + struct camu_post *post; + al_array_foreach(cache->cache, i, post) { + if (al_str_eq(&post->unique_id, unique_id)) { + return post; + } + if (al_str_eq(&post->author.unique_id, unique_id)) { + return post; + } + } + return NULL; +} diff --git a/src/portal/src/post_cache.h b/src/portal/src/post_cache.h new file mode 100644 index 0000000..a406c72 --- /dev/null +++ b/src/portal/src/post_cache.h @@ -0,0 +1,13 @@ +#pragma once + +#include <al/array.h> + +#include "post.h" + +struct camu_post_cache { + array(struct camu_post *) cache; +}; + +void camu_post_cache_init(struct camu_post_cache *cache); +void camu_post_cache_push(struct camu_post_cache *cache, struct camu_post *post); +struct camu_post *camu_post_cache_get(struct camu_post_cache *cache, str *unique_id); diff --git a/src/portal/src/search.c b/src/portal/src/search.c new file mode 100644 index 0000000..ec40d0e --- /dev/null +++ b/src/portal/src/search.c @@ -0,0 +1,141 @@ +#include <al/log.h> + +#include "../cpy/portal.c" + +#include "search.h" + +bool camu_python_init(void) +{ + PyPreConfig pre; + PyPreConfig_InitPythonConfig(&pre); + pre.utf8_mode = 1; + pre.dev_mode = 0; + Py_PreInitialize(&pre); + if (PyImport_AppendInittab("portal", PyInit_portal) == -1) { + al_log_error("portal", "Could not extend in-built modules table."); + camu_python_close(); + return false; + } + Py_Initialize(); + PyObject *module = PyImport_ImportModule("portal"); + if (!module) { + PyErr_Print(); + al_log_error("portal", "Could not import module."); + camu_python_close(); + return false; + } + return true; +} + +void camu_python_close(void) +{ + if (Py_IsInitialized()) Py_Finalize(); +} + +static struct camu_result_page *page_at_index(struct camu_search *search, u32 num) +{ + struct camu_result_page *page; + al_array_foreach_ptr(search->pages, i, page) { + if (page->num == num) return page; + } + al_array_push(search->pages, (struct camu_result_page){0}); + page = &al_array_last(search->pages); + page->num = num; + al_array_init(page->posts); + al_array_init(page->list); + return page; +} + + +void camu_portal_init(struct camu_portal *portal, struct camu_post_cache *cache) +{ + portal->cache = cache; + al_array_init(portal->searches); +} + +static void camu_search_init_internal(struct camu_search *search) +{ + search->page = 0; + al_array_init(search->pages); +} + +s32 camu_portal_create_search(struct camu_portal *portal, str *module, str *query) +{ + s32 id = portal_bridge_search(module, query); + if (id >= 0) { + struct camu_search *search = al_alloc_object(struct camu_search); + camu_search_init_internal(search); + search->id = id; + al_str_clone(&search->module, module); + al_str_clone(&search->query, query); + search->portal = portal; + al_array_push(portal->searches, search); + } + al_log_info("portal", "New search %i (%.*s).", id, AL_STR_PRINTF(query)); + return id; +} + +struct camu_search *camu_portal_get_search(struct camu_portal *portal, s32 id) +{ + struct camu_search *search; + al_array_foreach(portal->searches, i, search) { + if (search->id == id) return search; + } + return NULL; +} + +void camu_portal_discard_search(struct camu_portal *portal, s32 id) +{ + (void)portal; + (void)id; +} + +void camu_portal_close(struct camu_portal *portal) +{ + (void)portal; +} + +bool camu_search_get_page(struct camu_search *search, u32 num) +{ + struct camu_result_page *page; + al_array_foreach_ptr(search->pages, i, page) { + if (page->num == num) goto out; + } + al_log_info("portal", "Loading page %i (%.*s).", num, AL_STR_PRINTF(&search->query)); + if (portal_bridge_get_page(search, search->id, num) == -1) { + return false; + } + struct camu_portal *portal = search->portal; + if (portal->cache) { + struct camu_post *post; + al_array_foreach_ptr(al_array_at(search->pages, num).posts, i, post) { + camu_post_cache_push(portal->cache, post); + } + } +out: + search->page = num; + return true; +} + +void camu_search_free(struct camu_search *search) +{ + struct camu_result_page *page; + al_array_foreach_ptr(search->pages, i, page) { + // TODO: Free camu_post ? + al_array_free(page->posts); + al_array_free(page->list); + } + al_array_free(search->pages); +} + +void camu_search_add_post(struct camu_search *search, u32 num, struct camu_post *post) +{ + struct camu_result_page *page = page_at_index(search, num); + al_array_push(page->posts, *post); +} + +void camu_search_add_to_list(struct camu_search *search, u32 num, str *unique_id) +{ + struct camu_result_page *page = page_at_index(search, num); + al_array_push(page->list, *unique_id); +} diff --git a/src/portal/src/search.h b/src/portal/src/search.h new file mode 100644 index 0000000..4962170 --- /dev/null +++ b/src/portal/src/search.h @@ -0,0 +1,39 @@ +#pragma once + +#include "post.h" +#include "post_cache.h" + +struct camu_result_page { + u32 num; + array(struct camu_post) posts; + array(str) list; +}; + +struct camu_search { + s32 id; + str module; + str query; + u32 page; + array(struct camu_result_page) pages; + struct camu_portal *portal; +}; + +struct camu_portal { + struct camu_post_cache *cache; + array(struct camu_search *) searches; +}; + +bool camu_python_init(void); +void camu_python_close(void); + +void camu_portal_init(struct camu_portal *portal, struct camu_post_cache *cache); +s32 camu_portal_create_search(struct camu_portal *portal, str *module_str, str *search_str); +struct camu_search *camu_portal_get_search(struct camu_portal *portal, s32 id); +void camu_portal_discard_search(struct camu_portal *portal, s32 id); +void camu_portal_close(struct camu_portal *portal); + +bool camu_search_get_page(struct camu_search *search, u32 num); + +// Python internal. +void camu_search_add_post(struct camu_search *search, u32 num, struct camu_post *post); +void camu_search_add_to_list(struct camu_search *search, u32 num, str *unique_id); diff --git a/src/portal/tests/archive_query.py b/src/portal/tests/archive_query.py new file mode 100644 index 0000000..9f69c1f --- /dev/null +++ b/src/portal/tests/archive_query.py @@ -0,0 +1,262 @@ +import os +import sys +import signal +import json +import threading +import gzip +from datetime import datetime, timezone +sys.path.append('../py') +from base import Method +from post import PostEncoder, PostType, DateType, User +from modules import ALL_MODULES +from logger import Logger + +mode = 'instagram' +cmd = 'user' +module = ALL_MODULES[mode][0] +if not module.init(): + sys.exit(1) + +quit_mutex = threading.Lock() +queue_mutex = threading.Lock() +queue_cond = threading.Condition(queue_mutex) +is_running = True + +def default_media_download(obj, post, output_base): + for key in post.media.keys(): + download_params = module.get_download(post.unique_id, key) + if not download_params: + obj.log.write(f'Skipping media{key} for post {post.unique_id}.') + continue + for url in download_params['urls']: + media_path = f'{output_base}_media{key}.{url.ext}' + obj.log.write(f'Attempting to download file {media_path}.') + response = module.do_request(Method.GET, url.url) + if not response: + obj.log.write(f'ERROR: Downloading post media failed ({post.unique_id} #{key}).') + continue + if not obj.write_to_file(media_path, 'wb+', response.content, False): + return False + +class QueryDownloadThread(threading.Thread): + def __init__(self, log, complete, query, output_dir): + super().__init__() + self.log = log + self.complete = complete + self.query = query + self.output_dir = output_dir + self.media_download = default_media_download + + def make_path(self, post): + prefix: str + if post.type != PostType.TOMBSTONE: + prefix = post.author.unique_id.replace(':', '_') + else: + prefix = 'tombstones' + path = f'{self.output_dir}/{prefix}' + if not os.path.isdir(path): + os.mkdir(path) + return path + + def write_to_file(self, path, open_method, content, compress): + try: + if compress: + with gzip.open(path, open_method) as f: + f.write(content.encode('utf-8')) + else: + with open(path, open_method) as f: + f.write(content) + except: + self.log.write(f'ERROR: Writing file failed ({path}).') + return False + return True + + def write_post_to_disk(self, post): + if post.post.unique_id: + original = module.get_item(post.post.unique_id) + if original: + self.write_post_to_disk(original) + if post.quoted.unique_id: + quoted = module.get_item(post.quoted.unique_id) + if quoted: + self.write_post_to_disk(quoted) + message = f'Downloading post {post.unique_id}' + if 'detached_api' in post.raw_responses: + raw_response_path = f'{self.output_dir}/raw_responses' + if not os.path.isdir(raw_response_path): + os.mkdir(raw_response_path) + raw_response_path += f'/{post.raw_responses['hash']}.json.gz' + if not self.write_to_file(raw_response_path, 'wb+', json.dumps(json.loads(post.raw_responses['detached_api'])), True): + return False + del post.raw_responses['detached_api'] + if post.type != PostType.TOMBSTONE: + message += f':{datetime.fromtimestamp(post.dates[DateType.CREATED], tz=timezone.utc).isoformat()} from {post.author.username}({post.author.display_name}):{post.author.unique_id}.' + else: + message += '.' + output_path = self.make_path(post) + output_base = f'{output_path}/{post.unique_id.replace(':', '_')}' + output = output_base + '.json.gz' + ''' + if not os.path.isfile(output): + print('done') + sys.exit(1) + ''' + ''' + has_mp4 = False + for m in post.media.values(): + if m.url.ext == 'mp4': + self.log.write('Rewriting post with mp4.') + has_mp4 = True + break + if post.quoted.unique_id: + self.log.write('Rewriting post with quote.') + has_mp4 = True + ''' + if os.path.isfile(output): + skip_message = f'Skipping already downloaded post {post.unique_id}' + if post.type != PostType.TOMBSTONE: + skip_message += f' from {post.author.unique_id}.' + else: + skip_message += '.' + self.log.write(skip_message) + return True + self.log.write(message) + self.media_download(self, post, output_base) + if not self.write_to_file(output, 'wb+', json.dumps(post, cls=PostEncoder), True): + return False + return True + + def supply_post(self, post): + if not self.write_post_to_disk(post): + return False + return True + + def run(self): + global quit_mutex + global queue_cond + global is_running + search = module.search(self.query) + if not search: + return + completed = True + arg = self.query.split(':')[-1] + num = 0 + count = 0 + while completed: + self.log.write(f'Starting page {num} of {arg}.') + page = search.get_page(num) + if not page: + break + num += 1 + for unique_id in page: + if unique_id == 'twitter:t:1597637140833529856': + self.log.write('---------Hit target---------') + with quit_mutex: + if not is_running: + self.complete.write(f'{mode}:{cmd}:{arg}:i:{str(count)},{str(num)}') + completed = False + break + post = module.get_item(unique_id) + if not post: + self.log.write(f'Missing post with id {unique_id}.') + continue + if not self.write_post_to_disk(post): + continue + count += 1 + if completed: + self.complete.write(f'{mode}:{cmd}:{arg}:c:{str(count)},{str(num)}') + with queue_cond: + queue_cond.notify() + +RUNTIME_PATH = "/mnt/store/files/tmp/run" +LOG_FILE = f'{RUNTIME_PATH}/archive.log' +ARG_FILE = f'{RUNTIME_PATH}/completed_args.log' + +def read_completed_args(path): + args = [] + if os.path.isfile(path): + with open(path, 'r') as f: + for l in f.read().splitlines(): + if l[0] == '-': + continue + s = l.split(':') + if s[3] == 'c': + args.append(f'{s[0]}:{s[1]}:{s[2]}') + return args + +def download_args(): + global quit_mutex + global queue_cond + global compat_thread + + completed_args = read_completed_args(ARG_FILE) + + log = Logger(LOG_FILE) + complete = Logger(ARG_FILE) + + header = f'--------{datetime.now().isoformat()}---------' + log.write(header) + complete.write(header) + + output_dir = sys.argv[1] + if not os.path.isdir(output_dir): + os.mkdir(output_dir) + + #args = module.search(f'following:{22781328}') + args = ['byjoshuajamal'] + + threads = [] + start = True + + for arg in args: + #if mode == 'fanbox': + # user = module.get_item(arg) + # if not user or not isinstance(user, User): + # log.write(f'Failed to retrive information for user {arg}.') + # continue + # arg = user.username + with quit_mutex: + if not is_running: + break + if mode == 'pixiv_web' and not start: + if arg == '': + start = True + continue + if not start: + continue + if mode != 'instagram': + arg = f'{cmd}:{arg}' + if f'{mode}:{arg}' in completed_args: + log.write(f'Skipping already downloaded arg {arg}.') + continue + if len(threads) >= 1: + with queue_cond: + queue_cond.wait() + threads = [t for t in threads if t.is_alive()] + with quit_mutex: + if not is_running: + break + log.write(f'***********************\nStarting download of {arg}\n***********************\n') + arg_thread = QueryDownloadThread(log, complete, arg, output_dir) + threads.append(arg_thread) + arg_thread.start() + + for t in threads: + t.join() + + log.close() + complete.close() + +def signal_handler(sig, frame): + global quit_mutex + global is_running + print('Attempting to quit...') + with quit_mutex: + if not is_running: + sys.exit(1) + else: + is_running = False + +if __name__ == "__main__": + signal.signal(signal.SIGINT, signal_handler) + download_args() diff --git a/src/portal/tests/logger.py b/src/portal/tests/logger.py new file mode 100644 index 0000000..80e22da --- /dev/null +++ b/src/portal/tests/logger.py @@ -0,0 +1,15 @@ +import threading + +class Logger(): + def __init__(self, path): + self.file = open(path, 'a+') + self.mutex = threading.Lock() + + def write(self, msg): + with self.mutex: + print(msg) + self.file.write(msg + '\n') + self.file.flush() + + def close(self): + self.file.close() diff --git a/src/portal/tests/old_twitter_api.py b/src/portal/tests/old_twitter_api.py new file mode 100644 index 0000000..40d0346 --- /dev/null +++ b/src/portal/tests/old_twitter_api.py @@ -0,0 +1,271 @@ +import sys +import os +import shutil +import gzip +import json +import email.utils +sys.path.append('../py') +from base import Method +from post import PostEncoder, User, Post, PostRef, PostType, DateType, Image, Video, MediaUrl +from modules import ALL_MODULES +from archive_query import QueryDownloadThread +from logger import Logger + +module = ALL_MODULES['twitter'][0] +if not module.init(): + sys.exit(1) + +#origin_base = '/mnt/store/camu_db/twitter' +origin_base = '/mnt/store/files/camu_db/data/twitter_orig' + +#RUNTIME_PATH = '.' +RUNTIME_PATH = '/mnt/store/files/tmp/run' +LOG_FILE = f'{RUNTIME_PATH}/twitter_reparse3.log' + +log = Logger(LOG_FILE) + +compat_thread = QueryDownloadThread(log, None, None, sys.argv[1]) + +def compat_media_download(obj, post, output_base): + for i, key in enumerate(post.media.keys()): + media = post.media[key] + url = media.url + media_path = f'{output_base}_media{key}.{url.ext}' + if os.path.isfile(media_path): + obj.log.write(f'Skipping media {media_path}.') + continue + origin_path = f'{origin_base}/{post.author.unique_id.replace(':', '_')}/{post.unique_id.replace(':', '_')}_media{i}.{url.ext}' + if os.path.isfile(origin_path): + shutil.copyfile(origin_path, media_path) + #post.media[key].url.url = url.url.replace('name=orig', 'name=large') + obj.log.write(f'Used backup {origin_path}.') + continue + obj.log.write(f'Attempting to download file {media_path}.') + response = module.do_request(Method.GET, url.url, None, 1) + if not response: + return False + if not obj.write_to_file(media_path, 'wb+', response.content, False): + return False + +compat_thread.media_download = compat_media_download + +def process_post(post): + compat_thread.supply_post(post) + #print(json.dumps(post, indent=4, cls=PostEncoder)) + +def create_media_url(url, format=True): + question = url.rfind('?') + if question >= 0: + ext = url[0:question].rsplit('.')[-1] + else: + ext = url.rsplit('.')[-1] + if format: + return MediaUrl(f'{url}?format={ext}&name=orig', ext), MediaUrl(f'{url}?format={ext}&name=small', ext) + return MediaUrl(url, ext), None + +def create_media(m): + original, thumbnail = create_media_url(m['media_url_https']) + if m['type'] == 'photo': + return Image(url=original, thumbnail_url=thumbnail) + elif m['type'] == 'video' or m['type'] == 'animated_gif': + variants = sorted(m['video_info']['variants'], key=lambda x: x['bitrate'] if 'bitrate' in x else 0, reverse=True) + url = variants[0]['url'] + video_url, _ = create_media_url(url, False) + return Video(url=video_url, thumbnail_url=thumbnail) + return None + +def create_user(data): + unique_id = f'twitter:u:{data['id']}' + user = User(unique_id=unique_id, username=data['screen_name'], display_name=data['name']) + user.profile_picture_url = data['profile_image_url_https'] + return user + +def create_post_v1(data, user, mtime, hash): + kwargs = {} + kwargs['unique_id'] = f'twitter:t:{data['id']}' + kwargs['raw_responses'] = {} + kwargs['raw_responses']['hash'] = hash + kwargs['url'] = f'https://twitter.com/{user.username}/status/{data['id']}' + kwargs['dates'] = { + DateType.CREATED: email.utils.parsedate_to_datetime(data['created_at']).timestamp(), + DateType.RETRIEVED: mtime + } + kwargs['author'] = user + if 'retweeted_status_id' in data: + kwargs['type'] = PostType.REPOST + kwargs['post'] = PostRef(f'twitter:t:{data['retweeted_status_id']}') + process_post(Post(**kwargs)) + return + kwargs['type'] = PostType.POST + kwargs['text'] = data['full_text'] + kwargs['likes'] = data['favorite_count'] + kwargs['reposts'] = data['retweet_count'] + kwargs['quotes'] = data['quote_count'] + kwargs['comments'] = data['reply_count'] + media = {} + if 'media' in data['entities']: + for m in data['entities']['media']: + media_id = m['id_str'] if 'id_str' in m else str(m['id']) + media[media_id] = create_media(m) + if 'extended_entities' in data and 'media' in data['extended_entities']: + for m in data['extended_entities']['media']: + media_id = m['id_str'] if 'id_str' in m else str(m['id']) + media[media_id] = create_media(m) + kwargs['media'] = media + if data['in_reply_to_status_id']: + kwargs['in_reply_to'] = PostRef(f'twitter:t:{data['in_reply_to_status_id']}') + if 'quoted_status_id' in data: + kwargs['quoted'] = PostRef(f'twitter:t:{data['quoted_status_id']}') + process_post(Post(**kwargs)) + +def parse_v1_raw_response(obj, mtime, hash): + users = {} + for id, user in obj['globalObjects']['users'].items(): + users[id] = create_user(user) + for tweet in obj['globalObjects']['tweets'].values(): + create_post_v1(tweet, users[tweet['user_id_str']], mtime, hash) + +def create_post_graphql(data, users, tweet_id, mtime, hash): + if not tweet_id: + tweet_id = data.get('rest_id', None) + if data['__typename'] == 'TweetWithVisibilityResults': + data = data['tweet'] + if not tweet_id: + tweet_id = data.get('rest_id', None) + if not tweet_id: + log.write('no id') + return PostRef() + kwargs = {} + unique_id = f'twitter:t:{tweet_id}' + kwargs['unique_id'] = unique_id + kwargs['raw_responses'] = {} + kwargs['raw_responses']['hash'] = hash + if 'legacy' not in data or ('__typename' in data and (data['__typename'] == 'TweetTombstone' or data['__typename'] == 'TweetUnavailable')): + kwargs['type'] = PostType.TOMBSTONE + process_post(Post(**kwargs)) + return PostRef(unique_id) + tweet = data['legacy'] + user_id = tweet['user_id_str'] + if user_id not in users: + user_data = data['core']['user_results']['result']['legacy'] + user = User(unique_id=f'twitter:u:{user_id}', username=user_data['screen_name'], display_name=user_data['name']) + user.profile_picture_url, _ = create_media_url(user_data['profile_image_url_https'], False) + users[user_id] = user + else: + user = users[user_id] + kwargs['url'] = f'https://twitter.com/{user.username}/status/{tweet_id}' + kwargs['dates'] = { + DateType.CREATED: email.utils.parsedate_to_datetime(tweet['created_at']).timestamp(), + DateType.RETRIEVED: mtime + } + kwargs['author'] = user + if 'retweeted_status_result' in tweet: + if 'result' in tweet['retweeted_status_result']: + retweet_id = create_post_graphql(tweet['retweeted_status_result']['result'], users, None, mtime, hash) + elif 'retweeted_status_id' in tweet: + retweet_id = PostRef(f'twitter:t:{tweet['retweeted_status_id']}') + else: + retweet_id = PostRef() + kwargs['type'] = PostType.REPOST + kwargs['post'] = retweet_id + process_post(Post(**kwargs)) + return PostRef(unique_id) + kwargs['type'] = PostType.POST + kwargs['text'] = tweet['full_text'] + kwargs['likes'] = tweet['favorite_count'] + kwargs['reposts'] = tweet['retweet_count'] + kwargs['quotes'] = tweet['quote_count'] + kwargs['comments'] = tweet['reply_count'] + if 'views' in data and 'count' in data['views']: + kwargs['views'] = int(data['views']['count']) + if 'urls' in tweet['entities']: + urls = [] + for url in tweet['entities']['urls']: + urls.append(url['expanded_url']) + kwargs['links'] = urls + media = {} + if 'media' in tweet['entities']: + for m in tweet['entities']['media']: + media_id = m['id_str'] if 'id_str' in m else str(m['id']) + media[media_id] = create_media(m) + if 'extended_entities' in tweet and 'media' in tweet['extended_entities']: + for m in tweet['extended_entities']['media']: + media_id = m['id_str'] if 'id_str' in m else str(m['id']) + media[media_id] = create_media(m) + kwargs['media'] = media + if 'quoted_status_result' in data: + if 'result' not in data['quoted_status_result']: + quoted_id = PostRef(f'twitter:t:{tweet['quoted_status_id_str']}') + else: + quoted_id = create_post_graphql(data['quoted_status_result']['result'], users, None, mtime, hash) + kwargs['quoted'] = quoted_id + elif 'quoted_status_id_str' in tweet: + log.write('quote id no data') + quoted_id = PostRef(f'twitter:t:{tweet['quoted_status_id_str']}') + kwargs['quoted'] = quoted_id + elif data.get('quotedRefResult'): + log.write('ref quote') + quoted = data['quotedRefResult']['result'] + if quoted['__typename'] == 'TweetWithVisibilityResults': + quoted = quoted['tweet'] + quoted_id = PostRef(f'twitter:t:{quoted['rest_id']}') + kwargs['quoted'] = quoted_id + if 'in_reply_to_status_id_str' in tweet: + kwargs['in_reply_to'] = PostRef(f'twitter:t:{tweet['in_reply_to_status_id_str']}') + process_post(Post(**kwargs)) + return PostRef(unique_id) + +def parse_graphql_raw_response(obj, mtime, hash): + users = {} + if 'user' in obj['data']: + instructions = obj['data']['user']['result']['timeline_v2']['timeline']['instructions'] + elif 'search_by_raw_query' in obj['data']: + instructions = obj['data']['search_by_raw_query']['search_timeline']['timeline']['instructions'] + else: + instructions = [] + for inst in instructions: + if inst['type'] == 'TimelineAddEntries': + for entry in inst['entries']: + if entry['entryId'].startswith('tweet-'): + tweet_id = int(entry['entryId'].split('-', 1)[1]) + if entry['content']['entryType'] == 'TimelineTimelineItem' and entry['content']['itemContent']['itemType'] == 'TimelineTweet': + if 'result' not in entry['content']['itemContent']['tweet_results']: + continue + create_post_graphql(entry['content']['itemContent']['tweet_results']['result'], users, tweet_id, mtime, hash) + else: + log.write('Got unrecognised timeline tweet item(s)') # snscrape copy and paste + elif entry['entryId'].startswith(('homeConversation-', 'profile-conversation-')): + if entry['content']['entryType'] == 'TimelineTimelineModule': + for item in reversed(entry['content']['items']): + if not item['entryId'].startswith(entry['entryId'].split('ion-', 1)[0] + 'ion-') or '-tweet-' not in item['entryId']: + log.write(f'Unexpected conversation entry ID: {item['entryId']!r}') # snscrape copy and paste + continue + if item['item']['itemContent']['itemType'] == 'TimelineTweet': + tweet_id = int(item['entryId'].split('-tweet-', 1)[1]) + if 'result' in item['item']['itemContent']['tweet_results']: + create_post_graphql(item['item']['itemContent']['tweet_results']['result'], users, tweet_id, mtime, hash) + else: + kwargs = {} + unique_id = f'twitter:t:{tweet_id}' + kwargs['unique_id'] = unique_id + kwargs['raw_responses'] = {} + kwargs['raw_responses']['hash'] = hash + kwargs['type'] = PostType.TOMBSTONE + process_post(Post(**kwargs)) + +def parse_raw_response(path, hash): + mtime = os.path.getmtime(path) + with gzip.open(path, 'rb') as f: + obj = json.loads(f.read()) + if 'globalObjects' not in obj: + parse_graphql_raw_response(obj, mtime, hash) + else: + parse_v1_raw_response(obj, mtime, hash) + +#target = '/mnt/store/camu_db/twitter/raw_responses' +target = '/mnt/store/files/camu_db/data/twitter_orig/raw_responses' + +for file in os.listdir(target): + hash = file.split('.')[0] + print(hash) + parse_raw_response(f'{target}/{file}', hash) diff --git a/src/portal/tests/rewrite_post.py b/src/portal/tests/rewrite_post.py new file mode 100644 index 0000000..3bb152b --- /dev/null +++ b/src/portal/tests/rewrite_post.py @@ -0,0 +1,56 @@ +import sys +import os +import gzip +import json +import hashlib +sys.path.append('../py') +import config +#from post import PostEncoder, DateType +#from modules.pixiv_web import PixivWebBase, PixivWebModule +#from modules.twitter import TwitterScrapeBase, TwitterScrapeModule +from logger import Logger + +#module = PixivWebModule(config.PIXIV_SESSID) +#search = PixivWebBase(module) + +log = Logger('deleted_posts.log') + +''' +def rewrite_post(path): + print(f'Rewriting post @ {path}') + with gzip.open(path, 'rb') as f: + obj = json.loads(f.read()) + post = search.remake_post(obj['raw_responses'], obj['dates'][str(DateType.RETRIEVED.value)]) + if not post: + log.write(path) + return + #print(json.dumps(post, indent=4, cls=PostEncoder)) + with gzip.open(path, 'wb') as f: + f.write(json.dumps(post, cls=PostEncoder).encode('utf-8')) +''' + +def dump_raw_response(path, target): + print(f'Extracting raw_response from {path}') + with gzip.open(path, 'rb') as f: + obj = json.loads(f.read()) + if 'api' in obj['raw_responses']: + hasher = hashlib.new('sha256') + data = json.dumps(json.loads(obj['raw_responses']['api'])).encode('utf-8') + hasher.update(data) + output = f'{target}/{hasher.hexdigest()}.json.gz' + with gzip.open(output, 'wb+') as f: + f.write(data) + +target = sys.argv[1] +raw_responses = f'{target}/raw_responses' + +if not os.path.isdir(raw_responses): + os.mkdir(raw_responses) + +for user in os.listdir(target): + if user == 'raw_responses': + continue + for file in os.listdir(f'{target}/{user}'): + if file.split('.')[-1] == 'gz': + dump_raw_response(f'{target}/{user}/{file}', raw_responses) + #rewrite_post(f'{target}/{user}/{file}') diff --git a/src/portal/tests/test.py b/src/portal/tests/test.py new file mode 100644 index 0000000..cdf7178 --- /dev/null +++ b/src/portal/tests/test.py @@ -0,0 +1,47 @@ +import json +import httpx +import sys +sys.path.append('../py') +from base import Method +from post import DateType, PostEncoder +from modules import ALL_MODULES + +#ALL_MODULES['instagram'][0].init() +#search = ALL_MODULES['instagram'][0].search('opium_00pium') +#page = search.get_page(0) +#print(page) +#print(json.dumps(page, indent=4, cls=PostEncoder)) + +#module = ALL_MODULES['fanbox'][0] +#module = ALL_MODULES['pixiv_web'][0] +module = ALL_MODULES['instagram'][0] +#module = ALL_MODULES['twitter'][0] +module.init() + +search = module.search('opium_00pium') +page = search.get_page(1) +for unique_id in page: + post = module.get_item(unique_id) + print(json.dumps(post, indent=4, cls=PostEncoder)) + +#post = module.get_item(page[0]) +#for key in post.media.keys(): +# download_params = module.get_download(post.unique_id, key) +# for url in download_params['urls']: +# #response = module.do_request(Method.GET, url.url) +# r = httpx.get(url.url, headers=download_params['headers']) +# with open(f'test_{key}.{url.ext}', 'wb+') as f: +# f.write(r.content) + +#print(json.dumps(page, indent=4, cls=PostEncoder)) +#for i in range(0, len(post.media)): +# download_params = module.get_download(post.unique_id, i) +# r = httpx.get(download_params['url'], headers=download_params['headers']) +# if r.status_code != 200: +# print('error{}'.format(r.status_code)) +# else: +# with open('test{}.png'.format(i), 'wb+') as f: +# f.write(r.content) + +#print(json.dumps(page, indent=4, cls=PostEncoder)) +#print(page) diff --git a/src/portal/tests/unescape_json.py b/src/portal/tests/unescape_json.py new file mode 100644 index 0000000..eaf5194 --- /dev/null +++ b/src/portal/tests/unescape_json.py @@ -0,0 +1,10 @@ +import sys +import gzip +import json + +for path in sys.argv[1:]: + with gzip.open(path, 'rb') as f: + obj = json.loads(f.read().decode('utf-8')) + obj = json.loads(obj) + with gzip.open(path, 'wb') as f: + f.write(json.dumps(obj).encode('utf-8')) |