summaryrefslogtreecommitdiff
path: root/src/portal
diff options
context:
space:
mode:
Diffstat (limited to 'src/portal')
-rw-r--r--src/portal/cpy/portal.pyx2
-rw-r--r--src/portal/py/base.py6
-rw-r--r--src/portal/py/log.py22
-rw-r--r--src/portal/py/modules/__init__.py6
-rw-r--r--src/portal/py/modules/fanbox.py10
-rw-r--r--src/portal/py/modules/instagram.py4
-rw-r--r--src/portal/py/modules/patreon.py7
-rw-r--r--src/portal/py/modules/pixiv_app.py14
-rw-r--r--src/portal/py/modules/pixiv_web.py14
-rw-r--r--src/portal/py/modules/searx.py95
-rw-r--r--src/portal/py/modules/youtube.py23
-rw-r--r--src/portal/py/post.py6
-rw-r--r--src/portal/py/query_parser.py4
-rwxr-xr-xsrc/portal/scripts/create_vendor.sh10
-rw-r--r--src/portal/src/search.c2
15 files changed, 62 insertions, 163 deletions
diff --git a/src/portal/cpy/portal.pyx b/src/portal/cpy/portal.pyx
index 28af444..d6909f3 100644
--- a/src/portal/cpy/portal.pyx
+++ b/src/portal/cpy/portal.pyx
@@ -41,7 +41,7 @@ sys.stderr = hook
import log
-class PortalLogger():
+class PortalLogger(log.Logger):
def debug(self, message):
log_debug(encode_str(message))
diff --git a/src/portal/py/base.py b/src/portal/py/base.py
index 7869521..660843c 100644
--- a/src/portal/py/base.py
+++ b/src/portal/py/base.py
@@ -20,7 +20,7 @@ class Method(Enum):
POST = "POST"
class Search():
- def __init__(self):
+ def __init__(self) -> None:
self.pages: dict[int, list[str]] = {}
self.completed: bool = False
self.errored: bool = False
@@ -59,7 +59,7 @@ class Search():
return self.pages[num]
class Module():
- def __init__(self):
+ def __init__(self) -> None:
self.headers: dict[str, str] = {}
self.cookies: dict[str, str] = {}
self.mutex: threading.Lock = threading.Lock()
@@ -114,7 +114,7 @@ class Module():
self.raw_responses[hash_str] = dump
return hash_str
- def add_to_map(self, unique_id: str, item: Post | User):
+ def add_to_map(self, unique_id: str, item: Post | User) -> None:
with self.mutex:
self.unique_id_map[unique_id] = item
if len(self.unique_id_map) > 10240:
diff --git a/src/portal/py/log.py b/src/portal/py/log.py
index 8d52d0e..3109d5c 100644
--- a/src/portal/py/log.py
+++ b/src/portal/py/log.py
@@ -1,34 +1,34 @@
-class DefaultLogger():
- def debug(self, message):
+class Logger():
+ def debug(self, message: str) -> None:
print(message)
- def info(self, message):
+ def info(self, message: str) -> None:
print(message)
- def warn(self, message):
+ def warn(self, message: str) -> None:
print(message)
- def error(self, message):
+ def error(self, message: str) -> None:
print(message)
-LOGGER = DefaultLogger()
+LOGGER = Logger()
-def set_logger(logger):
+def set_logger(logger: Logger) -> None:
global LOGGER
LOGGER = logger
-def debug(message):
+def debug(message: str) -> None:
global LOGGER
LOGGER.debug(message)
-def info(message):
+def info(message: str) -> None:
global LOGGER
LOGGER.info(message)
-def warn(message):
+def warn(message: str) -> None:
global LOGGER
LOGGER.warn(message)
-def error(message):
+def error(message: str) -> None:
global LOGGER
LOGGER.error(message)
diff --git a/src/portal/py/modules/__init__.py b/src/portal/py/modules/__init__.py
index 1a9e9ca..7fe56b8 100644
--- a/src/portal/py/modules/__init__.py
+++ b/src/portal/py/modules/__init__.py
@@ -1,12 +1,12 @@
-import config
+#import config
from modules.youtube import YoutubeModule
#from modules.twitter import TwitterScrapeModule
#from modules.pixiv_app import PixivAppModule
#from modules.pixiv_web import PixivWebModule
-from modules.fanbox import FanboxModule
+#from modules.fanbox import FanboxModule
#from modules.instagram import InstagramModule
-from modules.patreon import PatreonModule
+#from modules.patreon import PatreonModule
ALL_MODULES = {
'youtube': (YoutubeModule(), []),
diff --git a/src/portal/py/modules/fanbox.py b/src/portal/py/modules/fanbox.py
index bf55b9d..fefe1c2 100644
--- a/src/portal/py/modules/fanbox.py
+++ b/src/portal/py/modules/fanbox.py
@@ -10,7 +10,7 @@ from modules.common import reformat_pixiv_date, get_current_utc_time
BASE_URL = "https://api.fanbox.cc"
class FanboxBase(Search):
- def __init__(self, userdata: Any):
+ def __init__(self, userdata: Any) -> None:
super().__init__()
self.module: FanboxModule = userdata
@@ -95,7 +95,7 @@ class FanboxBase(Search):
return user
class FanboxPost(FanboxBase):
- def __init__(self, userdata: Any, arg: str):
+ def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
try:
self.id: int = int(arg)
@@ -112,7 +112,7 @@ class FanboxPost(FanboxBase):
return True
class FanboxUser(FanboxBase):
- def __init__(self, userdata: Any, arg: str):
+ def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
self.id = arg
self.pagination = []
@@ -147,7 +147,7 @@ class FanboxUser(FanboxBase):
return True
class FanboxSupporting(FanboxBase):
- def __init__(self, userdata: Any, arg: str):
+ def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
def request_page(self, num: int) -> bool:
@@ -163,7 +163,7 @@ class FanboxSupporting(FanboxBase):
return True
class FanboxModule(Module):
- def __init__(self, sessid: str):
+ def __init__(self, sessid: str) -> None:
super().__init__()
self.parser: QueryParser = QueryParser(self, 'post')
self.parser.add_command('post', FanboxPost)
diff --git a/src/portal/py/modules/instagram.py b/src/portal/py/modules/instagram.py
index 28f7def..553ce56 100644
--- a/src/portal/py/modules/instagram.py
+++ b/src/portal/py/modules/instagram.py
@@ -11,7 +11,7 @@ from instagrapi.exceptions import UserNotFound, LoginRequired, ChallengeRequired
class InstagramSearch(Search):
REACH_LIMIT = 3
- def __init__(self, module: Any, pk: str):
+ def __init__(self, module: Any, pk: str) -> None:
super().__init__()
self.module: InstagramModule = module
self.pk: str = pk
@@ -92,7 +92,7 @@ class InstagramSearch(Search):
return request_satisfied
class InstagramModule(Module):
- def __init__(self, user_agent: str, settings: Path, session_id: str):
+ def __init__(self, user_agent: str, settings: Path, session_id: str) -> None:
super().__init__()
self.cl: Client = Client()
# https://specdevice.com/showspec.php?id=c318-0c39-0033-c5870033c587
diff --git a/src/portal/py/modules/patreon.py b/src/portal/py/modules/patreon.py
index f33ec18..3c0efd2 100644
--- a/src/portal/py/modules/patreon.py
+++ b/src/portal/py/modules/patreon.py
@@ -1,6 +1,5 @@
import log
import httpx
-import sys
from json import JSONDecodeError
from typing import Optional, Any
from datetime import datetime
@@ -14,7 +13,7 @@ BASE_URL = 'https://www.patreon.com/api'
class PatreonBase(Search):
REACH_LIMIT = 3
- def __init__(self, userdata: Any):
+ def __init__(self, userdata: Any) -> None:
super().__init__()
self.module: PatreonModule = userdata
self.page: int = 0
@@ -56,7 +55,7 @@ class PatreonBase(Search):
return None, None
class PatreonUser(PatreonBase):
- def __init__(self, userdata: Any, arg: str):
+ def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
self.username: str = arg
self.user: Optional[User] = None
@@ -161,7 +160,7 @@ class PatreonUser(PatreonBase):
return request_satisfied
class PatreonModule(Module):
- def __init__(self, session_id: str, uuid: str, user_id: str):
+ def __init__(self, session_id: str, uuid: str, user_id: str) -> None:
super().__init__()
self.user_id: str = user_id
self.parser: QueryParser = QueryParser(self, 'user')
diff --git a/src/portal/py/modules/pixiv_app.py b/src/portal/py/modules/pixiv_app.py
index ef5886d..02d5254 100644
--- a/src/portal/py/modules/pixiv_app.py
+++ b/src/portal/py/modules/pixiv_app.py
@@ -7,7 +7,7 @@ from pixivpy3 import AppPixivAPI
from modules.common import get_current_utc_time, reformat_pixiv_date
class PixivAppBase(Search):
- def __init__(self, userdata: Any):
+ def __init__(self, userdata: Any) -> None:
super().__init__()
self.module: PixivAppModule = userdata
self.page: int = 0
@@ -86,7 +86,7 @@ class PixivAppBase(Search):
return user
class PixivAppSearch(PixivAppBase):
- def __init__(self, userdata: Any, arg: str):
+ def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
self.next_qs: Optional[dict[str, Any]] = {}
self.next_qs['word'] = arg
@@ -106,7 +106,7 @@ class PixivAppSearch(PixivAppBase):
return True, self.module.api.parse_qs(obj['next_url'])
class PixivAppIllust(PixivAppBase):
- def __init__(self, userdata: Any, arg: str):
+ def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
self.next_qs: Optional[dict[str, Any]] = {}
self.next_qs['illust_id'] = arg
@@ -121,7 +121,7 @@ class PixivAppIllust(PixivAppBase):
return True, None
class PixivAppUser(PixivAppBase):
- def __init__(self, userdata: Any, arg: str):
+ def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
self.next_qs: Optional[dict[str, Any]] = {}
self.next_qs['user_id'] = arg
@@ -138,7 +138,7 @@ class PixivAppUser(PixivAppBase):
return True, self.module.api.parse_qs(obj['next_url'])
class PixivAppBookmarks(PixivAppBase):
- def __init__(self, userdata: Any, arg: str):
+ def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
self.next_qs: Optional[dict[str, Any]] = {}
self.next_qs['user_id'] = arg
@@ -155,7 +155,7 @@ class PixivAppBookmarks(PixivAppBase):
return True, self.module.api.parse_qs(obj['next_url'])
class PixivAppFollowing(PixivAppBase):
- def __init__(self, userdata: Any, arg: str):
+ def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
self.next_qs: Optional[dict[str, Any]] = {}
self.next_qs['user_id'] = arg
@@ -172,7 +172,7 @@ class PixivAppFollowing(PixivAppBase):
return True, self.module.api.parse_qs(obj['next_url'])
class PixivAppModule(Module):
- def __init__(self, refresh_token: str):
+ def __init__(self, refresh_token: str) -> None:
super().__init__()
self.parser: QueryParser = QueryParser(self, 'search')
self.parser.add_command('search', PixivAppSearch)
diff --git a/src/portal/py/modules/pixiv_web.py b/src/portal/py/modules/pixiv_web.py
index 7dbb8bb..33daeb3 100644
--- a/src/portal/py/modules/pixiv_web.py
+++ b/src/portal/py/modules/pixiv_web.py
@@ -14,7 +14,7 @@ LANG = 'en'
VERSION = '5f53980f6b59c9376220b6d86e8feb5ea9e22e10'
class PixivWebBase(Search):
- def __init__(self, userdata: Any):
+ def __init__(self, userdata: Any) -> None:
super().__init__()
self.module: PixivWebModule = userdata
@@ -62,7 +62,7 @@ class PixivWebBase(Search):
tag.alts['en'] = data['translation']['en']
return tag
- def create_post(self, data: ParsedJson, pages: Optional[ParsedJson]=None, ugoira: Optional[ParsedJson]=None) -> Optional[Post]:
+ def create_post(self, data: ParsedJson, pages: Optional[ParsedJson] = None, ugoira: Optional[ParsedJson] = None) -> Optional[Post]:
kwargs = {}
kwargs['type'] = PostType.POST
illust_id = data['illustId']
@@ -142,7 +142,7 @@ class PixivWebBase(Search):
return user
class PixivWebSearch(PixivWebBase):
- def __init__(self, userdata: Any, arg: str):
+ def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
self.query: str = arg
self.last_page: Optional[int] = None
@@ -169,7 +169,7 @@ class PixivWebSearch(PixivWebBase):
return True
class PixivWebIllust(PixivWebBase):
- def __init__(self, userdata: Any, arg: str):
+ def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
try:
self.id: int = int(arg)
@@ -186,7 +186,7 @@ class PixivWebIllust(PixivWebBase):
return True
class PixivWebUser(PixivWebBase):
- def __init__(self, userdata: Any, arg: str):
+ def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
try:
self.id: int = int(arg)
@@ -217,7 +217,7 @@ class PixivWebUser(PixivWebBase):
class PixivWebFollowing(PixivWebBase):
LIMIT = 24
- def __init__(self, userdata: Any, arg: str):
+ def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
try:
self.id: int = int(arg)
@@ -236,7 +236,7 @@ class PixivWebFollowing(PixivWebBase):
return True
class PixivWebModule(Module):
- def __init__(self, sessid: str, user_id: str):
+ def __init__(self, sessid: str, user_id: str) -> None:
super().__init__()
self.parser: QueryParser = QueryParser(self, 'search')
self.parser.add_command('search', PixivWebSearch)
diff --git a/src/portal/py/modules/searx.py b/src/portal/py/modules/searx.py
deleted file mode 100644
index 4cc0b04..0000000
--- a/src/portal/py/modules/searx.py
+++ /dev/null
@@ -1,95 +0,0 @@
-import sys
-import json
-
-from base import Module, Search
-from post import Post, Image
-
-sys.path.append('../../vendor/searxng')
-from searx import settings
-from searx.utils import gen_useragent
-from searx.engines import load_engine, register_engine
-
-# TODO: Put these on the Provider object.
-ENGINES = {}
-
-ENGINE_INITIAL_INDEX = {
- 'google images': 0,
- 'google': 1,
- 'bing images': 0
-}
-
-class SearxSearch(Search):
- def __init__(self, plugin, query, engine):
- super().__init__()
- global ENGINES
- self.query = query
- self.plugin = plugin
- self.key = engine.replace(' ', '_')
- self.engine = ENGINES[engine]
- self.index = ENGINE_INITIAL_INDEX[engine]
-
- def load_page(self, index):
- self.pages[index] = []
- res = self.plugin.send_http_request(self.engine.request(self.query, {
- 'language': 'en-US',
- 'safesearch': 0,
- 'time_range': None,
- 'pageno': self.index + index,
- 'cookies': self.plugin.cookies,
- 'headers': self.plugin.headers,
- 'data': None
- }))
- if not res:
- return False
- try:
- ret = self.engine.response(res)
- except:
- return False
- if len(ret) == 0:
- # TODO: Test.
- self.completed = True
- return False
- for i, p in enumerate(ret):
- if 'url' not in p:
- continue
- # Don't add this to the global map because it's not actually unique.
- unique_id = '{}:{}_{}:{}'.format(self.key, self.query, i, index)
- media = []
- if 'img_src' in p:
- media.append(Image(url=p['img_src'], thumbnail_url=''))
- elif 'image_url' in p:
- media.append(Image(url=p['image_url'], thumbnail_url=''))
- kwargs = {}
- kwargs['unique_id'] = unique_id
- kwargs['raw_responses'] = {}
- kwargs['raw_responses']['result'] = json.dumps(p)
- kwargs['url'] = p['url']
- kwargs['title'] = p['title']
- kwargs['text'] = p['content']
- kwargs['media'] = media
- self.pages[index].append(Post(**kwargs))
- return True
-
-class SearxModule(Module):
- def __init__(self):
- super().__init__()
- global ENGINES
- self.headers = {
- 'User-Agent': gen_useragent(),
- 'Accept-Language': 'en-US,en;q=0.5',
- }
- for engine_data in settings['engines']:
- engine = load_engine(engine_data)
- if engine:
- register_engine(engine)
- ENGINES[engine.name] = engine
-
- def send_http_request(self, params):
- self.cookies.update(params['cookies'])
- self.headers.update(params['headers'])
- return self.get(params['url'], params=params['data'])
-
- def search(self, query, *extra_args):
- if not extra_args:
- return None
- return SearxSearch(self, query, extra_args[0])
diff --git a/src/portal/py/modules/youtube.py b/src/portal/py/modules/youtube.py
index 9645ada..16d7f67 100644
--- a/src/portal/py/modules/youtube.py
+++ b/src/portal/py/modules/youtube.py
@@ -7,19 +7,19 @@ from query_parser import QueryParser
from yt_dlp import YoutubeDL
class YDLLogger():
- def debug(self, message):
+ def debug(self, message: str) -> None:
if message.startswith('[debug] '):
log.debug(message)
else:
log.info(message)
- def info(self, message):
+ def info(self, message: str) -> None:
log.info(message)
- def warning(self, message):
+ def warning(self, message: str) -> None:
log.warn(message)
- def error(self, message):
+ def error(self, message: str) -> None:
log.error(message)
ydl_opts = {
@@ -34,9 +34,8 @@ ydl_opts = {
ydl = YoutubeDL(ydl_opts)
-def get_playback_url(data, video=True) -> Optional[str]:
- data['formats'] = list(filter(
- lambda f: not f['protocol'].startswith('m3u8'), data['formats']))
+def get_playback_url(data: ParsedJson, video: bool = True) -> Optional[str]:
+ data['formats'] = list(filter(lambda f: not f['protocol'].startswith('m3u8'), data['formats']))
if len(data['formats']) == 0:
return None
@@ -69,7 +68,7 @@ def get_playback_url(data, video=True) -> Optional[str]:
return result
class YoutubeBase(Search):
- def __init__(self, userdata: Any):
+ def __init__(self, userdata: Any) -> None:
super().__init__()
self.module: YoutubeModule = userdata
self.guessed_index: int = 0
@@ -92,7 +91,7 @@ class YoutubeBase(Search):
info = self.get_info()
if not info:
return False
- if 'direct' in info and info['direct']:
+ if info.get('direct'):
url = info['url']
else:
url = get_playback_url(info, video=False)
@@ -105,7 +104,7 @@ class YoutubeBase(Search):
return True
class YoutubeSearch(YoutubeBase):
- def __init__(self, userdata: Any, arg: str):
+ def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
self.query: str = arg
@@ -119,7 +118,7 @@ class YoutubeSearch(YoutubeBase):
return info
class YoutubeLink(YoutubeBase):
- def __init__(self, userdata: Any, arg: str):
+ def __init__(self, userdata: Any, arg: str) -> None:
super().__init__(userdata)
self.link: str = arg
@@ -144,7 +143,7 @@ class YoutubeLink(YoutubeBase):
return info
class YoutubeModule(Module):
- def __init__(self):
+ def __init__(self) -> None:
super().__init__()
self.parser: QueryParser = QueryParser(self, 'search')
self.parser.add_command('search', YoutubeSearch)
diff --git a/src/portal/py/post.py b/src/portal/py/post.py
index 76e82b9..ca7cf4e 100644
--- a/src/portal/py/post.py
+++ b/src/portal/py/post.py
@@ -53,7 +53,7 @@ class TagType(IntEnum):
def df(c: Any) -> Any:
return field(default_factory=lambda: c)
-def default_url_ext_guess(url):
+def default_url_ext_guess(url: str) -> str:
# TODO: List of known extensions.
question = url.rfind('?')
if question >= 0:
@@ -67,7 +67,7 @@ class Url():
url: str
ext: str
- def __init__(self, url: str, ext: str = 'unknown'):
+ def __init__(self, url: str, ext: str = 'unknown') -> None:
self.url = url
ext_guess = default_url_ext_guess(url)
if ext_guess:
@@ -185,5 +185,5 @@ class Post():
in_reply_to: PostRef = df(PostRef())
class PostEncoder(JSONEncoder):
- def default(self, o):
+ def default(self, o: Any) -> Any:
return o.__dict__
diff --git a/src/portal/py/query_parser.py b/src/portal/py/query_parser.py
index c16ded2..cd6c1e6 100644
--- a/src/portal/py/query_parser.py
+++ b/src/portal/py/query_parser.py
@@ -4,13 +4,13 @@ from base import Search
type QueryCommand = Callable[[Any, Optional[str]], Search]
class QueryParser():
- def __init__(self, userdata: Any, default_command: str):
+ def __init__(self, userdata: Any, default_command: str) -> Any:
self.commands: dict[str, QueryCommand] = {}
self.escaped_commands: dict[str, str] = {}
self.userdata: Any = userdata
self.default_command: str = default_command
- def add_command(self, name, search) -> None:
+ def add_command(self, name: str, search: QueryCommand) -> None:
self.commands[name] = search
self.escaped_commands['\\' + name] = name
diff --git a/src/portal/scripts/create_vendor.sh b/src/portal/scripts/create_vendor.sh
index 6fdf9e0..414a4c2 100755
--- a/src/portal/scripts/create_vendor.sh
+++ b/src/portal/scripts/create_vendor.sh
@@ -12,10 +12,6 @@ cd yt-dlp/
pip3 install . --upgrade --target=../vendor
cd ../
-#cd snscrape/
-#pip3 install . --upgrade --target=../vendor
-#cd ../
-
cd twitter-api-client/
pip3 install . --upgrade --target=../vendor
cd ../
@@ -32,9 +28,9 @@ cd CyberDropDownloader/
pip3 install . --upgrade --target=../vendor
cd ../
-#cd gallery-dl/
-#pip3 install . --upgrade --target=../vendor
-#cd ../
+cd gallery-dl/
+pip3 install . --upgrade --target=../vendor
+cd ../
cd ../
rm -r venv
diff --git a/src/portal/src/search.c b/src/portal/src/search.c
index e6022f5..46ff0eb 100644
--- a/src/portal/src/search.c
+++ b/src/portal/src/search.c
@@ -58,7 +58,7 @@ static aki_thread_result AKI_THREADCALL queue_thread(void *userdata)
}
struct camu_portal_cmd *cmd;
al_array_foreach_ptr(bridge->queue, i, cmd) {
- struct camu_portal_result result;
+ struct camu_portal_result result = { 0 };
result.op = cmd->op;
result.callback = cmd->callback;
result.userdata = cmd->userdata;
generated by cgit v1.2.3-101-g0448 (git 2.54.0) at 2026-07-25 17:10:57 +0000