Cleanup python stuff

Signed-off-by: Andrew Opalach <andrew@akon.city>

12 files changed, 57 insertions, 154 deletions

diff --git a/src/portal/py/base.py b/src/portal/py/base.py
index 7869521..660843c 100644
--- a/src/portal/py/base.py
+++ b/src/portal/py/base.py
@@ -20,7 +20,7 @@ class Method(Enum):
     POST = "POST"
 
 class Search():
-    def __init__(self):
+    def __init__(self) -> None:
         self.pages: dict[int, list[str]] = {}
         self.completed: bool = False
         self.errored: bool = False
@@ -59,7 +59,7 @@ class Search():
         return self.pages[num]
 
 class Module():
-    def __init__(self):
+    def __init__(self) -> None:
         self.headers: dict[str, str] = {}
         self.cookies: dict[str, str] = {}
         self.mutex: threading.Lock = threading.Lock()
@@ -114,7 +114,7 @@ class Module():
             self.raw_responses[hash_str] = dump
             return hash_str
 
-    def add_to_map(self, unique_id: str, item: Post | User):
+    def add_to_map(self, unique_id: str, item: Post | User) -> None:
         with self.mutex:
             self.unique_id_map[unique_id] = item
             if len(self.unique_id_map) > 10240:
diff --git a/src/portal/py/log.py b/src/portal/py/log.py
index 8d52d0e..3109d5c 100644
--- a/src/portal/py/log.py
+++ b/src/portal/py/log.py
@@ -1,34 +1,34 @@
-class DefaultLogger():
-    def debug(self, message):
+class Logger():
+    def debug(self, message: str) -> None:
         print(message)
 
-    def info(self, message):
+    def info(self, message: str) -> None:
         print(message)
 
-    def warn(self, message):
+    def warn(self, message: str) -> None:
         print(message)
 
-    def error(self, message):
+    def error(self, message: str) -> None:
         print(message)
 
-LOGGER = DefaultLogger()
+LOGGER = Logger()
 
-def set_logger(logger):
+def set_logger(logger: Logger) -> None:
     global LOGGER
     LOGGER = logger
 
-def debug(message):
+def debug(message: str) -> None:
     global LOGGER
     LOGGER.debug(message)
 
-def info(message):
+def info(message: str) -> None:
     global LOGGER
     LOGGER.info(message)
 
-def warn(message):
+def warn(message: str) -> None:
     global LOGGER
     LOGGER.warn(message)
 
-def error(message):
+def error(message: str) -> None:
     global LOGGER
     LOGGER.error(message)
diff --git a/src/portal/py/modules/__init__.py b/src/portal/py/modules/__init__.py
index 1a9e9ca..7fe56b8 100644
--- a/src/portal/py/modules/__init__.py
+++ b/src/portal/py/modules/__init__.py
@@ -1,12 +1,12 @@
-import config
+#import config
 
 from modules.youtube import YoutubeModule
 #from modules.twitter import TwitterScrapeModule
 #from modules.pixiv_app import PixivAppModule
 #from modules.pixiv_web import PixivWebModule
-from modules.fanbox import FanboxModule
+#from modules.fanbox import FanboxModule
 #from modules.instagram import InstagramModule
-from modules.patreon import PatreonModule
+#from modules.patreon import PatreonModule
 
 ALL_MODULES = {
     'youtube': (YoutubeModule(), []),
diff --git a/src/portal/py/modules/fanbox.py b/src/portal/py/modules/fanbox.py
index bf55b9d..fefe1c2 100644
--- a/src/portal/py/modules/fanbox.py
+++ b/src/portal/py/modules/fanbox.py
@@ -10,7 +10,7 @@ from modules.common import reformat_pixiv_date, get_current_utc_time
 BASE_URL = "https://api.fanbox.cc"
 
 class FanboxBase(Search):
-    def __init__(self, userdata: Any):
+    def __init__(self, userdata: Any) -> None:
         super().__init__()
         self.module: FanboxModule = userdata
 
@@ -95,7 +95,7 @@ class FanboxBase(Search):
         return user
 
 class FanboxPost(FanboxBase):
-    def __init__(self, userdata: Any, arg: str):
+    def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
         try:
             self.id: int = int(arg)
@@ -112,7 +112,7 @@ class FanboxPost(FanboxBase):
         return True
 
 class FanboxUser(FanboxBase):
-    def __init__(self, userdata: Any, arg: str):
+    def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
         self.id = arg
         self.pagination = []
@@ -147,7 +147,7 @@ class FanboxUser(FanboxBase):
         return True
 
 class FanboxSupporting(FanboxBase):
-    def __init__(self, userdata: Any, arg: str):
+    def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
 
     def request_page(self, num: int) -> bool:
@@ -163,7 +163,7 @@ class FanboxSupporting(FanboxBase):
         return True
 
 class FanboxModule(Module):
-    def __init__(self, sessid: str):
+    def __init__(self, sessid: str) -> None:
         super().__init__()
         self.parser: QueryParser = QueryParser(self, 'post')
         self.parser.add_command('post', FanboxPost)
diff --git a/src/portal/py/modules/instagram.py b/src/portal/py/modules/instagram.py
index 28f7def..553ce56 100644
--- a/src/portal/py/modules/instagram.py
+++ b/src/portal/py/modules/instagram.py
@@ -11,7 +11,7 @@ from instagrapi.exceptions import UserNotFound, LoginRequired, ChallengeRequired
 class InstagramSearch(Search):
     REACH_LIMIT = 3
 
-    def __init__(self, module: Any, pk: str):
+    def __init__(self, module: Any, pk: str) -> None:
         super().__init__()
         self.module: InstagramModule = module
         self.pk: str = pk
@@ -92,7 +92,7 @@ class InstagramSearch(Search):
         return request_satisfied
 
 class InstagramModule(Module):
-    def __init__(self, user_agent: str, settings: Path, session_id: str):
+    def __init__(self, user_agent: str, settings: Path, session_id: str) -> None:
         super().__init__()
         self.cl: Client = Client()
         # https://specdevice.com/showspec.php?id=c318-0c39-0033-c5870033c587
diff --git a/src/portal/py/modules/patreon.py b/src/portal/py/modules/patreon.py
index f33ec18..3c0efd2 100644
--- a/src/portal/py/modules/patreon.py
+++ b/src/portal/py/modules/patreon.py
@@ -1,6 +1,5 @@
 import log
 import httpx
-import sys
 from json import JSONDecodeError
 from typing import Optional, Any
 from datetime import datetime
@@ -14,7 +13,7 @@ BASE_URL = 'https://www.patreon.com/api'
 class PatreonBase(Search):
     REACH_LIMIT = 3
 
-    def __init__(self, userdata: Any):
+    def __init__(self, userdata: Any) -> None:
         super().__init__()
         self.module: PatreonModule = userdata
         self.page: int = 0
@@ -56,7 +55,7 @@ class PatreonBase(Search):
         return None, None
 
 class PatreonUser(PatreonBase):
-    def __init__(self, userdata: Any, arg: str):
+    def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
         self.username: str = arg
         self.user: Optional[User] = None
@@ -161,7 +160,7 @@ class PatreonUser(PatreonBase):
         return request_satisfied
 
 class PatreonModule(Module):
-    def __init__(self, session_id: str, uuid: str, user_id: str):
+    def __init__(self, session_id: str, uuid: str, user_id: str) -> None:
         super().__init__()
         self.user_id: str = user_id
         self.parser: QueryParser = QueryParser(self, 'user')
diff --git a/src/portal/py/modules/pixiv_app.py b/src/portal/py/modules/pixiv_app.py
index ef5886d..02d5254 100644
--- a/src/portal/py/modules/pixiv_app.py
+++ b/src/portal/py/modules/pixiv_app.py
@@ -7,7 +7,7 @@ from pixivpy3 import AppPixivAPI
 from modules.common import get_current_utc_time, reformat_pixiv_date
 
 class PixivAppBase(Search):
-    def __init__(self, userdata: Any):
+    def __init__(self, userdata: Any) -> None:
         super().__init__()
         self.module: PixivAppModule = userdata
         self.page: int = 0
@@ -86,7 +86,7 @@ class PixivAppBase(Search):
         return user
 
 class PixivAppSearch(PixivAppBase):
-    def __init__(self, userdata: Any, arg: str):
+    def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
         self.next_qs: Optional[dict[str, Any]] = {}
         self.next_qs['word'] = arg
@@ -106,7 +106,7 @@ class PixivAppSearch(PixivAppBase):
         return True, self.module.api.parse_qs(obj['next_url'])
 
 class PixivAppIllust(PixivAppBase):
-    def __init__(self, userdata: Any, arg: str):
+    def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
         self.next_qs: Optional[dict[str, Any]] = {}
         self.next_qs['illust_id'] = arg
@@ -121,7 +121,7 @@ class PixivAppIllust(PixivAppBase):
         return True, None
 
 class PixivAppUser(PixivAppBase):
-    def __init__(self, userdata: Any, arg: str):
+    def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
         self.next_qs: Optional[dict[str, Any]] = {}
         self.next_qs['user_id'] = arg
@@ -138,7 +138,7 @@ class PixivAppUser(PixivAppBase):
         return True, self.module.api.parse_qs(obj['next_url'])
 
 class PixivAppBookmarks(PixivAppBase):
-    def __init__(self, userdata: Any, arg: str):
+    def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
         self.next_qs: Optional[dict[str, Any]] = {}
         self.next_qs['user_id'] = arg
@@ -155,7 +155,7 @@ class PixivAppBookmarks(PixivAppBase):
         return True, self.module.api.parse_qs(obj['next_url'])
 
 class PixivAppFollowing(PixivAppBase):
-    def __init__(self, userdata: Any, arg: str):
+    def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
         self.next_qs: Optional[dict[str, Any]] = {}
         self.next_qs['user_id'] = arg
@@ -172,7 +172,7 @@ class PixivAppFollowing(PixivAppBase):
         return True, self.module.api.parse_qs(obj['next_url'])
 
 class PixivAppModule(Module):
-    def __init__(self, refresh_token: str):
+    def __init__(self, refresh_token: str) -> None:
         super().__init__()
         self.parser: QueryParser = QueryParser(self, 'search')
         self.parser.add_command('search', PixivAppSearch)
diff --git a/src/portal/py/modules/pixiv_web.py b/src/portal/py/modules/pixiv_web.py
index 7dbb8bb..33daeb3 100644
--- a/src/portal/py/modules/pixiv_web.py
+++ b/src/portal/py/modules/pixiv_web.py
@@ -14,7 +14,7 @@ LANG = 'en'
 VERSION = '5f53980f6b59c9376220b6d86e8feb5ea9e22e10'
 
 class PixivWebBase(Search):
-    def __init__(self, userdata: Any):
+    def __init__(self, userdata: Any) -> None:
         super().__init__()
         self.module: PixivWebModule = userdata
 
@@ -62,7 +62,7 @@ class PixivWebBase(Search):
                 tag.alts['en'] = data['translation']['en']
         return tag
 
-    def create_post(self, data: ParsedJson, pages: Optional[ParsedJson]=None, ugoira: Optional[ParsedJson]=None) -> Optional[Post]:
+    def create_post(self, data: ParsedJson, pages: Optional[ParsedJson] = None, ugoira: Optional[ParsedJson] = None) -> Optional[Post]:
         kwargs = {}
         kwargs['type'] = PostType.POST
         illust_id = data['illustId']
@@ -142,7 +142,7 @@ class PixivWebBase(Search):
         return user
 
 class PixivWebSearch(PixivWebBase):
-    def __init__(self, userdata: Any, arg: str):
+    def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
         self.query: str = arg
         self.last_page: Optional[int] = None
@@ -169,7 +169,7 @@ class PixivWebSearch(PixivWebBase):
         return True
 
 class PixivWebIllust(PixivWebBase):
-    def __init__(self, userdata: Any, arg: str):
+    def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
         try:
             self.id: int = int(arg)
@@ -186,7 +186,7 @@ class PixivWebIllust(PixivWebBase):
         return True
 
 class PixivWebUser(PixivWebBase):
-    def __init__(self, userdata: Any, arg: str):
+    def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
         try:
             self.id: int = int(arg)
@@ -217,7 +217,7 @@ class PixivWebUser(PixivWebBase):
 class PixivWebFollowing(PixivWebBase):
     LIMIT = 24
 
-    def __init__(self, userdata: Any, arg: str):
+    def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
         try:
             self.id: int = int(arg)
@@ -236,7 +236,7 @@ class PixivWebFollowing(PixivWebBase):
         return True
 
 class PixivWebModule(Module):
-    def __init__(self, sessid: str, user_id: str):
+    def __init__(self, sessid: str, user_id: str) -> None:
         super().__init__()
         self.parser: QueryParser = QueryParser(self, 'search')
         self.parser.add_command('search', PixivWebSearch)
diff --git a/src/portal/py/modules/searx.py b/src/portal/py/modules/searx.py
deleted file mode 100644
index 4cc0b04..0000000
--- a/src/portal/py/modules/searx.py
+++ /dev/null
@@ -1,95 +0,0 @@
-import sys
-import json
-
-from base import Module, Search
-from post import Post, Image
-
-sys.path.append('../../vendor/searxng')
-from searx import settings
-from searx.utils import  gen_useragent
-from searx.engines import load_engine, register_engine
-
-# TODO: Put these on the Provider object.
-ENGINES = {}
-
-ENGINE_INITIAL_INDEX = {
-    'google images': 0,
-    'google': 1,
-    'bing images': 0
-}
-
-class SearxSearch(Search):
-    def __init__(self, plugin, query, engine):
-        super().__init__()
-        global ENGINES
-        self.query = query
-        self.plugin = plugin
-        self.key = engine.replace(' ', '_')
-        self.engine = ENGINES[engine]
-        self.index = ENGINE_INITIAL_INDEX[engine]
-
-    def load_page(self, index):
-        self.pages[index] = []
-        res = self.plugin.send_http_request(self.engine.request(self.query, {
-            'language': 'en-US',
-            'safesearch': 0,
-            'time_range': None,
-            'pageno': self.index + index,
-            'cookies': self.plugin.cookies,
-            'headers': self.plugin.headers,
-            'data': None
-        }))
-        if not res:
-            return False
-        try:
-            ret = self.engine.response(res)
-        except:
-            return False
-        if len(ret) == 0:
-            # TODO: Test.
-            self.completed = True
-            return False
-        for i, p in enumerate(ret):
-            if 'url' not in p:
-                continue
-            # Don't add this to the global map because it's not actually unique.
-            unique_id = '{}:{}_{}:{}'.format(self.key, self.query, i, index)
-            media = []
-            if 'img_src' in p:
-                media.append(Image(url=p['img_src'], thumbnail_url=''))
-            elif 'image_url' in p:
-                media.append(Image(url=p['image_url'], thumbnail_url=''))
-            kwargs = {}
-            kwargs['unique_id'] = unique_id
-            kwargs['raw_responses'] = {}
-            kwargs['raw_responses']['result'] = json.dumps(p)
-            kwargs['url'] = p['url']
-            kwargs['title'] = p['title']
-            kwargs['text'] = p['content']
-            kwargs['media'] = media
-            self.pages[index].append(Post(**kwargs))
-        return True
-
-class SearxModule(Module):
-    def __init__(self):
-        super().__init__()
-        global ENGINES
-        self.headers = {
-            'User-Agent': gen_useragent(),
-            'Accept-Language': 'en-US,en;q=0.5',
-        }
-        for engine_data in settings['engines']:
-            engine = load_engine(engine_data)
-            if engine:
-                register_engine(engine)
-                ENGINES[engine.name] = engine
-
-    def send_http_request(self, params):
-        self.cookies.update(params['cookies'])
-        self.headers.update(params['headers'])
-        return self.get(params['url'], params=params['data'])
-
-    def search(self, query, *extra_args):
-        if not extra_args:
-            return None
-        return SearxSearch(self, query, extra_args[0])
diff --git a/src/portal/py/modules/youtube.py b/src/portal/py/modules/youtube.py
index 9645ada..16d7f67 100644
--- a/src/portal/py/modules/youtube.py
+++ b/src/portal/py/modules/youtube.py
@@ -7,19 +7,19 @@ from query_parser import QueryParser
 from yt_dlp import YoutubeDL
 
 class YDLLogger():
-    def debug(self, message):
+    def debug(self, message: str) -> None:
         if message.startswith('[debug] '):
             log.debug(message)
         else:
             log.info(message)
 
-    def info(self, message):
+    def info(self, message: str) -> None:
         log.info(message)
 
-    def warning(self, message):
+    def warning(self, message: str) -> None:
         log.warn(message)
 
-    def error(self, message):
+    def error(self, message: str) -> None:
         log.error(message)
 
 ydl_opts = {
@@ -34,9 +34,8 @@ ydl_opts = {
 
 ydl = YoutubeDL(ydl_opts)
 
-def get_playback_url(data, video=True) -> Optional[str]:
-    data['formats'] = list(filter(
-        lambda f: not f['protocol'].startswith('m3u8'), data['formats']))
+def get_playback_url(data: ParsedJson, video: bool = True) -> Optional[str]:
+    data['formats'] = list(filter(lambda f: not f['protocol'].startswith('m3u8'), data['formats']))
 
     if len(data['formats']) == 0:
         return None
@@ -69,7 +68,7 @@ def get_playback_url(data, video=True) -> Optional[str]:
     return result
 
 class YoutubeBase(Search):
-    def __init__(self, userdata: Any):
+    def __init__(self, userdata: Any) -> None:
         super().__init__()
         self.module: YoutubeModule = userdata
         self.guessed_index: int = 0
@@ -92,7 +91,7 @@ class YoutubeBase(Search):
             info = self.get_info()
             if not info:
                 return False
-        if 'direct' in info and info['direct']:
+        if info.get('direct'):
             url = info['url']
         else:
             url = get_playback_url(info, video=False)
@@ -105,7 +104,7 @@ class YoutubeBase(Search):
         return True
 
 class YoutubeSearch(YoutubeBase):
-    def __init__(self, userdata: Any, arg: str):
+    def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
         self.query: str = arg
 
@@ -119,7 +118,7 @@ class YoutubeSearch(YoutubeBase):
         return info
 
 class YoutubeLink(YoutubeBase):
-    def __init__(self, userdata: Any, arg: str):
+    def __init__(self, userdata: Any, arg: str) -> None:
         super().__init__(userdata)
         self.link: str = arg
 
@@ -144,7 +143,7 @@ class YoutubeLink(YoutubeBase):
         return info
 
 class YoutubeModule(Module):
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__()
         self.parser: QueryParser = QueryParser(self, 'search')
         self.parser.add_command('search', YoutubeSearch)
diff --git a/src/portal/py/post.py b/src/portal/py/post.py
index 76e82b9..ca7cf4e 100644
--- a/src/portal/py/post.py
+++ b/src/portal/py/post.py
@@ -53,7 +53,7 @@ class TagType(IntEnum):
 def df(c: Any) -> Any:
     return field(default_factory=lambda: c)
 
-def default_url_ext_guess(url):
+def default_url_ext_guess(url: str) -> str:
     # TODO: List of known extensions.
     question = url.rfind('?')
     if question >= 0:
@@ -67,7 +67,7 @@ class Url():
     url: str
     ext: str
 
-    def __init__(self, url: str, ext: str = 'unknown'):
+    def __init__(self, url: str, ext: str = 'unknown') -> None:
         self.url = url
         ext_guess = default_url_ext_guess(url)
         if ext_guess:
@@ -185,5 +185,5 @@ class Post():
     in_reply_to: PostRef = df(PostRef())
 
 class PostEncoder(JSONEncoder):
-    def default(self, o):
+    def default(self, o: Any) -> Any:
         return o.__dict__
diff --git a/src/portal/py/query_parser.py b/src/portal/py/query_parser.py
index c16ded2..cd6c1e6 100644
--- a/src/portal/py/query_parser.py
+++ b/src/portal/py/query_parser.py
@@ -4,13 +4,13 @@ from base import Search
 type QueryCommand = Callable[[Any, Optional[str]], Search]
 
 class QueryParser():
-    def __init__(self, userdata: Any, default_command: str):
+    def __init__(self, userdata: Any, default_command: str) -> Any:
         self.commands: dict[str, QueryCommand] = {}
         self.escaped_commands: dict[str, str] = {}
         self.userdata: Any = userdata
         self.default_command: str = default_command
 
-    def add_command(self, name, search) -> None:
+    def add_command(self, name: str, search: QueryCommand) -> None:
         self.commands[name] = search
         self.escaped_commands['\\' + name] = name