import log import time import json import tls_client import binascii import threading import config from enum import Enum from blake3 import blake3 from collections import OrderedDict from typing import Optional, Self, Any from post import Post, User type ParsedJson = Any type HttpResponse = tls_client.Response USER_AGENT = config.USER_AGENT class Method(Enum): HEAD = 'HEAD' GET = 'GET' POST = 'POST' class Search(): def __init__(self) -> None: self.pages: dict[int, list[str]] = {} self.completed: bool = False self.errored: bool = False self.iter_page: int = 0 self.iter_index: int = 0 def __iter__(self) -> Self: self.iter_page = 0 self.iter_index = 0 return self def __next__(self) -> str: page = self.get_page(self.iter_page) if not page: raise StopIteration unique_id = page[self.iter_index] self.iter_index += 1 if self.iter_index >= len(page): self.iter_page += 1 self.iter_index = 0 return unique_id def request_page(self, num: int) -> bool: return False def get_page(self, num: int) -> Optional[list[str]]: if self.errored: return None if num not in self.pages: if self.completed: return None if not self.request_page(num): if not self.completed: self.errored = True return None return self.pages[num] class Module(): def __init__(self) -> None: self.headers: dict[str, str] = {} self.cookies: dict[str, str] = {} self.mutex: threading.Lock = threading.Lock() self.raw_responses: dict[str, str] = {} self.unique_id_map: OrderedDict = OrderedDict() self.session = tls_client.Session(client_identifier=config.TLS_CLIENT_IDENTIFIER, random_tls_extension_order=True) def init(self) -> bool: return True def do_request(self, method: Method, url: str, params: Optional[dict[str, str]] = None, retries: int = 2) -> Optional[HttpResponse]: log.debug(f'HTTP {method.value} request {url} {params}.') response: Optional[HttpResponse] = None for _ in range(0, retries + 1): try: response = self.session.execute_request(method.value, url, headers=self.headers, cookies=self.cookies, params=params, timeout=600) except Exception as e: log.info(f'Connection exception ({repr(e)}), retrying in 20 seconds...') time.sleep(20) continue if response is None: log.error('Invalid None response from requests library.') break if response.status_code == 429: log.info('Got too many requests, waiting for 4 minutes...') time.sleep(60 * 4) response = None retries += 1 continue if response.status_code == 404 or response.status_code == 403: # Shortcut 404, 403 log.warn(f'Returning None, {response.status_code}') return None if int(response.status_code / 100) != 2: # Non-200 response log.error(f'Non-200 status code ({response.status_code}), retrying in 5 seconds...') time.sleep(5) response = None continue ''' try: response.read() except Exception as e: log.info(f'Read exception ({e}), retrying in 20 seconds...') time.sleep(20) response = None continue else: break ''' break return response def add_raw_response(self, data: ParsedJson) -> str: with self.mutex: dump = json.dumps(data, separators=(',', ':'), indent=None) hash_str = binascii.hexlify(blake3(dump.encode('utf-8')).digest()).decode('ascii') self.raw_responses[hash_str] = dump return hash_str def add_to_map(self, unique_id: str, item: Post | User) -> None: with self.mutex: self.unique_id_map[unique_id] = item if len(self.unique_id_map) > 10240: self.unique_id_map.popitem(last=False) def search(self, query: str, *extra_args: Any) -> Optional[Search]: return None def get_item(self, unique_id: str) -> Optional[Post | User]: with self.mutex: return self.unique_id_map.get(unique_id, None) def get_download(self, unique_id: str, key: str) -> Optional[dict[str, Any]]: return None