diff options
Diffstat (limited to 'src/portal/py/base.py')
| -rw-r--r-- | src/portal/py/base.py | 32 |
1 files changed, 19 insertions, 13 deletions
diff --git a/src/portal/py/base.py b/src/portal/py/base.py index bd68925..a310cf5 100644 --- a/src/portal/py/base.py +++ b/src/portal/py/base.py @@ -1,7 +1,7 @@ import log import time import json -import httpx +import tls_client import binascii import threading import config @@ -12,13 +12,14 @@ from typing import Optional, Self, Any from post import Post, User type ParsedJson = Any +type HttpResponse = tls_client.Response USER_AGENT = config.USER_AGENT class Method(Enum): - HEAD = "HEAD" - GET = "GET" - POST = "POST" + HEAD = 'HEAD' + GET = 'GET' + POST = 'POST' class Search(): def __init__(self) -> None: @@ -66,27 +67,29 @@ class Module(): self.mutex: threading.Lock = threading.Lock() self.raw_responses: dict[str, str] = {} self.unique_id_map: OrderedDict = OrderedDict() - self.session: httpx.Client = httpx.Client(follow_redirects=True, timeout=20.0, http2=False) + self.session = tls_client.Session(client_identifier=config.TLS_CLIENT_IDENTIFIER, random_tls_extension_order=True) def init(self) -> bool: return True - def do_request(self, method: Method, url: str, params: Optional[dict[str, str]] = None, retries: int = 2) -> Optional[httpx.Response]: + def do_request(self, method: Method, url: str, params: Optional[dict[str, str]] = None, retries: int = 2) -> Optional[HttpResponse]: log.debug(f'HTTP {method.value} request {url} {params}.') - response: Optional[httpx.Response] = None + response: Optional[HttpResponse] = None for _ in range(0, retries + 1): try: - response = self.session.request(method.value, url, headers=self.headers, cookies=self.cookies, params=params) + response = self.session.execute_request(method.value, url, headers=self.headers, cookies=self.cookies, params=params, timeout=600) except Exception as e: - log.info(f'Connection exception ({e}), retrying in 20 seconds...') + log.info(f'Connection exception ({repr(e)}), retrying in 20 seconds...') time.sleep(20) continue - if not response: - continue + if response is None: + log.error('Invalid None response from requests library.') + break if response.status_code == 429: - log.info('Got too many requests, waiting for 3 minutes...') - time.sleep(60 * 3) + log.info('Got too many requests, waiting for 4 minutes...') + time.sleep(60 * 4) response = None + retries += 1 continue if response.status_code == 404 or response.status_code == 403: # Shortcut 404, 403 log.warn(f'Returning None, {response.status_code}') @@ -96,6 +99,7 @@ class Module(): time.sleep(5) response = None continue + ''' try: response.read() except Exception as e: @@ -105,6 +109,8 @@ class Module(): continue else: break + ''' + break return response def add_raw_response(self, data: ParsedJson) -> str: |