import json import http.cookiejar from typing import Optional, Any, Iterator from base import USER_AGENT, HttpResponse, Search, Module, ParsedJson, Method from post import Post from query_parser import QueryParser from modules.twitter_common import parse_graphql_raw_response class TwitterBase(Search): def __init__(self, userdata: Any) -> None: super().__init__() self.module: TwitterModule = userdata self.cursor: Optional[dict] = None def build_params(params: dict) -> dict: return json.dumps(params, separators=(',', ':'), indent=None) class TwitterUser(TwitterBase): def __init__(self, userdata: Any, arg: str) -> None: super().__init__(userdata) self.user_id = '' def check_api_response(self, response: Optional[HttpResponse]) -> Optional[ParsedJson]: if not response: return None try: obj = response.json() except JSONDecodeError as e: log.error(f'JSONDecodeError: {e}.') return None return obj def do_request(self) -> Optional[ParsedJson]: variables = { 'userId': f'{self.user_id}', 'count': 20, 'includePromotedContent': False, 'withClientEventToken': False, 'withBirdwatchNotes': False, 'withVoice': True } if self.cursor: variables['cursor'] = self.cursor features = { 'rweb_video_screen_enabled': False, 'payments_enabled': False, 'profile_label_improvements_pcf_label_in_post_enabled': True, 'rweb_tipjar_consumption_enabled': True, 'verified_phone_label_enabled': False, 'creator_subscriptions_tweet_preview_api_enabled': True, 'responsive_web_graphql_timeline_navigation_enabled': True, 'responsive_web_graphql_skip_user_profile_image_extensions_enabled': False, 'premium_content_api_read_enabled': False, 'communities_web_enable_tweet_community_results_fetch': True, 'c9s_tweet_anatomy_moderator_badge_enabled': True, 'responsive_web_grok_analyze_button_fetch_trends_enabled': False, 'responsive_web_grok_analyze_post_followups_enabled': True, 'responsive_web_jetfuel_frame': True, 'responsive_web_grok_share_attachment_enabled': True, 'articles_preview_enabled': True, 'responsive_web_edit_tweet_api_enabled': True, 'graphql_is_translatable_rweb_tweet_is_translatable_enabled': True, 'view_counts_everywhere_api_enabled': True, 'longform_notetweets_consumption_enabled': True, 'responsive_web_twitter_article_tweet_consumption_enabled': True, 'tweet_awards_web_tipping_enabled': False, 'responsive_web_grok_show_grok_translated_post': False, 'responsive_web_grok_analysis_button_from_backend': True, 'creator_subscriptions_quote_tweet_preview_enabled': False, 'freedom_of_speech_not_reach_fetch_enabled': True, 'standardized_nudges_misinfo': True, 'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled': True, 'longform_notetweets_rich_text_read_enabled': True, 'longform_notetweets_inline_media_enabled': True, 'responsive_web_grok_image_annotation_enabled': True, 'responsive_web_grok_community_note_auto_translation_is_enabled': False, 'responsive_web_enhance_cards_enabled': False } fieldToggles = { 'withArticlePlainText': False } params = { 'variables': build_params(variables), 'features': build_params(features), 'fieldToggles': build_params(fieldToggles) } url = 'https://x.com/i/api/graphql/DdW3L2nfGW6FZDcSSSG4yg/UserMedia' return self.check_api_response(self.module.do_request(Method.GET, url, params=params)) #print(json.dumps(media, indent=4)) def request_page(self, num: int) -> bool: media = self.do_request() #print(json.dumps(media, indent=4)) response_hash = self.module.add_raw_response(media) tps = parse_graphql_raw_response(media, response_hash, None, self.module) if not tps.cursor: self.completed = True if len(tps.page) > 0: self.cursor = tps.cursor else: self.completed = True return False self.pages[num] = tps.page return True class TwitterModule(Module): def __init__(self, cookies_path: str) -> None: super().__init__() self.parser: QueryParser = QueryParser(self, 'user') self.parser.add_command('user', TwitterUser) # @TODO: Cookie import from browser. self.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; rv:128.0) Gecko/20100101 Firefox/128.0' self.headers['Accept'] = '*/*' self.headers['Accept-Language'] = 'en-US,en;q=0.5' self.headers['Accept-Encoding'] = 'gzip, deflate, br, zstd' self.headers['content-type'] = 'application/json' self.headers['x-twitter-auth-type'] = 'OAuth2Session' self.headers['x-csrf-token'] = '' self.headers['x-twitter-client-language'] = 'en' self.headers['x-twitter-active-user'] = 'yes' self.headers['x-client-transaction-id'] = '' self.headers['x-xp-forwarded-for'] = '' self.headers['DNT'] = '1' self.headers['Sec-Fetch-Dest'] = 'empty' self.headers['Sec-Fetch-Mode'] = 'cors' self.headers['Sec-Fetch-Site'] = 'same-origin' self.headers['authorization'] = '' self.headers['Connection'] = 'keep-alive' self.headers['Cookie'] = '' self.headers['TE'] = 'trailers' #if cookies_path: # cookie_jar = http.cookiejar.MozillaCookieJar() # cookie_jar.load(filename=cookies_path, ignore_expires=True) # for c in cookie_jar: # if c.value: # self.cookies[c.name] = c.value def search(self, query: str, *extra_args: Any) -> Optional[Search]: return self.parser.parse_query(query) def get_download(self, unique_id: str, key: str) -> Optional[dict[str, Any]]: if unique_id not in self.unique_id_map: return None post = self.unique_id_map[unique_id] if key not in post.media: return None return { 'urls': [post.media[key].url], 'headers': { 'User-Agent': USER_AGENT } }