diff --git a/snscrape/modules/twitter.py b/snscrape/modules/twitter.py index d1719a8..5892340 100644 --- a/snscrape/modules/twitter.py +++ b/snscrape/modules/twitter.py @@ -92,6 +92,7 @@ import typing import urllib.parse import urllib3.util.ssl_ import warnings +import hashlib _logger = logging.getLogger(__name__) @@ -99,10 +100,12 @@ _API_AUTHORIZATION_HEADER = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOu _globalGuestTokenManager = None _GUEST_TOKEN_VALIDITY = 10800 _CIPHERS_CHROME = 'TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA:AES256-SHA' +_AGE_RESTRICTED_MSG = 'Age-restricted adult content. This content might not be appropriate for people under 18 years old. To view this media, you’ll need to log in to Twitter. Learn more' @dataclasses.dataclass class Tweet(snscrape.base.Item): + rawResponses: typing.Dict[str, str] url: str date: datetime.datetime rawContent: str @@ -793,17 +796,23 @@ class _TwitterAPIType(enum.Enum): V2 = 0 # Introduced with the redesign GRAPHQL = 1 +class _TwitterAPIAuthType(enum.Enum): + GUEST_TOKEN = 0 + OAUTH2 = 1 class _TwitterAPIScraper(snscrape.base.Scraper): - def __init__(self, baseUrl, *, guestTokenManager = None, maxEmptyPages = 0, **kwargs): + def __init__(self, baseUrl, *, guestTokenManager = None, maxEmptyPages = 0, cookies = None, **kwargs): super().__init__(**kwargs) self._baseUrl = baseUrl - if guestTokenManager is None: + self._syndicationUrl = 'https://cdn.syndication.twimg.com/tweet?id=' + self._authType = _TwitterAPIAuthType.OAUTH2 + if self._authType == _TwitterAPIAuthType.GUEST_TOKEN and guestTokenManager is None: global _globalGuestTokenManager if _globalGuestTokenManager is None: _globalGuestTokenManager = GuestTokenManager() guestTokenManager = _globalGuestTokenManager self._guestTokenManager = guestTokenManager + self._cookies = cookies self._maxEmptyPages = maxEmptyPages self._apiHeaders = { 'Authorization': _API_AUTHORIZATION_HEADER, @@ -813,6 +822,11 @@ class _TwitterAPIScraper(snscrape.base.Scraper): adapter = _TwitterTLSAdapter() self._session.mount('https://twitter.com', adapter) self._session.mount('https://api.twitter.com', adapter) + if self._authType == _TwitterAPIAuthType.OAUTH2: + self._ensure_oauth() + self._lastResponse = None + self._lastResponseHash = None + self._lastSentResponseHash = None def _check_guest_token_response(self, r): if r.status_code != 200: @@ -820,6 +834,8 @@ class _TwitterAPIScraper(snscrape.base.Scraper): return True, None def _ensure_guest_token(self, url = None): + if not self._guestTokenManager: + return if self._guestTokenManager.token is None: _logger.info('Retrieving guest token') r = self._get(self._baseUrl if url is None else url, responseOkCallback = self._check_guest_token_response) @@ -843,18 +859,42 @@ class _TwitterAPIScraper(snscrape.base.Scraper): self._apiHeaders['x-guest-token'] = self._guestTokenManager.token def _unset_guest_token(self, blockUntil): + if not self._guestTokenManager: + return self._guestTokenManager.reset(blockUntil = blockUntil) del self._session.cookies['gt'] del self._apiHeaders['x-guest-token'] + def _ensure_oauth(self): + if not self._cookies: + return + for k, v in self._cookies.items(): + self._session.cookies.set(k, v, domain = '.twitter.com', path = '/', secure = True) + self._apiHeaders['x-csrf-token'] = self._session.cookies.get('ct0') + self._apiHeaders['x-twitter-active-user'] = 'yes' + self._apiHeaders['x-twitter-auth-type'] = 'OAuth2Session' + self._apiHeaders['x-twitter-client-language'] = 'en' + + def _ensure_auth_updated(self, url = None): + if self._authType == _TwitterAPIAuthType.OAUTH2: + self._ensure_oauth() + else: + self._ensure_guest_token(url) + def _check_api_response(self, r, apiType, instructionsPath): if r.status_code in (403, 404, 429): + currentTime = time.time() if r.status_code == 429 and r.headers.get('x-rate-limit-remaining', '') == '0' and 'x-rate-limit-reset' in r.headers: - blockUntil = min(int(r.headers['x-rate-limit-reset']), int(time.time()) + 900) + blockUntil = min(int(r.headers['x-rate-limit-reset']), int(currentTime) + 900) + else: + blockUntil = int(currentTime) + 300 + if self._authType == _TwitterAPIAuthType.GUEST_TOKEN: + self._unset_guest_token(blockUntil) + self._ensure_guest_token() else: - blockUntil = int(time.time()) + 300 - self._unset_guest_token(blockUntil) - self._ensure_guest_token() + blockFor = (blockUntil - currentTime) * (int(r.headers.get('x-rate-limit-remaining', '0')) + 1) + _logger.warn(f'Got too many requests, blocking for {blockFor} seconds ({r.headers['x-rate-limit-reset']}) ({r.headers.get('x-rate-limit-remaining', '0')})') + time.sleep(blockFor) return False, f'blocked ({r.status_code})' if r.headers.get('content-type', '').replace(' ', '') != 'application/json;charset=utf-8': return False, 'content type is not JSON' @@ -868,6 +908,11 @@ class _TwitterAPIScraper(snscrape.base.Scraper): r._snscrapeObj = obj if apiType is _TwitterAPIType.GRAPHQL and 'errors' in obj: msg = 'Twitter responded with an error: ' + ', '.join(f'{e["name"]}: {e["message"]}' for e in obj['errors']) + blockFor = 10.0 + _logger.warn(f'Got errors waiting for {blockFor} seconds') + time.sleep(blockFor) + return False, msg + ''' instructions = obj for k in instructionsPath: instructions = instructions.get(k, {}) @@ -877,13 +922,23 @@ class _TwitterAPIScraper(snscrape.base.Scraper): return True, None else: return False, msg + ''' return True, None def _get_api_data(self, endpoint, apiType, params, instructionsPath = None): - self._ensure_guest_token() + self._ensure_auth_updated() if apiType is _TwitterAPIType.GRAPHQL: params = urllib.parse.urlencode({k: json.dumps(v, separators = (',', ':')) for k, v in params.items()}, quote_via = urllib.parse.quote) r = self._get(endpoint, params = params, headers = self._apiHeaders, responseOkCallback = functools.partial(self._check_api_response, apiType = apiType, instructionsPath = instructionsPath)) + if self._authType == _TwitterAPIAuthType.OAUTH2: + csrf = r.cookies.get('ct0') + if csrf: + _logger.warn('Response contained updated csrf token') + self._apiHeaders['x-csrf-token'] = csrf + self._lastResponse = r._snscrapeObj + hasher = hashlib.new('sha256') + hasher.update(json.dumps(self._lastResponse).encode('utf-8')) + self._lastResponseHash = hasher.hexdigest() return r._snscrapeObj def _iter_api_data(self, endpoint, apiType, params, paginationParams = None, cursor = None, direction = _ScrollDirection.BOTTOM, instructionsPath = None): @@ -988,8 +1043,50 @@ class _TwitterAPIScraper(snscrape.base.Scraper): def _get_tweet_id(self, tweet): return tweet['id'] if 'id' in tweet else int(tweet['id_str']) + def _entry_is_age_restricted(self, entry): + if entry['entryId'].startswith('tombstone-') and entry['content']['entryType'] == 'TimelineTimelineItem': + if entry['content']['itemContent']['tombstoneInfo']['richText']['text'] == _AGE_RESTRICTED_MSG: + return True + return False + + # Can we parse out more data here? + def _syndication_make_tweet(self, tweet): + tweetId = self._get_tweet_id(tweet) + kwargs = {} + kwargs['rawResponses'] = { + 'detached_api' : json.dumps(tweet) + } + kwargs['id'] = tweetId + kwargs['rawContent'] = tweet['text'] + kwargs['renderedContent'] = '' + username = tweet['user']['screen_name'] + kwargs['user'] = User(id = int(tweet['user']['id_str']), username = username, displayname = tweet['user']['name']) + kwargs['date'] = datetime.datetime.strptime(tweet['created_at'], '%Y-%m-%dT%H:%M:%S.%fZ') + kwargs['replyCount'] = tweet['conversation_count'] + kwargs['retweetCount'] = 0 + kwargs['quoteCount'] = 0 + kwargs['conversationId'] = 0 + kwargs['lang'] = tweet['lang'] + kwargs['source'] = '' + kwargs['url'] = f'https://twitter.com/{username}/status/{tweetId}' + kwargs['likeCount'] = tweet['favorite_count'] + media = [] + for p in tweet['photos']: + media.append(Photo(previewUrl = '', fullUrl = p['url'])) + kwargs['media'] = media + return Tweet(**kwargs) + + def _make_tweet(self, tweet, user, retweetedTweet = None, quotedTweet = None, card = None, noteTweet = None, **kwargs): tweetId = self._get_tweet_id(tweet) + kwargs['rawResponses'] = { + 'hash': self._lastResponseHash, + 'tweet': json.dumps(tweet), + 'user': user.json() + } + if not self._lastSentResponseHash or self._lastResponseHash != self._lastSentResponseHash: + kwargs['rawResponses']['detached_api'] = json.dumps(self._lastResponse) + self._lastSentResponseHash = self._lastResponseHash kwargs['id'] = tweetId if noteTweet and 'text' in noteTweet: kwargs['rawContent'] = noteTweet['text'] @@ -1005,9 +1102,9 @@ class _TwitterAPIScraper(snscrape.base.Scraper): kwargs['date'] = email.utils.parsedate_to_datetime(tweet['created_at']) if links: kwargs['links'] = [TextLink( - text = u.get('display_url'), - url = u['expanded_url'], - tcourl = u['url'], + text = u.get('display_url', ''), + url = u.get('expanded_url', ''), + tcourl = u.get('url', ''), indices = tuple(u['indices']), ) for u in links] kwargs['url'] = f'https://twitter.com/{getattr(user, "username", "i/web")}/status/{tweetId}' @@ -1789,7 +1886,7 @@ class TwitterUserScraper(TwitterSearchScraper): self._baseUrl = f'https://twitter.com/{self._user}' if not self._isUserId else f'https://twitter.com/i/user/{self._user}' def _get_entity(self): - self._ensure_guest_token() + self._ensure_auth_updated() if not self._isUserId: fieldName = 'screen_name' endpoint = 'https://twitter.com/i/api/graphql/pVrmNaXcxPjisIvKtLDMEA/UserByScreenName'