diff options
Diffstat (limited to 'src/portal/py/modules')
| -rw-r--r-- | src/portal/py/modules/fanbox.py | 4 | ||||
| -rw-r--r-- | src/portal/py/modules/patreon.py | 87 | ||||
| -rw-r--r-- | src/portal/py/modules/pixiv_web.py | 16 | ||||
| -rw-r--r-- | src/portal/py/modules/twitter.py | 2 |
4 files changed, 79 insertions, 30 deletions
diff --git a/src/portal/py/modules/fanbox.py b/src/portal/py/modules/fanbox.py index ba9a9f7..9d233bb 100644 --- a/src/portal/py/modules/fanbox.py +++ b/src/portal/py/modules/fanbox.py @@ -1,3 +1,4 @@ +import log import json import httpx from typing import Optional, Any @@ -19,7 +20,8 @@ class FanboxBase(Search): return None try: obj = response.json() - except JSONDecodeError: + except JSONDecodeError as e: + log.error(f'JSONDecodeError: {e}.') return None return obj['body'] diff --git a/src/portal/py/modules/patreon.py b/src/portal/py/modules/patreon.py index 3c0efd2..0e4345a 100644 --- a/src/portal/py/modules/patreon.py +++ b/src/portal/py/modules/patreon.py @@ -24,7 +24,8 @@ class PatreonBase(Search): return None try: obj = response.json() - except JSONDecodeError: + except JSONDecodeError as e: + log.error(f'JSONDecodeError: {e}.') return None return obj @@ -37,21 +38,27 @@ class PatreonBase(Search): 'json-api-use-default-includes': 'false', 'include': '[]' } + obj = self.check_api_response(self.module.do_request(Method.GET, url, params=params)) if not obj: return None, None + for result in obj['data']: username = result['attributes']['creator_name'] display_name = result['attributes']['name'] + url_name = result['attributes']['url'].split('/')[-1] + # Only accept exact matches. - if username.lower() != query.lower() and display_name.lower() != query.lower(): + if username.lower() != query.lower() and display_name.lower() != query.lower() and url_name != query.lower(): continue + prefix = result['id'].find('campaign_') if prefix >= 0: campaign = result['id'][prefix + len('campaign_'):] unique_id = 'patreon:c:' + campaign user = User(unique_id=unique_id, username=username, display_name=display_name) return user, campaign + return None, None class PatreonUser(PatreonBase): @@ -63,11 +70,16 @@ class PatreonUser(PatreonBase): self.cursor: Optional[str] = None self.attachments: dict[str, Media] = {} + # @TODO: # post_type # image_file: https://www.patreon.com/posts/emma-again-29391136 # text_only: https://www.patreon.com/posts/hello-update-29391184 + # tags def create_post(self, data: ParsedJson, hash: str) -> Optional[Post]: + #if 'access_rules' in data['relationships']: + # if not (len(data['relationships']['access_rules']['data']) == 1 and data['relationships']['access_rules']['data'][0]['id'] == '9816126'): + # return None kwargs = {} unique_id = f'patreon:p:{data['id']}' kwargs['unique_id'] = unique_id @@ -78,22 +90,24 @@ class PatreonUser(PatreonBase): current_time = get_current_utc_time().timestamp() kwargs['dates'] = [ Date(DateType.CREATED, (published_at, published_at), DateMeta.NONE), - Date(DateType.RETRIEVED, (current_time, current_time), DateMeta.NONE), + Date(DateType.RETRIEVED, (current_time, current_time), DateMeta.NONE) ] + if 'edited_at' in data['attributes']: + edited_at = datetime.strptime(data['attributes']['edited_at'], "%Y-%m-%dT%H:%M:%S.%f%z").timestamp() + kwargs['dates'].append(Date(DateType.EDITED, (edited_at, edited_at), DateMeta.NONE)) kwargs['author'] = self.user kwargs['title'] = data['attributes']['title'] kwargs['likes'] = data['attributes']['like_count'] kwargs['comments'] = data['attributes']['comment_count'] - kwargs['text'] = data['attributes']['content'] - post_type = data['attributes']['post_type'] + kwargs['text'] = '' + if 'content' in data['attributes']: + kwargs['text'] = data['attributes']['content'] + #post_type = data['attributes']['post_type'] media = {} - #if 'access_rules' in data['relationships']: - # if not (len(data['relationships']['access_rules']['data']) == 1 and data['relationships']['access_rules']['data'][0]['id'] == '9816126'): - # return None - if 'media' in data['relationships']: + if data['relationships']['media']['data']: for m in data['relationships']['media']['data']: media[m['id']] = self.attachments[m['id']] - if 'attachments' in data['relationships']: + if 'attachments' in data['relationships'] and data['relationships']['attachments']['data']: for a in data['relationships']['attachments']['data']: media[a['id']] = self.attachments[a['id']] kwargs['media'] = media @@ -104,59 +118,88 @@ class PatreonUser(PatreonBase): def request_page(self, num: int) -> bool: if num >= self.page + self.REACH_LIMIT: return False + if not self.user: self.user, self.campaign = self.search_user(self.username) if not self.user: self.errored = True return False + request_satisfied = False for i in range(self.page, num + 1): url = f'{BASE_URL}/posts' + #'filter[accessible_by_user_id]': self.module.user_id, params = { - 'include': 'campaign,access_rules,attachments,audio,audio_preview.null,drop,images,media,native_video_insights,poll.choices,poll.current_user_responses.user,poll.current_user_responses.choice,poll.current_user_responses.poll,user,user_defined_tags,ti_checks,content_unlock_options.product_variant.null', + 'include': 'campaign,access_rules,access_rules.tier.null,attachments_media,audio,audio_preview.null,drop,images,media,native_video_insights,poll.choices,poll.current_user_responses.user,poll.current_user_responses.choice,poll.current_user_responses.poll,user,user_defined_tags,ti_checks,video.null,content_unlock_options.product_variant.null', 'fields[campaign]': 'currency,show_audio_post_download_links,avatar_photo_url,avatar_photo_image_urls,earnings_visibility,is_nsfw,is_monthly,name,url', - 'fields[post]': 'change_visibility_at,comment_count,commenter_count,content,current_user_can_comment,current_user_can_delete,current_user_can_report,current_user_can_view,current_user_comment_disallowed_reason,current_user_has_liked,embed,image,insights_last_updated_at,is_paid,like_count,meta_image_url,min_cents_pledged_to_view,post_file,post_metadata,published_at,patreon_url,post_type,pledge_url,preview_asset_type,thumbnail,share_images,thumbnail_url,teaser_text,title,upgrade_url,url,was_posted_by_campaign_owner,has_ti_violation,moderation_status,post_level_suspension_removal_date,pls_one_liners_by_category,video_preview,view_count', + 'fields[post]': 'change_visibility_at,comment_count,commenter_count,content,created_at,current_user_can_comment,current_user_can_delete,current_user_can_report,current_user_can_view,current_user_comment_disallowed_reason,current_user_has_liked,embed,image,insights_last_updated_at,is_paid,like_count,meta_image_url,min_cents_pledged_to_view,monetization_ineligibility_reason,post_file,post_metadata,published_at,patreon_url,post_type,pledge_url,preview_asset_type,thumbnail,thumbnail_url,teaser_text,title,upgrade_url,url,was_posted_by_campaign_owner,has_ti_violation,moderation_status,post_level_suspension_removal_date,pls_one_liners_by_category,video,video_preview,view_count,content_unlock_options,is_new_to_current_user,watch_state', 'fields[post_tag]': 'tag_type,value', 'fields[user]': 'image_url,full_name,url', 'fields[access_rule]': 'access_rule_type,amount_cents', 'fields[media]': 'id,image_urls,display,download_url,metadata,file_name', 'fields[native_video_insights]': 'average_view_duration,average_view_pct,has_preview,id,last_updated_at,num_views,preview_views,video_duration', 'fields[content-unlock-option]': 'content_unlock_type', - 'fields[product-variant]': 'price_cents,currency_code,checkout_url,is_hidden,published_at_datetime,orders_count,access_metadata', + 'fields[product-variant]': 'price_cents,currency_code,checkout_url,is_hidden,published_at_datetime,content_type,orders_count,access_metadata', 'filter[campaign_id]': self.campaign, 'filter[contains_exclusive_posts]': 'true', 'filter[is_draft]': 'false', - 'filter[accessible_by_user_id]': self.module.user_id, - 'sort': 'published_at', - 'json-api-version': '1.0', - 'json-api-use-default-includes': 'false' + 'sort': '-published_at', + 'json-api-use-default-includes': 'false', + 'json-api-version': '1.0' } + if self.cursor: params['page[cursor]'] = self.cursor + obj = self.check_api_response(self.module.do_request(Method.GET, url, params=params)) if not obj: self.errored = True return False + hash = self.module.add_raw_response(obj) - self.cursor = obj['meta']['pagination']['cursors']['next'] - self.pages[i] = [] + + try: + self.cursor = obj['meta']['pagination']['cursors']['next'] + except KeyError: + self.cursor = None + for inc in obj['included']: if inc['type'] == 'attachment': self.attachments[inc['id']] = File(url=Url(inc['attributes']['url']), name=inc['attributes']['name']) elif inc['type'] == 'media': - self.attachments[inc['id']] = Media(url=Url(inc['attributes']['download_url']), thumbnail_url=Url(inc['attributes']['image_urls']['thumbnail'])) + if 'download_url' in inc['attributes']: + media_url = inc['attributes']['download_url'] + elif inc['attributes']['image_urls']: + media_url = inc['attributes']['image_urls']['original'] + elif 'default_thumbnail' in inc['attributes']['display']: + media_url = inc['attributes']['display']['default_thumbnail']['url'] + else: + continue + + if 'default_thumbnail' in inc['attributes']['display']: + thumbnail_url = inc['attributes']['display']['default_thumbnail']['url'] + elif inc['attributes']['image_urls']: + thumbnail_url = inc['attributes']['image_urls']['thumbnail'] + else: + thumbnail_url = '' + + self.attachments[inc['id']] = Media(url=Url(media_url), thumbnail_url=Url(thumbnail_url)) else: - log.error(f'Unhandled inc type {inc['type']}.') + log.warn(f'Unhandled include type {inc['type']}.') + + self.pages[i] = [] for entry in obj['data']: if entry['type'] != 'post': - log.error(f'Unhandled entry type {entry['type']}.') + log.warn(f'Unhandled entry type {entry['type']}.') continue post = self.create_post(entry, hash) if post: self.pages[i].append(post.unique_id) + self.page += 1 if i == num: request_satisfied = True + return request_satisfied class PatreonModule(Module): diff --git a/src/portal/py/modules/pixiv_web.py b/src/portal/py/modules/pixiv_web.py index 60fa812..c6a9c82 100644 --- a/src/portal/py/modules/pixiv_web.py +++ b/src/portal/py/modules/pixiv_web.py @@ -23,7 +23,8 @@ class PixivWebBase(Search): return None try: obj = response.json() - except JSONDecodeError: + except JSONDecodeError as e: + log.error(f'JSONDecodeError: {e}.') return None if obj['error']: log.error(obj['message']) @@ -51,13 +52,16 @@ class PixivWebBase(Search): return None def create_tag(self, data: ParsedJson) -> Tag: - tag = Tag(type=TagType.GENERAL, name=data['tag']) + kwargs= {} + kwargs['type'] = TagType.GENERAL + kwargs['name'] = data['tag'] + kwargs['alts'] = {} if 'romaji' in data: - tag.alts['romaji'] = data['romaji'] + kwargs['alts']['romaji'] = data['romaji'] if 'translation' in data: if 'en' in data['translation']: - tag.alts['en'] = data['translation']['en'] - return tag + kwargs['alts']['en'] = data['translation']['en'] + return Tag(**kwargs) def create_post(self, data: ParsedJson, pages: Optional[ParsedJson] = None, ugoira: Optional[ParsedJson] = None) -> Optional[Post]: kwargs = {} @@ -88,7 +92,7 @@ class PixivWebBase(Search): kwargs['bookmarks'] = data['bookmarkCount'] kwargs['comments'] = data['commentCount'] kwargs['views'] = data['viewCount'] - kwargs['tags'] = [self.create_tag(tag) for tag in data['tags']['tags']] + kwargs['tags'] = [self.create_tag(t) for t in data['tags']['tags']] if data['illustType'] == 2: # Ugoira if not ugoira: url = f'{BASE_URL}/illust/{illust_id}/ugoira_meta?lang={LANG}&version={VERSION}' diff --git a/src/portal/py/modules/twitter.py b/src/portal/py/modules/twitter.py index 386bc5b..2f45ec1 100644 --- a/src/portal/py/modules/twitter.py +++ b/src/portal/py/modules/twitter.py @@ -55,7 +55,7 @@ class TwitterScrapeBase(Search): return Post(**kwargs) kwargs['raw_responses'] = tweet.rawResponses kwargs['url'] = tweet.url - # TODO: replace correct? + # @TODO: Replace correct? kwargs['dates'] = { DateType.CREATED: tweet.date.replace(tzinfo=timezone.utc).timestamp(), DateType.RETRIEVED: get_current_utc_time().timestamp() |