diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/portal/py/base.py | 2 | ||||
| -rw-r--r-- | src/portal/py/modules/fanbox.py | 4 | ||||
| -rw-r--r-- | src/portal/py/modules/patreon.py | 87 | ||||
| -rw-r--r-- | src/portal/py/modules/pixiv_web.py | 16 | ||||
| -rw-r--r-- | src/portal/py/modules/twitter.py | 2 | ||||
| -rw-r--r-- | src/portal/py/post.py | 2 | ||||
| -rw-r--r-- | src/portal/py/tests/archive_query.py | 108 | ||||
| -rw-r--r-- | src/portal/py/tests/test.py | 21 |
8 files changed, 179 insertions, 63 deletions
diff --git a/src/portal/py/base.py b/src/portal/py/base.py index 18dd961..f40bc00 100644 --- a/src/portal/py/base.py +++ b/src/portal/py/base.py @@ -109,7 +109,7 @@ class Module(): def add_raw_response(self, data: ParsedJson) -> str: with self.mutex: - dump = json.dumps(data) + dump = json.dumps(data, separators=(',', ':'), indent=None) hash = blake3(dump.encode('utf-8')).digest() hash_str = binascii.hexlify(hash).decode('ascii') self.raw_responses[hash_str] = dump diff --git a/src/portal/py/modules/fanbox.py b/src/portal/py/modules/fanbox.py index ba9a9f7..9d233bb 100644 --- a/src/portal/py/modules/fanbox.py +++ b/src/portal/py/modules/fanbox.py @@ -1,3 +1,4 @@ +import log import json import httpx from typing import Optional, Any @@ -19,7 +20,8 @@ class FanboxBase(Search): return None try: obj = response.json() - except JSONDecodeError: + except JSONDecodeError as e: + log.error(f'JSONDecodeError: {e}.') return None return obj['body'] diff --git a/src/portal/py/modules/patreon.py b/src/portal/py/modules/patreon.py index 3c0efd2..0e4345a 100644 --- a/src/portal/py/modules/patreon.py +++ b/src/portal/py/modules/patreon.py @@ -24,7 +24,8 @@ class PatreonBase(Search): return None try: obj = response.json() - except JSONDecodeError: + except JSONDecodeError as e: + log.error(f'JSONDecodeError: {e}.') return None return obj @@ -37,21 +38,27 @@ class PatreonBase(Search): 'json-api-use-default-includes': 'false', 'include': '[]' } + obj = self.check_api_response(self.module.do_request(Method.GET, url, params=params)) if not obj: return None, None + for result in obj['data']: username = result['attributes']['creator_name'] display_name = result['attributes']['name'] + url_name = result['attributes']['url'].split('/')[-1] + # Only accept exact matches. - if username.lower() != query.lower() and display_name.lower() != query.lower(): + if username.lower() != query.lower() and display_name.lower() != query.lower() and url_name != query.lower(): continue + prefix = result['id'].find('campaign_') if prefix >= 0: campaign = result['id'][prefix + len('campaign_'):] unique_id = 'patreon:c:' + campaign user = User(unique_id=unique_id, username=username, display_name=display_name) return user, campaign + return None, None class PatreonUser(PatreonBase): @@ -63,11 +70,16 @@ class PatreonUser(PatreonBase): self.cursor: Optional[str] = None self.attachments: dict[str, Media] = {} + # @TODO: # post_type # image_file: https://www.patreon.com/posts/emma-again-29391136 # text_only: https://www.patreon.com/posts/hello-update-29391184 + # tags def create_post(self, data: ParsedJson, hash: str) -> Optional[Post]: + #if 'access_rules' in data['relationships']: + # if not (len(data['relationships']['access_rules']['data']) == 1 and data['relationships']['access_rules']['data'][0]['id'] == '9816126'): + # return None kwargs = {} unique_id = f'patreon:p:{data['id']}' kwargs['unique_id'] = unique_id @@ -78,22 +90,24 @@ class PatreonUser(PatreonBase): current_time = get_current_utc_time().timestamp() kwargs['dates'] = [ Date(DateType.CREATED, (published_at, published_at), DateMeta.NONE), - Date(DateType.RETRIEVED, (current_time, current_time), DateMeta.NONE), + Date(DateType.RETRIEVED, (current_time, current_time), DateMeta.NONE) ] + if 'edited_at' in data['attributes']: + edited_at = datetime.strptime(data['attributes']['edited_at'], "%Y-%m-%dT%H:%M:%S.%f%z").timestamp() + kwargs['dates'].append(Date(DateType.EDITED, (edited_at, edited_at), DateMeta.NONE)) kwargs['author'] = self.user kwargs['title'] = data['attributes']['title'] kwargs['likes'] = data['attributes']['like_count'] kwargs['comments'] = data['attributes']['comment_count'] - kwargs['text'] = data['attributes']['content'] - post_type = data['attributes']['post_type'] + kwargs['text'] = '' + if 'content' in data['attributes']: + kwargs['text'] = data['attributes']['content'] + #post_type = data['attributes']['post_type'] media = {} - #if 'access_rules' in data['relationships']: - # if not (len(data['relationships']['access_rules']['data']) == 1 and data['relationships']['access_rules']['data'][0]['id'] == '9816126'): - # return None - if 'media' in data['relationships']: + if data['relationships']['media']['data']: for m in data['relationships']['media']['data']: media[m['id']] = self.attachments[m['id']] - if 'attachments' in data['relationships']: + if 'attachments' in data['relationships'] and data['relationships']['attachments']['data']: for a in data['relationships']['attachments']['data']: media[a['id']] = self.attachments[a['id']] kwargs['media'] = media @@ -104,59 +118,88 @@ class PatreonUser(PatreonBase): def request_page(self, num: int) -> bool: if num >= self.page + self.REACH_LIMIT: return False + if not self.user: self.user, self.campaign = self.search_user(self.username) if not self.user: self.errored = True return False + request_satisfied = False for i in range(self.page, num + 1): url = f'{BASE_URL}/posts' + #'filter[accessible_by_user_id]': self.module.user_id, params = { - 'include': 'campaign,access_rules,attachments,audio,audio_preview.null,drop,images,media,native_video_insights,poll.choices,poll.current_user_responses.user,poll.current_user_responses.choice,poll.current_user_responses.poll,user,user_defined_tags,ti_checks,content_unlock_options.product_variant.null', + 'include': 'campaign,access_rules,access_rules.tier.null,attachments_media,audio,audio_preview.null,drop,images,media,native_video_insights,poll.choices,poll.current_user_responses.user,poll.current_user_responses.choice,poll.current_user_responses.poll,user,user_defined_tags,ti_checks,video.null,content_unlock_options.product_variant.null', 'fields[campaign]': 'currency,show_audio_post_download_links,avatar_photo_url,avatar_photo_image_urls,earnings_visibility,is_nsfw,is_monthly,name,url', - 'fields[post]': 'change_visibility_at,comment_count,commenter_count,content,current_user_can_comment,current_user_can_delete,current_user_can_report,current_user_can_view,current_user_comment_disallowed_reason,current_user_has_liked,embed,image,insights_last_updated_at,is_paid,like_count,meta_image_url,min_cents_pledged_to_view,post_file,post_metadata,published_at,patreon_url,post_type,pledge_url,preview_asset_type,thumbnail,share_images,thumbnail_url,teaser_text,title,upgrade_url,url,was_posted_by_campaign_owner,has_ti_violation,moderation_status,post_level_suspension_removal_date,pls_one_liners_by_category,video_preview,view_count', + 'fields[post]': 'change_visibility_at,comment_count,commenter_count,content,created_at,current_user_can_comment,current_user_can_delete,current_user_can_report,current_user_can_view,current_user_comment_disallowed_reason,current_user_has_liked,embed,image,insights_last_updated_at,is_paid,like_count,meta_image_url,min_cents_pledged_to_view,monetization_ineligibility_reason,post_file,post_metadata,published_at,patreon_url,post_type,pledge_url,preview_asset_type,thumbnail,thumbnail_url,teaser_text,title,upgrade_url,url,was_posted_by_campaign_owner,has_ti_violation,moderation_status,post_level_suspension_removal_date,pls_one_liners_by_category,video,video_preview,view_count,content_unlock_options,is_new_to_current_user,watch_state', 'fields[post_tag]': 'tag_type,value', 'fields[user]': 'image_url,full_name,url', 'fields[access_rule]': 'access_rule_type,amount_cents', 'fields[media]': 'id,image_urls,display,download_url,metadata,file_name', 'fields[native_video_insights]': 'average_view_duration,average_view_pct,has_preview,id,last_updated_at,num_views,preview_views,video_duration', 'fields[content-unlock-option]': 'content_unlock_type', - 'fields[product-variant]': 'price_cents,currency_code,checkout_url,is_hidden,published_at_datetime,orders_count,access_metadata', + 'fields[product-variant]': 'price_cents,currency_code,checkout_url,is_hidden,published_at_datetime,content_type,orders_count,access_metadata', 'filter[campaign_id]': self.campaign, 'filter[contains_exclusive_posts]': 'true', 'filter[is_draft]': 'false', - 'filter[accessible_by_user_id]': self.module.user_id, - 'sort': 'published_at', - 'json-api-version': '1.0', - 'json-api-use-default-includes': 'false' + 'sort': '-published_at', + 'json-api-use-default-includes': 'false', + 'json-api-version': '1.0' } + if self.cursor: params['page[cursor]'] = self.cursor + obj = self.check_api_response(self.module.do_request(Method.GET, url, params=params)) if not obj: self.errored = True return False + hash = self.module.add_raw_response(obj) - self.cursor = obj['meta']['pagination']['cursors']['next'] - self.pages[i] = [] + + try: + self.cursor = obj['meta']['pagination']['cursors']['next'] + except KeyError: + self.cursor = None + for inc in obj['included']: if inc['type'] == 'attachment': self.attachments[inc['id']] = File(url=Url(inc['attributes']['url']), name=inc['attributes']['name']) elif inc['type'] == 'media': - self.attachments[inc['id']] = Media(url=Url(inc['attributes']['download_url']), thumbnail_url=Url(inc['attributes']['image_urls']['thumbnail'])) + if 'download_url' in inc['attributes']: + media_url = inc['attributes']['download_url'] + elif inc['attributes']['image_urls']: + media_url = inc['attributes']['image_urls']['original'] + elif 'default_thumbnail' in inc['attributes']['display']: + media_url = inc['attributes']['display']['default_thumbnail']['url'] + else: + continue + + if 'default_thumbnail' in inc['attributes']['display']: + thumbnail_url = inc['attributes']['display']['default_thumbnail']['url'] + elif inc['attributes']['image_urls']: + thumbnail_url = inc['attributes']['image_urls']['thumbnail'] + else: + thumbnail_url = '' + + self.attachments[inc['id']] = Media(url=Url(media_url), thumbnail_url=Url(thumbnail_url)) else: - log.error(f'Unhandled inc type {inc['type']}.') + log.warn(f'Unhandled include type {inc['type']}.') + + self.pages[i] = [] for entry in obj['data']: if entry['type'] != 'post': - log.error(f'Unhandled entry type {entry['type']}.') + log.warn(f'Unhandled entry type {entry['type']}.') continue post = self.create_post(entry, hash) if post: self.pages[i].append(post.unique_id) + self.page += 1 if i == num: request_satisfied = True + return request_satisfied class PatreonModule(Module): diff --git a/src/portal/py/modules/pixiv_web.py b/src/portal/py/modules/pixiv_web.py index 60fa812..c6a9c82 100644 --- a/src/portal/py/modules/pixiv_web.py +++ b/src/portal/py/modules/pixiv_web.py @@ -23,7 +23,8 @@ class PixivWebBase(Search): return None try: obj = response.json() - except JSONDecodeError: + except JSONDecodeError as e: + log.error(f'JSONDecodeError: {e}.') return None if obj['error']: log.error(obj['message']) @@ -51,13 +52,16 @@ class PixivWebBase(Search): return None def create_tag(self, data: ParsedJson) -> Tag: - tag = Tag(type=TagType.GENERAL, name=data['tag']) + kwargs= {} + kwargs['type'] = TagType.GENERAL + kwargs['name'] = data['tag'] + kwargs['alts'] = {} if 'romaji' in data: - tag.alts['romaji'] = data['romaji'] + kwargs['alts']['romaji'] = data['romaji'] if 'translation' in data: if 'en' in data['translation']: - tag.alts['en'] = data['translation']['en'] - return tag + kwargs['alts']['en'] = data['translation']['en'] + return Tag(**kwargs) def create_post(self, data: ParsedJson, pages: Optional[ParsedJson] = None, ugoira: Optional[ParsedJson] = None) -> Optional[Post]: kwargs = {} @@ -88,7 +92,7 @@ class PixivWebBase(Search): kwargs['bookmarks'] = data['bookmarkCount'] kwargs['comments'] = data['commentCount'] kwargs['views'] = data['viewCount'] - kwargs['tags'] = [self.create_tag(tag) for tag in data['tags']['tags']] + kwargs['tags'] = [self.create_tag(t) for t in data['tags']['tags']] if data['illustType'] == 2: # Ugoira if not ugoira: url = f'{BASE_URL}/illust/{illust_id}/ugoira_meta?lang={LANG}&version={VERSION}' diff --git a/src/portal/py/modules/twitter.py b/src/portal/py/modules/twitter.py index 386bc5b..2f45ec1 100644 --- a/src/portal/py/modules/twitter.py +++ b/src/portal/py/modules/twitter.py @@ -55,7 +55,7 @@ class TwitterScrapeBase(Search): return Post(**kwargs) kwargs['raw_responses'] = tweet.rawResponses kwargs['url'] = tweet.url - # TODO: replace correct? + # @TODO: Replace correct? kwargs['dates'] = { DateType.CREATED: tweet.date.replace(tzinfo=timezone.utc).timestamp(), DateType.RETRIEVED: get_current_utc_time().timestamp() diff --git a/src/portal/py/post.py b/src/portal/py/post.py index e926c01..1a285c2 100644 --- a/src/portal/py/post.py +++ b/src/portal/py/post.py @@ -54,7 +54,7 @@ def df(c: Any) -> Any: return field(default_factory=lambda: c) def default_url_ext_guess(url: str) -> str: - # TODO: List of known extensions. + # @TODO: List of known extensions. question = url.rfind('?') if question >= 0: guess = url[0:question].rsplit('.')[-1] diff --git a/src/portal/py/tests/archive_query.py b/src/portal/py/tests/archive_query.py index 1ed9db4..9c4cf41 100644 --- a/src/portal/py/tests/archive_query.py +++ b/src/portal/py/tests/archive_query.py @@ -11,10 +11,12 @@ from post import PostEncoder, PostType, DateType, User from modules import ALL_MODULES from tests.logger import Logger -#mode = 'pixiv_web' -mode = 'fanbox' +mode = 'pixiv_web' +#mode = 'fanbox' +#mode = 'patreon' cmd = 'user' #cmd = 'bookmarks' +#cmd = 'search' module = ALL_MODULES[mode][0] if not module.init(): sys.exit(1) @@ -41,13 +43,14 @@ def default_media_download(obj, post, output_base): return False class QueryDownloadThread(threading.Thread): - def __init__(self, log, complete, query, output_dir): + def __init__(self, log, complete, query, output_dir, start_page): super().__init__() self.log = log self.complete = complete self.query = query self.output_dir = output_dir self.media_download = default_media_download + self.start_page = start_page def make_path(self, post): prefix: str @@ -142,22 +145,23 @@ class QueryDownloadThread(threading.Thread): return completed = True arg = self.query.split(':')[-1] - num = 0 + num = self.start_page count = 0 while completed: self.log.write(f'Starting page {num} of {arg}.') page = search.get_page(num) + for hash, response in module.raw_responses.items(): + raw_response_path = f'{self.output_dir}/raw_responses/{hash}.json.gz' + if not os.path.isfile(raw_response_path): + if not self.write_to_file(raw_response_path, 'wb+', response, True): + completed = False + break if not page: break num += 1 for unique_id in page: if unique_id == 'twitter:t:1597637140833529856': self.log.write('---------Hit target---------') - with quit_mutex: - if not is_running: - self.complete.write(f'{mode}:{cmd}:{arg}:i:{str(count)},{str(num)}') - completed = False - break post = module.get_item(unique_id) if not post: self.log.write(f'Missing post with id {unique_id}.') @@ -165,6 +169,11 @@ class QueryDownloadThread(threading.Thread): if not self.write_post_to_disk(post): continue count += 1 + with quit_mutex: + if not is_running: + self.complete.write(f'{mode}:{cmd}:{arg}:i:{str(count)},{str(num)}') + completed = False + break if completed: self.complete.write(f'{mode}:{cmd}:{arg}:c:{str(count)},{str(num)}') with queue_cond: @@ -204,25 +213,72 @@ def download_args(): output_dir = sys.argv[1] if not os.path.isdir(output_dir): os.mkdir(output_dir) + os.mkdir(f'{output_dir}/raw_responses') #args = module.search(f'following:{22781328}') - #args = ['22781328'] - #args = ['anoh223'] - #args = ['532891', '1324074',] - #args = ['6600030', '163436'] - #args = ['85117546', '29506', '3298353', '21674944', '2034628', '3085731'] - #args = ['6827013', '11742', '4671', '36323560', '5128785', '30970895'] - #args = ['mugi49'] - #args = ['71598064', '9204502', '689283', '19956185', '42987864', '103682732', '4675776', '7780', '160755', '250142', '3444594', '1995476', '357647'] - #args = ['10473280'] - #args = ['178217', '106414', '882896', '3040749'] - #args = ['21245269'] - #args = ['1969907', '292644', '746841'] - #args = ['1319954', '43718238', '2721319', '53459337'] + #args = [ + # '5838770', + # '44194940' + #] # TODO: - args = ['yzr'] - #args = ['11187954'] - # twitter test: butcha_u, sep__rina + #args = [ + # '68839843', + # '16105069', + # '67090304', + # '188984', + # '15041022', + # '3948', + # '2232374', + # '194668', + # '7835', + # '3067312', + # '59375709', + # '9077832', + # '199750', + # '5806160', + # '33726', + # '15261563', + # '7246', + # '8146', + # '40259810', + # '28865', + # '18362', + # '2750098', + # '2157729', + # '65523978', + # '154858', + # '37625681', + # '34642665', + # '24023', + # '3564336', + # '2658856', + # '990017', + # '55392960', + # '8967182', + # '8965979', + # '14305327', + # '469378', + # '8885157', + # '1107939', + # '18159122', + # '57304923', + # '6534119', + # '91521', + # '1074517', + # '14394', + # '46563991', + # '33694842', + # '84663', + # '4028970', + # '8876470', + # '1829995', + # '14113571', + # '56349311', + # '143555', + # '36388895', + # '14188490' + #] + # twitter test: butcha_u, sep__rina, nagayori000, sattinittas, threads = [] start = True @@ -256,7 +312,7 @@ def download_args(): if not is_running: break log.write(f'***********************\nStarting download of {arg}\n***********************\n') - arg_thread = QueryDownloadThread(log, complete, arg, output_dir) + arg_thread = QueryDownloadThread(log, complete, arg, output_dir, 0) threads.append(arg_thread) arg_thread.start() diff --git a/src/portal/py/tests/test.py b/src/portal/py/tests/test.py index 15bde0a..2fe2eda 100644 --- a/src/portal/py/tests/test.py +++ b/src/portal/py/tests/test.py @@ -1,3 +1,4 @@ +import sys import json import httpx import os.path @@ -5,11 +6,11 @@ from base import Method from typing import Optional, Any from post import DateType, DateMeta, Date, PostEncoder, Post, MediaType from modules import ALL_MODULES -#from twitter.scraper import Scraper +from twitter.scraper import Scraper #scraper = Scraper(cookies = { -# 'ct0': '5a3c98b7b50a74dc556fa497f497932e8958b5b2ef7ba74440ce3f341bb4753f663d3fe7633222d62a52d4ce121673234b927d97eeef806129d42bbfba7ab4d4cfbd2749c20fe6f8da480d0a1e6629e1', -# 'auth_token': 'ea66b4c2ebd04e22de855dce6f1366221635bf2e' +# 'ct0': '', +# 'auth_token': '' #}) # #media = scraper.media([1557919548904419328], limit=10) @@ -55,14 +56,24 @@ from modules import ALL_MODULES # else: # print('Skipping file') -module = ALL_MODULES['instagram'][0] + +module = ALL_MODULES['pixiv_web'][0] module.init() -search = module.search('plottttwistttttt') +search = module.search('illust:91352621') page = search.get_page(0) for unique_id in page: post = module.get_item(unique_id) print(json.dumps(post, indent=4, cls=PostEncoder)) +#module = ALL_MODULES['instagram'][0] +#module.init() +#search = module.search('plottttwistttttt') +#page = search.get_page(0) +#for unique_id in page: +# post = module.get_item(unique_id) +# print(json.dumps(post, indent=4, cls=PostEncoder)) + + #d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.EDITED_AT_UNKNOWN_TIME) #d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.NONE) #print(json.dumps(d, indent=4, cls=PostEncoder)) |