diff options
Diffstat (limited to 'src/portal/py')
| -rw-r--r-- | src/portal/py/modules/pixiv_web.py | 15 | ||||
| -rw-r--r-- | src/portal/py/tests/archive_query.py | 70 | ||||
| -rw-r--r-- | src/portal/py/tests/test.py | 19 |
3 files changed, 22 insertions, 82 deletions
diff --git a/src/portal/py/modules/pixiv_web.py b/src/portal/py/modules/pixiv_web.py index c6a9c82..3c04202 100644 --- a/src/portal/py/modules/pixiv_web.py +++ b/src/portal/py/modules/pixiv_web.py @@ -97,16 +97,15 @@ class PixivWebBase(Search): if not ugoira: url = f'{BASE_URL}/illust/{illust_id}/ugoira_meta?lang={LANG}&version={VERSION}' ugoira = self.check_api_response(self.module.do_request(Method.GET, url)) - if not ugoira: - return None - kwargs['raw_responses']['ugoira'] = json.dumps(ugoira) - kwargs['media'] = { '0': self.parse_ugoira(ugoira, data['urls']['original']) } elif data['pageCount'] > 1: if not pages: url = f'{BASE_URL}/illust/{illust_id}/pages?lang={LANG}&version={VERSION}' pages = self.check_api_response(self.module.do_request(Method.GET, url)) - if not pages: - return None + # Retrieving ugoria or pages could return 404. + if ugoira: + kwargs['raw_responses']['ugoira'] = json.dumps(ugoira) + kwargs['media'] = { '0': self.parse_ugoira(ugoira, data['urls']['original']) } + elif pages: kwargs['raw_responses']['pages'] = json.dumps(pages) kwargs['media'] = self.parse_pages(pages) else: @@ -142,7 +141,7 @@ class PixivWebBase(Search): url = f'{BASE_URL}/illust/{illust_id}?lang={LANG}&version={VERSION}' obj = self.check_api_response(self.module.do_request(Method.GET, url)) if not obj: - return None + return self.create_tombstone({ 'id': illust_id }) return self.create_post(obj) def create_user(self, data: ParsedJson) -> User: @@ -249,6 +248,8 @@ class PixivWebBookmarks(PixivWebBase): self.pages[num] = [] for illust in obj['works']: if illust['userId'] == 0: + # Is it worth assuming this is a tombstone? Should run some tests + # to see if an illust object is ever returned anyway. post = self.create_tombstone(illust) else: post = self.request_illust(int(illust['id'])) diff --git a/src/portal/py/tests/archive_query.py b/src/portal/py/tests/archive_query.py index 9c4cf41..4cc1e1e 100644 --- a/src/portal/py/tests/archive_query.py +++ b/src/portal/py/tests/archive_query.py @@ -14,8 +14,9 @@ from tests.logger import Logger mode = 'pixiv_web' #mode = 'fanbox' #mode = 'patreon' -cmd = 'user' -#cmd = 'bookmarks' +#mode = 'instagram' +#cmd = 'user' +cmd = 'bookmarks' #cmd = 'search' module = ALL_MODULES[mode][0] if not module.init(): @@ -216,70 +217,9 @@ def download_args(): os.mkdir(f'{output_dir}/raw_responses') #args = module.search(f'following:{22781328}') - #args = [ - # '5838770', - # '44194940' - #] - # TODO: - #args = [ - # '68839843', - # '16105069', - # '67090304', - # '188984', - # '15041022', - # '3948', - # '2232374', - # '194668', - # '7835', - # '3067312', - # '59375709', - # '9077832', - # '199750', - # '5806160', - # '33726', - # '15261563', - # '7246', - # '8146', - # '40259810', - # '28865', - # '18362', - # '2750098', - # '2157729', - # '65523978', - # '154858', - # '37625681', - # '34642665', - # '24023', - # '3564336', - # '2658856', - # '990017', - # '55392960', - # '8967182', - # '8965979', - # '14305327', - # '469378', - # '8885157', - # '1107939', - # '18159122', - # '57304923', - # '6534119', - # '91521', - # '1074517', - # '14394', - # '46563991', - # '33694842', - # '84663', - # '4028970', - # '8876470', - # '1829995', - # '14113571', - # '56349311', - # '143555', - # '36388895', - # '14188490' - #] + # @TODO: + args = [] # twitter test: butcha_u, sep__rina, nagayori000, sattinittas, - threads = [] start = True diff --git a/src/portal/py/tests/test.py b/src/portal/py/tests/test.py index 2fe2eda..ef166d9 100644 --- a/src/portal/py/tests/test.py +++ b/src/portal/py/tests/test.py @@ -57,22 +57,21 @@ from twitter.scraper import Scraper # print('Skipping file') -module = ALL_MODULES['pixiv_web'][0] -module.init() -search = module.search('illust:91352621') -page = search.get_page(0) -for unique_id in page: - post = module.get_item(unique_id) - print(json.dumps(post, indent=4, cls=PostEncoder)) - -#module = ALL_MODULES['instagram'][0] +#module = ALL_MODULES['pixiv_web'][0] #module.init() -#search = module.search('plottttwistttttt') +#search = module.search('illust:91352621') #page = search.get_page(0) #for unique_id in page: # post = module.get_item(unique_id) # print(json.dumps(post, indent=4, cls=PostEncoder)) +module = ALL_MODULES['instagram'][0] +module.init() +search = module.search('plottttwistttttt') +page = search.get_page(0) +for unique_id in page: + post = module.get_item(unique_id) + print(json.dumps(post, indent=4, cls=PostEncoder)) #d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.EDITED_AT_UNKNOWN_TIME) #d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.NONE) |