Portal updates

Signed-off-by: Andrew Opalach <andrew@akon.city>

8 files changed, 179 insertions, 63 deletions

diff --git a/src/portal/py/base.py b/src/portal/py/base.py
index 18dd961..f40bc00 100644
--- a/src/portal/py/base.py
+++ b/src/portal/py/base.py
@@ -109,7 +109,7 @@ class Module():
 
     def add_raw_response(self, data: ParsedJson) -> str:
         with self.mutex:
-            dump = json.dumps(data)
+            dump = json.dumps(data, separators=(',', ':'), indent=None)
             hash = blake3(dump.encode('utf-8')).digest()
             hash_str = binascii.hexlify(hash).decode('ascii')
             self.raw_responses[hash_str] = dump
diff --git a/src/portal/py/modules/fanbox.py b/src/portal/py/modules/fanbox.py
index ba9a9f7..9d233bb 100644
--- a/src/portal/py/modules/fanbox.py
+++ b/src/portal/py/modules/fanbox.py
@@ -1,3 +1,4 @@
+import log
 import json
 import httpx
 from typing import Optional, Any
@@ -19,7 +20,8 @@ class FanboxBase(Search):
             return None
         try:
             obj = response.json()
-        except JSONDecodeError:
+        except JSONDecodeError as e:
+            log.error(f'JSONDecodeError: {e}.')
             return None
         return obj['body']
 
diff --git a/src/portal/py/modules/patreon.py b/src/portal/py/modules/patreon.py
index 3c0efd2..0e4345a 100644
--- a/src/portal/py/modules/patreon.py
+++ b/src/portal/py/modules/patreon.py
@@ -24,7 +24,8 @@ class PatreonBase(Search):
             return None
         try:
             obj = response.json()
-        except JSONDecodeError:
+        except JSONDecodeError as e:
+            log.error(f'JSONDecodeError: {e}.')
             return None
         return obj
 
@@ -37,21 +38,27 @@ class PatreonBase(Search):
             'json-api-use-default-includes': 'false',
             'include': '[]'
         }
+
         obj = self.check_api_response(self.module.do_request(Method.GET, url, params=params))
         if not obj:
             return None, None
+
         for result in obj['data']:
             username = result['attributes']['creator_name']
             display_name = result['attributes']['name']
+            url_name = result['attributes']['url'].split('/')[-1]
+
             # Only accept exact matches.
-            if username.lower() != query.lower() and display_name.lower() != query.lower():
+            if username.lower() != query.lower() and display_name.lower() != query.lower() and url_name != query.lower():
                 continue
+
             prefix = result['id'].find('campaign_')
             if prefix >= 0:
                 campaign = result['id'][prefix + len('campaign_'):]
                 unique_id = 'patreon:c:' + campaign
                 user = User(unique_id=unique_id, username=username, display_name=display_name)
                 return user, campaign
+
         return None, None
 
 class PatreonUser(PatreonBase):
@@ -63,11 +70,16 @@ class PatreonUser(PatreonBase):
         self.cursor: Optional[str] = None
         self.attachments: dict[str, Media] = {}
 
+    # @TODO:
     # post_type
     #  image_file: https://www.patreon.com/posts/emma-again-29391136
     #  text_only: https://www.patreon.com/posts/hello-update-29391184
+    # tags
 
     def create_post(self, data: ParsedJson, hash: str) -> Optional[Post]:
+        #if 'access_rules' in data['relationships']:
+        #    if not (len(data['relationships']['access_rules']['data']) == 1 and data['relationships']['access_rules']['data'][0]['id'] == '9816126'):
+        #        return None
         kwargs = {}
         unique_id = f'patreon:p:{data['id']}'
         kwargs['unique_id'] = unique_id
@@ -78,22 +90,24 @@ class PatreonUser(PatreonBase):
         current_time = get_current_utc_time().timestamp()
         kwargs['dates'] = [
             Date(DateType.CREATED, (published_at, published_at), DateMeta.NONE),
-            Date(DateType.RETRIEVED, (current_time, current_time), DateMeta.NONE),
+            Date(DateType.RETRIEVED, (current_time, current_time), DateMeta.NONE)
         ]
+        if 'edited_at' in data['attributes']:
+            edited_at = datetime.strptime(data['attributes']['edited_at'], "%Y-%m-%dT%H:%M:%S.%f%z").timestamp()
+            kwargs['dates'].append(Date(DateType.EDITED, (edited_at, edited_at), DateMeta.NONE))
         kwargs['author'] = self.user
         kwargs['title'] = data['attributes']['title']
         kwargs['likes'] = data['attributes']['like_count']
         kwargs['comments'] = data['attributes']['comment_count']
-        kwargs['text'] = data['attributes']['content']
-        post_type = data['attributes']['post_type']
+        kwargs['text'] = ''
+        if 'content' in data['attributes']:
+            kwargs['text'] = data['attributes']['content']
+        #post_type = data['attributes']['post_type']
         media = {}
-        #if 'access_rules' in data['relationships']:
-        #    if not (len(data['relationships']['access_rules']['data']) == 1 and data['relationships']['access_rules']['data'][0]['id'] == '9816126'):
-        #        return None
-        if 'media' in data['relationships']:
+        if data['relationships']['media']['data']:
             for m in data['relationships']['media']['data']:
                 media[m['id']] = self.attachments[m['id']]
-        if 'attachments' in data['relationships']:
+        if 'attachments' in data['relationships'] and data['relationships']['attachments']['data']:
             for a in data['relationships']['attachments']['data']:
                 media[a['id']] = self.attachments[a['id']]
         kwargs['media'] = media
@@ -104,59 +118,88 @@ class PatreonUser(PatreonBase):
     def request_page(self, num: int) -> bool:
         if num >= self.page + self.REACH_LIMIT:
             return False
+
         if not self.user:
             self.user, self.campaign = self.search_user(self.username)
             if not self.user:
                 self.errored = True
                 return False
+
         request_satisfied = False
         for i in range(self.page, num + 1):
             url = f'{BASE_URL}/posts'
+            #'filter[accessible_by_user_id]': self.module.user_id,
             params = {
-                'include': 'campaign,access_rules,attachments,audio,audio_preview.null,drop,images,media,native_video_insights,poll.choices,poll.current_user_responses.user,poll.current_user_responses.choice,poll.current_user_responses.poll,user,user_defined_tags,ti_checks,content_unlock_options.product_variant.null',
+                'include': 'campaign,access_rules,access_rules.tier.null,attachments_media,audio,audio_preview.null,drop,images,media,native_video_insights,poll.choices,poll.current_user_responses.user,poll.current_user_responses.choice,poll.current_user_responses.poll,user,user_defined_tags,ti_checks,video.null,content_unlock_options.product_variant.null',
                 'fields[campaign]': 'currency,show_audio_post_download_links,avatar_photo_url,avatar_photo_image_urls,earnings_visibility,is_nsfw,is_monthly,name,url',
-                'fields[post]': 'change_visibility_at,comment_count,commenter_count,content,current_user_can_comment,current_user_can_delete,current_user_can_report,current_user_can_view,current_user_comment_disallowed_reason,current_user_has_liked,embed,image,insights_last_updated_at,is_paid,like_count,meta_image_url,min_cents_pledged_to_view,post_file,post_metadata,published_at,patreon_url,post_type,pledge_url,preview_asset_type,thumbnail,share_images,thumbnail_url,teaser_text,title,upgrade_url,url,was_posted_by_campaign_owner,has_ti_violation,moderation_status,post_level_suspension_removal_date,pls_one_liners_by_category,video_preview,view_count',
+                'fields[post]': 'change_visibility_at,comment_count,commenter_count,content,created_at,current_user_can_comment,current_user_can_delete,current_user_can_report,current_user_can_view,current_user_comment_disallowed_reason,current_user_has_liked,embed,image,insights_last_updated_at,is_paid,like_count,meta_image_url,min_cents_pledged_to_view,monetization_ineligibility_reason,post_file,post_metadata,published_at,patreon_url,post_type,pledge_url,preview_asset_type,thumbnail,thumbnail_url,teaser_text,title,upgrade_url,url,was_posted_by_campaign_owner,has_ti_violation,moderation_status,post_level_suspension_removal_date,pls_one_liners_by_category,video,video_preview,view_count,content_unlock_options,is_new_to_current_user,watch_state',
                 'fields[post_tag]': 'tag_type,value',
                 'fields[user]': 'image_url,full_name,url',
                 'fields[access_rule]': 'access_rule_type,amount_cents',
                 'fields[media]': 'id,image_urls,display,download_url,metadata,file_name',
                 'fields[native_video_insights]': 'average_view_duration,average_view_pct,has_preview,id,last_updated_at,num_views,preview_views,video_duration',
                 'fields[content-unlock-option]': 'content_unlock_type',
-                'fields[product-variant]': 'price_cents,currency_code,checkout_url,is_hidden,published_at_datetime,orders_count,access_metadata',
+                'fields[product-variant]': 'price_cents,currency_code,checkout_url,is_hidden,published_at_datetime,content_type,orders_count,access_metadata',
                 'filter[campaign_id]': self.campaign,
                 'filter[contains_exclusive_posts]': 'true',
                 'filter[is_draft]': 'false',
-                'filter[accessible_by_user_id]': self.module.user_id,
-                'sort': 'published_at',
-                'json-api-version': '1.0',
-                'json-api-use-default-includes': 'false'
+                'sort': '-published_at',
+                'json-api-use-default-includes': 'false',
+                'json-api-version': '1.0'
             }
+
             if self.cursor:
                 params['page[cursor]'] = self.cursor
+
             obj = self.check_api_response(self.module.do_request(Method.GET, url, params=params))
             if not obj:
                 self.errored = True
                 return False
+
             hash = self.module.add_raw_response(obj)
-            self.cursor = obj['meta']['pagination']['cursors']['next']
-            self.pages[i] = []
+
+            try:
+                self.cursor = obj['meta']['pagination']['cursors']['next']
+            except KeyError:
+                self.cursor = None
+
             for inc in obj['included']:
                 if inc['type'] == 'attachment':
                     self.attachments[inc['id']] = File(url=Url(inc['attributes']['url']), name=inc['attributes']['name'])
                 elif inc['type'] == 'media':
-                    self.attachments[inc['id']] = Media(url=Url(inc['attributes']['download_url']), thumbnail_url=Url(inc['attributes']['image_urls']['thumbnail']))
+                    if 'download_url' in inc['attributes']:
+                        media_url = inc['attributes']['download_url']
+                    elif inc['attributes']['image_urls']:
+                        media_url = inc['attributes']['image_urls']['original']
+                    elif 'default_thumbnail' in inc['attributes']['display']:
+                        media_url = inc['attributes']['display']['default_thumbnail']['url']
+                    else:
+                        continue
+
+                    if 'default_thumbnail' in inc['attributes']['display']:
+                        thumbnail_url = inc['attributes']['display']['default_thumbnail']['url']
+                    elif inc['attributes']['image_urls']:
+                        thumbnail_url = inc['attributes']['image_urls']['thumbnail']
+                    else:
+                        thumbnail_url = ''
+
+                    self.attachments[inc['id']] = Media(url=Url(media_url), thumbnail_url=Url(thumbnail_url))
                 else:
-                    log.error(f'Unhandled inc type {inc['type']}.')
+                    log.warn(f'Unhandled include type {inc['type']}.')
+
+            self.pages[i] = []
             for entry in obj['data']:
                 if entry['type'] != 'post':
-                    log.error(f'Unhandled entry type {entry['type']}.')
+                    log.warn(f'Unhandled entry type {entry['type']}.')
                     continue
                 post = self.create_post(entry, hash)
                 if post:
                     self.pages[i].append(post.unique_id)
+
             self.page += 1
             if i == num:
                 request_satisfied = True
+
         return request_satisfied
 
 class PatreonModule(Module):
diff --git a/src/portal/py/modules/pixiv_web.py b/src/portal/py/modules/pixiv_web.py
index 60fa812..c6a9c82 100644
--- a/src/portal/py/modules/pixiv_web.py
+++ b/src/portal/py/modules/pixiv_web.py
@@ -23,7 +23,8 @@ class PixivWebBase(Search):
             return None
         try:
             obj = response.json()
-        except JSONDecodeError:
+        except JSONDecodeError as e:
+            log.error(f'JSONDecodeError: {e}.')
             return None
         if obj['error']:
             log.error(obj['message'])
@@ -51,13 +52,16 @@ class PixivWebBase(Search):
         return None
 
     def create_tag(self, data: ParsedJson) -> Tag:
-        tag = Tag(type=TagType.GENERAL, name=data['tag'])
+        kwargs= {}
+        kwargs['type'] = TagType.GENERAL
+        kwargs['name'] = data['tag']
+        kwargs['alts'] = {}
         if 'romaji' in data:
-            tag.alts['romaji'] = data['romaji']
+            kwargs['alts']['romaji'] = data['romaji']
         if 'translation' in data:
             if 'en' in data['translation']:
-                tag.alts['en'] = data['translation']['en']
-        return tag
+                kwargs['alts']['en'] = data['translation']['en']
+        return Tag(**kwargs)
 
     def create_post(self, data: ParsedJson, pages: Optional[ParsedJson] = None, ugoira: Optional[ParsedJson] = None) -> Optional[Post]:
         kwargs = {}
@@ -88,7 +92,7 @@ class PixivWebBase(Search):
         kwargs['bookmarks'] = data['bookmarkCount']
         kwargs['comments'] = data['commentCount']
         kwargs['views'] = data['viewCount']
-        kwargs['tags'] = [self.create_tag(tag) for tag in data['tags']['tags']]
+        kwargs['tags'] = [self.create_tag(t) for t in data['tags']['tags']]
         if data['illustType'] == 2: # Ugoira
             if not ugoira:
                 url = f'{BASE_URL}/illust/{illust_id}/ugoira_meta?lang={LANG}&version={VERSION}'
diff --git a/src/portal/py/modules/twitter.py b/src/portal/py/modules/twitter.py
index 386bc5b..2f45ec1 100644
--- a/src/portal/py/modules/twitter.py
+++ b/src/portal/py/modules/twitter.py
@@ -55,7 +55,7 @@ class TwitterScrapeBase(Search):
             return Post(**kwargs)
         kwargs['raw_responses'] = tweet.rawResponses
         kwargs['url'] = tweet.url
-        # TODO: replace correct?
+        # @TODO: Replace correct?
         kwargs['dates'] = {
             DateType.CREATED: tweet.date.replace(tzinfo=timezone.utc).timestamp(),
             DateType.RETRIEVED: get_current_utc_time().timestamp()
diff --git a/src/portal/py/post.py b/src/portal/py/post.py
index e926c01..1a285c2 100644
--- a/src/portal/py/post.py
+++ b/src/portal/py/post.py
@@ -54,7 +54,7 @@ def df(c: Any) -> Any:
     return field(default_factory=lambda: c)
 
 def default_url_ext_guess(url: str) -> str:
-    # TODO: List of known extensions.
+    # @TODO: List of known extensions.
     question = url.rfind('?')
     if question >= 0:
         guess = url[0:question].rsplit('.')[-1]
diff --git a/src/portal/py/tests/archive_query.py b/src/portal/py/tests/archive_query.py
index 1ed9db4..9c4cf41 100644
--- a/src/portal/py/tests/archive_query.py
+++ b/src/portal/py/tests/archive_query.py
@@ -11,10 +11,12 @@ from post import PostEncoder, PostType, DateType, User
 from modules import ALL_MODULES
 from tests.logger import Logger
 
-#mode = 'pixiv_web'
-mode = 'fanbox'
+mode = 'pixiv_web'
+#mode = 'fanbox'
+#mode = 'patreon'
 cmd = 'user'
 #cmd = 'bookmarks'
+#cmd = 'search'
 module = ALL_MODULES[mode][0]
 if not module.init():
     sys.exit(1)
@@ -41,13 +43,14 @@ def default_media_download(obj, post, output_base):
                 return False
 
 class QueryDownloadThread(threading.Thread):
-    def __init__(self, log, complete, query, output_dir):
+    def __init__(self, log, complete, query, output_dir, start_page):
         super().__init__()
         self.log = log
         self.complete = complete
         self.query = query
         self.output_dir = output_dir
         self.media_download = default_media_download
+        self.start_page = start_page
 
     def make_path(self, post):
         prefix: str
@@ -142,22 +145,23 @@ class QueryDownloadThread(threading.Thread):
             return
         completed = True
         arg = self.query.split(':')[-1]
-        num = 0
+        num = self.start_page
         count = 0
         while completed:
             self.log.write(f'Starting page {num} of {arg}.')
             page = search.get_page(num)
+            for hash, response in module.raw_responses.items():
+                raw_response_path = f'{self.output_dir}/raw_responses/{hash}.json.gz'
+                if not os.path.isfile(raw_response_path):
+                    if not self.write_to_file(raw_response_path, 'wb+', response, True):
+                        completed = False
+                        break
             if not page:
                 break
             num += 1
             for unique_id in page:
                 if unique_id == 'twitter:t:1597637140833529856':
                     self.log.write('---------Hit target---------')
-                with quit_mutex:
-                    if not is_running:
-                        self.complete.write(f'{mode}:{cmd}:{arg}:i:{str(count)},{str(num)}')
-                        completed = False
-                        break
                 post = module.get_item(unique_id)
                 if not post:
                     self.log.write(f'Missing post with id {unique_id}.')
@@ -165,6 +169,11 @@ class QueryDownloadThread(threading.Thread):
                 if not self.write_post_to_disk(post):
                     continue
                 count += 1
+            with quit_mutex:
+                if not is_running:
+                    self.complete.write(f'{mode}:{cmd}:{arg}:i:{str(count)},{str(num)}')
+                    completed = False
+                    break
         if completed:
             self.complete.write(f'{mode}:{cmd}:{arg}:c:{str(count)},{str(num)}')
         with queue_cond:
@@ -204,25 +213,72 @@ def download_args():
     output_dir = sys.argv[1]
     if not os.path.isdir(output_dir):
         os.mkdir(output_dir)
+        os.mkdir(f'{output_dir}/raw_responses')
 
     #args = module.search(f'following:{22781328}')
-    #args = ['22781328']
-    #args = ['anoh223']
-    #args = ['532891', '1324074',]
-    #args = ['6600030', '163436']
-    #args = ['85117546', '29506', '3298353', '21674944', '2034628', '3085731']
-    #args = ['6827013', '11742', '4671', '36323560', '5128785', '30970895']
-    #args = ['mugi49']
-    #args = ['71598064', '9204502', '689283', '19956185', '42987864', '103682732', '4675776', '7780', '160755', '250142', '3444594', '1995476', '357647']
-    #args = ['10473280']
-    #args = ['178217', '106414', '882896', '3040749']
-    #args = ['21245269']
-    #args = ['1969907', '292644', '746841']
-    #args = ['1319954', '43718238', '2721319', '53459337']
+    #args = [
+    #    '5838770',
+    #    '44194940'
+    #]
     # TODO:
-    args = ['yzr']
-    #args = ['11187954']
-    # twitter test: butcha_u, sep__rina
+    #args = [
+    #    '68839843',
+    #    '16105069',
+    #    '67090304',
+    #    '188984',
+    #    '15041022',
+    #    '3948',
+    #    '2232374',
+    #    '194668',
+    #    '7835',
+    #    '3067312',
+    #    '59375709',
+    #    '9077832',
+    #    '199750',
+    #    '5806160',
+    #    '33726',
+    #    '15261563',
+    #    '7246',
+    #    '8146',
+    #    '40259810',
+    #    '28865',
+    #    '18362',
+    #    '2750098',
+    #    '2157729',
+    #    '65523978',
+    #    '154858',
+    #    '37625681',
+    #    '34642665',
+    #    '24023',
+    #    '3564336',
+    #    '2658856',
+    #    '990017',
+    #    '55392960',
+    #    '8967182',
+    #    '8965979',
+    #    '14305327',
+    #    '469378',
+    #    '8885157',
+    #    '1107939',
+    #    '18159122',
+    #    '57304923',
+    #    '6534119',
+    #    '91521',
+    #    '1074517',
+    #    '14394',
+    #    '46563991',
+    #    '33694842',
+    #    '84663',
+    #    '4028970',
+    #    '8876470',
+    #    '1829995',
+    #    '14113571',
+    #    '56349311',
+    #    '143555',
+    #    '36388895',
+    #    '14188490'
+    #]
+    # twitter test: butcha_u, sep__rina, nagayori000, sattinittas,
 
     threads = []
     start = True
@@ -256,7 +312,7 @@ def download_args():
             if not is_running:
                 break
         log.write(f'***********************\nStarting download of {arg}\n***********************\n')
-        arg_thread = QueryDownloadThread(log, complete, arg, output_dir)
+        arg_thread = QueryDownloadThread(log, complete, arg, output_dir, 0)
         threads.append(arg_thread)
         arg_thread.start()
 
diff --git a/src/portal/py/tests/test.py b/src/portal/py/tests/test.py
index 15bde0a..2fe2eda 100644
--- a/src/portal/py/tests/test.py
+++ b/src/portal/py/tests/test.py
@@ -1,3 +1,4 @@
+import sys
 import json
 import httpx
 import os.path
@@ -5,11 +6,11 @@ from base import Method
 from typing import Optional, Any
 from post import DateType, DateMeta, Date, PostEncoder, Post, MediaType
 from modules import ALL_MODULES
-#from twitter.scraper import Scraper
+from twitter.scraper import Scraper
 
 #scraper = Scraper(cookies = {
-#  'ct0': '5a3c98b7b50a74dc556fa497f497932e8958b5b2ef7ba74440ce3f341bb4753f663d3fe7633222d62a52d4ce121673234b927d97eeef806129d42bbfba7ab4d4cfbd2749c20fe6f8da480d0a1e6629e1',
-#  'auth_token': 'ea66b4c2ebd04e22de855dce6f1366221635bf2e'
+#  'ct0': '',
+#  'auth_token': ''
 #})
 #
 #media = scraper.media([1557919548904419328], limit=10)
@@ -55,14 +56,24 @@ from modules import ALL_MODULES
 #                else:
 #                    print('Skipping file')
 
-module = ALL_MODULES['instagram'][0]
+
+module = ALL_MODULES['pixiv_web'][0]
 module.init()
-search = module.search('plottttwistttttt')
+search = module.search('illust:91352621')
 page = search.get_page(0)
 for unique_id in page:
     post = module.get_item(unique_id)
     print(json.dumps(post, indent=4, cls=PostEncoder))
 
+#module = ALL_MODULES['instagram'][0]
+#module.init()
+#search = module.search('plottttwistttttt')
+#page = search.get_page(0)
+#for unique_id in page:
+#    post = module.get_item(unique_id)
+#    print(json.dumps(post, indent=4, cls=PostEncoder))
+
+
 #d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.EDITED_AT_UNKNOWN_TIME)
 #d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.NONE)
 #print(json.dumps(d, indent=4, cls=PostEncoder))