Portal update

Signed-off-by: Andrew Opalach <andrew@akon.city>

5 files changed, 100 insertions, 127 deletions

diff --git a/src/portal/py/tests/archive_query.py b/src/portal/py/tests/archive_query.py
index 4cc1e1e..3de4329 100644
--- a/src/portal/py/tests/archive_query.py
+++ b/src/portal/py/tests/archive_query.py
@@ -15,8 +15,8 @@ mode = 'pixiv_web'
 #mode = 'fanbox'
 #mode = 'patreon'
 #mode = 'instagram'
-#cmd = 'user'
-cmd = 'bookmarks'
+cmd = 'user'
+#cmd = 'bookmarks'
 #cmd = 'search'
 module = ALL_MODULES[mode][0]
 if not module.init():
@@ -180,8 +180,8 @@ class QueryDownloadThread(threading.Thread):
         with queue_cond:
             queue_cond.notify()
 
-#RUNTIME_PATH = "./run"
-RUNTIME_PATH = "/mnt/store/files/tmp/run"
+#RUNTIME_PATH = './run'
+RUNTIME_PATH = '/mnt/store/files/tmp/run'
 LOG_FILE = f'{RUNTIME_PATH}/archive4.log'
 ARG_FILE = f'{RUNTIME_PATH}/completed_args4.log'
 
@@ -216,10 +216,7 @@ def download_args():
         os.mkdir(output_dir)
         os.mkdir(f'{output_dir}/raw_responses')
 
-    #args = module.search(f'following:{22781328}')
-    # @TODO:
     args = []
-    # twitter test: butcha_u, sep__rina, nagayori000, sattinittas,
     threads = []
     start = True
 
diff --git a/src/portal/py/tests/old_twitter_api.py b/src/portal/py/tests/old_twitter_api.py
index 40d0346..4427dfd 100644
--- a/src/portal/py/tests/old_twitter_api.py
+++ b/src/portal/py/tests/old_twitter_api.py
@@ -27,14 +27,14 @@ log = Logger(LOG_FILE)
 compat_thread = QueryDownloadThread(log, None, None, sys.argv[1])
 
 def compat_media_download(obj, post, output_base):
-    for i, key in enumerate(post.media.keys()):
+    for index, key in enumerate(post.media.keys()):
         media = post.media[key]
         url = media.url
         media_path = f'{output_base}_media{key}.{url.ext}'
         if os.path.isfile(media_path):
             obj.log.write(f'Skipping media {media_path}.')
             continue
-        origin_path = f'{origin_base}/{post.author.unique_id.replace(':', '_')}/{post.unique_id.replace(':', '_')}_media{i}.{url.ext}'
+        origin_path = f'{origin_base}/{post.author.unique_id.replace(':', '_')}/{post.unique_id.replace(':', '_')}_media{index}.{url.ext}'
         if os.path.isfile(origin_path):
             shutil.copyfile(origin_path, media_path)
             #post.media[key].url.url = url.url.replace('name=orig', 'name=large')
diff --git a/src/portal/py/tests/rewrite_post.py b/src/portal/py/tests/rewrite_post.py
index 3bb152b..f00cfc7 100644
--- a/src/portal/py/tests/rewrite_post.py
+++ b/src/portal/py/tests/rewrite_post.py
@@ -2,20 +2,24 @@ import sys
 import os
 import gzip
 import json
-import hashlib
 sys.path.append('../py')
-import config
-#from post import PostEncoder, DateType
-#from modules.pixiv_web import PixivWebBase, PixivWebModule
-#from modules.twitter import TwitterScrapeBase, TwitterScrapeModule
-from logger import Logger
+from modules import ALL_MODULES
+from post import PostEncoder
+from modules.pixiv_app import PixivAppBase
 
-#module = PixivWebModule(config.PIXIV_SESSID)
-#search = PixivWebBase(module)
+module = ALL_MODULES['pixiv_app'][0]
+module.init()
+search = PixivAppBase(module)
 
-log = Logger('deleted_posts.log')
+target = sys.argv[1]
+with gzip.open(target, 'rb') as f:
+    obj = json.loads(f.read())
+    mtime = os.path.getmtime(target)
+    print(json.dumps(search.remake_post(obj['raw_responses'], mtime), cls=PostEncoder))
 
 '''
+log = Logger('deleted_posts.log')
+
 def rewrite_post(path):
     print(f'Rewriting post @ {path}')
     with gzip.open(path, 'rb') as f:
@@ -27,7 +31,6 @@ def rewrite_post(path):
     #print(json.dumps(post, indent=4, cls=PostEncoder))
     with gzip.open(path, 'wb') as f:
         f.write(json.dumps(post, cls=PostEncoder).encode('utf-8'))
-'''
 
 def dump_raw_response(path, target):
     print(f'Extracting raw_response from {path}')
@@ -54,3 +57,4 @@ for user in os.listdir(target):
         if file.split('.')[-1] == 'gz':
             dump_raw_response(f'{target}/{user}/{file}', raw_responses)
             #rewrite_post(f'{target}/{user}/{file}')
+'''
diff --git a/src/portal/py/tests/test.py b/src/portal/py/tests/test.py
index ef166d9..d3e5640 100644
--- a/src/portal/py/tests/test.py
+++ b/src/portal/py/tests/test.py
@@ -1,13 +1,4 @@
-import sys
-import json
-import httpx
-import os.path
-from base import Method
-from typing import Optional, Any
-from post import DateType, DateMeta, Date, PostEncoder, Post, MediaType
-from modules import ALL_MODULES
-from twitter.scraper import Scraper
-
+#from twitter.scraper import Scraper
 #scraper = Scraper(cookies = {
 #  'ct0': '',
 #  'auth_token': ''
@@ -15,101 +6,3 @@ from twitter.scraper import Scraper
 #
 #media = scraper.media([1557919548904419328], limit=10)
 #print(media)
-
-#output = '/mnt/store/files/ext/patreon_tmp/FPSBlyck'
-#
-#module = ALL_MODULES['patreon'][0]
-#module.init()
-#search = module.search('FPSBlyck')
-#page_num = 0
-#mark = False
-#while not mark:
-#    page = search.get_page(page_num)
-#    if not page:
-#        break
-#    page_num += 1
-#    for unique_id in page:
-#        if unique_id == 'patreon:p:29707872':
-#            print('hit mark')
-#            mark = True
-#        post = module.get_item(unique_id)
-#        print(json.dumps(post, indent=4, cls=PostEncoder))
-#        for key, m in post.media.items():
-#            if m.type == MediaType.FILE:
-#                path = f'{output}/{post.unique_id.replace(':', '_')}_{m.name}'
-#                if not os.path.isfile(path):
-#                    dl = module.get_download(post.unique_id, key)
-#                    retries = 3
-#                    r: Optional[httpx.Response] = None
-#                    while retries >= 0:
-#                        try:
-#                            r = httpx.get(dl['urls'][0].url, headers=dl['headers'], cookies=dl['cookies'], follow_redirects=True)
-#                        except:
-#                            retries -= 1
-#                        else:
-#                            break
-#                    if r:
-#                        with open(path, 'wb+') as f:
-#                            f.write(r.content)
-#                    else:
-#                        print('Download failed')
-#                else:
-#                    print('Skipping file')
-
-
-#module = ALL_MODULES['pixiv_web'][0]
-#module.init()
-#search = module.search('illust:91352621')
-#page = search.get_page(0)
-#for unique_id in page:
-#    post = module.get_item(unique_id)
-#    print(json.dumps(post, indent=4, cls=PostEncoder))
-
-module = ALL_MODULES['instagram'][0]
-module.init()
-search = module.search('plottttwistttttt')
-page = search.get_page(0)
-for unique_id in page:
-    post = module.get_item(unique_id)
-    print(json.dumps(post, indent=4, cls=PostEncoder))
-
-#d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.EDITED_AT_UNKNOWN_TIME)
-#d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.NONE)
-#print(json.dumps(d, indent=4, cls=PostEncoder))
-#sys.exit(1)
-
-#module = ALL_MODULES['fanbox'][0]
-#module = ALL_MODULES['pixiv_web'][0]
-#module = ALL_MODULES['youtube'][0]
-#module = ALL_MODULES['twitter'][0]
-#module.init()
-
-#search = module.search('opium_00pium')
-#search = module.search('search:ddd')
-#page = search.get_page(0)
-#for unique_id in page:
-#    post = module.get_item(unique_id)
-#    print(json.dumps(post, indent=4, cls=PostEncoder))
-#    break
-
-#post = module.get_item(page[0])
-#for key in post.media.keys():
-#  download_params = module.get_download(post.unique_id, key)
-#  for url in download_params['urls']:
-#      #response = module.do_request(Method.GET, url.url)
-#      r = httpx.get(url.url, headers=download_params['headers'])
-#      with open(f'test_{key}.{url.ext}', 'wb+') as f:
-#          f.write(r.content)
-
-#print(json.dumps(page, indent=4, cls=PostEncoder))
-#for i in range(0, len(post.media)):
-#    download_params = module.get_download(post.unique_id, i)
-#    r = httpx.get(download_params['url'], headers=download_params['headers'])
-#    if r.status_code != 200:
-#        print('error{}'.format(r.status_code))
-#    else:
-#        with open('test{}.png'.format(i), 'wb+') as f:
-#            f.write(r.content)
-
-#print(json.dumps(page, indent=4, cls=PostEncoder))
-#print(page)
diff --git a/src/portal/py/tests/touch_all.py b/src/portal/py/tests/touch_all.py
new file mode 100644
index 0000000..6d6bfa1
--- /dev/null
+++ b/src/portal/py/tests/touch_all.py
@@ -0,0 +1,79 @@
+import json
+from typing import Optional, Any
+from post import PostEncoder
+from modules import ALL_MODULES
+
+'''
+post = module.get_item(page[0])
+for key in post.media.keys():
+    download_params = module.get_download(post.unique_id, key)
+    for url in download_params['urls']:
+        #response = module.do_request(Method.GET, url.url)
+        r = httpx.get(url.url, headers=download_params['headers'])
+        if r.status_code != 200:
+            print('error{}'.format(r.status_code))
+        else:
+            with open(f'test_{key}.{url.ext}', 'wb+') as f:
+                f.write(r.content)
+'''
+
+def print_page(module: Any, page: Optional[list[str]]) -> None:
+    if not page:
+        print('Page is None')
+        return
+    count = 0
+    for unique_id in page:
+        post = module.get_item(unique_id)
+        print(json.dumps(post, indent=4, cls=PostEncoder))
+        count = count + 1
+        if count == 2:
+            break
+
+''' Blocked on certain endpoints, snscrape is outdated.
+module = ALL_MODULES['twitter'][0]
+module.init()
+
+search = module.search('search:#小関麗奈誕生祭2024')
+print_page(module, search.get_page(0))
+search = module.search('user:xuuikie')
+print_page(module, search.get_page(0))
+search = module.search('profile:@barackobama')
+print_page(module, search.get_page(0))
+search = module.search('tweet:1905966372288434680')
+print_page(module, search.get_page(0))
+'''
+
+'''
+module = ALL_MODULES['pixiv_app'][0]
+module.init()
+search = module.search('search:サムス・アラン')
+print_page(module, search.get_page(0))
+search = module.search('user:757584')
+print_page(module, search.get_page(0))
+search = module.search('bookmarks:2034628')
+print_page(module, search.get_page(0))
+search = module.search('following:22781328')
+print_page(module, search.get_page(0))
+search = module.search('illust:91352621')
+print_page(module, search.get_page(0))
+search = module.search('illust:26497574') # Deleted.
+print_page(module, search.get_page(0))
+'''
+
+''' No on-demand loading (Will load entire user before returning).
+module = ALL_MODULES['pixiv_web'][0]
+module.init()
+
+search = module.search('search:サムス・アラン')
+print_page(module, search.get_page(0))
+search = module.search('user:757584')
+print_page(module, search.get_page(0))
+search = module.search('bookmarks:2034628')
+print_page(module, search.get_page(0))
+search = module.search('following:22781328')
+print_page(module, search.get_page(0))
+search = module.search('illust:91352621')
+print_page(module, search.get_page(0))
+search = module.search('illust:100455669') # Deleted.
+print_page(module, search.get_page(0))
+'''