From 01f852081b1291da7fc342976bf5228ffb618736 Mon Sep 17 00:00:00 2001 From: Andrew Opalach Date: Sun, 18 May 2025 12:00:19 -0400 Subject: Portal update Signed-off-by: Andrew Opalach --- src/portal/py/tests/archive_query.py | 11 ++-- src/portal/py/tests/old_twitter_api.py | 4 +- src/portal/py/tests/rewrite_post.py | 24 +++++--- src/portal/py/tests/test.py | 109 +-------------------------------- src/portal/py/tests/touch_all.py | 79 ++++++++++++++++++++++++ 5 files changed, 100 insertions(+), 127 deletions(-) create mode 100644 src/portal/py/tests/touch_all.py (limited to 'src/portal/py/tests') diff --git a/src/portal/py/tests/archive_query.py b/src/portal/py/tests/archive_query.py index 4cc1e1e..3de4329 100644 --- a/src/portal/py/tests/archive_query.py +++ b/src/portal/py/tests/archive_query.py @@ -15,8 +15,8 @@ mode = 'pixiv_web' #mode = 'fanbox' #mode = 'patreon' #mode = 'instagram' -#cmd = 'user' -cmd = 'bookmarks' +cmd = 'user' +#cmd = 'bookmarks' #cmd = 'search' module = ALL_MODULES[mode][0] if not module.init(): @@ -180,8 +180,8 @@ class QueryDownloadThread(threading.Thread): with queue_cond: queue_cond.notify() -#RUNTIME_PATH = "./run" -RUNTIME_PATH = "/mnt/store/files/tmp/run" +#RUNTIME_PATH = './run' +RUNTIME_PATH = '/mnt/store/files/tmp/run' LOG_FILE = f'{RUNTIME_PATH}/archive4.log' ARG_FILE = f'{RUNTIME_PATH}/completed_args4.log' @@ -216,10 +216,7 @@ def download_args(): os.mkdir(output_dir) os.mkdir(f'{output_dir}/raw_responses') - #args = module.search(f'following:{22781328}') - # @TODO: args = [] - # twitter test: butcha_u, sep__rina, nagayori000, sattinittas, threads = [] start = True diff --git a/src/portal/py/tests/old_twitter_api.py b/src/portal/py/tests/old_twitter_api.py index 40d0346..4427dfd 100644 --- a/src/portal/py/tests/old_twitter_api.py +++ b/src/portal/py/tests/old_twitter_api.py @@ -27,14 +27,14 @@ log = Logger(LOG_FILE) compat_thread = QueryDownloadThread(log, None, None, sys.argv[1]) def compat_media_download(obj, post, output_base): - for i, key in enumerate(post.media.keys()): + for index, key in enumerate(post.media.keys()): media = post.media[key] url = media.url media_path = f'{output_base}_media{key}.{url.ext}' if os.path.isfile(media_path): obj.log.write(f'Skipping media {media_path}.') continue - origin_path = f'{origin_base}/{post.author.unique_id.replace(':', '_')}/{post.unique_id.replace(':', '_')}_media{i}.{url.ext}' + origin_path = f'{origin_base}/{post.author.unique_id.replace(':', '_')}/{post.unique_id.replace(':', '_')}_media{index}.{url.ext}' if os.path.isfile(origin_path): shutil.copyfile(origin_path, media_path) #post.media[key].url.url = url.url.replace('name=orig', 'name=large') diff --git a/src/portal/py/tests/rewrite_post.py b/src/portal/py/tests/rewrite_post.py index 3bb152b..f00cfc7 100644 --- a/src/portal/py/tests/rewrite_post.py +++ b/src/portal/py/tests/rewrite_post.py @@ -2,20 +2,24 @@ import sys import os import gzip import json -import hashlib sys.path.append('../py') -import config -#from post import PostEncoder, DateType -#from modules.pixiv_web import PixivWebBase, PixivWebModule -#from modules.twitter import TwitterScrapeBase, TwitterScrapeModule -from logger import Logger +from modules import ALL_MODULES +from post import PostEncoder +from modules.pixiv_app import PixivAppBase -#module = PixivWebModule(config.PIXIV_SESSID) -#search = PixivWebBase(module) +module = ALL_MODULES['pixiv_app'][0] +module.init() +search = PixivAppBase(module) -log = Logger('deleted_posts.log') +target = sys.argv[1] +with gzip.open(target, 'rb') as f: + obj = json.loads(f.read()) + mtime = os.path.getmtime(target) + print(json.dumps(search.remake_post(obj['raw_responses'], mtime), cls=PostEncoder)) ''' +log = Logger('deleted_posts.log') + def rewrite_post(path): print(f'Rewriting post @ {path}') with gzip.open(path, 'rb') as f: @@ -27,7 +31,6 @@ def rewrite_post(path): #print(json.dumps(post, indent=4, cls=PostEncoder)) with gzip.open(path, 'wb') as f: f.write(json.dumps(post, cls=PostEncoder).encode('utf-8')) -''' def dump_raw_response(path, target): print(f'Extracting raw_response from {path}') @@ -54,3 +57,4 @@ for user in os.listdir(target): if file.split('.')[-1] == 'gz': dump_raw_response(f'{target}/{user}/{file}', raw_responses) #rewrite_post(f'{target}/{user}/{file}') +''' diff --git a/src/portal/py/tests/test.py b/src/portal/py/tests/test.py index ef166d9..d3e5640 100644 --- a/src/portal/py/tests/test.py +++ b/src/portal/py/tests/test.py @@ -1,13 +1,4 @@ -import sys -import json -import httpx -import os.path -from base import Method -from typing import Optional, Any -from post import DateType, DateMeta, Date, PostEncoder, Post, MediaType -from modules import ALL_MODULES -from twitter.scraper import Scraper - +#from twitter.scraper import Scraper #scraper = Scraper(cookies = { # 'ct0': '', # 'auth_token': '' @@ -15,101 +6,3 @@ from twitter.scraper import Scraper # #media = scraper.media([1557919548904419328], limit=10) #print(media) - -#output = '/mnt/store/files/ext/patreon_tmp/FPSBlyck' -# -#module = ALL_MODULES['patreon'][0] -#module.init() -#search = module.search('FPSBlyck') -#page_num = 0 -#mark = False -#while not mark: -# page = search.get_page(page_num) -# if not page: -# break -# page_num += 1 -# for unique_id in page: -# if unique_id == 'patreon:p:29707872': -# print('hit mark') -# mark = True -# post = module.get_item(unique_id) -# print(json.dumps(post, indent=4, cls=PostEncoder)) -# for key, m in post.media.items(): -# if m.type == MediaType.FILE: -# path = f'{output}/{post.unique_id.replace(':', '_')}_{m.name}' -# if not os.path.isfile(path): -# dl = module.get_download(post.unique_id, key) -# retries = 3 -# r: Optional[httpx.Response] = None -# while retries >= 0: -# try: -# r = httpx.get(dl['urls'][0].url, headers=dl['headers'], cookies=dl['cookies'], follow_redirects=True) -# except: -# retries -= 1 -# else: -# break -# if r: -# with open(path, 'wb+') as f: -# f.write(r.content) -# else: -# print('Download failed') -# else: -# print('Skipping file') - - -#module = ALL_MODULES['pixiv_web'][0] -#module.init() -#search = module.search('illust:91352621') -#page = search.get_page(0) -#for unique_id in page: -# post = module.get_item(unique_id) -# print(json.dumps(post, indent=4, cls=PostEncoder)) - -module = ALL_MODULES['instagram'][0] -module.init() -search = module.search('plottttwistttttt') -page = search.get_page(0) -for unique_id in page: - post = module.get_item(unique_id) - print(json.dumps(post, indent=4, cls=PostEncoder)) - -#d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.EDITED_AT_UNKNOWN_TIME) -#d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.NONE) -#print(json.dumps(d, indent=4, cls=PostEncoder)) -#sys.exit(1) - -#module = ALL_MODULES['fanbox'][0] -#module = ALL_MODULES['pixiv_web'][0] -#module = ALL_MODULES['youtube'][0] -#module = ALL_MODULES['twitter'][0] -#module.init() - -#search = module.search('opium_00pium') -#search = module.search('search:ddd') -#page = search.get_page(0) -#for unique_id in page: -# post = module.get_item(unique_id) -# print(json.dumps(post, indent=4, cls=PostEncoder)) -# break - -#post = module.get_item(page[0]) -#for key in post.media.keys(): -# download_params = module.get_download(post.unique_id, key) -# for url in download_params['urls']: -# #response = module.do_request(Method.GET, url.url) -# r = httpx.get(url.url, headers=download_params['headers']) -# with open(f'test_{key}.{url.ext}', 'wb+') as f: -# f.write(r.content) - -#print(json.dumps(page, indent=4, cls=PostEncoder)) -#for i in range(0, len(post.media)): -# download_params = module.get_download(post.unique_id, i) -# r = httpx.get(download_params['url'], headers=download_params['headers']) -# if r.status_code != 200: -# print('error{}'.format(r.status_code)) -# else: -# with open('test{}.png'.format(i), 'wb+') as f: -# f.write(r.content) - -#print(json.dumps(page, indent=4, cls=PostEncoder)) -#print(page) diff --git a/src/portal/py/tests/touch_all.py b/src/portal/py/tests/touch_all.py new file mode 100644 index 0000000..6d6bfa1 --- /dev/null +++ b/src/portal/py/tests/touch_all.py @@ -0,0 +1,79 @@ +import json +from typing import Optional, Any +from post import PostEncoder +from modules import ALL_MODULES + +''' +post = module.get_item(page[0]) +for key in post.media.keys(): + download_params = module.get_download(post.unique_id, key) + for url in download_params['urls']: + #response = module.do_request(Method.GET, url.url) + r = httpx.get(url.url, headers=download_params['headers']) + if r.status_code != 200: + print('error{}'.format(r.status_code)) + else: + with open(f'test_{key}.{url.ext}', 'wb+') as f: + f.write(r.content) +''' + +def print_page(module: Any, page: Optional[list[str]]) -> None: + if not page: + print('Page is None') + return + count = 0 + for unique_id in page: + post = module.get_item(unique_id) + print(json.dumps(post, indent=4, cls=PostEncoder)) + count = count + 1 + if count == 2: + break + +''' Blocked on certain endpoints, snscrape is outdated. +module = ALL_MODULES['twitter'][0] +module.init() + +search = module.search('search:#小関麗奈誕生祭2024') +print_page(module, search.get_page(0)) +search = module.search('user:xuuikie') +print_page(module, search.get_page(0)) +search = module.search('profile:@barackobama') +print_page(module, search.get_page(0)) +search = module.search('tweet:1905966372288434680') +print_page(module, search.get_page(0)) +''' + +''' +module = ALL_MODULES['pixiv_app'][0] +module.init() +search = module.search('search:サムス・アラン') +print_page(module, search.get_page(0)) +search = module.search('user:757584') +print_page(module, search.get_page(0)) +search = module.search('bookmarks:2034628') +print_page(module, search.get_page(0)) +search = module.search('following:22781328') +print_page(module, search.get_page(0)) +search = module.search('illust:91352621') +print_page(module, search.get_page(0)) +search = module.search('illust:26497574') # Deleted. +print_page(module, search.get_page(0)) +''' + +''' No on-demand loading (Will load entire user before returning). +module = ALL_MODULES['pixiv_web'][0] +module.init() + +search = module.search('search:サムス・アラン') +print_page(module, search.get_page(0)) +search = module.search('user:757584') +print_page(module, search.get_page(0)) +search = module.search('bookmarks:2034628') +print_page(module, search.get_page(0)) +search = module.search('following:22781328') +print_page(module, search.get_page(0)) +search = module.search('illust:91352621') +print_page(module, search.get_page(0)) +search = module.search('illust:100455669') # Deleted. +print_page(module, search.get_page(0)) +''' -- cgit v1.2.3-101-g0448