From 2cc385b71e161eae39915bf24de53d5b704e7ea0 Mon Sep 17 00:00:00 2001 From: Andrew Opalach Date: Tue, 5 Aug 2025 10:50:08 -0400 Subject: Twitter module work, various cleanups and comment clarifications Signed-off-by: Andrew Opalach --- src/portal/py/tests/archive_query.py | 32 +++++++++++++++++++++-------- src/portal/py/tests/logger.py | 15 -------------- src/portal/py/tests/test.py | 8 -------- src/portal/py/tests/touch_all.py | 40 +++++++++++++++++++----------------- 4 files changed, 45 insertions(+), 50 deletions(-) delete mode 100644 src/portal/py/tests/logger.py delete mode 100644 src/portal/py/tests/test.py (limited to 'src/portal/py/tests') diff --git a/src/portal/py/tests/archive_query.py b/src/portal/py/tests/archive_query.py index 3de4329..8ecbb52 100644 --- a/src/portal/py/tests/archive_query.py +++ b/src/portal/py/tests/archive_query.py @@ -1,20 +1,35 @@ import os import sys import signal -import json import threading +import json import gzip from datetime import datetime, timezone sys.path.append('../') from base import Method from post import PostEncoder, PostType, DateType, User from modules import ALL_MODULES -from tests.logger import Logger + +class Logger(): + def __init__(self, path): + self.file = open(path, 'a+') + self.mutex = threading.Lock() + + def write(self, message): + with self.mutex: + print(message) + self.file.write(message + '\n') + self.file.flush() + + def close(self): + self.file.close() mode = 'pixiv_web' #mode = 'fanbox' #mode = 'patreon' #mode = 'instagram' +#mode = 'twitter' +#cmd = 'profile' cmd = 'user' #cmd = 'bookmarks' #cmd = 'search' @@ -151,8 +166,8 @@ class QueryDownloadThread(threading.Thread): while completed: self.log.write(f'Starting page {num} of {arg}.') page = search.get_page(num) - for hash, response in module.raw_responses.items(): - raw_response_path = f'{self.output_dir}/raw_responses/{hash}.json.gz' + for hash_str, response in module.raw_responses.items(): + raw_response_path = f'{self.output_dir}/raw_responses/{hash_str}.json.gz' if not os.path.isfile(raw_response_path): if not self.write_to_file(raw_response_path, 'wb+', response, True): completed = False @@ -161,8 +176,8 @@ class QueryDownloadThread(threading.Thread): break num += 1 for unique_id in page: - if unique_id == 'twitter:t:1597637140833529856': - self.log.write('---------Hit target---------') + #if unique_id == '': + # self.log.write('---------Hit target---------') post = module.get_item(unique_id) if not post: self.log.write(f'Missing post with id {unique_id}.') @@ -182,8 +197,8 @@ class QueryDownloadThread(threading.Thread): #RUNTIME_PATH = './run' RUNTIME_PATH = '/mnt/store/files/tmp/run' -LOG_FILE = f'{RUNTIME_PATH}/archive4.log' -ARG_FILE = f'{RUNTIME_PATH}/completed_args4.log' +LOG_FILE = f'{RUNTIME_PATH}/archive5.log' +ARG_FILE = f'{RUNTIME_PATH}/completed_args5.log' def read_completed_args(path): args = [] @@ -214,6 +229,7 @@ def download_args(): output_dir = sys.argv[1] if not os.path.isdir(output_dir): os.mkdir(output_dir) + if not os.path.isdir(f'{output_dir}/raw_responses'): os.mkdir(f'{output_dir}/raw_responses') args = [] diff --git a/src/portal/py/tests/logger.py b/src/portal/py/tests/logger.py deleted file mode 100644 index 3adf792..0000000 --- a/src/portal/py/tests/logger.py +++ /dev/null @@ -1,15 +0,0 @@ -import threading - -class Logger(): - def __init__(self, path): - self.file = open(path, 'a+') - self.mutex = threading.Lock() - - def write(self, message): - with self.mutex: - print(message) - self.file.write(message + '\n') - self.file.flush() - - def close(self): - self.file.close() diff --git a/src/portal/py/tests/test.py b/src/portal/py/tests/test.py deleted file mode 100644 index d3e5640..0000000 --- a/src/portal/py/tests/test.py +++ /dev/null @@ -1,8 +0,0 @@ -#from twitter.scraper import Scraper -#scraper = Scraper(cookies = { -# 'ct0': '', -# 'auth_token': '' -#}) -# -#media = scraper.media([1557919548904419328], limit=10) -#print(media) diff --git a/src/portal/py/tests/touch_all.py b/src/portal/py/tests/touch_all.py index 6d6bfa1..158c7dd 100644 --- a/src/portal/py/tests/touch_all.py +++ b/src/portal/py/tests/touch_all.py @@ -3,36 +3,24 @@ from typing import Optional, Any from post import PostEncoder from modules import ALL_MODULES -''' -post = module.get_item(page[0]) -for key in post.media.keys(): - download_params = module.get_download(post.unique_id, key) - for url in download_params['urls']: - #response = module.do_request(Method.GET, url.url) - r = httpx.get(url.url, headers=download_params['headers']) - if r.status_code != 200: - print('error{}'.format(r.status_code)) - else: - with open(f'test_{key}.{url.ext}', 'wb+') as f: - f.write(r.content) -''' - def print_page(module: Any, page: Optional[list[str]]) -> None: if not page: print('Page is None') return - count = 0 for unique_id in page: post = module.get_item(unique_id) print(json.dumps(post, indent=4, cls=PostEncoder)) - count = count + 1 - if count == 2: - break -''' Blocked on certain endpoints, snscrape is outdated. module = ALL_MODULES['twitter'][0] module.init() +search = module.search('user:de7_7_7e') +print_page(module, search.get_page(0)) + +''' Blocked on most endpoints, snscrape is outdated. +module = ALL_MODULES['twitter_scrape'][0] +module.init() + search = module.search('search:#小関麗奈誕生祭2024') print_page(module, search.get_page(0)) search = module.search('user:xuuikie') @@ -77,3 +65,17 @@ print_page(module, search.get_page(0)) search = module.search('illust:100455669') # Deleted. print_page(module, search.get_page(0)) ''' + +''' +post = module.get_item(page[0]) +for key in post.media.keys(): + download_params = module.get_download(post.unique_id, key) + for url in download_params['urls']: + #response = module.do_request(Method.GET, url.url) + r = httpx.get(url.url, headers=download_params['headers']) + if r.status_code != 200: + print('error{}'.format(r.status_code)) + else: + with open(f'test_{key}.{url.ext}', 'wb+') as f: + f.write(r.content) +''' -- cgit v1.2.3-101-g0448