Twitter module work, various cleanups

 and comment clarifications

Signed-off-by: Andrew Opalach <andrew@akon.city>

4 files changed, 45 insertions, 50 deletions

diff --git a/src/portal/py/tests/archive_query.py b/src/portal/py/tests/archive_query.py
index 3de4329..8ecbb52 100644
--- a/src/portal/py/tests/archive_query.py
+++ b/src/portal/py/tests/archive_query.py
@@ -1,20 +1,35 @@
 import os
 import sys
 import signal
-import json
 import threading
+import json
 import gzip
 from datetime import datetime, timezone
 sys.path.append('../')
 from base import Method
 from post import PostEncoder, PostType, DateType, User
 from modules import ALL_MODULES
-from tests.logger import Logger
+
+class Logger():
+    def __init__(self, path):
+        self.file = open(path, 'a+')
+        self.mutex = threading.Lock()
+
+    def write(self, message):
+        with self.mutex:
+            print(message)
+            self.file.write(message + '\n')
+            self.file.flush()
+
+    def close(self):
+        self.file.close()
 
 mode = 'pixiv_web'
 #mode = 'fanbox'
 #mode = 'patreon'
 #mode = 'instagram'
+#mode = 'twitter'
+#cmd = 'profile'
 cmd = 'user'
 #cmd = 'bookmarks'
 #cmd = 'search'
@@ -151,8 +166,8 @@ class QueryDownloadThread(threading.Thread):
         while completed:
             self.log.write(f'Starting page {num} of {arg}.')
             page = search.get_page(num)
-            for hash, response in module.raw_responses.items():
-                raw_response_path = f'{self.output_dir}/raw_responses/{hash}.json.gz'
+            for hash_str, response in module.raw_responses.items():
+                raw_response_path = f'{self.output_dir}/raw_responses/{hash_str}.json.gz'
                 if not os.path.isfile(raw_response_path):
                     if not self.write_to_file(raw_response_path, 'wb+', response, True):
                         completed = False
@@ -161,8 +176,8 @@ class QueryDownloadThread(threading.Thread):
                 break
             num += 1
             for unique_id in page:
-                if unique_id == 'twitter:t:1597637140833529856':
-                    self.log.write('---------Hit target---------')
+                #if unique_id == '':
+                #    self.log.write('---------Hit target---------')
                 post = module.get_item(unique_id)
                 if not post:
                     self.log.write(f'Missing post with id {unique_id}.')
@@ -182,8 +197,8 @@ class QueryDownloadThread(threading.Thread):
 
 #RUNTIME_PATH = './run'
 RUNTIME_PATH = '/mnt/store/files/tmp/run'
-LOG_FILE = f'{RUNTIME_PATH}/archive4.log'
-ARG_FILE = f'{RUNTIME_PATH}/completed_args4.log'
+LOG_FILE = f'{RUNTIME_PATH}/archive5.log'
+ARG_FILE = f'{RUNTIME_PATH}/completed_args5.log'
 
 def read_completed_args(path):
     args = []
@@ -214,6 +229,7 @@ def download_args():
     output_dir = sys.argv[1]
     if not os.path.isdir(output_dir):
         os.mkdir(output_dir)
+    if not os.path.isdir(f'{output_dir}/raw_responses'):
         os.mkdir(f'{output_dir}/raw_responses')
 
     args = []
diff --git a/src/portal/py/tests/logger.py b/src/portal/py/tests/logger.py
deleted file mode 100644
index 3adf792..0000000
--- a/src/portal/py/tests/logger.py
+++ /dev/null
@@ -1,15 +0,0 @@
-import threading
-
-class Logger():
-    def __init__(self, path):
-        self.file = open(path, 'a+')
-        self.mutex = threading.Lock()
-
-    def write(self, message):
-        with self.mutex:
-            print(message)
-            self.file.write(message + '\n')
-            self.file.flush()
-
-    def close(self):
-        self.file.close()
diff --git a/src/portal/py/tests/test.py b/src/portal/py/tests/test.py
deleted file mode 100644
index d3e5640..0000000
--- a/src/portal/py/tests/test.py
+++ /dev/null
@@ -1,8 +0,0 @@
-#from twitter.scraper import Scraper
-#scraper = Scraper(cookies = {
-#  'ct0': '',
-#  'auth_token': ''
-#})
-#
-#media = scraper.media([1557919548904419328], limit=10)
-#print(media)
diff --git a/src/portal/py/tests/touch_all.py b/src/portal/py/tests/touch_all.py
index 6d6bfa1..158c7dd 100644
--- a/src/portal/py/tests/touch_all.py
+++ b/src/portal/py/tests/touch_all.py
@@ -3,36 +3,24 @@ from typing import Optional, Any
 from post import PostEncoder
 from modules import ALL_MODULES
 
-'''
-post = module.get_item(page[0])
-for key in post.media.keys():
-    download_params = module.get_download(post.unique_id, key)
-    for url in download_params['urls']:
-        #response = module.do_request(Method.GET, url.url)
-        r = httpx.get(url.url, headers=download_params['headers'])
-        if r.status_code != 200:
-            print('error{}'.format(r.status_code))
-        else:
-            with open(f'test_{key}.{url.ext}', 'wb+') as f:
-                f.write(r.content)
-'''
-
 def print_page(module: Any, page: Optional[list[str]]) -> None:
     if not page:
         print('Page is None')
         return
-    count = 0
     for unique_id in page:
         post = module.get_item(unique_id)
         print(json.dumps(post, indent=4, cls=PostEncoder))
-        count = count + 1
-        if count == 2:
-            break
 
-''' Blocked on certain endpoints, snscrape is outdated.
 module = ALL_MODULES['twitter'][0]
 module.init()
 
+search = module.search('user:de7_7_7e')
+print_page(module, search.get_page(0))
+
+''' Blocked on most endpoints, snscrape is outdated.
+module = ALL_MODULES['twitter_scrape'][0]
+module.init()
+
 search = module.search('search:#小関麗奈誕生祭2024')
 print_page(module, search.get_page(0))
 search = module.search('user:xuuikie')
@@ -77,3 +65,17 @@ print_page(module, search.get_page(0))
 search = module.search('illust:100455669') # Deleted.
 print_page(module, search.get_page(0))
 '''
+
+'''
+post = module.get_item(page[0])
+for key in post.media.keys():
+    download_params = module.get_download(post.unique_id, key)
+    for url in download_params['urls']:
+        #response = module.do_request(Method.GET, url.url)
+        r = httpx.get(url.url, headers=download_params['headers'])
+        if r.status_code != 200:
+            print('error{}'.format(r.status_code))
+        else:
+            with open(f'test_{key}.{url.ext}', 'wb+') as f:
+                f.write(r.content)
+'''