Everything before initial synced list

Signed-off-by: Andrew Opalach <andrew@akon.city>

6 files changed, 0 insertions, 665 deletions

diff --git a/src/portal/tests/archive_query.py b/src/portal/tests/archive_query.py
deleted file mode 100644
index 9f69c1f..0000000
--- a/src/portal/tests/archive_query.py
+++ /dev/null
@@ -1,262 +0,0 @@
-import os
-import sys
-import signal
-import json
-import threading
-import gzip
-from datetime import datetime, timezone
-sys.path.append('../py')
-from base import Method
-from post import PostEncoder, PostType, DateType, User
-from modules import ALL_MODULES
-from logger import Logger
-
-mode = 'instagram'
-cmd = 'user'
-module = ALL_MODULES[mode][0]
-if not module.init():
-    sys.exit(1)
-
-quit_mutex = threading.Lock()
-queue_mutex = threading.Lock()
-queue_cond = threading.Condition(queue_mutex)
-is_running = True
-
-def default_media_download(obj, post, output_base):
-    for key in post.media.keys():
-        download_params = module.get_download(post.unique_id, key)
-        if not download_params:
-            obj.log.write(f'Skipping media{key} for post {post.unique_id}.')
-            continue
-        for url in download_params['urls']:
-            media_path = f'{output_base}_media{key}.{url.ext}'
-            obj.log.write(f'Attempting to download file {media_path}.')
-            response = module.do_request(Method.GET, url.url)
-            if not response:
-                obj.log.write(f'ERROR: Downloading post media failed ({post.unique_id} #{key}).')
-                continue
-            if not obj.write_to_file(media_path, 'wb+', response.content, False):
-                return False
-
-class QueryDownloadThread(threading.Thread):
-    def __init__(self, log, complete, query, output_dir):
-        super().__init__()
-        self.log = log
-        self.complete = complete
-        self.query = query
-        self.output_dir = output_dir
-        self.media_download = default_media_download
-
-    def make_path(self, post):
-        prefix: str
-        if post.type != PostType.TOMBSTONE:
-            prefix = post.author.unique_id.replace(':', '_')
-        else:
-            prefix = 'tombstones'
-        path = f'{self.output_dir}/{prefix}'
-        if not os.path.isdir(path):
-            os.mkdir(path)
-        return path
-
-    def write_to_file(self, path, open_method, content, compress):
-        try:
-            if compress:
-                with gzip.open(path, open_method) as f:
-                    f.write(content.encode('utf-8'))
-            else:
-                with open(path, open_method) as f:
-                    f.write(content)
-        except:
-            self.log.write(f'ERROR: Writing file failed ({path}).')
-            return False
-        return True
-
-    def write_post_to_disk(self, post):
-        if post.post.unique_id:
-            original = module.get_item(post.post.unique_id)
-            if original:
-                self.write_post_to_disk(original)
-        if post.quoted.unique_id:
-            quoted = module.get_item(post.quoted.unique_id)
-            if quoted:
-                self.write_post_to_disk(quoted)
-        message = f'Downloading post {post.unique_id}'
-        if 'detached_api' in post.raw_responses:
-            raw_response_path = f'{self.output_dir}/raw_responses'
-            if not os.path.isdir(raw_response_path):
-                os.mkdir(raw_response_path)
-            raw_response_path += f'/{post.raw_responses['hash']}.json.gz'
-            if not self.write_to_file(raw_response_path, 'wb+', json.dumps(json.loads(post.raw_responses['detached_api'])), True):
-                return False
-            del post.raw_responses['detached_api']
-        if post.type != PostType.TOMBSTONE:
-            message += f':{datetime.fromtimestamp(post.dates[DateType.CREATED], tz=timezone.utc).isoformat()} from {post.author.username}({post.author.display_name}):{post.author.unique_id}.'
-        else:
-            message += '.'
-        output_path = self.make_path(post)
-        output_base = f'{output_path}/{post.unique_id.replace(':', '_')}'
-        output = output_base + '.json.gz'
-        '''
-        if not os.path.isfile(output):
-            print('done')
-            sys.exit(1)
-        '''
-        '''
-        has_mp4 = False
-        for m in post.media.values():
-            if m.url.ext == 'mp4':
-                self.log.write('Rewriting post with mp4.')
-                has_mp4 = True
-                break
-        if post.quoted.unique_id:
-            self.log.write('Rewriting post with quote.')
-            has_mp4 = True
-        '''
-        if os.path.isfile(output):
-            skip_message = f'Skipping already downloaded post {post.unique_id}'
-            if post.type != PostType.TOMBSTONE:
-                skip_message += f' from {post.author.unique_id}.'
-            else:
-                skip_message += '.'
-            self.log.write(skip_message)
-            return True
-        self.log.write(message)
-        self.media_download(self, post, output_base)
-        if not self.write_to_file(output, 'wb+', json.dumps(post, cls=PostEncoder), True):
-            return False
-        return True
-
-    def supply_post(self, post):
-        if not self.write_post_to_disk(post):
-            return False
-        return True
-
-    def run(self):
-        global quit_mutex
-        global queue_cond
-        global is_running
-        search = module.search(self.query)
-        if not search:
-            return
-        completed = True
-        arg = self.query.split(':')[-1]
-        num = 0
-        count = 0
-        while completed:
-            self.log.write(f'Starting page {num} of {arg}.')
-            page = search.get_page(num)
-            if not page:
-                break
-            num += 1
-            for unique_id in page:
-                if unique_id == 'twitter:t:1597637140833529856':
-                    self.log.write('---------Hit target---------')
-                with quit_mutex:
-                    if not is_running:
-                        self.complete.write(f'{mode}:{cmd}:{arg}:i:{str(count)},{str(num)}')
-                        completed = False
-                        break
-                post = module.get_item(unique_id)
-                if not post:
-                    self.log.write(f'Missing post with id {unique_id}.')
-                    continue
-                if not self.write_post_to_disk(post):
-                    continue
-                count += 1
-        if completed:
-            self.complete.write(f'{mode}:{cmd}:{arg}:c:{str(count)},{str(num)}')
-        with queue_cond:
-            queue_cond.notify()
-
-RUNTIME_PATH = "/mnt/store/files/tmp/run"
-LOG_FILE = f'{RUNTIME_PATH}/archive.log'
-ARG_FILE = f'{RUNTIME_PATH}/completed_args.log'
-
-def read_completed_args(path):
-    args = []
-    if os.path.isfile(path):
-        with open(path, 'r') as f:
-            for l in f.read().splitlines():
-                if l[0] == '-':
-                    continue
-                s = l.split(':')
-                if s[3] == 'c':
-                    args.append(f'{s[0]}:{s[1]}:{s[2]}')
-    return args
-
-def download_args():
-    global quit_mutex
-    global queue_cond
-    global compat_thread
-
-    completed_args = read_completed_args(ARG_FILE)
-
-    log = Logger(LOG_FILE)
-    complete = Logger(ARG_FILE)
-
-    header = f'--------{datetime.now().isoformat()}---------'
-    log.write(header)
-    complete.write(header)
-
-    output_dir = sys.argv[1]
-    if not os.path.isdir(output_dir):
-        os.mkdir(output_dir)
-
-    #args = module.search(f'following:{22781328}')
-    args = ['byjoshuajamal']
-
-    threads = []
-    start = True
-
-    for arg in args:
-        #if mode == 'fanbox':
-        #    user = module.get_item(arg)
-        #    if not user or not isinstance(user, User):
-        #        log.write(f'Failed to retrive information for user {arg}.')
-        #        continue
-        #    arg = user.username
-        with quit_mutex:
-            if not is_running:
-                break
-        if mode == 'pixiv_web' and not start:
-            if arg == '':
-                start = True
-                continue
-        if not start:
-            continue
-        if mode != 'instagram':
-            arg = f'{cmd}:{arg}'
-        if f'{mode}:{arg}' in completed_args:
-            log.write(f'Skipping already downloaded arg {arg}.')
-            continue
-        if len(threads) >= 1:
-            with queue_cond:
-                queue_cond.wait()
-            threads = [t for t in threads if t.is_alive()]
-        with quit_mutex:
-            if not is_running:
-                break
-        log.write(f'***********************\nStarting download of {arg}\n***********************\n')
-        arg_thread = QueryDownloadThread(log, complete, arg, output_dir)
-        threads.append(arg_thread)
-        arg_thread.start()
-
-    for t in threads:
-        t.join()
-
-    log.close()
-    complete.close()
-
-def signal_handler(sig, frame):
-    global quit_mutex
-    global is_running
-    print('Attempting to quit...')
-    with quit_mutex:
-        if not is_running:
-            sys.exit(1)
-        else:
-            is_running = False
-
-if __name__ == "__main__":
-    signal.signal(signal.SIGINT, signal_handler)
-    download_args()
diff --git a/src/portal/tests/logger.py b/src/portal/tests/logger.py
deleted file mode 100644
index 80e22da..0000000
--- a/src/portal/tests/logger.py
+++ /dev/null
@@ -1,15 +0,0 @@
-import threading
-
-class Logger():
-    def __init__(self, path):
-        self.file = open(path, 'a+')
-        self.mutex = threading.Lock()
-
-    def write(self, msg):
-        with self.mutex:
-            print(msg)
-            self.file.write(msg + '\n')
-            self.file.flush()
-
-    def close(self):
-        self.file.close()
diff --git a/src/portal/tests/old_twitter_api.py b/src/portal/tests/old_twitter_api.py
deleted file mode 100644
index 40d0346..0000000
--- a/src/portal/tests/old_twitter_api.py
+++ /dev/null
@@ -1,271 +0,0 @@
-import sys
-import os
-import shutil
-import gzip
-import json
-import email.utils
-sys.path.append('../py')
-from base import Method
-from post import PostEncoder, User, Post, PostRef, PostType, DateType, Image, Video, MediaUrl
-from modules import ALL_MODULES
-from archive_query import QueryDownloadThread
-from logger import Logger
-
-module = ALL_MODULES['twitter'][0]
-if not module.init():
-    sys.exit(1)
-
-#origin_base = '/mnt/store/camu_db/twitter'
-origin_base = '/mnt/store/files/camu_db/data/twitter_orig'
-
-#RUNTIME_PATH = '.'
-RUNTIME_PATH = '/mnt/store/files/tmp/run'
-LOG_FILE = f'{RUNTIME_PATH}/twitter_reparse3.log'
-
-log = Logger(LOG_FILE)
-
-compat_thread = QueryDownloadThread(log, None, None, sys.argv[1])
-
-def compat_media_download(obj, post, output_base):
-    for i, key in enumerate(post.media.keys()):
-        media = post.media[key]
-        url = media.url
-        media_path = f'{output_base}_media{key}.{url.ext}'
-        if os.path.isfile(media_path):
-            obj.log.write(f'Skipping media {media_path}.')
-            continue
-        origin_path = f'{origin_base}/{post.author.unique_id.replace(':', '_')}/{post.unique_id.replace(':', '_')}_media{i}.{url.ext}'
-        if os.path.isfile(origin_path):
-            shutil.copyfile(origin_path, media_path)
-            #post.media[key].url.url = url.url.replace('name=orig', 'name=large')
-            obj.log.write(f'Used backup {origin_path}.')
-            continue
-        obj.log.write(f'Attempting to download file {media_path}.')
-        response = module.do_request(Method.GET, url.url, None, 1)
-        if not response:
-            return False
-        if not obj.write_to_file(media_path, 'wb+', response.content, False):
-            return False
-
-compat_thread.media_download = compat_media_download
-
-def process_post(post):
-    compat_thread.supply_post(post)
-    #print(json.dumps(post, indent=4, cls=PostEncoder))
-
-def create_media_url(url, format=True):
-    question = url.rfind('?')
-    if question >= 0:
-        ext = url[0:question].rsplit('.')[-1]
-    else:
-        ext = url.rsplit('.')[-1]
-    if format:
-        return MediaUrl(f'{url}?format={ext}&name=orig', ext), MediaUrl(f'{url}?format={ext}&name=small', ext)
-    return MediaUrl(url, ext), None
-
-def create_media(m):
-    original, thumbnail = create_media_url(m['media_url_https'])
-    if m['type'] == 'photo':
-        return Image(url=original, thumbnail_url=thumbnail)
-    elif m['type'] == 'video' or m['type'] == 'animated_gif':
-        variants = sorted(m['video_info']['variants'], key=lambda x: x['bitrate'] if 'bitrate' in x else 0, reverse=True)
-        url = variants[0]['url']
-        video_url, _ = create_media_url(url, False)
-        return Video(url=video_url, thumbnail_url=thumbnail)
-    return None
-
-def create_user(data):
-    unique_id = f'twitter:u:{data['id']}'
-    user = User(unique_id=unique_id, username=data['screen_name'], display_name=data['name'])
-    user.profile_picture_url = data['profile_image_url_https']
-    return user
-
-def create_post_v1(data, user, mtime, hash):
-    kwargs = {}
-    kwargs['unique_id'] = f'twitter:t:{data['id']}'
-    kwargs['raw_responses'] = {}
-    kwargs['raw_responses']['hash'] = hash
-    kwargs['url'] = f'https://twitter.com/{user.username}/status/{data['id']}'
-    kwargs['dates'] = {
-        DateType.CREATED: email.utils.parsedate_to_datetime(data['created_at']).timestamp(),
-        DateType.RETRIEVED: mtime
-    }
-    kwargs['author'] = user
-    if 'retweeted_status_id' in data:
-        kwargs['type'] = PostType.REPOST
-        kwargs['post'] = PostRef(f'twitter:t:{data['retweeted_status_id']}')
-        process_post(Post(**kwargs))
-        return
-    kwargs['type'] = PostType.POST
-    kwargs['text'] = data['full_text']
-    kwargs['likes'] = data['favorite_count']
-    kwargs['reposts'] = data['retweet_count']
-    kwargs['quotes'] = data['quote_count']
-    kwargs['comments'] = data['reply_count']
-    media = {}
-    if 'media' in data['entities']:
-        for m in data['entities']['media']:
-            media_id = m['id_str'] if 'id_str' in m else str(m['id'])
-            media[media_id] = create_media(m)
-    if 'extended_entities' in data and 'media' in data['extended_entities']:
-        for m in data['extended_entities']['media']:
-            media_id = m['id_str'] if 'id_str' in m else str(m['id'])
-            media[media_id] = create_media(m)
-    kwargs['media'] = media
-    if data['in_reply_to_status_id']:
-        kwargs['in_reply_to'] = PostRef(f'twitter:t:{data['in_reply_to_status_id']}')
-    if 'quoted_status_id' in data:
-        kwargs['quoted'] = PostRef(f'twitter:t:{data['quoted_status_id']}')
-    process_post(Post(**kwargs))
-
-def parse_v1_raw_response(obj, mtime, hash):
-    users = {}
-    for id, user in obj['globalObjects']['users'].items():
-        users[id] = create_user(user)
-    for tweet in obj['globalObjects']['tweets'].values():
-        create_post_v1(tweet, users[tweet['user_id_str']], mtime, hash)
-
-def create_post_graphql(data, users, tweet_id, mtime, hash):
-    if not tweet_id:
-        tweet_id = data.get('rest_id', None)
-    if data['__typename'] == 'TweetWithVisibilityResults':
-        data = data['tweet']
-        if not tweet_id:
-            tweet_id = data.get('rest_id', None)
-    if not tweet_id:
-        log.write('no id')
-        return PostRef()
-    kwargs = {}
-    unique_id = f'twitter:t:{tweet_id}'
-    kwargs['unique_id'] = unique_id
-    kwargs['raw_responses'] = {}
-    kwargs['raw_responses']['hash'] = hash
-    if 'legacy' not in data or ('__typename' in data and (data['__typename'] == 'TweetTombstone' or data['__typename'] == 'TweetUnavailable')):
-        kwargs['type'] = PostType.TOMBSTONE
-        process_post(Post(**kwargs))
-        return PostRef(unique_id)
-    tweet = data['legacy']
-    user_id = tweet['user_id_str']
-    if user_id not in users:
-        user_data = data['core']['user_results']['result']['legacy']
-        user = User(unique_id=f'twitter:u:{user_id}', username=user_data['screen_name'], display_name=user_data['name'])
-        user.profile_picture_url, _ = create_media_url(user_data['profile_image_url_https'], False)
-        users[user_id] = user
-    else:
-        user = users[user_id]
-    kwargs['url'] = f'https://twitter.com/{user.username}/status/{tweet_id}'
-    kwargs['dates'] = {
-        DateType.CREATED: email.utils.parsedate_to_datetime(tweet['created_at']).timestamp(),
-        DateType.RETRIEVED: mtime
-    }
-    kwargs['author'] = user
-    if 'retweeted_status_result' in tweet:
-        if 'result' in tweet['retweeted_status_result']:
-            retweet_id = create_post_graphql(tweet['retweeted_status_result']['result'], users, None, mtime, hash)
-        elif 'retweeted_status_id' in tweet:
-            retweet_id = PostRef(f'twitter:t:{tweet['retweeted_status_id']}')
-        else:
-            retweet_id = PostRef()
-        kwargs['type'] = PostType.REPOST
-        kwargs['post'] = retweet_id
-        process_post(Post(**kwargs))
-        return PostRef(unique_id)
-    kwargs['type'] = PostType.POST
-    kwargs['text'] = tweet['full_text']
-    kwargs['likes'] = tweet['favorite_count']
-    kwargs['reposts'] = tweet['retweet_count']
-    kwargs['quotes'] = tweet['quote_count']
-    kwargs['comments'] = tweet['reply_count']
-    if 'views' in data and 'count' in data['views']:
-        kwargs['views'] = int(data['views']['count'])
-    if 'urls' in tweet['entities']:
-        urls = []
-        for url in tweet['entities']['urls']:
-            urls.append(url['expanded_url'])
-        kwargs['links'] = urls
-    media = {}
-    if 'media' in tweet['entities']:
-        for m in tweet['entities']['media']:
-            media_id = m['id_str'] if 'id_str' in m else str(m['id'])
-            media[media_id] = create_media(m)
-    if 'extended_entities' in tweet and 'media' in tweet['extended_entities']:
-        for m in tweet['extended_entities']['media']:
-            media_id = m['id_str'] if 'id_str' in m else str(m['id'])
-            media[media_id] = create_media(m)
-    kwargs['media'] = media
-    if 'quoted_status_result' in data:
-        if 'result' not in data['quoted_status_result']:
-            quoted_id = PostRef(f'twitter:t:{tweet['quoted_status_id_str']}')
-        else:
-            quoted_id = create_post_graphql(data['quoted_status_result']['result'], users, None, mtime, hash)
-        kwargs['quoted'] = quoted_id
-    elif 'quoted_status_id_str' in tweet:
-        log.write('quote id no data')
-        quoted_id = PostRef(f'twitter:t:{tweet['quoted_status_id_str']}')
-        kwargs['quoted'] = quoted_id
-    elif data.get('quotedRefResult'):
-        log.write('ref quote')
-        quoted = data['quotedRefResult']['result']
-        if quoted['__typename'] == 'TweetWithVisibilityResults':
-            quoted = quoted['tweet']
-        quoted_id = PostRef(f'twitter:t:{quoted['rest_id']}')
-        kwargs['quoted'] = quoted_id
-    if 'in_reply_to_status_id_str' in tweet:
-        kwargs['in_reply_to'] = PostRef(f'twitter:t:{tweet['in_reply_to_status_id_str']}')
-    process_post(Post(**kwargs))
-    return PostRef(unique_id)
-
-def parse_graphql_raw_response(obj, mtime, hash):
-    users = {}
-    if 'user' in obj['data']:
-        instructions = obj['data']['user']['result']['timeline_v2']['timeline']['instructions']
-    elif 'search_by_raw_query' in obj['data']:
-        instructions = obj['data']['search_by_raw_query']['search_timeline']['timeline']['instructions']
-    else:
-        instructions = []
-    for inst in instructions:
-        if inst['type'] == 'TimelineAddEntries':
-            for entry in inst['entries']:
-                if entry['entryId'].startswith('tweet-'):
-                    tweet_id = int(entry['entryId'].split('-', 1)[1])
-                    if entry['content']['entryType'] == 'TimelineTimelineItem' and entry['content']['itemContent']['itemType'] == 'TimelineTweet':
-                        if 'result' not in entry['content']['itemContent']['tweet_results']:
-                            continue
-                        create_post_graphql(entry['content']['itemContent']['tweet_results']['result'], users, tweet_id, mtime, hash)
-                    else:
-                        log.write('Got unrecognised timeline tweet item(s)') # snscrape copy and paste
-                elif entry['entryId'].startswith(('homeConversation-', 'profile-conversation-')):
-                    if entry['content']['entryType'] == 'TimelineTimelineModule':
-                        for item in reversed(entry['content']['items']):
-                            if not item['entryId'].startswith(entry['entryId'].split('ion-', 1)[0] + 'ion-') or '-tweet-' not in item['entryId']:
-                                log.write(f'Unexpected conversation entry ID: {item['entryId']!r}') # snscrape copy and paste
-                                continue
-                            if item['item']['itemContent']['itemType'] == 'TimelineTweet':
-                                tweet_id = int(item['entryId'].split('-tweet-', 1)[1])
-                                if 'result' in item['item']['itemContent']['tweet_results']:
-                                    create_post_graphql(item['item']['itemContent']['tweet_results']['result'], users, tweet_id, mtime, hash)
-                                else:
-                                    kwargs = {}
-                                    unique_id = f'twitter:t:{tweet_id}'
-                                    kwargs['unique_id'] = unique_id
-                                    kwargs['raw_responses'] = {}
-                                    kwargs['raw_responses']['hash'] = hash
-                                    kwargs['type'] = PostType.TOMBSTONE
-                                    process_post(Post(**kwargs))
-
-def parse_raw_response(path, hash):
-    mtime = os.path.getmtime(path)
-    with gzip.open(path, 'rb') as f:
-        obj = json.loads(f.read())
-    if 'globalObjects' not in obj:
-        parse_graphql_raw_response(obj, mtime, hash)
-    else:
-        parse_v1_raw_response(obj, mtime, hash)
-
-#target = '/mnt/store/camu_db/twitter/raw_responses'
-target = '/mnt/store/files/camu_db/data/twitter_orig/raw_responses'
-
-for file in os.listdir(target):
-    hash = file.split('.')[0]
-    print(hash)
-    parse_raw_response(f'{target}/{file}', hash)
diff --git a/src/portal/tests/rewrite_post.py b/src/portal/tests/rewrite_post.py
deleted file mode 100644
index 3bb152b..0000000
--- a/src/portal/tests/rewrite_post.py
+++ /dev/null
@@ -1,56 +0,0 @@
-import sys
-import os
-import gzip
-import json
-import hashlib
-sys.path.append('../py')
-import config
-#from post import PostEncoder, DateType
-#from modules.pixiv_web import PixivWebBase, PixivWebModule
-#from modules.twitter import TwitterScrapeBase, TwitterScrapeModule
-from logger import Logger
-
-#module = PixivWebModule(config.PIXIV_SESSID)
-#search = PixivWebBase(module)
-
-log = Logger('deleted_posts.log')
-
-'''
-def rewrite_post(path):
-    print(f'Rewriting post @ {path}')
-    with gzip.open(path, 'rb') as f:
-        obj = json.loads(f.read())
-        post = search.remake_post(obj['raw_responses'], obj['dates'][str(DateType.RETRIEVED.value)])
-    if not post:
-        log.write(path)
-        return
-    #print(json.dumps(post, indent=4, cls=PostEncoder))
-    with gzip.open(path, 'wb') as f:
-        f.write(json.dumps(post, cls=PostEncoder).encode('utf-8'))
-'''
-
-def dump_raw_response(path, target):
-    print(f'Extracting raw_response from {path}')
-    with gzip.open(path, 'rb') as f:
-        obj = json.loads(f.read())
-    if 'api' in obj['raw_responses']:
-        hasher = hashlib.new('sha256')
-        data = json.dumps(json.loads(obj['raw_responses']['api'])).encode('utf-8')
-        hasher.update(data)
-        output = f'{target}/{hasher.hexdigest()}.json.gz'
-        with gzip.open(output, 'wb+') as f:
-            f.write(data)
-
-target = sys.argv[1]
-raw_responses = f'{target}/raw_responses'
-
-if not os.path.isdir(raw_responses):
-    os.mkdir(raw_responses)
-
-for user in os.listdir(target):
-    if user == 'raw_responses':
-        continue
-    for file in os.listdir(f'{target}/{user}'):
-        if file.split('.')[-1] == 'gz':
-            dump_raw_response(f'{target}/{user}/{file}', raw_responses)
-            #rewrite_post(f'{target}/{user}/{file}')
diff --git a/src/portal/tests/test.py b/src/portal/tests/test.py
deleted file mode 100644
index a5139f6..0000000
--- a/src/portal/tests/test.py
+++ /dev/null
@@ -1,51 +0,0 @@
-import json
-import httpx
-import sys
-sys.path.append('../py')
-from base import Method
-from post import DateType, PostEncoder, Media
-from modules import ALL_MODULES
-
-#ALL_MODULES['instagram'][0].init()
-#search = ALL_MODULES['instagram'][0].search('opium_00pium')
-#page = search.get_page(0)
-#print(page)
-#print(json.dumps(page, indent=4, cls=PostEncoder))
-
-d = Media()
-print(d)
-sys.exit(1)
-
-#module = ALL_MODULES['fanbox'][0]
-#module = ALL_MODULES['pixiv_web'][0]
-module = ALL_MODULES['instagram'][0]
-#module = ALL_MODULES['twitter'][0]
-module.init()
-
-search = module.search('opium_00pium')
-page = search.get_page(1)
-for unique_id in page:
-    post = module.get_item(unique_id)
-    print(json.dumps(post, indent=4, cls=PostEncoder))
-
-#post = module.get_item(page[0])
-#for key in post.media.keys():
-#  download_params = module.get_download(post.unique_id, key)
-#  for url in download_params['urls']:
-#      #response = module.do_request(Method.GET, url.url)
-#      r = httpx.get(url.url, headers=download_params['headers'])
-#      with open(f'test_{key}.{url.ext}', 'wb+') as f:
-#          f.write(r.content)
-
-#print(json.dumps(page, indent=4, cls=PostEncoder))
-#for i in range(0, len(post.media)):
-#    download_params = module.get_download(post.unique_id, i)
-#    r = httpx.get(download_params['url'], headers=download_params['headers'])
-#    if r.status_code != 200:
-#        print('error{}'.format(r.status_code))
-#    else:
-#        with open('test{}.png'.format(i), 'wb+') as f:
-#            f.write(r.content)
-
-#print(json.dumps(page, indent=4, cls=PostEncoder))
-#print(page)
diff --git a/src/portal/tests/unescape_json.py b/src/portal/tests/unescape_json.py
deleted file mode 100644
index eaf5194..0000000
--- a/src/portal/tests/unescape_json.py
+++ /dev/null
@@ -1,10 +0,0 @@
-import sys
-import gzip
-import json
-
-for path in sys.argv[1:]:
-    with gzip.open(path, 'rb') as f:
-        obj = json.loads(f.read().decode('utf-8'))
-    obj = json.loads(obj)
-    with gzip.open(path, 'wb') as f:
-        f.write(json.dumps(obj).encode('utf-8'))