diff --git a/apis/api_helper.py b/apis/api_helper.py index 32dab2c0f..c4fa9e6eb 100644 --- a/apis/api_helper.py +++ b/apis/api_helper.py @@ -157,7 +157,7 @@ async def json_request( method: str = "GET", stream: bool = False, json_format: bool = True, - payload: Dict[str, str] = {}, + payload: dict[str, str] = {}, ) -> Any: headers = {} custom_session = False diff --git a/apis/onlyfans/classes/create_auth.py b/apis/onlyfans/classes/create_auth.py index 86b42300a..fac521720 100644 --- a/apis/onlyfans/classes/create_auth.py +++ b/apis/onlyfans/classes/create_auth.py @@ -185,10 +185,12 @@ async def get_lists(self, refresh=True, limit=100, offset=0): self.lists = results return results - async def get_user(self, identifier: Union[str, int]) -> Union[create_user, dict]: + async def get_user( + self, identifier: Union[str, int] + ) -> Union[create_user, error_details]: link = endpoint_links(identifier).users response = await self.session_manager.json_request(link) - if isinstance(response, dict): + if not isinstance(response, error_details): response["session_manager"] = self.session_manager response = create_user(response, self) return response @@ -257,7 +259,8 @@ async def get_subscriptions( self.pool = temp_pool self.paid_content = temp_paid_content temp_auth = await self.get_user(self.username) - json_authed = json_authed | temp_auth.__dict__ + if isinstance(json_authed, dict): + json_authed = json_authed | temp_auth.__dict__ subscription = create_user(json_authed, self) subscription.subscriber = self diff --git a/classes/prepare_metadata.py b/classes/prepare_metadata.py index e2cdc9363..a3afe1259 100644 --- a/classes/prepare_metadata.py +++ b/classes/prepare_metadata.py @@ -408,14 +408,14 @@ def __iter__(self): for attr, value in self.__dict__.items(): yield attr, value - def reformat(self, unformatted_list) -> list[str]: + async def reformat(self, unformatted_list) -> list[str]: x = [] format_variables2 = format_variables() for key, unformatted_item in unformatted_list.items(): if "filename_format" == key: unformatted_item = os.path.join(x[1], unformatted_item) print - string = main_helper.reformat(self, unformatted_item) + string = await main_helper.reformat(self, unformatted_item) final_path = [] paths = string.split(os.sep) for path in paths: diff --git a/extras/OFRenamer/start.py b/extras/OFRenamer/start_ofr.py similarity index 93% rename from extras/OFRenamer/start.py rename to extras/OFRenamer/start_ofr.py index ce8a4b686..569c52736 100644 --- a/extras/OFRenamer/start.py +++ b/extras/OFRenamer/start_ofr.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +import asyncio import os +from random import randint import shutil import traceback import urllib.parse as urlparse @@ -68,7 +70,7 @@ async def fix_directories2(post: api_table, media_db: list[media_table]): option["preview"] = media.preview option["archived"] = post.archived prepared_format = prepare_reformat(option) - file_directory = main_helper.reformat( + file_directory = await main_helper.reformat( prepared_format, file_directory_format ) prepared_format.directory = file_directory @@ -87,7 +89,9 @@ async def fix_directories2(post: api_table, media_db: list[media_table]): old_filepaths = [x for x in old_filepaths if "linked_" not in x] if old_filepaths: old_filepath = old_filepaths[0] - new_filepath = main_helper.reformat(prepared_format, filename_format) + # a = randint(0,1) + # await asyncio.sleep(a) + new_filepath = await main_helper.reformat(prepared_format, filename_format) if old_filepath and old_filepath != new_filepath: if os.path.exists(new_filepath): os.remove(new_filepath) @@ -133,7 +137,8 @@ async def fix_directories2(post: api_table, media_db: list[media_table]): result = database_session.query(folder.media_table) media_db = result.all() pool = api.pool - tasks = pool.starmap(fix_directories2, product(posts, [media_db])) + # tasks = pool.starmap(fix_directories2, product(posts, [media_db])) + tasks = [asyncio.ensure_future(fix_directories2(post,media_db)) for post in posts] settings = {"colour": "MAGENTA", "disable": False} delete_rows = await tqdm.gather(tasks, **settings) delete_rows = list(chain(*delete_rows)) @@ -186,7 +191,7 @@ async def start( reformats[key] = value.split(key2, 1)[0] + key2 print print - a, base_directory, c = prepare_reformat(option, keep_vars=True).reformat(reformats) + a, base_directory, c = await prepare_reformat(option, keep_vars=True).reformat(reformats) download_info["base_directory"] = base_directory print all_files = [] diff --git a/helpers/main_helper.py b/helpers/main_helper.py index 2f76cd66a..08c9206da 100644 --- a/helpers/main_helper.py +++ b/helpers/main_helper.py @@ -212,7 +212,7 @@ def legacy_database_fixer(database_path, database, database_name, database_exist export_sqlite(old_database_path, datas, database_name, legacy_fixer=True) -def fix_sqlite( +async def fix_sqlite( profile_directory, download_directory, metadata_directory, @@ -229,7 +229,7 @@ def fix_sqlite( mandatory_directories["profile_directory"] = profile_directory mandatory_directories["download_directory"] = download_directory mandatory_directories["metadata_directory"] = metadata_directory - formatted_directories = format_directories( + formatted_directories = await format_directories( mandatory_directories, authed, site_name, @@ -375,7 +375,7 @@ def format_paths(j_directories, site_name): return paths -def reformat(prepared_format: prepare_reformat, unformatted): +async def reformat(prepared_format: prepare_reformat, unformatted): post_id = prepared_format.post_id media_id = prepared_format.media_id date = prepared_format.date diff --git a/modules/onlyfans.py b/modules/onlyfans.py index 177ac72d9..0f6d5a4c9 100644 --- a/modules/onlyfans.py +++ b/modules/onlyfans.py @@ -11,7 +11,7 @@ from urllib.parse import urlparse import extras.OFLogin.start_ofl as oflogin -import extras.OFRenamer.start as ofrenamer +import extras.OFRenamer.start_ofr as ofrenamer import helpers.db_helper as db_helper import helpers.main_helper as main_helper from apis.onlyfans import onlyfans as OnlyFans @@ -145,7 +145,7 @@ async def start_datascraper( username, metadata_directory_format, ] - fix_sqlite(*some_list) + await fix_sqlite(*some_list) api_array = scrape_choice(authed, subscription) api_array = format_options(api_array, "apis") apis = api_array[0] @@ -275,7 +275,7 @@ async def profile_scraper( option["date_format"] = date_format option["maximum_length"] = text_length option["directory"] = base_directory - a, b, c = prepare_reformat(option, keep_vars=True).reformat(reformats) + a, b, c = await prepare_reformat(option, keep_vars=True).reformat(reformats) print y = await authed.get_subscription(identifier=model_username) override_media_types = [] @@ -368,7 +368,7 @@ async def paid_content_scraper(api: start, identifiers=[]): mandatory_directories["profile_directory"] = profile_directory mandatory_directories["download_directory"] = download_directory mandatory_directories["metadata_directory"] = metadata_directory - formatted_directories = format_directories( + formatted_directories = await format_directories( mandatory_directories, authed, site_name, @@ -689,7 +689,7 @@ async def process_metadata( os.remove(old_metadata) -def format_directories( +async def format_directories( directories: dict[str, Any], authed: create_auth, site_name: str, @@ -725,7 +725,7 @@ def format_directories( legacy_model_directory, "Metadata" ) if key == "metadata_directory": - x["metadata_directory"] = main_helper.reformat(prepared_format, unformatted) + x["metadata_directory"] = await main_helper.reformat(prepared_format, unformatted) x["locations"] = [] for location in locations: directories = {} @@ -759,7 +759,7 @@ async def prepare_scraper(authed: create_auth, site_name, item): mandatory_directories["profile_directory"] = profile_directory mandatory_directories["download_directory"] = download_directory mandatory_directories["metadata_directory"] = metadata_directory - formatted_directories = format_directories( + formatted_directories = await format_directories( mandatory_directories, authed, site_name, @@ -1147,11 +1147,11 @@ async def media_scraper( option["archived"] = new_post["archived"] prepared_format = prepare_reformat(option) - file_directory = main_helper.reformat( + file_directory = await main_helper.reformat( prepared_format, file_directory_format ) prepared_format.directory = file_directory - file_path = main_helper.reformat(prepared_format, filename_format) + file_path = await main_helper.reformat(prepared_format, filename_format) new_media["directory"] = os.path.join(file_directory) new_media["filename"] = os.path.basename(file_path) if file_directory not in directories: diff --git a/requirements.txt b/requirements.txt index 0b474f075..2233eff97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ lxml mergedeep jsonpickle ujson -sqlalchemy==1.4.15 +sqlalchemy==1.4.18 alembic tqdm selenium