Config and text update

Config will update automatically. text will now be cleaned so it can be a valid filename format. Don't use "text" in the filename format, I've warned people multiple times. Don't use it. I don't wanna hear complaints that you have to redownload 10TB worth of data when you've already been warned. I don't care lol.
UltimaHoarder · May 31, 2020 · 7638e71 · 7638e71
1 parent 9c452be
commit 7638e71
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 23 deletions.
diff --git a/classes/make_config.py b/classes/make_config.py
@@ -20,9 +20,10 @@ def __init__(self, onlyfans={}, starsavn={}, fourchan={}, bbwchan={}):
 
             class OnlyFans:
                 def __init__(self, module):
-                    self.auth = module.get('auth', self.Auth())
-                    self.settings = module.get('settings', self.Settings())
-                    self.extra_auth_settings = module.get('extra_auth_settings', self.ExtraAuthSettings())
+                    self.auth = self.Auth(module.get('auth', {}))
+                    self.settings = self.Settings(module.get('settings', {}))
+                    self.extra_auth_settings = self.ExtraAuthSettings(
+                        module.get('extra_auth_settings', {}))
 
                 class Auth:
                     def __init__(self, option={}):
@@ -46,7 +47,7 @@ def __init__(self, option={}):
                         self.directory = option.get('directory', "")
                         self.file_name_format = option.get(
                             'file_name_format', "{file_name}.{ext}")
-                        self.text_length = option.get('text_length', "")
+                        self.text_length = option.get('text_length', "255")
                         self.overwrite_files = option.get(
                             'overwrite_files', False)
                         self.date_format = option.get(
@@ -72,9 +73,10 @@ def __init__(self, option={}):
 
             class StarsAvn:
                 def __init__(self, module):
-                    self.auth = module.get('auth', self.Auth())
-                    self.settings = module.get('settings', self.Settings())
-                    self.extra_auth_settings = module.get('extra_auth_settings', self.ExtraAuthSettings())
+                    self.auth = self.Auth(module.get('auth', {}))
+                    self.settings = self.Settings(module.get('settings', {}))
+                    self.extra_auth_settings = self.ExtraAuthSettings(
+                        module.get('extra_auth_settings', {}))
 
                 class Auth:
                     def __init__(self, option={}):
@@ -92,7 +94,7 @@ def __init__(self, option={}):
                         self.directory = option.get('directory', "")
                         self.file_name_format = option.get(
                             'file_name_format', "{file_name}.{ext}")
-                        self.text_length = option.get('text_length', "")
+                        self.text_length = option.get('text_length', "255")
                         self.overwrite_files = option.get(
                             'overwrite_files', False)
                         self.date_format = option.get(
@@ -114,8 +116,8 @@ def __init__(self, option={}):
 
             class FourChan:
                 def __init__(self, module):
-                    self.auth = module.get('auth', self.Auth())
-                    self.settings = module.get('settings', self.Settings())
+                    self.auth = self.Auth(module.get('auth', {}))
+                    self.settings = self.Settings(module.get('settings', {}))
 
                 class Auth:
                     def __init__(self, option={}):
@@ -129,7 +131,7 @@ def __init__(self, option={}):
                         self.directory = option.get('directory', "")
                         self.file_name_format = option.get(
                             'file_name_format', "{file_name}.{ext}")
-                        self.text_length = option.get('text_length', "")
+                        self.text_length = option.get('text_length', "255")
                         self.overwrite_files = option.get(
                             'overwrite_files', False)
                         self.date_format = option.get(
@@ -141,8 +143,8 @@ def __init__(self, option={}):
 
             class BBWChan:
                 def __init__(self, module):
-                    self.auth = module.get('auth', self.Auth())
-                    self.settings = module.get('settings', self.Settings())
+                    self.auth = self.Auth(module.get('auth', {}))
+                    self.settings = self.Settings(module.get('settings', {}))
 
                 class Auth:
                     def __init__(self, option={}):
@@ -156,7 +158,7 @@ def __init__(self, option={}):
                         self.directory = option.get('directory', "")
                         self.file_name_format = option.get(
                             'file_name_format', "{file_name}.{ext}")
-                        self.text_length = option.get('text_length', "")
+                        self.text_length = option.get('text_length', "255")
                         self.overwrite_files = option.get(
                             'overwrite_files', False)
                         self.date_format = option.get(

diff --git a/modules/onlyfans.py b/modules/onlyfans.py
@@ -1,5 +1,5 @@
 import requests
-from helpers.main_helper import get_directory, json_request, reformat, format_directory, format_media_set, export_archive, format_image, check_for_dupe_file, setup_logger, log_error
+from helpers.main_helper import clean_text, get_directory, json_request, reformat, format_directory, format_media_set, export_archive, format_image, check_for_dupe_file, setup_logger, log_error
 
 import os
 import json
@@ -115,7 +115,7 @@ def start_datascraper(session, identifier, site_name, app_token, choice_type=Non
                 directory = results[1]
                 location = result["type"]
                 prep_download.append(
-                    [media_set["valid"], session, directory, username, post_count, location,api_type])
+                    [media_set["valid"], session, directory, username, post_count, location, api_type])
     # When profile is done scraping, this function will return True
     print("Scrape Completed"+"\n")
     return [True, prep_download]
@@ -325,22 +325,23 @@ def media_scraper(link, session, directory, username, api_type):
             if media["type"] not in media_type:
                 x += 1
                 continue
-            if "text" not in media_api:
-                media_api["text"] = ""
-            new_dict["text"] = media_api["text"] if media_api["text"] else ""
-            matches = [s for s in ignored_keywords if s in new_dict["text"]]
+            if "rawText" not in media_api:
+                media_api["rawText"] = ""
+            text = media_api["rawText"] if media_api["rawText"] else ""
+            matches = [s for s in ignored_keywords if s in text]
             if matches:
                 print("Matches: ", matches)
                 continue
+            text = clean_text(text)
             new_dict["postedAt"] = date_string
             media_id = media["id"] if "id" in media else None
             media_id = media_id if isinstance(media_id, int) else None
             file_name = link.rsplit('/', 1)[-1]
             file_name, ext = os.path.splitext(file_name)
             ext = ext.__str__().replace(".", "").split('?')[0]
             file_path = reformat(directory[0][1], media_id, file_name,
-                                 new_dict["text"], ext, date_object, username, format_path, date_format, maximum_length)
-
+                                 text, ext, date_object, username, format_path, date_format, maximum_length)
+            new_dict["text"] = text
             new_dict["paid"] = False
             if new_dict["price"]:
                 if api_type in ["Messages", "Mass Messages"]:
@@ -585,7 +586,8 @@ def choose_link(session, links):
                 break
         return return_bool
     string = "Download Processing\n"
-    string += "Name: "+username+" | Type: "+api_type+" | Directory: " + directory+"\n"
+    string += "Name: "+username+" | Type: " + \
+        api_type+" | Directory: " + directory+"\n"
     string += "Downloading "+str(len(media_set))+" "+location+"\n"
     print(string)
     if multithreading: