Fixes: OFRenamer

Script no longer downloads files that are already downloaded unless overwritten is set to true.

Fixed renaming.

OnlyFans decided to let creators send posts to messages which would cause errors such as the FileNotFoundError. I added the "linked" column in the database. Any file that is linked will have "linked" in it's filename.

apis/onlyfans/onlyfans.py

@@ -81,9 +81,6 @@ def __init__(self, option={}, assign_states=False) -> None:
         self.Videos = option.get("Videos", [])
         self.Audios = option.get("Audios", [])
         self.Texts = option.get("Texts", [])
-        if assign_states:
-            for k, v in self:
-                setattr(self, k, assign_states())
 
     def remove_empty(self):
         copied = copy.deepcopy(self)
@@ -644,7 +641,7 @@ def get_subscriptions(self, resume=None, refresh=True, identifiers: list = [], e
         # Following logic is unique to creators only
         results = []
         if authed.isPerformer:
-            delattr(authed,"session_manager")
+            delattr(authed, "session_manager")
             json_authed = jsonpickle.encode(
                 authed, unpicklable=False)
             json_authed = jsonpickle.decode(json_authed)

database/databases/messages/alembic/versions/aeb9fe314556_content.py

@@ -0,0 +1,32 @@
+"""content
+
+Revision ID: aeb9fe314556
+Revises: d0118d8ec0b4
+Create Date: 2021-02-14 19:56:59.175268
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'aeb9fe314556'
+down_revision = 'd0118d8ec0b4'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('medias', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('linked', sa.String(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('medias', schema=None) as batch_op:
+        batch_op.drop_column('linked')
+
+    # ### end Alembic commands ###

database/databases/posts/alembic/versions/6b1b10eb67de_content.py

@@ -0,0 +1,32 @@
+"""content
+
+Revision ID: 6b1b10eb67de
+Revises: 5b4bea08c27f
+Create Date: 2021-02-14 19:56:56.267261
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '6b1b10eb67de'
+down_revision = '5b4bea08c27f'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('medias', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('linked', sa.String(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('medias', schema=None) as batch_op:
+        batch_op.drop_column('linked')
+
+    # ### end Alembic commands ###

database/databases/stories/alembic/versions/ebc3f4bb0782_content.py

@@ -0,0 +1,32 @@
+"""content
+
+Revision ID: ebc3f4bb0782
+Revises: 29f675c35eee
+Create Date: 2021-02-14 19:56:54.040372
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'ebc3f4bb0782'
+down_revision = '29f675c35eee'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('medias', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('linked', sa.String(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('medias', schema=None) as batch_op:
+        batch_op.drop_column('linked')
+
+    # ### end Alembic commands ###

database/models/media_table.py

@@ -14,6 +14,7 @@ class media_table():
     size = sqlalchemy.Column(sqlalchemy.Integer, default=None)
     media_type = sqlalchemy.Column(sqlalchemy.String)
     preview = sqlalchemy.Column(sqlalchemy.Integer, default=0)
+    linked = sqlalchemy.Column(sqlalchemy.String, default=None)
     downloaded = sqlalchemy.Column(sqlalchemy.Integer, default=0)
     created_at = sqlalchemy.Column(sqlalchemy.DATETIME)
 

extras/OFRenamer/start.py

@@ -28,7 +28,7 @@ def fix_directories(post: api_table, media_db: list[media_table]):
             else:
                 path: str = media.filename
             new_filename = os.path.basename(path)
-            filename, ext = os.path.splitext(new_filename)
+            original_filename, ext = os.path.splitext(new_filename)
             ext = ext.replace(".", "")
             file_directory_format = json_settings["file_directory_format"]
             filename_format = json_settings["filename_format"]
@@ -44,7 +44,7 @@ def fix_directories(post: api_table, media_db: list[media_table]):
             option["username"] = username
             option["api_type"] = final_type if parent_type else api_type
             option["media_type"] = media.media_type
-            option["filename"] = filename
+            option["filename"] = original_filename
             option["ext"] = ext
             option["text"] = post.text
             option["postedAt"] = media.created_at
@@ -58,12 +58,16 @@ def fix_directories(post: api_table, media_db: list[media_table]):
                 prepared_format, file_directory_format)
             prepared_format.directory = file_directory
             old_filepath = ""
+            if media.linked:
+                filename_format = f"linked_{filename_format}"
             old_filepaths = [
-                x for x in all_files if media.filename in os.path.basename(x)]
+                x for x in all_files if original_filename in os.path.basename(x)]
             if not old_filepaths:
                 old_filepaths = [
                     x for x in all_files if str(media_id) in os.path.basename(x)]
                 print
+            if not media.linked:
+                old_filepaths = [x for x in old_filepaths if "linked_" not in x]
             if old_filepaths:
                 old_filepath = old_filepaths[0]
             new_filepath = main_helper.reformat(
@@ -78,11 +82,17 @@ def fix_directories(post: api_table, media_db: list[media_table]):
                             if media.size:
                                 media.downloaded = True
                             found_dupes = [
-                                x for x in media_db if x.filename == new_filename and not x.id != media.id]
+                                x for x in media_db if x.filename == new_filename and x.id != media.id]
                             delete_rows.extend(found_dupes)
                             os.makedirs(os.path.dirname(
                                 new_filepath), exist_ok=True)
-                            moved = shutil.move(old_filepath, new_filepath)
+                            if media.linked:
+                                if os.path.dirname(old_filepath) == os.path.dirname(new_filepath):
+                                    moved = shutil.move(old_filepath, new_filepath)
+                                else:
+                                    moved = shutil.copy(old_filepath, new_filepath)
+                            else:
+                                moved = shutil.move(old_filepath, new_filepath)
                         else:
                             break
                     except OSError as e:
@@ -93,8 +103,6 @@ def fix_directories(post: api_table, media_db: list[media_table]):
             if os.path.exists(new_filepath):
                 if media.size:
                     media.downloaded = True
-            else:
-                media.downloaded = False
             if prepared_format.text:
                 pass
             media.directory = file_directory

helpers/main_helper.py

@@ -276,6 +276,7 @@ def export_sqlite(archive_path, datas, parent_type, legacy_fixer=False, api=None
             media_db.directory = media["directory"]
             media_db.filename = media["filename"]
             media_db.media_type = media["media_type"]
+            media_db.linked = media["linked"]
             if date_object:
                 media_db.created_at = date_object
             database_session.add(media_db)

modules/onlyfans.py

@@ -350,7 +350,7 @@ def paid_content_scraper(apis: list[start], identifiers=[]):
                 formatted_metadata_directory = formatted_directories["metadata_directory"]
                 metadata_path = os.path.join(
                     formatted_metadata_directory, api_type+".db")
-                unrefined_set = media_scraper(paid_content, api,
+                unrefined_set = media_scraper(paid_content, api, subscription,
                                               formatted_directories, username, api_type)
                 unrefined_set = [x for x in [unrefined_set]]
                 new_metadata = main_helper.format_media_set(unrefined_set)
@@ -561,7 +561,6 @@ def process_legacy_metadata(api: start, new_metadata_set, formatted_directories,
             print
         print
     print
-    subscription.set_scraped(api_type, old_metadata_object)
     if old_metadata_set:
         delete_metadatas.append(archive_path)
     final_set = []
@@ -698,12 +697,12 @@ def prepare_scraper(api: start, site_name, item):
             parent_type = master_set3["type"]
             results = master_set3["results"]
             unrefined_result = pool.starmap(media_scraper, product(
-                results, [api], [formatted_directories], [username], [api_type], [parent_type]))
+                results, [api], [subscription], [formatted_directories], [username], [api_type], [parent_type]))
             unrefined_set.append(unrefined_result)
         unrefined_set = list(chain(*unrefined_set))
     else:
         unrefined_set = pool.starmap(media_scraper, product(
-            master_set2, [api], [formatted_directories], [username], [api_type], [parent_type]))
+            master_set2, [api], [subscription], [formatted_directories], [username], [api_type], [parent_type]))
         unrefined_set = [x for x in unrefined_set]
     new_metadata = main_helper.format_media_set(unrefined_set)
     if new_metadata:
@@ -714,8 +713,11 @@ def prepare_scraper(api: start, site_name, item):
         old_metadata, delete_metadatas = process_legacy_metadata(
             api, new_metadata, formatted_directories, subscription, api_type, api_path, metadata_path, site_name)
         new_metadata = new_metadata + old_metadata
+        subscription.set_scraped(api_type, new_metadata)
+        print
         w = process_metadata(metadata_path, formatted_directories, new_metadata,
                              site_name, parent_type, api_path, subscription, delete_metadatas)
+        print
     else:
         print("No "+api_type+" Found.")
         delattr(subscription.scraped, api_type)
@@ -864,7 +866,7 @@ def compare_metadata(new_metadata: create_metadata, old_metadata: create_metadat
 # Scrapes the API for content
 
 
-def media_scraper(results, api: start, formatted_directories, username, api_type, parent_type=""):
+def media_scraper(results, api: start, subscription: create_subscription, formatted_directories, username, api_type, parent_type="", print_output=True):
     new_set = {}
     new_set["content"] = []
     directories = []
@@ -908,9 +910,11 @@ def media_scraper(results, api: start, formatted_directories, username, api_type
                 print
             print
         seperator = " | "
-        print(
-            f"Scraping [{seperator.join(alt_media_type)}]. Should take less than a minute.")
+        if print_output:
+            print(
+                f"Scraping [{seperator.join(alt_media_type)}]. Should take less than a minute.")
         for media_api in results:
+            post_id = media_api["id"]
             new_post = {}
             new_post["medias"] = []
             rawText = media_api.get("rawText", "")
@@ -949,6 +953,7 @@ def media_scraper(results, api: start, formatted_directories, username, api_type
             new_post["postedAt"] = date_string
             new_post["paid"] = False
             new_post["preview_media_ids"] = previews
+            new_post["api_type"] = api_type
             price = new_post["price"] = media_api["price"]if "price" in media_api else None
             if price == None:
                 price = 0
@@ -1055,8 +1060,31 @@ def media_scraper(results, api: start, formatted_directories, username, api_type
                 new_media["filename"] = os.path.basename(file_path)
                 if file_directory not in directories:
                     directories.append(file_directory)
+                new_media["linked"] = None
+                for k, v in subscription.scraped:
+                    if k == api_type or k == "Archived":
+                        continue
+                    if v:
+                        for post in v:
+                            found_medias = [x for x in post["medias"]
+                                            if x["filename"] == new_media["filename"]]
+                            if found_medias:
+                                for found_media in found_medias:
+                                    found_media["linked"] = api_type
+                                new_media["linked"] = post["api_type"]
+                                new_media["filename"] = f"linked_{new_media['filename']}"
+                                print
+                            print
+                        print
+                    print
                 new_post["medias"].append(new_media)
-            new_set["content"].append(new_post)
+            found_post = [x for x in new_set["content"]
+                          if x["post_id"] == post_id]
+            if found_post:
+                found_post = found_post[0]
+                found_post["medias"] += new_post["medias"]
+            else:
+                new_set["content"].append(new_post)
     new_set["directories"] = directories
     return new_set
 
@@ -1155,8 +1183,8 @@ def choose_link(session, links):
                 media_type = format_media_types()
                 formatted_directories = format_directories(
                     mandatory_directories, site_name, subscription.username, metadata_directory_format, media_type, api_type)
-                unrefined_set = media_scraper(result_list, api,
-                                              formatted_directories, subscription.username, api_type)
+                unrefined_set = media_scraper(result_list, api, subscription,
+                                              formatted_directories, subscription.username, api_type, print_output=False)
                 unrefined_set = [x for x in [unrefined_set]]
                 new_metadata = main_helper.format_media_set(unrefined_set)
                 new_metadata = new_metadata["content"]