Export to CSV

Added the option to export to CSV.
Check the readme for the "export_type" options
Update config :^)

README.md

@@ -87,6 +87,15 @@ auto_choice:
     c = Videos
 
     You can automatically choose what you want to scrape if you add it in the config file.
+
+|**NEW**| export_type:
+
+    Default = "json"
+
+    a = "json"
+    b = "csv"
+
+    You can export an archive to different formats.
 
 overwrite_files:
 

config.json

@@ -2,6 +2,7 @@
   "settings": {
     "auto_site_choice": "",
     "auto_choice": "",
+    "export_type": "json",
     "multithreading": true,
     "user-agent": ""
   },

modules/four_chan.py

@@ -1,6 +1,6 @@
 import requests
 from bs4 import BeautifulSoup
-from modules.helpers import reformat
+from modules.helpers import *
 
 import os
 import json

modules/helpers.py

@@ -2,8 +2,14 @@
 import os
 from bs4 import BeautifulSoup
 import platform
+import csv
+import itertools
+import json
 
-
+# Open config.json and fill in OPTIONAL information
+json_config = json.load(open('config.json'))
+json_global_settings = json_config["settings"]
+export_type = json_global_settings["export_type"]
 def parse_links(site_name, input_link):
     if site_name in {"onlyfans", "justforfans"}:
         username = input_link.rsplit('/', 1)[-1]
@@ -65,9 +71,31 @@ def format_media_set(media_set):
         x["invalid"].extend(y[1])
     return x
 
+
 def format_image(directory, timestamp):
     os_name = platform.system()
     if os_name == "Windows":
         from win32_setctime import setctime
         setctime(directory, timestamp)
 
+
+def export_archive(data, archive_directory):
+    # Not Finished
+    if export_type == "json":
+        with open(archive_directory+".json", 'w') as outfile:
+            json.dump(data, outfile)
+    if export_type == "csv":
+        with open(archive_directory+'.csv', mode='w', newline='') as csv_file:
+            fieldnames = []
+            if data["valid"]:
+                fieldnames.extend(data["valid"][0].keys())
+            elif data["invalid"]:
+                fieldnames.extend(data["invalid"][0].keys())
+            header = [""]+fieldnames
+            if len(fieldnames) > 1:
+                writer = csv.DictWriter(csv_file, fieldnames=header)
+                writer.writeheader()
+                for item in data["valid"]:
+                    writer.writerow({**{"": "valid"}, **item})
+                for item in data["invalid"]:
+                    writer.writerow({**{"": "invalid"}, **item})

modules/justforfans.py

@@ -1,7 +1,6 @@
 import requests
 from bs4 import BeautifulSoup
-from modules.helpers import reformat
-from modules.helpers import format_media_set
+from modules.helpers import *
 
 import os
 import json
@@ -238,9 +237,8 @@ def media_scraper(session, site_name, only_links, link, location, media_type, di
             print("DIRECTORY - " + directory)
             os.makedirs(directory, exist_ok=True)
         os.makedirs(metadata_directory, exist_ok=True)
-
-        with open(metadata_directory+location+".json", 'w') as outfile:
-            json.dump(media_set, outfile)
+        archive_directory = metadata_directory+location
+        export_archive(media_set, archive_directory)
     return [media_set, directory]
 
 

modules/onlyfans.py

@@ -1,8 +1,6 @@
 import requests
 from bs4 import BeautifulSoup
-from modules.helpers import reformat
-from modules.helpers import format_media_set
-from modules.helpers import format_image
+from modules.helpers import *
 
 import os
 import json
@@ -219,9 +217,8 @@ def media_scraper(session, site_name, only_links, link, location, media_type, di
             print("DIRECTORY - " + directory)
             os.makedirs(directory, exist_ok=True)
         os.makedirs(metadata_directory, exist_ok=True)
-
-        with open(metadata_directory+location+".json", 'w') as outfile:
-            json.dump(media_set, outfile)
+        archive_directory = metadata_directory+location
+        export_archive(media_set, archive_directory)
     return [media_set, directory]