From ed3c071746948dc6ee083509eeefaf2636ec7e69 Mon Sep 17 00:00:00 2001 From: William East Date: Mon, 27 Jun 2022 12:28:50 +0200 Subject: [PATCH] added argparse handling --- .envrc | 2 + connectionhandler.py | 164 ++++++++++++++++++++++++++++++++----------- functions.py | 154 +++++++++++++++++++++++++++++++++++----- main.py | 94 +++++++++++++++++++------ shell.nix | 8 +++ 5 files changed, 345 insertions(+), 77 deletions(-) create mode 100644 .envrc create mode 100644 shell.nix diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..7dcc6bd --- /dev/null +++ b/.envrc @@ -0,0 +1,2 @@ +use nix +use nix diff --git a/connectionhandler.py b/connectionhandler.py index a5d31a5..ca0c4b3 100644 --- a/connectionhandler.py +++ b/connectionhandler.py @@ -3,6 +3,8 @@ from pathlib import Path import os import sys +import re +import time ############################################################################### # https://sftptogo.com/blog/python-sftp/ # @@ -55,46 +57,128 @@ def testUpload(self, localFile): print(ftpResponseMessage) fileObject.close() - def testDownload(self, remoteFile): - """Download file to location of execution. this needs to be amended to allow for choice of location.""" - fileObject = open(remoteFile, "wb") - ftpcommand = "RETR %s" % remoteFile - ftpResponseMessage = self.connection.retrbinary(ftpcommand, fileObject.write) - print(ftpResponseMessage) - fileObject.close() + # def testDownload(self, remoteFile): + # """Download file to location of execution. this needs to be amended to allow for choice of location.""" + # fileObject = open(remoteFile, "wb") + # ftpcommand = "RETR %s" % remoteFile + # ftpResponseMessage = self.connection.retrbinary(ftpcommand, fileObject.write) + # print(ftpResponseMessage) + # fileObject.close() + + def isFtpDir(self, name, guess_by_extension=True): + """simply determines if an item listed on the ftp server is a valid directory or not""" - def downloadFiles(self, path, destination): - # path & destination are str of the form "/dir/folder/something/" - # path should be the abs path to the root FOLDER of the file tree to download + # if the name has a "." in the fourth or fifth to last position, its probably a file extension + # this is MUCH faster than trying to set every file to a working directory, and will work 99% of time. + # if guess_by_extension is True: + # if len(name) >= 4: + # if name[-4] or name[-5] == ".": + # return False + if guess_by_extension is True: + if os.path.splitext(name) == "": + return True + + original_cwd = self.connection.pwd() # remember the current working directory try: - self.connection.cwd(path) - # clone path to destination - os.chdir(destination) - os.mkdir(destination[0 : len(destination) - 1] + path) - print(destination[0 : len(destination) - 1] + path + " built") - except OSError: - # folder already exists at destination - pass - except ftplib.error_perm: - # invalid entry (ensure input form: "/dir/folder/something/") - print("error: could not change to " + path) - sys.exit("ending session") - - # list children: - filelist = self.connection.nlst() - - for file in filelist: + self.connection.cwd(name) # try to set directory to new name + self.connection.cwd(original_cwd) # set it back to what it was + return True + + except ftplib.error_perm as e: + print(e) + return False + + except Exception as e: + print(e) + return False + + def makeParentDir(self, fpath): + """ensures the parent directory of a filepath exists""" + dirname = os.path.dirname(fpath) + print(f"Trying to create a directory called: {fpath}") + print(f"Trying to create a directory called: {dirname}") + + while not os.path.exists(dirname) or dirname == "": try: - # this will check if file is folder: - self.connection.cwd(path + file + "/") - # if so, explore it: - downloadFiles(path + file + "/", destination) - except ftplib.error_perm: - # not a folder with accessible content - # download & return - os.chdir(destination[0 : len(destination) - 1] + path) - # possibly need a permission exception catch: - with open(os.path.join(destination, file), "wb") as f: - self.connection.retrbinary("RETR " + file, f.write) - print(file + " downloaded") - return + os.makedirs(fpath) + print("created {0}".format(dirname)) + except FileNotFoundError as error: + print(f"file not found.trying to create {dirname}") + time.sleep(1) + + # self.makeParentDir(dirname) + except OSError as e: + print(e) + self.makeParentDir(fpath) + + def downloadRemoteFile(self, name, dest, overwrite): + """downloads a single file from an ftp server""" + self.makeParentDir(dest.lstrip("/")) + if not os.path.exists(dest) or overwrite is True: + try: + with open(dest, "wb") as f: + self.connection.retrbinary("RETR {0}".format(name), f.write) + print("downloaded: {0}".format(dest)) + except FileNotFoundError: + print("FAILED: {0}".format(dest)) + else: + print("already exists: {0}".format(dest)) + + def fileNameMatchPattern(self, pattern, name): + """returns True if filename matches the pattern""" + if pattern is None: + return True + else: + return bool(re.match(pattern, name)) + + def cloneRemoteDir(self, name, overwrite, guess_by_extension, pattern): + """replicates a directory on an ftp server recursively""" + for item in self.connection.nlst(name): + print("I am processing", item) + if self.isFtpDir(item, guess_by_extension): + print("I think this is a dir") + self.cloneRemoteDir(item, overwrite, guess_by_extension, pattern) + print("cloning remote dir") + else: + print("I think this is a file") + if self.fileNameMatchPattern(pattern, name): + self.downloadRemoteFile(item, item, overwrite) + else: + print("I have skipped this") + # quietly skip the file + pass + + def downloadRemoteDir( + self, + path, + destination, + pattern=None, + overwrite=False, + guess_by_extension=True, + ): + """ + Downloads an entire directory tree from an ftp server to the local destination + :param path: the folder on the ftp server to download + :param destination: the local directory to store the copied folder + :param pattern: Python regex pattern, only files that match this pattern will be downloaded. + :param overwrite: set to True to force re-download of all files, even if they appear to exist already + :param guess_by_extension: It takes a while to explicitly check if every item is a directory or a file. + if this flag is set to True, it will assume any file ending with a three character extension ".???" is + a file and not a directory. Set to False if some folders may have a "." in their names -4th position. + """ + path = path.lstrip("/") + original_directory = ( + os.getcwd() + ) # remember working directory before function is executed + os.chdir(destination) # change working directory to ftp mirror directory + + self.cloneRemoteDir( + path, + pattern=pattern, + overwrite=overwrite, + guess_by_extension=guess_by_extension, + ) + + os.chdir( + original_directory + ) # reset working directory to what it was before function exec diff --git a/functions.py b/functions.py index 8eb9821..cd44344 100644 --- a/functions.py +++ b/functions.py @@ -3,6 +3,9 @@ import os import shutil import yaml +import libtorrent as lt +import ftplib +import re def checkMimes(file, allowed_extensions): @@ -14,23 +17,15 @@ def checkMimes(file, allowed_extensions): return file_valid -# def handleLocalTorrents(torrent_dir, initial_dir, move_file=False): -# """move .torrent files to initial_dir""" -# torrents_found = 0 +def checkIfFolder(path): + if os.path.splitext(path)[1] == "": + is_Folder = True + else: + is_Folder = False + return is_Folder + -# with os.scandir(initial_dir) as localdir: -# for entry in localdir: -# # check if file is correct type -# if entry.is_file() and checkMimes(entry, ".torrent"): -# print(f"Found torrent file ::: {entry.name}") -# torrents_found += 1 -# if move_file: -# shutil.move(entry, torrent_dir) -# print(f"Moved to {torrent_dir}") -# if torrents_found == 0: -# print(f"No torrent files found to process in {initial_dir}") -# else: -# print(f"proceesed {os.path(initial_dir)}") +print(checkIfFolder("/home/weast/Downloads/cover")) def torrentIdentifier(directory): @@ -50,9 +45,13 @@ def getDiffList(a, b): return difflist +# Todo: handle file already exists def moveManager(torrents, torrent_dir): for torrent in torrents: - shutil.move(torrent, torrent_dir) + try: + shutil.move(torrent, torrent_dir) + except Exception as e: + raise Exception(e) def yamlDataExtract(config_file="config.yaml"): @@ -70,3 +69,124 @@ def getTorrentDiffList(torrent_dir, target_dir): torrentIdentifier(torrent_dir), torrentIdentifier(target_dir) ) return torrents + + +def getFileNamefromTorrent(torrent): + """must be a direntry item. Gets the name of the torrent's finished folder from the .torrent file.""" + torrent_info = lt.torrent_info(torrent.path) + return torrent_info.name() + + +############################################################################### +# Below needs integration. # +############################################################################### + + +def _is_ftp_dir(ftp_handle, name, guess_by_extension=True): + """simply determines if an item listed on the ftp server is a valid directory or not""" + + # if the name has a "." in the fourth to last position, its probably a file extension + # this is MUCH faster than trying to set every file to a working directory, and will work 99% of time. + if guess_by_extension is True: + if len(name) >= 4: + if name[-4] == ".": + return False + + original_cwd = ftp_handle.pwd() # remember the current working directory + try: + ftp_handle.cwd(name) # try to set directory to new name + ftp_handle.cwd(original_cwd) # set it back to what it was + return True + + except ftplib.error_perm as e: + print(e) + return False + + except Exception as e: + print(e) + return False + + +def _make_parent_dir(fpath): + """ensures the parent directory of a filepath exists""" + dirname = os.path.dirname(fpath) + while not os.path.exists(dirname): + try: + os.makedirs(dirname) + print("created {0}".format(dirname)) + except OSError as e: + print(e) + _make_parent_dir(dirname) + + +def _download_ftp_file(ftp_handle, name, dest, overwrite): + """downloads a single file from an ftp server""" + _make_parent_dir(dest.lstrip("/")) + if not os.path.exists(dest) or overwrite is True: + try: + with open(dest, "wb") as f: + ftp_handle.retrbinary("RETR {0}".format(name), f.write) + print("downloaded: {0}".format(dest)) + except FileNotFoundError: + print("FAILED: {0}".format(dest)) + else: + print("already exists: {0}".format(dest)) + + +def _file_name_match_patern(pattern, name): + """returns True if filename matches the pattern""" + if pattern is None: + return True + else: + return bool(re.match(pattern, name)) + + +def _mirror_ftp_dir(ftp_handle, name, overwrite, guess_by_extension, pattern): + """replicates a directory on an ftp server recursively""" + for item in ftp_handle.nlst(name): + if _is_ftp_dir(ftp_handle, item, guess_by_extension): + _mirror_ftp_dir(ftp_handle, item, overwrite, guess_by_extension, pattern) + else: + if _file_name_match_patern(pattern, name): + _download_ftp_file(ftp_handle, item, item, overwrite) + else: + # quietly skip the file + pass + + +def download_ftp_tree( + ftp_handle, + path, + destination, + pattern=None, + overwrite=False, + guess_by_extension=True, +): + """ + Downloads an entire directory tree from an ftp server to the local destination + :param ftp_handle: an authenticated ftplib.FTP instance + :param path: the folder on the ftp server to download + :param destination: the local directory to store the copied folder + :param pattern: Python regex pattern, only files that match this pattern will be downloaded. + :param overwrite: set to True to force re-download of all files, even if they appear to exist already + :param guess_by_extension: It takes a while to explicitly check if every item is a directory or a file. + if this flag is set to True, it will assume any file ending with a three character extension ".???" is + a file and not a directory. Set to False if some folders may have a "." in their names -4th position. + """ + path = path.lstrip("/") + original_directory = ( + os.getcwd() + ) # remember working directory before function is executed + os.chdir(destination) # change working directory to ftp mirror directory + + _mirror_ftp_dir( + ftp_handle, + path, + pattern=pattern, + overwrite=overwrite, + guess_by_extension=guess_by_extension, + ) + + os.chdir( + original_directory + ) # reset working directory to what it was before function exec diff --git a/main.py b/main.py index 34aafc1..281f014 100644 --- a/main.py +++ b/main.py @@ -1,13 +1,53 @@ #!/usr/bin/env python3 +import sys +import argparse import functions import connectionhandler +# from time import sleep + + +# handle argument parsing +parser = argparse.ArgumentParser( + description="This tool finds .torrent files in your download folder\ + and gives the option to move them directly into a shared seedbox via\ + FTP. it also allows for download of that file, as well as moving the\ + .torrent files somewhere convenient after processing." +) +# only upload files to seedbox.io, +parser.add_argument( + "--upload", + "-u", + action="store_true", + help="upload files to seedbox.io using the\ + credentials in the config", +) +parser.add_argument( + "--print", + "-p", + action="store_true", + help="print files in inbound torrent folder", +) +parser.add_argument( + "--move", + "-m", + action="store_true", + help="move files in inbound folder to outbound torrent folder", +) # move torrent files locally +parser.add_argument( + "--download", + "-d", + action="store_true", + help="Download torrents from seedbox.io to the specified folder.", +) # download torrents +args = parser.parse_args() + # Handle the config file config = functions.yamlDataExtract() -TORRENT_DIR = config["torrent_dir"] # this +TORRENT_DIR = config["torrent_dir"] TARGET_DIR = config["target_dir"] SEEDBOX_ADDR = config["seedbox_addr"] SEEDBOX_LOGIN = config["seedbox_login"] @@ -19,30 +59,44 @@ num_torrents = len(torrents) -if num_torrents > 0: - print(f"Found {num_torrents} new torrents") - # functions.moveManager(torrents, TORRENT_DIR) -else: - print("No new torrents found.") +if num_torrents == 0: + print(f"Aborting because no torrents were found in {TARGET_DIR}") + sys.exit() + +if args.print or args.download or args.upload is True: + for entry in torrents: + print(f"Found torrent file ::: {entry.name}") + +if args.upload is True: + sftp = connectionhandler.SeedboxFTP(SEEDBOX_ADDR, SEEDBOX_LOGIN, SEEDBOX_PW) -for entry in torrents: - print(f"Found torrent file ::: {entry.name}") + print(f"attempting to connect to {SEEDBOX_ADDR}") + sftp.connect() + print(f"attempting to upload {num_torrents} to {SEEDBOX_ADDR}.") + # set CWD to Watch folder + sftp.changeWorkingDirectory(remotePath="watch") -# initiate the connection -sftp = connectionhandler.SeedboxFTP(SEEDBOX_ADDR, SEEDBOX_LOGIN, SEEDBOX_PW) + # loop through torrent list, and send them to the seedbox + for torrent in torrents: + sftp.testUpload(torrent) + # disconnect + sftp.disconnect() -sftp.connect() +if args.download is True: -# set CWD to Watch folder -sftp.changeWorkingDirectory(remotePath="watch") + sftp.changeWorkingDirectory(remotePath="/files/Completed Downloads") -# loop through torrent list, and send them to the seedbox -for torrent in torrents: - sftp.testUpload(torrent) -# disconnect -sftp.disconnect() + for torrent in torrents: + torrentfile = functions.getFileNamefromTorrent(torrent) + print(f"processing {torrentfile}") + sftp.downloadRemoteDir( + torrentfile, + destination=TARGET_DIR, + ) + sftp.disconnect() -# move torrents into the torrent dir -functions.moveManager(torrents, TORRENT_DIR) +if args.move is True: + print(f"moving torrents to {TORRENT_DIR}") + functions.moveManager(torrents, TORRENT_DIR) diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..06055bf --- /dev/null +++ b/shell.nix @@ -0,0 +1,8 @@ +with import { }; +let + my-python-packages = python-packages: [ + python-packages.pyyaml + python-packages.libtorrent-rasterbar + ]; + my-python = python37.withPackages my-python-packages; +in mkShell { buildInputs = [ bashInteractive my-python ]; }