From fdc7eeea03a45bab055d5a3625a90526c877ed67 Mon Sep 17 00:00:00 2001 From: William East Date: Mon, 11 Jul 2022 21:39:00 +0200 Subject: [PATCH] added semistable download functionality --- .envrc | 1 - config_example.yaml | 3 ++ connectionhandler.py | 66 +++++++++++++++++--------------------------- functions.py | 11 +++++--- main.py | 33 ++++++++++++---------- 5 files changed, 55 insertions(+), 59 deletions(-) diff --git a/.envrc b/.envrc index 7dcc6bd..1d953f4 100644 --- a/.envrc +++ b/.envrc @@ -1,2 +1 @@ use nix -use nix diff --git a/config_example.yaml b/config_example.yaml index bc5f789..f34be85 100644 --- a/config_example.yaml +++ b/config_example.yaml @@ -8,3 +8,6 @@ torrent_dir: "/place/to/where/torrent/files/should/go" # place where .torrent files land (likely your DL folder) target_dir: "/downloads/" +# place where the finished torrents go in the seedbox directory +# this needs to be configured manually in RuTorrent. + ftp_remote_directory_for_completed_downloads: "/files/Completed Downloads/" diff --git a/connectionhandler.py b/connectionhandler.py index 2063093..b3b941b 100644 --- a/connectionhandler.py +++ b/connectionhandler.py @@ -1,14 +1,8 @@ #!/usr/bin/env python3 import ftplib -from pathlib import Path import os -import sys import re -import time - -############################################################################### -# https://sftptogo.com/blog/python-sftp/ # -############################################################################### +from functions import getDiffList class SeedboxFTP: @@ -45,10 +39,22 @@ def changeWorkingDirectory(self, remotePath): def printWorkingDirectory(self): self.connection.dir() + def nlstSafe(self, directory): + """ + creates a directory tree for all directories and file in directory. + Note that this replaces the nlst() command in the ftplib, because this + was not able to parse square brackets, and the ftp server could not handle + escaped strings, for whatever reason. this function basically mimics nlst() + """ + out = [] + for i in self.connection.mlsd(directory): + out.append(os.path.join(directory, i[0])) + return out + # This is the upload function. Very unstable as it stands, and has no proper error handling. # The issue I am having is that there is a both a list of posix files (localFile) and the directory # in which those files are to be found. it seems odd to set both. Ideally you just feed it the main file. - + # TODO rename this!!!! def testUpload(self, localFile): fileObject = open(localFile, "rb") file2BeSavedAs = localFile.name @@ -57,26 +63,19 @@ def testUpload(self, localFile): print(ftpResponseMessage) fileObject.close() - # def testDownload(self, remoteFile): - # """Download file to location of execution. this needs to be amended to allow for choice of location.""" - # fileObject = open(remoteFile, "wb") - # ftpcommand = "RETR %s" % remoteFile - # ftpResponseMessage = self.connection.retrbinary(ftpcommand, fileObject.write) - # print(ftpResponseMessage) - # fileObject.close() + def checkTorrentDownloaded(self, ftpdirectory, torrents): + remote = self.nlstSafe(ftpdirectory) + return getDiffList(torrents, remote) def isFtpDir(self, name, guess_by_extension=True): """simply determines if an item listed on the ftp server is a valid directory or not""" # if the name has a "." in the fourth or fifth to last position, its probably a file extension # this is MUCH faster than trying to set every file to a working directory, and will work 99% of time. - # if guess_by_extension is True: - # if len(name) >= 4: - # if name[-4] or name[-5] == ".": - # return False if guess_by_extension is True: - if os.path.splitext(name) == "": - return True + if len(name) >= 4: + if name[-4] == ".": + return False original_cwd = self.connection.pwd() # remember the current working directory try: @@ -95,21 +94,13 @@ def isFtpDir(self, name, guess_by_extension=True): def makeParentDir(self, fpath): """ensures the parent directory of a filepath exists""" dirname = os.path.dirname(fpath) - print(f"Trying to create a directory called: {fpath}") - print(f"Trying to create a directory called: {dirname}") - - while not os.path.exists(dirname) or dirname == "": + while not os.path.exists(dirname): try: - os.makedirs(fpath) - print("created {0}".format(dirname)) - except FileNotFoundError as error: - print(f"file not found.trying to create {dirname}") - time.sleep(1) - - # self.makeParentDir(dirname) + os.makedirs(dirname) + print("created directory {0}".format(dirname)) except OSError as e: print(e) - self.makeParentDir(fpath) + self.makeParentDir(dirname) def downloadRemoteFile(self, name, dest, overwrite): """downloads a single file from an ftp server""" @@ -133,19 +124,14 @@ def fileNameMatchPattern(self, pattern, name): def cloneRemoteDir(self, name, overwrite, guess_by_extension, pattern): """replicates a directory on an ftp server recursively""" - for item in self.connection.nlst(name): - print("I am processing", item) + + for item in self.nlstSafe(name): if self.isFtpDir(item, guess_by_extension): - print("I think this is a dir") self.cloneRemoteDir(item, overwrite, guess_by_extension, pattern) - print("cloning remote dir") else: - print("I think this is a file") if self.fileNameMatchPattern(pattern, name): self.downloadRemoteFile(item, item, overwrite) else: - print("I have skipped this") - # quietly skip the file pass def downloadRemoteDir( diff --git a/functions.py b/functions.py index 886cd51..4e7176b 100644 --- a/functions.py +++ b/functions.py @@ -48,10 +48,13 @@ def getDiffList(a, b): def moveManager(torrents, torrent_dir): """move files (expects posix) from target directory to torrent directory""" for torrent in torrents: - try: - shutil.move(torrent.path, torrent_dir) - except Exception as e: - raise Exception(e) + if not os.path.exists(torrent): + try: + shutil.move(torrent.path, torrent_dir) + except Exception as e: + raise Exception(e) + else: + print(f"already exists: {torrent}") def yamlDataExtract(config_file="config.yaml"): diff --git a/main.py b/main.py index 530913f..248080b 100755 --- a/main.py +++ b/main.py @@ -64,13 +64,15 @@ for entry in torrents: print(f"Found torrent file ::: {entry.name}") - if args.upload is True: + if args.upload or args.download is True: sftp = connectionhandler.SeedboxFTP(SEEDBOX_ADDR, SEEDBOX_LOGIN, SEEDBOX_PW) print(f"attempting to connect to {SEEDBOX_ADDR}") sftp.connect() + if args.upload is True: + print(f"attempting to upload {num_torrents} to {SEEDBOX_ADDR}.") # set CWD to Watch folder sftp.changeWorkingDirectory(remotePath="watch") @@ -78,22 +80,25 @@ # loop through torrent list, and send them to the seedbox for torrent in torrents: sftp.testUpload(torrent) - # disconnect - sftp.disconnect() - # if args.download is True: + if args.download is True: + sftp.changeWorkingDirectory(remotePath="/files/") - # sftp.changeWorkingDirectory(remotePath="/files/Completed Downloads") - - # for torrent in torrents: - # torrentfile = functions.getFileNamefromTorrent(torrent) - # print(f"processing {torrentfile}") - # sftp.downloadRemoteDir( - # torrentfile, - # destination=TARGET_DIR, - # ) - # sftp.disconnect() + for torrent in torrents: + torrentfile = functions.getFileNamefromTorrent(torrent) + print(f"processing {torrentfile}") + sftp.downloadRemoteDir( + torrentfile, + destination=TARGET_DIR, + ) if args.move is True: print(f"moving torrents to {TORRENT_DIR}") functions.moveManager(torrents, TORRENT_DIR) + +# this just sees if an ftp instance was created, and if it does exist, disconnects. +try: + sftp.disconnect() + print("disconnected successfully.") +except NameError: + pass