From bbb4bec85c84e21d5380192e82de29c1ac40a87b Mon Sep 17 00:00:00 2001 From: onioneffect Date: Wed, 17 Aug 2022 16:26:39 -0400 Subject: [PATCH] Adds single-processing Adds option to download one thread at a time. Related to Exceen#35. --- inb4404.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/inb4404.py b/inb4404.py index 8ed2acb..407dc85 100755 --- a/inb4404.py +++ b/inb4404.py @@ -21,7 +21,7 @@ def main(): parser.add_argument('-s', '--sleep', action='store', help='sleep `N` seconds between every download', metavar='N') # TODO parser.add_argument('-t', '--title', action='store_true', help='save original filenames') - parser.add_argument('--single-process', action='store_true', help='disable multiprocessing, download one thread at a time') # TODO + parser.add_argument('--single-process', action='store_true', help='disable multiprocessing, download one thread at a time') args = parser.parse_args() @@ -145,21 +145,32 @@ def download_thread(thread_link, args): log.info('Checking ' + board + '/' + thread) time.sleep(20) +def run_single_process(links_list): + for l in links_list: + call_download_thread(l, args) + +def run_multiple_process(links_list, processes_list): + for l in links_list: + process = Process(target=call_download_thread, args=(l, args, )) + process.start() + processes_list.append([process, l]) + + if len(processes_list) == 0: + log.warning(filename + ' empty') + def download_from_file(filename): running_links = [] while True: processes = [] - for link in [_f for _f in [line.strip() for line in open(filename) if line[:4] == 'http'] if _f]: + for link in [_f for _f in [line.strip() for line in open(filename) if line[:4] == 'http'] if _f]: # Fix this line if link not in running_links: running_links.append(link) log.info('Added ' + link) - process = Process(target=call_download_thread, args=(link, args, )) - process.start() - processes.append([process, link]) - - if len(processes) == 0: - log.warning(filename + ' empty') + if args.single_process: + run_single_process(running_links) + else: + run_multiple_process(running_links, processes) if args.reload: time.sleep(60 * 5) # 5 minutes