From f423bbb414dd7d9ea5a16e47f9c1ebc647e23460 Mon Sep 17 00:00:00 2001
From: Christian Emmer <10749361+emmercm@users.noreply.github.com>
Date: Wed, 25 Oct 2023 18:06:10 -0700
Subject: [PATCH] Keep an overall thread limit

---
 src/constants.ts      |  8 ++++----
 src/driveSemaphore.ts | 22 +++++++++++++++++-----
 2 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/src/constants.ts b/src/constants.ts
index 3df174981..40d7b6f88 100644
--- a/src/constants.ts
+++ b/src/constants.ts
@@ -73,14 +73,14 @@ export default class Constants {
   static readonly DAT_DEFAULT_THREADS = 3;
 
   /**
-   * Max number of archive entries to process (possibly extract & MD5/SHA1 checksum) at once.
+   * A reasonable max number of files to write at once.
    */
-  static readonly ARCHIVE_ENTRY_SCANNER_THREADS_PER_ARCHIVE = 5;
+  static readonly FILE_READER_DEFAULT_THREADS = 10;
 
   /**
-   * A reasonable max number of files to write at once.
+   * Max number of archive entries to process (possibly extract & MD5/SHA1 checksum) at once.
    */
-  static readonly FILE_READER_DEFAULT_THREADS = 10;
+  static readonly ARCHIVE_ENTRY_SCANNER_THREADS_PER_ARCHIVE = 5;
 
   /**
    * A reasonable max number of ROM release candidates to write at once. This will be the limiting
diff --git a/src/driveSemaphore.ts b/src/driveSemaphore.ts
index fd8eedacf..e3f5793c0 100644
--- a/src/driveSemaphore.ts
+++ b/src/driveSemaphore.ts
@@ -16,10 +16,13 @@ export default class DriveSemaphore {
 
   private readonly keySemaphoresMutex = new Mutex();
 
-  private readonly defaultThreads: number;
+  private readonly threads: number;
 
-  constructor(defaultThreads = 1) {
-    this.defaultThreads = defaultThreads;
+  private readonly threadsSemaphore: Semaphore;
+
+  constructor(threads = 1) {
+    this.threads = threads;
+    this.threadsSemaphore = new Semaphore(threads);
   }
 
   /**
@@ -74,8 +77,10 @@ export default class DriveSemaphore {
 
     const keySemaphore = await this.keySemaphoresMutex.runExclusive(async () => {
       if (!this.keySemaphores.has(filePathDisk)) {
-        let threads = this.defaultThreads;
+        let { threads } = this;
         if (await FsPoly.isSamba(filePathDisk)) {
+          // Forcefully limit the number of files to be processed concurrently from a single
+          // Samba network share
           threads = 1;
         }
         this.keySemaphores.set(filePathDisk, new Semaphore(threads));
@@ -83,6 +88,13 @@ export default class DriveSemaphore {
       return this.keySemaphores.get(filePathDisk) as Semaphore;
     });
 
-    return keySemaphore.runExclusive(async () => runnable(file));
+    // First, limit the number of threads per drive, which will better balance the processing of
+    // files on different drives vs. processing files sequentially
+    return keySemaphore.runExclusive(
+      // Second, limit the overall number of threads
+      async () => this.threadsSemaphore.runExclusive(
+        async () => runnable(file),
+      ),
+    );
   }
 }