Keep an overall thread limit

src/constants.ts

@@ -73,14 +73,14 @@ export default class Constants {
   static readonly DAT_DEFAULT_THREADS = 3;
 
   /**
-   * Max number of archive entries to process (possibly extract & MD5/SHA1 checksum) at once.
+   * A reasonable max number of files to write at once.
    */
-  static readonly ARCHIVE_ENTRY_SCANNER_THREADS_PER_ARCHIVE = 5;
+  static readonly FILE_READER_DEFAULT_THREADS = 10;
 
   /**
-   * A reasonable max number of files to write at once.
+   * Max number of archive entries to process (possibly extract & MD5/SHA1 checksum) at once.
    */
-  static readonly FILE_READER_DEFAULT_THREADS = 10;
+  static readonly ARCHIVE_ENTRY_SCANNER_THREADS_PER_ARCHIVE = 5;
 
   /**
    * A reasonable max number of ROM release candidates to write at once. This will be the limiting

src/driveSemaphore.ts

@@ -16,10 +16,13 @@ export default class DriveSemaphore {
 
   private readonly keySemaphoresMutex = new Mutex();
 
-  private readonly defaultThreads: number;
+  private readonly threads: number;
 
-  constructor(defaultThreads = 1) {
-    this.defaultThreads = defaultThreads;
+  private readonly threadsSemaphore: Semaphore;
+
+  constructor(threads = 1) {
+    this.threads = threads;
+    this.threadsSemaphore = new Semaphore(threads);
   }
 
   /**
@@ -74,15 +77,24 @@ export default class DriveSemaphore {
 
     const keySemaphore = await this.keySemaphoresMutex.runExclusive(async () => {
       if (!this.keySemaphores.has(filePathDisk)) {
-        let threads = this.defaultThreads;
+        let { threads } = this;
         if (await FsPoly.isSamba(filePathDisk)) {
+          // Forcefully limit the number of files to be processed concurrently from a single
+          // Samba network share
           threads = 1;
         }
         this.keySemaphores.set(filePathDisk, new Semaphore(threads));
       }
       return this.keySemaphores.get(filePathDisk) as Semaphore;
     });
 
-    return keySemaphore.runExclusive(async () => runnable(file));
+    // First, limit the number of threads per drive, which will better balance the processing of
+    // files on different drives vs. processing files sequentially
+    return keySemaphore.runExclusive(
+      // Second, limit the overall number of threads
+      async () => this.threadsSemaphore.runExclusive(
+        async () => runnable(file),
+      ),
+    );
   }
 }