settings option to specify dns server + fixed import/export art

CHANGELOG.md

@@ -1,6 +1,12 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+## 2.5.9
+### Added
+* Settings option to manually set DNS Server (fixes many timeout issues on the Steam Deck)
+### Changed
+* Manual Import / Export of Image Choices now matches on image pool.
+
 ## 2.5.8
 ### Fixed
 * Provider workers not returning images under certain conditions

src/lang/en-US/langStrings.json

@@ -304,6 +304,7 @@
         "fuzzy_verbose": "Log matching results",
         "fuzzy_filter": "Filter images",
         "enabledProviders": "Enabled image providers",
+        "dnsServers": "DNS manual override",
         "selectLanguage": "Select language",
         "selectTheme": "Select theme",
         "resetFuzzy_desc": "fuzzy list",

src/lang/en-US/markdown/settings.md

@@ -22,7 +22,9 @@ Clears the cache of titles that fuzzy matching has already seen (try this if cha
 ### Preload retrieved images `[Recommend disabled]`
 When enabled, SRM will pull all available artwork for every game, rather than pulling one piece of artwork at a time as the user flips through the images. Don't enable this unless you have a good reason and a very small library of games, otherwise it could result in very large (slow) network requests.
 ### Enabled providers
-Global setting to disable certain providers. Currently the only image provider is `SteamGridDB` since ConsoleGrid and RetroGaming.cloud are defunct.
+Global setting to enable/disable particular image providers. Current options are `SteamGridDB` and `Steam Official`.
+### DNS manual override
+Set this if you want SRM to do DNS resolution internally, as opposed to relying on your system's default DNS server. This solves many timeout issues on the Steam Deck.
 
 ## Community Variables and Presets
 ### Force download custom variables.

src/lib/helpers/url/image-downloader.ts

@@ -1,32 +1,45 @@
 import * as fs from 'fs-extra'
 import fetch, { AbortError } from 'node-fetch';
-// import { Resolver } from 'dns';
+import { Resolver } from 'dns';
 import { decodeFile } from './encode-file'
 
 export class ImageDownloader {
-  // private dnsResolver = new Resolver();
+  private dnsResolver;
   private timeout: number = 10000;
-  // private dnsCache: {[host: string]: string} = {};
+  private dnsCache: {[host: string]: string};
 
-  constructor() {
-    // this.dnsResolver.setServers(['1.1.1.1', '8.8.8.8']);
+  constructor(dnsServers: string[]) {
+    if(dnsServers && dnsServers.length) {
+      this.dnsResolver = new Resolver();
+      this.dnsResolver.setServers(dnsServers);
+      this.dnsCache = {};
+    }
   }
 
-  async downloadAndSaveImage(imageUrl: string, filePath: string, retryCount?: number, secondaryPath?: string): Promise<void> {
+  async downloadAndSaveImage(imageUrl: string, filePath: string, retryCount?: number, secondaryPath?: string, externalDNS?: string[]): Promise<void> {
     if(imageUrl.startsWith('file://')) {
       await fs.copyFile(decodeFile(imageUrl), filePath);
     } else {
       const controller = new AbortController();
       const timeoutId = setTimeout(() => controller.abort(), this.timeout);
       try {
-        // const {resolved, host} = await this.resolveDNS(imageUrl)
-        const res = await fetch(imageUrl, {
-          signal: controller.signal,
-          method: 'GET',
-          // headers: {
-          //   Host: host
-          // }
-        });
+        let res;
+        if(this.dnsResolver) {
+          const {resolved, host} = await this.resolveDNS(imageUrl);
+          res = await fetch(resolved, {
+            signal: controller.signal,
+            method: 'GET',
+            headers: {
+              Host: host
+            }
+          });
+        } else {
+          res = await fetch(imageUrl, {
+            signal: controller.signal,
+            method: 'GET'
+          });
+        }
+
         const arrayBuff = Buffer.from(await res.arrayBuffer());
         await fs.outputFile(filePath, arrayBuff);
         if(secondaryPath) {
@@ -37,14 +50,38 @@ export class ImageDownloader {
           if(retryCount && retryCount > 0) {
             return this.downloadAndSaveImage(imageUrl, filePath, retryCount - 1);
           } else {
-            throw `Request timed out after ${this.timeout} milliseconds.`
+            throw `Request timed out after ${this.timeout} milliseconds. URL: ${imageUrl}`
           }
         } else {
           throw error;
         }
       }
     }
   }
+
+  resolveDNS(imageUrl: string) {
+    return new Promise<{resolved: string, host: string}>((resolve,reject)=> {
+      const { host, pathname, protocol } = new URL(imageUrl);
+      if(this.dnsCache[host]) {
+        resolve({
+          resolved: `${protocol}//${this.dnsCache[host]}${pathname}`,
+          host: host
+        })
+      } else {
+        this.dnsResolver.resolve(host, (err, addresses) => {
+          if(err || !addresses.length) {
+            reject(err)
+          } else {
+            this.dnsCache[host] = addresses[0];
+            resolve({
+              resolved: `${protocol}//${addresses[0]}${pathname}`,
+              host: host
+            })
+          }
+        })
+      }
+    })
+  }
 }
 
 

src/lib/image-providers/available-providers.ts

@@ -120,4 +120,12 @@ export const providersSelect = onlineProviders.map((provider)=>{
   return {value: provider, displayValue: imageProviderNames[provider]}
 })
 
+export const dnsSelect = [
+  {value: '1.1.1.1', displayValue: 'Cloudflare'},
+  {value: '1.0.0.1', displayValue: 'Cloudflare (Alt)'},
+  {value: '8.8.8.8', displayValue: 'Google'},
+  {value: '8.8.4.4', displayValue: 'Google (Alt)'},
+
+]
+
 export const sgdbIdRegex: RegExp = /^\$\{gameid\:([0-9]*?)\}$/;

src/lib/vdf-manager.ts

@@ -141,7 +141,7 @@ export class VDF_Manager {
     })
   }
 
-  write(batch: boolean, batchSize?: number, options?: { shortcuts?: boolean, addedItems?: boolean, screenshots?: boolean}) {
+  write(batch: boolean, batchSize?: number, dnsServers?: string[], options?: { shortcuts?: boolean, addedItems?: boolean, screenshots?: boolean}) {
     return new Promise<{nonFatal: VDF_Error, outcomes: VDF_AllScreenshotsOutcomes}>((resolve,reject)=>{
       let shortcutPromises: Promise<void>[] = [];
       let addedItemsPromises: Promise<void>[] = [];
@@ -160,7 +160,7 @@ export class VDF_Manager {
             addedItemsPromises.push(this.data[steamDirectory][userId].addedItems.write());
           }
           if (writeScreenshots) {
-            screenshotPromises.push(this.data[steamDirectory][userId].screenshots.write(batch, batchSize).then((outcome: VDF_ScreenshotsOutcome)=>{
+            screenshotPromises.push(this.data[steamDirectory][userId].screenshots.write(batch, batchSize, dnsServers).then((outcome: VDF_ScreenshotsOutcome)=>{
               screenshotsOutcomes[steamDirectory][userId] = outcome;
               return outcome.error
             }))

src/lib/vdf-screenshots-file.ts

@@ -128,12 +128,12 @@ export class VDF_ScreenshotsFile {
     return vdfErrors;
   }
 
-  async write(batch: boolean, batchSizeInput?: number) {
+  async write(batch: boolean, batchSizeInput?: number, dnsServers?: string[]) {
     let addErrors: (VDF_Error|void)[] = [];
     let screenshotsData: VDF_ScreenshotsData = this.data;
     const extraneous = [...this.extraneous, ...Object.keys(screenshotsData).filter(appId=> !screenshotsData[appId])];
     const batchSize = batchSizeInput || 50;
-    const imageDownloader: ImageDownloader = new ImageDownloader();
+    const imageDownloader: ImageDownloader = new ImageDownloader(dnsServers);
     const addableAppIds = Object.keys(screenshotsData).filter((appId)=>{
       return screenshotsData[appId] !== undefined && (typeof screenshotsData[appId] !== 'string')
     });

src/models/language.model.ts

@@ -377,6 +377,7 @@ export interface languageStruct {
         fuzzy_verbose: string,
         fuzzy_filter: string,
         enabledProviders: string,
+        dnsServers: string,
         selectLanguage: string,
         selectTheme: string,
         resetFuzzy_desc: string,

src/models/settings.model.ts

@@ -32,6 +32,7 @@ export interface AppSettings {
   offlineMode: boolean,
   enabledProviders: OnlineProviderType[],
   batchDownloadSize: number,
+  dnsServers: string[],
   previewSettings: PreviewSettings,
   navigationWidth: number,
   clearLogOnTest: boolean

src/renderer/components/settings.component.ts

@@ -12,7 +12,7 @@ import { fstat } from 'fs';
 import * as fs from 'fs';
 import * as path from 'path';
 import * as paths from '../../paths';
-import { providersSelect } from '../../lib/image-providers/available-providers';
+import { providersSelect, dnsSelect } from '../../lib/image-providers/available-providers';
 
 @Component({
   selector: 'settings',
@@ -62,7 +62,11 @@ export class SettingsComponent implements OnDestroy {
   }
 
   get availableProviders() {
-    return providersSelect
+    return providersSelect;
+  }
+
+  get availableDNS() {
+    return dnsSelect;
   }
 
   get lang() {

src/renderer/modifiers/app-settings.modifier.ts

@@ -5,7 +5,7 @@ let versionUp = (version: number) => { return version + 1 };
 
 export const appSettings: ValidatorModifier<AppSettings> = {
   controlProperty: 'version',
-  latestVersion: 7,
+  latestVersion: 8,
   fields: {
     undefined: {
       'version': { method: () => 0 },
@@ -89,6 +89,12 @@ export const appSettings: ValidatorModifier<AppSettings> = {
       'enabledProviders': {method: (oldValue) => {
         return oldValue.length ? ['sgdb', 'steamCDN'] : []
       }}
+    },
+    7: {
+      'version': {method: versionUp},
+      'dnsServers': {method: (oldValue) => {
+        return [];
+      }}
     }
   }
 };

src/renderer/schemas/app-settings.schema.ts

@@ -60,6 +60,13 @@ export const appSettings = {
       }
     },
     batchDownloadSize: { type: 'number', default: 50 },
+    dnsServers: {
+      type: 'array',
+      default: [] as string[],
+      items: {
+        type: "string"
+      }
+    },
     language: { type: 'string', default: languageManager.getDefaultLanguage(), enum: languageManager.getAvailableLanguages() },
     theme: {type:'string', default: 'Deck', enum: availableThemes},
     emudeckInstall: {type: 'boolean', default: false},

src/renderer/services/preview.service.ts

@@ -257,7 +257,7 @@ export class PreviewService {
           }
         })
       }
-      return vdfManager.write(batchWrite, this.appSettings.batchDownloadSize);
+      return vdfManager.write(batchWrite, this.appSettings.batchDownloadSize, this.appSettings.dnsServers);
     })
     .then(({nonFatal, outcomes}: {nonFatal: VDF_Error, outcomes: VDF_AllScreenshotsOutcomes})=> {
       if(nonFatal) {
@@ -817,7 +817,7 @@ export class PreviewService {
   }
 
   async exportSelection() {
-    const imageDownloader = new url.ImageDownloader();
+    const imageDownloader = new url.ImageDownloader(this.appSettings.dnsServers);
     async function saveImage(imageUrl: string, temporaryDir: string, append: string) {
       const extension = imageUrl.split(/[#?]/)[0].split('.').pop().trim();
       const filename = `${append}.${extension}`;
@@ -869,13 +869,10 @@ export class PreviewService {
                 const currentImage = appImage.getCurrentImage(app.images[artworkType], this.onlineImages[artworkType]);
                 if(currentImage) {
                   const imageUrl = currentImage.imageUrl;
-                  const nintendoSucks = imageUrl.slice(-1) == '?';
-                  if(!nintendoSucks) {
-                    selection.images[artworkType] = {
-                      pool: app.images[artworkType].imagePool,
-                      filename: await saveImage(imageUrl, packagePath,`${saveId}.${artworkType}`)
-                    };
-                  }
+                  selection.images[artworkType] = {
+                    pool: app.images[artworkType].imagePool,
+                    filename: await saveImage(imageUrl, packagePath,`${saveId}.${artworkType}`)
+                  };
                 }
               }
               apps.push(selection);
@@ -930,18 +927,20 @@ export class PreviewService {
                 imageUrl: url.encodeFile(path.join(packagePath, selection.images[artworkType].filename)),
                 loadStatus: 'done'
               }, artworkType, 'imported')
+              if(!importedApps.includes(selection.images[artworkType].pool)) {
+                importedApps.push(selection.images[artworkType].pool)
+              }
             }
           }
-          importedApps.push(selection.title);
         }
-
         for (const directory in this.previewData) {
           for (const userId in this.previewData[directory]) {
             for (const appId in this.previewData[directory][userId].apps) {
               const app: PreviewDataApp = this.previewData[directory][userId].apps[appId];
-              if (importedApps.includes(app.extractedTitle)) {
-                for(const artworkType of artworkTypes) {
-                  this.setImageIndex(app, this.getTotalLengthOfImages(app, artworkType, true) -1, artworkType, true);
+              for(const artworkType of artworkTypes) {
+                const imageRanges = appImage.getImageRanges(app.images[artworkType], this.onlineImages[artworkType]);
+                if(imageRanges.imported.start < imageRanges.imported.end) {
+                  this.setImageIndex(app, imageRanges.imported.end - 1, artworkType, true);
                 }
               }
             }

src/renderer/templates/settings.component.html

@@ -103,6 +103,15 @@
         <div class=text style="margin-left: 0.25em;">{{lang.text.enabledProviders}}</div>
       </div>
     </div>
+    <div class="inputContainer">
+      <div class="inlineGroup">
+        <ng-select style="font-size: 0.85em;" class="ngSelect"
+        [ngModel]="settings.dnsServers" (ngModelChange)="settings.dnsServers = $event; onSettingsChange();"
+        [multiple]="true" [allowEmpty]="true" [placeholder]="lang.placeholder.noProviders" [values]="availableDNS">
+        </ng-select>
+        <div class=text style="margin-left: 0.25em;">{{lang.text.dnsServers}}</div>
+      </div>
+    </div>
     <div class="inputContainer">
       <div class="inlineGroup">
         <ng-select style="font-size: 0.85em;" class="ngSelect"