General fixes

src/helper/proxies/index.js

@@ -0,0 +1,19 @@
+import { getProxiesList as getProxiesListWebshare } from "./webshare.js";
+import { getProxiesList as getProxiesListProxyscrape } from "./proxyscrape.js";
+
+let proxyList = [];
+
+async function getProxiesList() {
+  if (process.env.WEBSHARE_TOKEN !== undefined) {
+    proxyList = await getProxiesListWebshare();
+  }
+  else if (process.env.PROXYSCRAPE_TOKEN !== undefined) {
+    proxyList = await getProxiesListProxyscrape();
+  }
+}
+
+getProxiesList();
+
+export default function getProxy(index) {
+  return proxyList[index];
+}

src/helper/proxies/proxyscrape.js

@@ -0,0 +1,32 @@
+const PROXYSCRAPE_TOKEN = process.env.PROXYSCRAPE_TOKEN;
+import { UploadStatus } from "../../data/upload.js";
+import { CronJob } from "cron";
+import axios from "axios";
+
+async function getProxies() {
+  try {
+    const url = `https://api.proxyscrape.com/v2/account/datacenter_shared/proxy-list?protocol=http&auth=${PROXYSCRAPE_TOKEN}&type=getproxies&country[]=de&format=normal&status=all`;
+    const response = await axios.get(url);
+    const { data } = response;
+
+    const results = data.split("\r\n").filter((proxy) => proxy.length > 0);
+    const proxyList = results.map((proxy) => {
+      const [ip, port] = proxy.split(":");
+      return {
+        ip,
+        port,
+      };
+    });
+    return proxyList;
+  } catch (error) {
+    return []
+  }
+}
+
+export async function getProxiesList() {
+  return await getProxies();
+}
+
+export default function getProxy(index) {
+  return proxyList[index];
+}

src/helper/proxy.js → src/helper/proxies/webshare.js

@@ -1,6 +1,6 @@
 const WEBSHARE_TOKEN = process.env.WEBSHARE_TOKEN;
 const WEBSHARE_PROXYAMOUNT = process.env.WEBSHARE_PROXYAMOUNT;
-import { UploadStatus } from "../data/upload.js";
+import { UploadStatus } from "../../data/upload.js";
 import { CronJob } from "cron";
 import axios from "axios";
 
@@ -19,25 +19,19 @@ async function getProxies(page) {
   }
 }
 
-let proxyList = [];
-let promises = [];
-
-for (let i = 1; i <= Math.ceil(Number(WEBSHARE_PROXYAMOUNT) / 100); i += 1) {
-  promises.push(getProxies(i));
-}
+export async function getProxiesList() {
+  let promises = [];
+  for (let i = 1; i <= Math.ceil(Number(WEBSHARE_PROXYAMOUNT) / 100); i += 1) {
+    promises.push(getProxies(i));
+  }
 
-Promise.all(promises).then((results) => {
+  let proxyList = [];
+  const results = await Promise.all(promises)
   results.forEach((proxies) => {
     proxyList = proxyList.concat(proxies);
   });
-  // Now you can use the proxyList
-  console.log(`Loaded ${proxyList.length} proxies from Webshare`);
-}).catch((error) => {
-  console.error(error);
-});
 
-export default function getProxy(index) {
-  return proxyList[index];
+  return proxyList;
 }
 
 async function downloadAndUploadProxyStatistics() {

src/process/index.js

@@ -1,6 +1,6 @@
 import { GetRoomHistory } from "./screepsApi.js";
 import { sleep } from "../helper/index.js";
-import getProxy from "../helper/proxy.js";
+import getProxy from "../helper/proxies/index.js";
 import ProcessDataBroker from "../data/broker/processData.js";
 
 const validData = {}

src/process/screepsApi.js

@@ -47,8 +47,7 @@ const lastTickCache = {
 }
 export async function GetGameTime(shard) {
   try {
-    if (lastTickCache[shard] && lastTickCache[shard].lastUpdate + 10 * 1000 > Date.now()) {
-      logger.info(`GetGameTime: ${shard}/${lastTickCache[shard].tick}`);
+    if (lastTickCache[shard] && lastTickCache[shard].lastUpdate + 600 * 1000 > Date.now()) {
       return lastTickCache[shard].tick;
     }
     await sleep(500);
@@ -79,7 +78,7 @@ async function getHistory(proxy, room, tick, shard) {
       logger.info(`GetHistory - Success: ${url}`);
       return { status: "Success", result: response.data };
     } catch (error) {
-      if (error.message && error.message.includes("404 Not Found")) {
+      if (error.response && error.response.status === 404) {
         // logger.info(`GetHistory - ${url} / ${error.message}`)
         return { status: "Not found", message: error };
       }
@@ -92,7 +91,7 @@ async function getHistory(proxy, room, tick, shard) {
   const proxySettings = proxy;
 
   const timeoutPromise = new Promise((resolve) => {
-    setTimeout(resolve, 10 * 1000, () => {
+    setTimeout(resolve, 15 * 1000, () => {
       logger.info('Failed: Proxy Timeout')
       return { status: "Timeout" }
     });
@@ -101,7 +100,13 @@ async function getHistory(proxy, room, tick, shard) {
   const getHistoryPromise = new Promise((resolve) => {
 
     // eslint-disable-next-line max-len
-    const proxyUrl = `http://${proxySettings.username}:${proxySettings.password}@${proxySettings.proxy_address}:${proxySettings.ports.http}`;
+    let proxyUrl = "";
+    if (proxySettings.username && proxySettings.password) {
+      proxyUrl = `http://${proxySettings.username}:${proxySettings.password}@${proxySettings.proxy_address}:${proxySettings.ports.http}`;
+    }
+    if (proxySettings.ip) {
+      proxyUrl = `http://${proxySettings.ip}:${proxySettings.port}`;
+    }
     const agent = new HttpsProxyAgent(
       proxyUrl
     );
@@ -114,7 +119,7 @@ async function getHistory(proxy, room, tick, shard) {
         resolve({ status: "Success", result: response.data });
       })
       .catch((error) => {
-        if (error.message && error.message.includes("404 Not Found")) {
+        if (error.response && error.response.status === 404) {
           // apiLogger.info(`${url} / ${error.message}`)
           resolve({ status: "Not found", message: error });
         }

src/requests/index.js

@@ -5,7 +5,7 @@ import { getCycle, cycleStatus } from "../helper/requests.js";
 import processOpts from "../process/index.js";
 import { GetUserData } from "../helper/users.js"
 
-const useProxy = process.env.WEBSHARE_TOKEN !== undefined;
+const useProxy = process.env.WEBSHARE_TOKEN !== undefined || process.env.PROXYSCRAPE_TOKEN !== undefined;
 
 let proxyCycles = [];
 
@@ -27,7 +27,11 @@ export default class Requests {
 
       if (useProxy) {
         proxyCycles = [];
-        const proxies = Array.from({ length: Number(process.env.WEBSHARE_PROXYAMOUNT) }, (_, index) => proxy(cycle, index));
+        let proxyAmount = Number(process.env.WEBSHARE_PROXYAMOUNT);
+        if (!proxyAmount) {
+          proxyAmount = Number(process.env.PROXYSCRAPE_PROXYAMOUNT);
+        }
+        const proxies = Array.from({ length: proxyAmount }, (_, index) => proxy(cycle, index));
         await Promise.all(proxies);
         cycle = proxyCycles;
       }