Upgraded performance with specific timed sleep

screepers · Feb 13, 2024 · e42a90b · e42a90b
1 parent 124903e
commit e42a90b
Show file tree

Hide file tree

Showing 8 changed files with 96 additions and 116 deletions.
diff --git a/.gitignore b/.gitignore
@@ -9,4 +9,6 @@ docker-compose.override.yml
 ips.json
 users.json
 requests.json
-roomsBeingChecked.json
+roomsBeingChecked.json
+newrelic_agent.log
+newrelic.cjs
diff --git a/files/defaultActions.unknown.json b/files/defaultActions.unknown.json
@@ -338,5 +338,40 @@
     "path": "countByType.constructionSites.storage",
     "value": 0,
     "type": 1
+  },
+  {
+    "path": "countByType.constructionSites.rampart",
+    "value": 0,
+    "type": 1
+  },
+  {
+    "path": "resourcesStored.energy",
+    "value": 0,
+    "type": 1
+  },
+  {
+    "path": "resourcesStored.UH",
+    "value": 0,
+    "type": 1
+  },
+  {
+    "path": "resourcesStored.ZH",
+    "value": 0,
+    "type": 1
+  },
+  {
+    "path": "resourcesStored.KO",
+    "value": 0,
+    "type": 1
+  },
+  {
+    "path": "resourcesStored.GO",
+    "value": 0,
+    "type": 1
+  },
+  {
+    "path": "countByType.intents.upgradeController",
+    "value": 0,
+    "type": 0
   }
 ]
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -4,10 +4,10 @@
 	"description": "",
 	"main": "index.js",
 	"scripts": {
-		"testLoop": "node testHelper/index.js",
+		"start": "node src/index.js",
 		"test": "jest",
 		"test:cov": "jest --coverage",
-		"start": "node src/index.js",
+		"testLoop": "node testHelper/index.js",
 		"lint": "eslint controller/**/*.js && eslint getter/**/*.js && eslint terminal/**/*.js",
 		"lint:fix": "eslint controller/**/*.js --fix && eslint getter/**/*.js --fix && eslint terminal/**/*.js --fix",
 		"generate:docs": "node helper/generateDataMD.js"
@@ -33,7 +33,6 @@
 		"https-proxy-agent": "^7.0.2",
 		"jest": "^29.7.0",
 		"postgres": "^3.4.3",
-		"poolifier": "^3.1.19",
 		"screeps-advanced-api": "^1.4.3",
 		"screeps-api": "^1.16.0",
 		"winston": "^3.11.0",

diff --git a/src/process/index.js b/src/process/index.js
@@ -1,28 +1,10 @@
 import { GetRoomHistory } from "./screepsApi.js";
 import { sleep } from "../helper/index.js";
 import getProxy from "../helper/proxy.js";
-import { FixedThreadPool } from "poolifier";
-import { dirname } from 'node:path';
-import { fileURLToPath } from 'node:url';
 import { GetUsernameById } from "../helper/users.js";
 import ProcessDataBroker from "../data/broker/processData.js";
 
-const __dirname = dirname(fileURLToPath(import.meta.url));
-
-let pool = null;
-if (process.env.USE_MULTITHREADING_CORES) {
-  pool = new FixedThreadPool(parseInt(process.env.USE_MULTITHREADING_CORES), __dirname + "/processThreadWorker.js",
-    {
-      errorHandler: (e) => {
-        console.error(e);
-      },
-      onlineHandler: () => {
-        console.log("A thread came online");
-      }
-    });
-}
-
-let validData = {}
+const validData = {}
 function shouldFail(opts) {
   const key = `${opts.shard}-${opts.room}`
   const data = validData[key]
@@ -34,13 +16,13 @@ function shouldFail(opts) {
 }
 
 const lastRequest = {}
-async function waitMax500ms(proxyIndex) {
+async function waitMax500ms(proxyIndex = -1) {
   const last = lastRequest[proxyIndex] || 0;
   const diff = Date.now() - last;
-  if (diff < 500) {
-    return sleep(500 - diff);
-  }
-  return Promise.resolve();
+  lastRequest[proxyIndex] = Date.now();
+  const sleepTime = Math.min(500 - diff, 500);
+  if (sleepTime < 0) Promise.resolve();
+  return sleep(sleepTime);
 }
 
 export default async function processData(opts, proxyIndex) {
@@ -54,62 +36,56 @@ export default async function processData(opts, proxyIndex) {
     const { data } = dataResult;
     if (opts.username) {
       opts.timestamp = data.timestamp;
-      if (process.env.USE_MULTITHREADING_CORES) {
-        opts.data = await pool.execute({ roomData: data, opts });
-      }
-      else {
-        if (opts.username === "Unknown") {
-          const userIds = {};
-          const tick = Object.values(data.ticks)[0];
-          if (tick) {
-            const tickObjects = Object.values(tick);
-            for (let i = 0; i < tickObjects.length; i += 1) {
-              const tickData = tickObjects[i];
-              if (tickData.controller) {
-                opts.userId = tickData.controller.user;
-                opts.username = await GetUsernameById(opts.userId);
-                opts.type = "owned"
-                break;
-              }
-              if (tickData.reservation) {
-                opts.userId = tickData.reservation.user;
-                opts.username = await GetUsernameById(opts.userId);
-                opts.type = "reserved"
-                break;
-              }
-              if (tickData.user) {
-                userIds[tickData.user] = userIds[tickData.user] || 0;
-                userIds[tickData.user] += 1;
-              }
+      if (opts.username === "Unknown") {
+        const userIds = {};
+        const tick = Object.values(data.ticks)[0];
+        if (tick) {
+          const tickObjects = Object.values(tick);
+          for (let i = 0; i < tickObjects.length; i += 1) {
+            const tickData = tickObjects[i];
+            if (tickData.controller) {
+              opts.userId = tickData.controller.user;
+              opts.username = await GetUsernameById(opts.userId);
+              opts.type = "owned"
+              break;
             }
-
-            if (opts.username === "Unknown") {
-              let max = 0;
-              let maxId = "Unknown";
-              const userIdsKeys = Object.keys(userIds);
-              for (let u = 0; u < userIdsKeys.length; u += 1) {
-                const id = userIdsKeys[u];
-                if (userIds[id] > max) {
-                  max = userIds[id];
-                  maxId = id;
-                }
-              }
-
-              opts.userId = maxId;
-              opts.username = await GetUsernameById(maxId);
+            if (tickData.reservation) {
+              opts.userId = tickData.reservation.user;
+              opts.username = await GetUsernameById(opts.userId);
+              opts.type = "reserved"
+              break;
+            }
+            if (tickData.user) {
+              userIds[tickData.user] = userIds[tickData.user] || 0;
+              userIds[tickData.user] += 1;
             }
           }
 
-          if (!opts.username || opts.username === "Unknown") {
-            return {
-              status: "No user",
+          if (opts.username === "Unknown") {
+            let max = 0;
+            let maxId = "Unknown";
+            const userIdsKeys = Object.keys(userIds);
+            for (let u = 0; u < userIdsKeys.length; u += 1) {
+              const id = userIdsKeys[u];
+              if (userIds[id] > max) {
+                max = userIds[id];
+                maxId = id;
+              }
             }
+
+            opts.userId = maxId;
+            opts.username = await GetUsernameById(maxId);
           }
         }
 
-        opts.data = await ProcessDataBroker.single({ roomData: data, opts });
+        if (!opts.username || opts.username === "Unknown") {
+          return {
+            status: "No user",
+          }
+        }
       }
 
+      opts.data = await ProcessDataBroker.single({ roomData: data, opts });
       validData[`${opts.shard}-${opts.room}`] = { data: opts.data, tick: opts.tick }
     }
     return {

diff --git a/src/process/processThreadWorker.js b/src/process/processThreadWorker.js
diff --git a/src/process/screepsApi.js b/src/process/screepsApi.js
@@ -80,11 +80,11 @@ async function getHistory(proxy, room, tick, shard) {
       return { status: "Success", result: response.data };
     } catch (error) {
       if (error.message && error.message.includes("404 Not Found")) {
-        logger.info(`GetHistory - ${url} / ${error.message}`)
+        // logger.info(`GetHistory - ${url} / ${error.message}`)
         return { status: "Not found", message: error };
       }
 
-      logger.error(`GetHistory - ${url} / ${error.message} / ${error.stack}`)
+      // logger.error(`GetHistory - ${url} / ${error.message} / ${error.stack}`)
       return { status: "Error", message: error };
     }
   }

diff --git a/src/requests/index.js b/src/requests/index.js
@@ -14,7 +14,6 @@ async function proxy(cycle, proxyIndex) {
   while (cycle.length > 0) {
     const opts = cycle.pop();
     await processOpts(opts, proxyIndex);
-    console.log(`${cycle.length} left out of ${cycle.length + proxyCycles.length} of cycle ${opts.shard}/${opts.tick}`)
     proxyCycles.push(opts);
   }
 }
@@ -36,8 +35,6 @@ export default class Requests {
       else {
         for (let i = cycleLength - 1; i >= 0; i -= 1) {
           const opts = cycle[i];
-          console.log(`${cycleLength - i} / ${cycleLength} of cycle ${opts.shard}/${opts.tick}`)
-          await sleep(500)
           await processOpts(opts, undefined);
         }
       }
@@ -67,6 +64,7 @@ export default class Requests {
       }
       await UploadStats(stats, timestamp)
       const timeTaken = Date.now() - start;
+
       const percentageOnTarget = (cycleLength * 500 * 60) / timeTaken;
       const timePerRoom = (Date.now() - start) / cycleLength
       await UploadStatus({