From 96ac6989e3c63661491bd58c77438480db61fd09 Mon Sep 17 00:00:00 2001
From: Zachary Brown <z.brown@chia.net>
Date: Thu, 29 Feb 2024 14:50:03 -0800
Subject: [PATCH 1/5] fix: mirror logic fixes for url and scheduled tasks

---
 src/datalayer/persistance.js | 23 +++++++++++++++--------
 src/tasks/mirror-check.js    | 15 ++++++++++-----
 src/utils/datalayer-utils.js | 15 ++++++++-------
 3 files changed, 33 insertions(+), 20 deletions(-)

diff --git a/src/datalayer/persistance.js b/src/datalayer/persistance.js
index 14d170b6..badc4d21 100644
--- a/src/datalayer/persistance.js
+++ b/src/datalayer/persistance.js
@@ -3,12 +3,11 @@ import fs from 'fs';
 import path from 'path';
 import superagent from 'superagent';
 import { getConfig } from '../utils/config-loader';
-import fullNode from './fullNode';
-import { publicIpv4 } from '../utils/ip-tools';
 import wallet from './wallet';
 import { Organization } from '../models';
 import { logger } from '../config/logger.cjs';
 import { getChiaRoot } from '../utils/chia-root.js';
+import { getMirrorUrl } from '../utils/datalayer-utils';
 
 process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;
 
@@ -117,6 +116,17 @@ const addMirror = async (storeId, url, forceAddMirror = false) => {
   await wallet.waitForAllTransactionsToConfirm();
   const homeOrg = await Organization.getHomeOrg();
 
+  logger.info(
+    `Checking mirrors for storeID is ${storeId} with mirror URL ${url}`,
+  );
+
+  if (!url) {
+    logger.info(
+      `No DATALAYER_FILE_SERVER_URL specified so skipping mirror for ${storeId}`,
+    );
+    return false;
+  }
+
   if (!homeOrg && !forceAddMirror) {
     logger.info(`No home org detected so skipping mirror for ${storeId}`);
     return false;
@@ -130,7 +140,7 @@ const addMirror = async (storeId, url, forceAddMirror = false) => {
   );
 
   if (mirror) {
-    logger.info(`Mirror already available for ${storeId}`);
+    logger.info(`Mirror already available for ${storeId} at ${url}`);
     return true;
   }
 
@@ -553,12 +563,9 @@ const subscribeToStoreOnDataLayer = async (storeId) => {
     if (Object.keys(data).includes('success') && data.success) {
       logger.info(`Successfully Subscribed: ${storeId}`);
 
-      const chiaConfig = fullNode.getChiaConfig();
+      const mirrorUrl = await getMirrorUrl();
 
-      await addMirror(
-        storeId,
-        `http://${await publicIpv4()}:${chiaConfig.data_layer.host_port}`,
-      );
+      await addMirror(storeId, mirrorUrl);
 
       return data;
     }
diff --git a/src/tasks/mirror-check.js b/src/tasks/mirror-check.js
index b101e5a6..809a6cf9 100644
--- a/src/tasks/mirror-check.js
+++ b/src/tasks/mirror-check.js
@@ -43,18 +43,23 @@ const job = new SimpleIntervalJob(
 );
 
 const runMirrorCheck = async () => {
-  const homeOrg = Organization.getHomeOrg();
+  const homeOrg = await Organization.getHomeOrg();
 
   if (homeOrg) {
-    const organizations = Organization.getOrgsMap();
+    const organizations = await Organization.getOrgsMap();
     const orgs = Object.keys(organizations);
     for (const org of orgs) {
       const orgData = organizations[org];
       const mirrorUrl = await getMirrorUrl();
 
-      // There is logic within the addMirror function to check if the mirror already exists
-      await Organization.addMirror(orgData.orgUid, mirrorUrl);
-      await Organization.addMirror(orgData.registryId, mirrorUrl);
+      if (mirrorUrl) {
+        // There is logic within the addMirror function to check if the mirror already exists
+        await Organization.addMirror(orgData.orgUid, mirrorUrl);
+        await Organization.addMirror(orgData.registryId, mirrorUrl);
+      } else {
+      logger.error(
+        'DATALAYER_FILE_SERVER_URL not set, skipping mirror announcement',
+      );
     }
   }
 };
diff --git a/src/utils/datalayer-utils.js b/src/utils/datalayer-utils.js
index 22ca9505..22c4c776 100644
--- a/src/utils/datalayer-utils.js
+++ b/src/utils/datalayer-utils.js
@@ -1,6 +1,5 @@
 import { getConfig } from './config-loader';
-import fullNode from '../datalayer/fullNode';
-import { publicIpv4 } from './ip-tools';
+import { logger } from '../logger.js';
 
 export const encodeHex = (str) => {
   return Buffer.from(str).toString('hex');
@@ -113,10 +112,12 @@ export const optimizeAndSortKvDiff = (kvDiff) => {
 };
 
 export const getMirrorUrl = async () => {
+  try {
   const { DATALAYER_FILE_SERVER_URL } = getConfig().APP;
-  const chiaConfig = fullNode.getChiaConfig();
-  return (
-    DATALAYER_FILE_SERVER_URL ||
-    `http://${await publicIpv4()}:${chiaConfig.data_layer.host_port}`
-  );
+  logger.debug(`Resolved Mirror Url: ${DATALAYER_FILE_SERVER_URL}`);
+  return DATALAYER_FILE_SERVER_URL;
+  } catch (error) {
+    logger.error('Error getting DATALAYER_FILE_SERVER_URL: ${error}');
+    return null;
+  }
 };

From 46ba5aebbbf4ecf84230fb100bfd2d2834216aa5 Mon Sep 17 00:00:00 2001
From: Zachary Brown <z.brown@chia.net>
Date: Fri, 1 Mar 2024 10:20:11 -0800
Subject: [PATCH 2/5] fix: mirror url logic

---
 src/tasks/mirror-check.js | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/tasks/mirror-check.js b/src/tasks/mirror-check.js
index 809a6cf9..f115f35d 100644
--- a/src/tasks/mirror-check.js
+++ b/src/tasks/mirror-check.js
@@ -57,9 +57,10 @@ const runMirrorCheck = async () => {
         await Organization.addMirror(orgData.orgUid, mirrorUrl);
         await Organization.addMirror(orgData.registryId, mirrorUrl);
       } else {
-      logger.error(
-        'DATALAYER_FILE_SERVER_URL not set, skipping mirror announcement',
-      );
+        logger.error(
+          'DATALAYER_FILE_SERVER_URL not set, skipping mirror announcement',
+        );
+      }
     }
   }
 };

From 63b74b6666227055522292c43f033b3593f3622b Mon Sep 17 00:00:00 2001
From: Zachary Brown <z.brown@chia.net>
Date: Fri, 1 Mar 2024 10:52:08 -0800
Subject: [PATCH 3/5] fix: import logger correctly for mirror-check

---
 .gitignore                   | 1 +
 src/utils/datalayer-utils.js | 8 ++++----
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.gitignore b/.gitignore
index d5f13559..9ea2058a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
 data.sqlite3
 test.sqlite3
+test.sqlite3*
 testMirror.sqlite3
 
 # dependencies
diff --git a/src/utils/datalayer-utils.js b/src/utils/datalayer-utils.js
index 22c4c776..c9f39af3 100644
--- a/src/utils/datalayer-utils.js
+++ b/src/utils/datalayer-utils.js
@@ -1,5 +1,5 @@
 import { getConfig } from './config-loader';
-import { logger } from '../logger.js';
+import { logger } from '../config/logger.cjs';
 
 export const encodeHex = (str) => {
   return Buffer.from(str).toString('hex');
@@ -113,9 +113,9 @@ export const optimizeAndSortKvDiff = (kvDiff) => {
 
 export const getMirrorUrl = async () => {
   try {
-  const { DATALAYER_FILE_SERVER_URL } = getConfig().APP;
-  logger.debug(`Resolved Mirror Url: ${DATALAYER_FILE_SERVER_URL}`);
-  return DATALAYER_FILE_SERVER_URL;
+    const { DATALAYER_FILE_SERVER_URL } = getConfig().APP;
+    logger.debug(`Resolved Mirror Url: ${DATALAYER_FILE_SERVER_URL}`);
+    return DATALAYER_FILE_SERVER_URL;
   } catch (error) {
     logger.error('Error getting DATALAYER_FILE_SERVER_URL: ${error}');
     return null;

From 6c561c5a87056245872382a329aebeec14dc276a Mon Sep 17 00:00:00 2001
From: Zachary Brown <z.brown@chia.net>
Date: Thu, 7 Mar 2024 12:04:50 -0800
Subject: [PATCH 4/5] fix: wip debugging mirror logic

---
 package-lock.json                             |   4 +-
 package.json                                  |   2 +-
 src/controllers/organization.controller.js    |   2 +
 src/datalayer/persistance.js                  |   7 ++
 .../organizations/organizations.model.js      |   7 +-
 src/tasks/mirror-check.js                     |  35 +++---
 src/utils/datalayer-utils.js                  |   2 +-
 src/utils/ip-tools.js                         | 101 ------------------
 8 files changed, 35 insertions(+), 125 deletions(-)
 delete mode 100644 src/utils/ip-tools.js

diff --git a/package-lock.json b/package-lock.json
index 9f894cce..d5cbbf35 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "cadt",
-  "version": "1.7.9",
+  "version": "1.7.9b1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "cadt",
-      "version": "1.7.9",
+      "version": "1.7.9b1",
       "dependencies": {
         "@babel/eslint-parser": "^7.23.10",
         "async-mutex": "^0.4.1",
diff --git a/package.json b/package.json
index 3c30d897..bcdde061 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "cadt",
-  "version": "1.7.9",
+  "version": "1.7.9b1",
   "_comment": "DONT CHANGE MAJOR UNLESS DATAMODEL CHANGES: The major version corresponds to the datamodel version your using, so 2.0.0 means it'll use datamodel v2",
   "private": true,
   "bin": "build/server.js",
diff --git a/src/controllers/organization.controller.js b/src/controllers/organization.controller.js
index e90c247f..14bc433c 100644
--- a/src/controllers/organization.controller.js
+++ b/src/controllers/organization.controller.js
@@ -403,6 +403,8 @@ export const addMetadata = async (req, res) => {
 
 export const addMirror = async (req, res) => {
   try {
+    console.info(`ZGB: We are in the organization.controller.js file now`);
+
     await assertIfReadOnlyMode();
     await assertWalletIsSynced();
     await assertHomeOrgExists();
diff --git a/src/datalayer/persistance.js b/src/datalayer/persistance.js
index badc4d21..52da2fb4 100644
--- a/src/datalayer/persistance.js
+++ b/src/datalayer/persistance.js
@@ -113,8 +113,15 @@ const clearPendingRoots = async (storeId) => {
 };
 
 const addMirror = async (storeId, url, forceAddMirror = false) => {
+  logger.info(
+    `ZGB: in persistance.js, forceAddMirror is ${forceAddMirror}, url is ${url}, storeId is ${storeId}`,
+  );
+
   await wallet.waitForAllTransactionsToConfirm();
   const homeOrg = await Organization.getHomeOrg();
+  logger.info(
+    `ZGB: homeorg is ${homeOrg}, forceAddMirror is ${forceAddMirror}, url is ${url}, storeId is ${storeId}`,
+  );
 
   logger.info(
     `Checking mirrors for storeID is ${storeId} with mirror URL ${url}`,
diff --git a/src/models/organizations/organizations.model.js b/src/models/organizations/organizations.model.js
index ead372cb..90c5b4ff 100644
--- a/src/models/organizations/organizations.model.js
+++ b/src/models/organizations/organizations.model.js
@@ -222,8 +222,11 @@ class Organization extends Model {
     return registryVersionId;
   }
 
-  static async addMirror(storeId, url) {
-    await datalayer.addMirror(storeId, url);
+  static async addMirror(storeId, url, force = false) {
+    logger.info(
+      `ZGB: in organizations.model.js, storeId is ${storeId}, force is ${force}, url is ${url}`,
+    );
+    await datalayer.addMirror(storeId, url, force);
   }
 
   static async importHomeOrg(orgUid) {
diff --git a/src/tasks/mirror-check.js b/src/tasks/mirror-check.js
index f115f35d..ad5be9ca 100644
--- a/src/tasks/mirror-check.js
+++ b/src/tasks/mirror-check.js
@@ -19,10 +19,12 @@ const task = new Task('mirror-check', async () => {
     await assertDataLayerAvailable();
     await assertWalletIsSynced();
 
+    logger.info(`ZGB: In the loop to do the mirror-check - no logic run yet`);
     // Default AUTO_MIRROR_EXTERNAL_STORES to true if it is null or undefined
     const shouldMirror = CONFIG?.AUTO_MIRROR_EXTERNAL_STORES ?? true;
 
     if (!CONFIG.USE_SIMULATOR && shouldMirror) {
+      logger.info(`ZGB: Conditions met for mirror-check - trying now`);
       runMirrorCheck();
     }
   } catch (error) {
@@ -43,24 +45,21 @@ const job = new SimpleIntervalJob(
 );
 
 const runMirrorCheck = async () => {
-  const homeOrg = await Organization.getHomeOrg();
-
-  if (homeOrg) {
-    const organizations = await Organization.getOrgsMap();
-    const orgs = Object.keys(organizations);
-    for (const org of orgs) {
-      const orgData = organizations[org];
-      const mirrorUrl = await getMirrorUrl();
-
-      if (mirrorUrl) {
-        // There is logic within the addMirror function to check if the mirror already exists
-        await Organization.addMirror(orgData.orgUid, mirrorUrl);
-        await Organization.addMirror(orgData.registryId, mirrorUrl);
-      } else {
-        logger.error(
-          'DATALAYER_FILE_SERVER_URL not set, skipping mirror announcement',
-        );
-      }
+  const organizations = await Organization.getOrgsMap();
+  const orgs = Object.keys(organizations);
+  logger.info(`ZGB: orgs available: ${orgs}`);
+  for (const org of orgs) {
+    const orgData = organizations[org];
+    const mirrorUrl = await getMirrorUrl();
+    logger.info(`ZGB: Working on mirror for ${org} with URL ${mirrorUrl}`);
+    if (mirrorUrl) {
+      // There is logic within the addMirror function to check if the mirror already exists
+      await Organization.addMirror(orgData.orgUid, mirrorUrl, true);
+      await Organization.addMirror(orgData.registryId, mirrorUrl, true);
+    } else {
+      logger.error(
+        'DATALAYER_FILE_SERVER_URL not set, skipping mirror announcement',
+      );
     }
   }
 };
diff --git a/src/utils/datalayer-utils.js b/src/utils/datalayer-utils.js
index c9f39af3..f83553af 100644
--- a/src/utils/datalayer-utils.js
+++ b/src/utils/datalayer-utils.js
@@ -114,7 +114,7 @@ export const optimizeAndSortKvDiff = (kvDiff) => {
 export const getMirrorUrl = async () => {
   try {
     const { DATALAYER_FILE_SERVER_URL } = getConfig().APP;
-    logger.debug(`Resolved Mirror Url: ${DATALAYER_FILE_SERVER_URL}`);
+    logger.info(`Resolved Mirror Url: ${DATALAYER_FILE_SERVER_URL}`);
     return DATALAYER_FILE_SERVER_URL;
   } catch (error) {
     logger.error('Error getting DATALAYER_FILE_SERVER_URL: ${error}');
diff --git a/src/utils/ip-tools.js b/src/utils/ip-tools.js
deleted file mode 100644
index 036cf4b8..00000000
--- a/src/utils/ip-tools.js
+++ /dev/null
@@ -1,101 +0,0 @@
-import _ from 'lodash';
-import superagent from 'superagent';
-
-const defaults = {
-  timeout: 5000,
-  onlyHttps: false,
-};
-
-const dnsServers = [
-  {
-    v4: {
-      servers: [
-        '208.67.222.222',
-        '208.67.220.220',
-        '208.67.222.220',
-        '208.67.220.222',
-      ],
-      name: 'myip.opendns.com',
-      type: 'A',
-    },
-    v6: {
-      servers: ['2620:0:ccc::2', '2620:0:ccd::2'],
-      name: 'myip.opendns.com',
-      type: 'AAAA',
-    },
-  },
-  {
-    v4: {
-      servers: [
-        '216.239.32.10',
-        '216.239.34.10',
-        '216.239.36.10',
-        '216.239.38.10',
-      ],
-      name: 'o-o.myaddr.l.google.com',
-      type: 'TXT',
-      transform: (ip) => ip.replace(/"/g, ''),
-    },
-    v6: {
-      servers: [
-        '2001:4860:4802:32::a',
-        '2001:4860:4802:34::a',
-        '2001:4860:4802:36::a',
-        '2001:4860:4802:38::a',
-      ],
-      name: 'o-o.myaddr.l.google.com',
-      type: 'TXT',
-      transform: (ip) => ip.replace(/"/g, ''),
-    },
-  },
-];
-
-const type = {
-  v4: {
-    dnsServers: dnsServers.map(({ v4: { servers, ...question } }) => ({
-      servers,
-      question,
-    })),
-    httpsUrls: ['https://icanhazip.com/', 'https://api.ipify.org/'],
-  },
-  v6: {
-    dnsServers: dnsServers.map(({ v6: { servers, ...question } }) => ({
-      servers,
-      question,
-    })),
-    httpsUrls: ['https://icanhazip.com/', 'https://api6.ipify.org/'],
-  },
-};
-
-const queryHttps = (version, options) => {
-  let cancel;
-
-  const promise = (async () => {
-    const urls = [...type[version].httpsUrls, ...(options.fallbackUrls ?? [])];
-
-    for (const url of urls) {
-      const response = await superagent.get(url);
-      const ip = (response.text || '').trim();
-      if (!_.isEmpty(ip)) {
-        return ip;
-      }
-    }
-
-    throw new Error('IP Not found');
-  })();
-
-  promise.cancel = function () {
-    return cancel.apply(this);
-  };
-
-  return promise;
-};
-
-export function publicIpv4(options) {
-  options = {
-    ...defaults,
-    ...options,
-  };
-
-  return queryHttps('v4', options);
-}

From 06bfe79db35fd3997b233164e2d0418f911e5e43 Mon Sep 17 00:00:00 2001
From: Zachary Brown <z.brown@chia.net>
Date: Thu, 7 Mar 2024 20:56:51 -0800
Subject: [PATCH 5/5] fix: mirror logic when no homeorg

---
 src/datalayer/persistance.js                    | 9 +--------
 src/datalayer/writeService.js                   | 4 ++--
 src/models/organizations/organizations.model.js | 3 ---
 src/tasks/mirror-check.js                       | 4 ----
 4 files changed, 3 insertions(+), 17 deletions(-)

diff --git a/src/datalayer/persistance.js b/src/datalayer/persistance.js
index 52da2fb4..9ece8832 100644
--- a/src/datalayer/persistance.js
+++ b/src/datalayer/persistance.js
@@ -113,15 +113,8 @@ const clearPendingRoots = async (storeId) => {
 };
 
 const addMirror = async (storeId, url, forceAddMirror = false) => {
-  logger.info(
-    `ZGB: in persistance.js, forceAddMirror is ${forceAddMirror}, url is ${url}, storeId is ${storeId}`,
-  );
-
   await wallet.waitForAllTransactionsToConfirm();
   const homeOrg = await Organization.getHomeOrg();
-  logger.info(
-    `ZGB: homeorg is ${homeOrg}, forceAddMirror is ${forceAddMirror}, url is ${url}, storeId is ${storeId}`,
-  );
 
   logger.info(
     `Checking mirrors for storeID is ${storeId} with mirror URL ${url}`,
@@ -572,7 +565,7 @@ const subscribeToStoreOnDataLayer = async (storeId) => {
 
       const mirrorUrl = await getMirrorUrl();
 
-      await addMirror(storeId, mirrorUrl);
+      await addMirror(storeId, mirrorUrl, true);
 
       return data;
     }
diff --git a/src/datalayer/writeService.js b/src/datalayer/writeService.js
index 9e1c42ea..bbe8d917 100644
--- a/src/datalayer/writeService.js
+++ b/src/datalayer/writeService.js
@@ -38,8 +38,8 @@ const createDataLayerStore = async () => {
   return storeId;
 };
 
-const addMirror = async (storeId, url) => {
-  return dataLayer.addMirror(storeId, url);
+const addMirror = async (storeId, url, force = false) => {
+  return dataLayer.addMirror(storeId, url, force);
 };
 
 const waitForStoreToBeConfirmed = async (storeId, retry = 0) => {
diff --git a/src/models/organizations/organizations.model.js b/src/models/organizations/organizations.model.js
index 90c5b4ff..b593967d 100644
--- a/src/models/organizations/organizations.model.js
+++ b/src/models/organizations/organizations.model.js
@@ -223,9 +223,6 @@ class Organization extends Model {
   }
 
   static async addMirror(storeId, url, force = false) {
-    logger.info(
-      `ZGB: in organizations.model.js, storeId is ${storeId}, force is ${force}, url is ${url}`,
-    );
     await datalayer.addMirror(storeId, url, force);
   }
 
diff --git a/src/tasks/mirror-check.js b/src/tasks/mirror-check.js
index ad5be9ca..7ebb1965 100644
--- a/src/tasks/mirror-check.js
+++ b/src/tasks/mirror-check.js
@@ -19,12 +19,10 @@ const task = new Task('mirror-check', async () => {
     await assertDataLayerAvailable();
     await assertWalletIsSynced();
 
-    logger.info(`ZGB: In the loop to do the mirror-check - no logic run yet`);
     // Default AUTO_MIRROR_EXTERNAL_STORES to true if it is null or undefined
     const shouldMirror = CONFIG?.AUTO_MIRROR_EXTERNAL_STORES ?? true;
 
     if (!CONFIG.USE_SIMULATOR && shouldMirror) {
-      logger.info(`ZGB: Conditions met for mirror-check - trying now`);
       runMirrorCheck();
     }
   } catch (error) {
@@ -47,11 +45,9 @@ const job = new SimpleIntervalJob(
 const runMirrorCheck = async () => {
   const organizations = await Organization.getOrgsMap();
   const orgs = Object.keys(organizations);
-  logger.info(`ZGB: orgs available: ${orgs}`);
   for (const org of orgs) {
     const orgData = organizations[org];
     const mirrorUrl = await getMirrorUrl();
-    logger.info(`ZGB: Working on mirror for ${org} with URL ${mirrorUrl}`);
     if (mirrorUrl) {
       // There is logic within the addMirror function to check if the mirror already exists
       await Organization.addMirror(orgData.orgUid, mirrorUrl, true);