diff --git a/.gitignore b/.gitignore index d5f13559..9ea2058a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. data.sqlite3 test.sqlite3 +test.sqlite3* testMirror.sqlite3 # dependencies diff --git a/package-lock.json b/package-lock.json index 9f894cce..d5cbbf35 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "cadt", - "version": "1.7.9", + "version": "1.7.9b1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cadt", - "version": "1.7.9", + "version": "1.7.9b1", "dependencies": { "@babel/eslint-parser": "^7.23.10", "async-mutex": "^0.4.1", diff --git a/package.json b/package.json index 3c30d897..bcdde061 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cadt", - "version": "1.7.9", + "version": "1.7.9b1", "_comment": "DONT CHANGE MAJOR UNLESS DATAMODEL CHANGES: The major version corresponds to the datamodel version your using, so 2.0.0 means it'll use datamodel v2", "private": true, "bin": "build/server.js", diff --git a/src/controllers/organization.controller.js b/src/controllers/organization.controller.js index e90c247f..14bc433c 100644 --- a/src/controllers/organization.controller.js +++ b/src/controllers/organization.controller.js @@ -403,6 +403,8 @@ export const addMetadata = async (req, res) => { export const addMirror = async (req, res) => { try { + console.info(`ZGB: We are in the organization.controller.js file now`); + await assertIfReadOnlyMode(); await assertWalletIsSynced(); await assertHomeOrgExists(); diff --git a/src/datalayer/persistance.js b/src/datalayer/persistance.js index 14d170b6..9ece8832 100644 --- a/src/datalayer/persistance.js +++ b/src/datalayer/persistance.js @@ -3,12 +3,11 @@ import fs from 'fs'; import path from 'path'; import superagent from 'superagent'; import { getConfig } from '../utils/config-loader'; -import fullNode from './fullNode'; -import { publicIpv4 } from '../utils/ip-tools'; import wallet from './wallet'; import { Organization } from '../models'; import { logger } from '../config/logger.cjs'; import { getChiaRoot } from '../utils/chia-root.js'; +import { getMirrorUrl } from '../utils/datalayer-utils'; process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0; @@ -117,6 +116,17 @@ const addMirror = async (storeId, url, forceAddMirror = false) => { await wallet.waitForAllTransactionsToConfirm(); const homeOrg = await Organization.getHomeOrg(); + logger.info( + `Checking mirrors for storeID is ${storeId} with mirror URL ${url}`, + ); + + if (!url) { + logger.info( + `No DATALAYER_FILE_SERVER_URL specified so skipping mirror for ${storeId}`, + ); + return false; + } + if (!homeOrg && !forceAddMirror) { logger.info(`No home org detected so skipping mirror for ${storeId}`); return false; @@ -130,7 +140,7 @@ const addMirror = async (storeId, url, forceAddMirror = false) => { ); if (mirror) { - logger.info(`Mirror already available for ${storeId}`); + logger.info(`Mirror already available for ${storeId} at ${url}`); return true; } @@ -553,12 +563,9 @@ const subscribeToStoreOnDataLayer = async (storeId) => { if (Object.keys(data).includes('success') && data.success) { logger.info(`Successfully Subscribed: ${storeId}`); - const chiaConfig = fullNode.getChiaConfig(); + const mirrorUrl = await getMirrorUrl(); - await addMirror( - storeId, - `http://${await publicIpv4()}:${chiaConfig.data_layer.host_port}`, - ); + await addMirror(storeId, mirrorUrl, true); return data; } diff --git a/src/datalayer/writeService.js b/src/datalayer/writeService.js index 9e1c42ea..bbe8d917 100644 --- a/src/datalayer/writeService.js +++ b/src/datalayer/writeService.js @@ -38,8 +38,8 @@ const createDataLayerStore = async () => { return storeId; }; -const addMirror = async (storeId, url) => { - return dataLayer.addMirror(storeId, url); +const addMirror = async (storeId, url, force = false) => { + return dataLayer.addMirror(storeId, url, force); }; const waitForStoreToBeConfirmed = async (storeId, retry = 0) => { diff --git a/src/models/organizations/organizations.model.js b/src/models/organizations/organizations.model.js index ead372cb..b593967d 100644 --- a/src/models/organizations/organizations.model.js +++ b/src/models/organizations/organizations.model.js @@ -222,8 +222,8 @@ class Organization extends Model { return registryVersionId; } - static async addMirror(storeId, url) { - await datalayer.addMirror(storeId, url); + static async addMirror(storeId, url, force = false) { + await datalayer.addMirror(storeId, url, force); } static async importHomeOrg(orgUid) { diff --git a/src/tasks/mirror-check.js b/src/tasks/mirror-check.js index b101e5a6..7ebb1965 100644 --- a/src/tasks/mirror-check.js +++ b/src/tasks/mirror-check.js @@ -43,18 +43,19 @@ const job = new SimpleIntervalJob( ); const runMirrorCheck = async () => { - const homeOrg = Organization.getHomeOrg(); - - if (homeOrg) { - const organizations = Organization.getOrgsMap(); - const orgs = Object.keys(organizations); - for (const org of orgs) { - const orgData = organizations[org]; - const mirrorUrl = await getMirrorUrl(); - + const organizations = await Organization.getOrgsMap(); + const orgs = Object.keys(organizations); + for (const org of orgs) { + const orgData = organizations[org]; + const mirrorUrl = await getMirrorUrl(); + if (mirrorUrl) { // There is logic within the addMirror function to check if the mirror already exists - await Organization.addMirror(orgData.orgUid, mirrorUrl); - await Organization.addMirror(orgData.registryId, mirrorUrl); + await Organization.addMirror(orgData.orgUid, mirrorUrl, true); + await Organization.addMirror(orgData.registryId, mirrorUrl, true); + } else { + logger.error( + 'DATALAYER_FILE_SERVER_URL not set, skipping mirror announcement', + ); } } }; diff --git a/src/utils/datalayer-utils.js b/src/utils/datalayer-utils.js index 22ca9505..f83553af 100644 --- a/src/utils/datalayer-utils.js +++ b/src/utils/datalayer-utils.js @@ -1,6 +1,5 @@ import { getConfig } from './config-loader'; -import fullNode from '../datalayer/fullNode'; -import { publicIpv4 } from './ip-tools'; +import { logger } from '../config/logger.cjs'; export const encodeHex = (str) => { return Buffer.from(str).toString('hex'); @@ -113,10 +112,12 @@ export const optimizeAndSortKvDiff = (kvDiff) => { }; export const getMirrorUrl = async () => { - const { DATALAYER_FILE_SERVER_URL } = getConfig().APP; - const chiaConfig = fullNode.getChiaConfig(); - return ( - DATALAYER_FILE_SERVER_URL || - `http://${await publicIpv4()}:${chiaConfig.data_layer.host_port}` - ); + try { + const { DATALAYER_FILE_SERVER_URL } = getConfig().APP; + logger.info(`Resolved Mirror Url: ${DATALAYER_FILE_SERVER_URL}`); + return DATALAYER_FILE_SERVER_URL; + } catch (error) { + logger.error('Error getting DATALAYER_FILE_SERVER_URL: ${error}'); + return null; + } }; diff --git a/src/utils/ip-tools.js b/src/utils/ip-tools.js deleted file mode 100644 index 036cf4b8..00000000 --- a/src/utils/ip-tools.js +++ /dev/null @@ -1,101 +0,0 @@ -import _ from 'lodash'; -import superagent from 'superagent'; - -const defaults = { - timeout: 5000, - onlyHttps: false, -}; - -const dnsServers = [ - { - v4: { - servers: [ - '208.67.222.222', - '208.67.220.220', - '208.67.222.220', - '208.67.220.222', - ], - name: 'myip.opendns.com', - type: 'A', - }, - v6: { - servers: ['2620:0:ccc::2', '2620:0:ccd::2'], - name: 'myip.opendns.com', - type: 'AAAA', - }, - }, - { - v4: { - servers: [ - '216.239.32.10', - '216.239.34.10', - '216.239.36.10', - '216.239.38.10', - ], - name: 'o-o.myaddr.l.google.com', - type: 'TXT', - transform: (ip) => ip.replace(/"/g, ''), - }, - v6: { - servers: [ - '2001:4860:4802:32::a', - '2001:4860:4802:34::a', - '2001:4860:4802:36::a', - '2001:4860:4802:38::a', - ], - name: 'o-o.myaddr.l.google.com', - type: 'TXT', - transform: (ip) => ip.replace(/"/g, ''), - }, - }, -]; - -const type = { - v4: { - dnsServers: dnsServers.map(({ v4: { servers, ...question } }) => ({ - servers, - question, - })), - httpsUrls: ['https://icanhazip.com/', 'https://api.ipify.org/'], - }, - v6: { - dnsServers: dnsServers.map(({ v6: { servers, ...question } }) => ({ - servers, - question, - })), - httpsUrls: ['https://icanhazip.com/', 'https://api6.ipify.org/'], - }, -}; - -const queryHttps = (version, options) => { - let cancel; - - const promise = (async () => { - const urls = [...type[version].httpsUrls, ...(options.fallbackUrls ?? [])]; - - for (const url of urls) { - const response = await superagent.get(url); - const ip = (response.text || '').trim(); - if (!_.isEmpty(ip)) { - return ip; - } - } - - throw new Error('IP Not found'); - })(); - - promise.cancel = function () { - return cancel.apply(this); - }; - - return promise; -}; - -export function publicIpv4(options) { - options = { - ...defaults, - ...options, - }; - - return queryHttps('v4', options); -}