From 273aac7fc04018f407d29313b10f77f432433360 Mon Sep 17 00:00:00 2001 From: William Wills Date: Mon, 16 Dec 2024 17:33:36 -0500 Subject: [PATCH] feat: create organization working fix: bugs --- src/controllers/organization.controller.js | 11 +-- src/datalayer/persistance.js | 15 ++-- src/datalayer/syncService.js | 8 ++- src/datalayer/writeService.js | 18 ++--- .../organizations/organizations.model.js | 68 +++++++++++++------ src/tasks/clean-up-failed-org.js | 2 +- src/tasks/mirror-check.js | 19 +----- 7 files changed, 78 insertions(+), 63 deletions(-) diff --git a/src/controllers/organization.controller.js b/src/controllers/organization.controller.js index 3c354067..6577a76f 100644 --- a/src/controllers/organization.controller.js +++ b/src/controllers/organization.controller.js @@ -350,14 +350,15 @@ export const resyncOrganization = async (req, res) => { try { await assertIfReadOnlyMode(); await assertWalletIsSynced(); - await assertHomeOrgExists(); + const orgUid = req.body.orgUid; transaction = await sequelize.transaction(); - await Organization.update( - { registryHash: '0' }, - { where: { orgUid: req?.body?.orgUid } }, - ); + const organization = await Organization.findOne({ where: { orgUid } }); + + await Organization.reconcileOrganization(organization); + + await Organization.update({ registryHash: '0' }, { where: { orgUid } }); await Promise.all([ ...Object.keys(ModelKeys).map( diff --git a/src/datalayer/persistance.js b/src/datalayer/persistance.js index e0b8f1b1..f3a57016 100644 --- a/src/datalayer/persistance.js +++ b/src/datalayer/persistance.js @@ -382,7 +382,7 @@ const getStoreData = async (storeId, rootHash) => { return false; }; -const getRoot = async (storeId, ignoreEmptyStore = false) => { +const getRoot = async (storeId) => { const url = `${CONFIG.DATALAYER_URL}/get_root`; const { cert, key, timeout } = getBaseOptions(); @@ -394,21 +394,22 @@ const getRoot = async (storeId, ignoreEmptyStore = false) => { .timeout(timeout) .send({ id: storeId }); - const { confirmed, hash } = response.body; logger.debug( `the current root data for store ${storeId} is ${JSON.stringify(response.body)}`, ); - if (confirmed && (ignoreEmptyStore || !hash.includes('0x00000000000'))) { - return response.body; + const { success, error, traceback } = response.body; + + if (!success || error || traceback) { + throw new Error(`${error}, ${traceback}`); } - return false; + return response.body; } catch (error) { logger.error( - `failed to get root for store ${storeId}. error: ${error.message}`, + `could not get root data for store ${storeId}. this could be due to the store being in the process of confirming. error: ${error.message}`, ); - return false; + return {}; } }; diff --git a/src/datalayer/syncService.js b/src/datalayer/syncService.js index ad4ac016..5089e728 100644 --- a/src/datalayer/syncService.js +++ b/src/datalayer/syncService.js @@ -55,8 +55,8 @@ const getSubscribedStoreData = async (storeId) => { logger.debug( `syncService getSubscribedData() checking that data is available for ${storeId}.`, ); - const storeExistAndIsConfirmed = await dataLayer.getRoot(storeId, true); - if (!storeExistAndIsConfirmed) { + const { confirmed } = await dataLayer.getRoot(storeId); + if (!confirmed) { throw new Error(`Store not found in DataLayer: ${storeId}.`); } else { logger.debug( @@ -192,7 +192,9 @@ const getStoreIfUpdated = async (storeId, lastRootHash, callback, onFail) => { ); } } catch (error) { - logger.error(error.message); + logger.error( + `getStoreIfUpdated() failed to get updated store data. Error: ${error.message}`, + ); onFail(error.message); } }; diff --git a/src/datalayer/writeService.js b/src/datalayer/writeService.js index 1820f981..37bec34c 100644 --- a/src/datalayer/writeService.js +++ b/src/datalayer/writeService.js @@ -22,7 +22,7 @@ const createDataLayerStore = async () => { logger.info( `Created storeId: ${storeId}, waiting for this to be confirmed on the blockchain.`, ); - await waitForStoreToBeConfirmed(storeId); + await waitForNewStoreToBeConfirmed(storeId); await wallet.waitForAllTransactionsToConfirm(); // Default AUTO_MIRROR_EXTERNAL_STORES to true if it is null or undefined @@ -42,23 +42,23 @@ const addMirror = async (storeId, url, force = false) => { return dataLayer.addMirror(storeId, url, force); }; -const waitForStoreToBeConfirmed = async (storeId, retry = 0) => { +const waitForNewStoreToBeConfirmed = async (storeId, retry = 0) => { if (retry > 120) { throw new Error( `Creating storeId: ${storeId} timed out. Its possible the transaction is stuck.`, ); } - const storeExistAndIsConfirmed = await dataLayer.getRoot(storeId); + const { confirmed } = await dataLayer.getRoot(storeId, true); - if (!storeExistAndIsConfirmed) { + if (!confirmed) { logger.info(`Still waiting for ${storeId} to confirm`); await new Promise((resolve) => { setTimeout(() => { resolve(); }, 30000); }); - return waitForStoreToBeConfirmed(storeId, retry + 1); + return waitForNewStoreToBeConfirmed(storeId, retry + 1); } logger.info(`StoreId: ${storeId} has been confirmed. Congrats!`); }; @@ -109,7 +109,7 @@ const retry = (storeId, changeList, failedCallback, retryAttempts) => { logger.info(`Retrying pushing to store ${storeId}: ${retryAttempts}`); if (retryAttempts >= 60) { logger.info( - 'Could not push changelist to datalayer after retrying 10 times', + 'Could not push changelist to datalayer after retrying 60 times', ); failedCallback(); return; @@ -137,9 +137,9 @@ export const pushChangesWhenStoreIsAvailable = async ( const hasUnconfirmedTransactions = await wallet.hasUnconfirmedTransactions(); - const storeExistAndIsConfirmed = await dataLayer.getRoot(storeId); + const { confirmed } = await dataLayer.getRoot(storeId); - if (!hasUnconfirmedTransactions && storeExistAndIsConfirmed) { + if (!hasUnconfirmedTransactions && confirmed) { logger.info(`pushing to datalayer ${storeId}`); const success = await dataLayer.pushChangeListToDataLayer( @@ -148,7 +148,7 @@ export const pushChangesWhenStoreIsAvailable = async ( ); if (!success) { - logger.info( + logger.error( `RPC failed when pushing to store ${storeId}, attempting retry.`, ); retry(storeId, changeList, failedCallback, retryAttempts); diff --git a/src/models/organizations/organizations.model.js b/src/models/organizations/organizations.model.js index 2488e04d..a02a0462 100644 --- a/src/models/organizations/organizations.model.js +++ b/src/models/organizations/organizations.model.js @@ -71,6 +71,8 @@ class Organization extends Model { 'registryId', 'registryHash', 'sync_remaining', + 'dataModelVersionStoreId', + 'dataModelVersionStoreHash', ], }); @@ -118,18 +120,18 @@ class Organization extends Model { icon: '', }); - logger.debug('createHomeOrg() is creating organization (orgUid) store'); + logger.verbose('createHomeOrg() is creating organization (orgUid) store'); const newOrganizationId = USE_SIMULATOR ? 'f1c54511-865e-4611-976c-7c3c1f704662' : await datalayer.createDataLayerStore(); - logger.debug('createHomeOrg() is creating registryId store'); + logger.verbose('createHomeOrg() is creating registryId store'); const registryStoreId = await datalayer.createDataLayerStore(); - logger.debug('createHomeOrg() is creating dataModelVersionId store'); + logger.verbose('createHomeOrg() is creating dataModelVersionId store'); const dataModelVersionStoreId = await datalayer.createDataLayerStore(); - logger.debug('createHomeOrg() is creating file store'); + logger.verbose('createHomeOrg() is creating file store'); const fileStoreId = await datalayer.createDataLayerStore(); const revertOrganizationIfFailed = async () => { @@ -150,7 +152,7 @@ class Organization extends Model { await datalayer.waitForAllTransactionsToConfirm(); } - logger.debug( + logger.verbose( `the blockchain reported new organization stores orgUid: ${newOrganizationId}, ` + `dataModelVersionStore: ${dataModelVersionStoreId}, registryId: ${registryStoreId} have confirmed. `, ); @@ -161,7 +163,7 @@ class Organization extends Model { await datalayer.syncDataLayer( newOrganizationId, { - registryId: registryStoreId, + registryId: dataModelVersionStoreId, // registryId is the key named here, but this the DATA MODEL VERSION store id fileStoreId, name, icon, @@ -181,9 +183,9 @@ class Organization extends Model { `commiting registry data model version data to data model version store ${dataModelVersionStoreId}`, ); await datalayer.syncDataLayer( - registryStoreId, + dataModelVersionStoreId, { - [dataVersion]: dataModelVersionStoreId, + [dataVersion]: registryStoreId, }, revertOrganizationIfFailed, ); @@ -220,8 +222,8 @@ class Organization extends Model { if (!USE_SIMULATOR) { logger.info('Waiting for New Organization to be confirmed'); - datalayer.getStoreData( - registryStoreId, + await datalayer.getStoreData( + newOrganizationId, onConfirm, revertOrganizationIfFailed, ); @@ -329,7 +331,7 @@ class Organization extends Model { } } - const updatedOrganizationData = { subscribed: true }; + const updatedOrganizationData = {}; if (dataModelVersionStoreId !== datalayerDataModelVersionStoreId) { const message = @@ -358,21 +360,44 @@ class Organization extends Model { const dataModelVersionStoreSyncStatus = await getSyncStatus( datalayerDataModelVersionStoreId, ); + if (isDlSynced(dataModelVersionStoreSyncStatus)) { - const { hash } = await getRoot(datalayerDataModelVersionStoreId); - if (hash !== organization.dataModelVersionStoreHash) { + const { confirmed, hash } = await getRoot( + datalayerDataModelVersionStoreId, + ); + if (confirmed && hash !== organization.dataModelVersionStoreHash) { logger.info( - `updating data model version store ${datalayerDataModelVersionStoreId} root hash to ${hash} from ${organization.dataModelVersionStoreHash}`, + `data model version store ${datalayerDataModelVersionStoreId} root hash needs to be updated ${hash} ` + + `from ${organization.dataModelVersionStoreHash} to ${hash}`, ); updatedOrganizationData.dataModelVersionStoreHash = hash; + } else if (!confirmed) { + logger.warn( + `data model version store ${datalayerDataModelVersionStoreId} has not been confirmed yet. cannot validate or update hash.`, + ); } } - try { - await Organization.update(updatedOrganizationData, { where: { orgUid } }); - } catch { - throw new Error( - 'failed to write updated organization data to organization table', + if (!organization.subscribed) { + updatedOrganizationData.subscribed = true; + } + + if (!_.isEmpty(updatedOrganizationData)) { + logger.info( + `reconcile organization task is updating organization ${organization.orgUid} with the following data ${JSON.stringify(updatedOrganizationData)}`, + ); + try { + await Organization.update(updatedOrganizationData, { + where: { orgUid }, + }); + } catch { + throw new Error( + 'failed to write updated organization data to organization table', + ); + } + } else { + logger.info( + `reconcile organization task found organization ${organization.orgUid} required no updates or corrections`, ); } } @@ -650,7 +675,10 @@ class Organization extends Model { await Organization.update( { - ..._.omit(updateData, ['registryId']), + ..._.omit(updateData, [ + 'registryId', + 'dataModelVersionStoreId', + ]), prefix: updateData.prefix || '0', metadata: JSON.stringify(metadata), }, diff --git a/src/tasks/clean-up-failed-org.js b/src/tasks/clean-up-failed-org.js index db697c42..47ae1403 100644 --- a/src/tasks/clean-up-failed-org.js +++ b/src/tasks/clean-up-failed-org.js @@ -32,7 +32,7 @@ const job = new SimpleIntervalJob( runImmediately: true, }, task, - { id: 'clean-up failed-org', preventOverrun: true }, + { id: 'clean-up-failed-org', preventOverrun: true }, ); export default job; diff --git a/src/tasks/mirror-check.js b/src/tasks/mirror-check.js index e66cd636..04c4348c 100644 --- a/src/tasks/mirror-check.js +++ b/src/tasks/mirror-check.js @@ -8,10 +8,8 @@ import { logger } from '../config/logger.js'; import { getConfig } from '../utils/config-loader'; import { getMirrorUrl } from '../utils/datalayer-utils'; import dotenv from 'dotenv'; -import datalayer from '../datalayer'; const APP_CONFIG = getConfig().APP; -const GOVERNANCE_CONFIG = getConfig().GOVERNANCE; dotenv.config(); // This task checks if there are any mirrors that have not been properly mirrored and then mirrors them if not @@ -82,22 +80,6 @@ const runMirrorCheck = async () => { mirrorUrl, true, ); - } else if (GOVERNANCE_CONFIG?.GOVERNANCE_BODY_ID) { - const governanceStoreValue = await datalayer.getSubscribedStoreData( - GOVERNANCE_CONFIG.GOVERNANCE_BODY_ID, - ); - - if (governanceStoreValue?.v1) { - // add governance mirrors if non-governance instance - await Organization.addMirror( - GOVERNANCE_CONFIG.GOVERNANCE_BODY_ID, - mirrorUrl, - true, - ); - await Organization.addMirror(governanceStoreValue.v1, mirrorUrl, true); - } else { - logger.warn('error adding governance mirrors'); - } } const organizations = await Organization.getOrgsMap(); @@ -105,6 +87,7 @@ const runMirrorCheck = async () => { for (const org of orgs) { const orgData = organizations[org]; await Organization.addMirror(orgData.orgUid, mirrorUrl, true); + await Organization.addMirror(orgData.dataModelVersionStoreId, true); await Organization.addMirror(orgData.registryId, mirrorUrl, true); } };