From fc4ccb58b9fc408edfaef5a33589f3c345f8d31d Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Fri, 9 Feb 2024 13:34:08 -0500 Subject: [PATCH 1/6] fix: sync remaining bug --- src/tasks/sync-registries.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tasks/sync-registries.js b/src/tasks/sync-registries.js index a3b49a19..c01d3011 100644 --- a/src/tasks/sync-registries.js +++ b/src/tasks/sync-registries.js @@ -219,7 +219,7 @@ const syncOrganizationAudit = async (organization) => { ); } - const rootHistoryCount = rootHistory.length - 1; + const rootHistoryCount = rootHistory.length; const syncRemaining = rootHistoryCount - historyIndex; const isSynced = syncRemaining === 0; From 192a9e5447c3213bfd6309374bda04218523e689 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Fri, 9 Feb 2024 14:35:59 -0500 Subject: [PATCH 2/6] fix: sync remaining bug --- src/tasks/sync-registries.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tasks/sync-registries.js b/src/tasks/sync-registries.js index c01d3011..8b7d6683 100644 --- a/src/tasks/sync-registries.js +++ b/src/tasks/sync-registries.js @@ -221,7 +221,7 @@ const syncOrganizationAudit = async (organization) => { const rootHistoryCount = rootHistory.length; const syncRemaining = rootHistoryCount - historyIndex; - const isSynced = syncRemaining === 0; + const isSynced = Boolean(rootHistory?.[historyIndex + 1]) === false; await Organization.update( { From 64dacdf7ea376c6ec45fb13a0aee72bd2a004842 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Mon, 12 Feb 2024 17:14:42 -0500 Subject: [PATCH 3/6] fix: sync remaining bug --- src/tasks/sync-registries.js | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/tasks/sync-registries.js b/src/tasks/sync-registries.js index 8b7d6683..f077e7cb 100644 --- a/src/tasks/sync-registries.js +++ b/src/tasks/sync-registries.js @@ -219,9 +219,9 @@ const syncOrganizationAudit = async (organization) => { ); } - const rootHistoryCount = rootHistory.length; - const syncRemaining = rootHistoryCount - historyIndex; - const isSynced = Boolean(rootHistory?.[historyIndex + 1]) === false; + const rootHistoryZeroBasedCount = rootHistory.length - 1; + const syncRemaining = rootHistoryZeroBasedCount - historyIndex; + const isSynced = syncRemaining === 0; await Organization.update( { @@ -235,9 +235,13 @@ const syncOrganizationAudit = async (organization) => { return; } + const toBeProcessedIndex = historyIndex + 1; + // Organization not synced, sync it logger.info(' '); - logger.info(`Syncing ${organization.name} generation ${historyIndex}`); + logger.info( + `Syncing ${organization.name} generation ${toBeProcessedIndex}`, + ); logger.info( `${organization.name} is ${syncRemaining} DataLayer generations away from being fully synced.`, ); @@ -247,7 +251,7 @@ const syncOrganizationAudit = async (organization) => { } const root1 = _.get(rootHistory, `[${historyIndex}]`); - const root2 = _.get(rootHistory, `[${historyIndex + 1}]`); + const root2 = _.get(rootHistory, `[${toBeProcessedIndex}]`); logger.info(`ROOT 1 ${JSON.stringify(root1)}`); logger.info(`ROOT 2', ${JSON.stringify(root2)}`); @@ -299,7 +303,7 @@ const syncOrganizationAudit = async (organization) => { const updateTransaction = async (transaction, mirrorTransaction) => { logger.info( - `Syncing ${organization.name} generation ${historyIndex + 1}`, + `Syncing ${organization.name} generation ${toBeProcessedIndex}`, ); for (const diff of optimizedKvDiff) { const key = decodeHex(diff.key); @@ -313,7 +317,7 @@ const syncOrganizationAudit = async (organization) => { table: modelKey, change: decodeHex(diff.value), onchainConfirmationTimeStamp: root2.timestamp, - generation: historyIndex + 1, + generation: toBeProcessedIndex, comment: _.get( tryParseJSON( decodeHex(_.get(comment, '[0].value', encodeHex('{}'))), From dd4c289f33e789a9556d40478d5a03d34acf0828 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Mon, 12 Feb 2024 17:20:09 -0500 Subject: [PATCH 4/6] fix: sync remaining bug --- src/models/governance/governance.model.js | 5 ++--- src/tasks/sync-registries.js | 10 +++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/models/governance/governance.model.js b/src/models/governance/governance.model.js index afc2a592..e70cabd8 100644 --- a/src/models/governance/governance.model.js +++ b/src/models/governance/governance.model.js @@ -126,9 +126,8 @@ class Governance extends Model { return; } - const governanceData = await datalayer.getSubscribedStoreData( - GOVERNANCE_BODY_ID, - ); + const governanceData = + await datalayer.getSubscribedStoreData(GOVERNANCE_BODY_ID); // Check if there is v1, v2, v3 ..... and if not, then we assume this is a legacy governance table that isnt versioned const shouldSyncLegacy = !Object.keys(governanceData).some((key) => diff --git a/src/tasks/sync-registries.js b/src/tasks/sync-registries.js index f077e7cb..2d1bcb6f 100644 --- a/src/tasks/sync-registries.js +++ b/src/tasks/sync-registries.js @@ -211,16 +211,16 @@ const syncOrganizationAudit = async (organization) => { currentGeneration = lastRootSaved; } - const historyIndex = currentGeneration.generation; + const lastProcessedIndex = currentGeneration.generation; - if (historyIndex > rootHistory.length) { + if (lastProcessedIndex > rootHistory.length) { logger.error( `Could not find root history for ${organization.name} with timestamp ${currentGeneration.timestamp}, something is wrong and the sync for this organization will be paused until this is resolved.`, ); } const rootHistoryZeroBasedCount = rootHistory.length - 1; - const syncRemaining = rootHistoryZeroBasedCount - historyIndex; + const syncRemaining = rootHistoryZeroBasedCount - lastProcessedIndex; const isSynced = syncRemaining === 0; await Organization.update( @@ -235,7 +235,7 @@ const syncOrganizationAudit = async (organization) => { return; } - const toBeProcessedIndex = historyIndex + 1; + const toBeProcessedIndex = lastProcessedIndex + 1; // Organization not synced, sync it logger.info(' '); @@ -250,7 +250,7 @@ const syncOrganizationAudit = async (organization) => { await new Promise((resolve) => setTimeout(resolve, 30000)); } - const root1 = _.get(rootHistory, `[${historyIndex}]`); + const root1 = _.get(rootHistory, `[${lastProcessedIndex}]`); const root2 = _.get(rootHistory, `[${toBeProcessedIndex}]`); logger.info(`ROOT 1 ${JSON.stringify(root1)}`); From 8528fb3d43d20de2b0f82df730506fa533ee86fa Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Thu, 15 Feb 2024 12:21:32 -0500 Subject: [PATCH 5/6] test: add logging to sync --- src/tasks/sync-registries.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/tasks/sync-registries.js b/src/tasks/sync-registries.js index 2d1bcb6f..f017dda3 100644 --- a/src/tasks/sync-registries.js +++ b/src/tasks/sync-registries.js @@ -212,6 +212,7 @@ const syncOrganizationAudit = async (organization) => { } const lastProcessedIndex = currentGeneration.generation; + logger.debug(`1 Last processed index: ${lastProcessedIndex}`); if (lastProcessedIndex > rootHistory.length) { logger.error( @@ -232,10 +233,13 @@ const syncOrganizationAudit = async (organization) => { ); if (process.env.NODE_ENV !== 'test' && isSynced) { + logger.debug(`3 Last processed index: ${lastProcessedIndex}`); return; } const toBeProcessedIndex = lastProcessedIndex + 1; + logger.debug(`3 Last processed index: ${lastProcessedIndex}`); + logger.debug(`4 To be processed index: ${toBeProcessedIndex}`); // Organization not synced, sync it logger.info(' '); @@ -250,7 +254,9 @@ const syncOrganizationAudit = async (organization) => { await new Promise((resolve) => setTimeout(resolve, 30000)); } + logger.debug(`5 Last processed index: ${lastProcessedIndex}`); const root1 = _.get(rootHistory, `[${lastProcessedIndex}]`); + logger.debug(`6 To be processed index: ${toBeProcessedIndex}`); const root2 = _.get(rootHistory, `[${toBeProcessedIndex}]`); logger.info(`ROOT 1 ${JSON.stringify(root1)}`); @@ -263,6 +269,9 @@ const syncOrganizationAudit = async (organization) => { return; } + logger.debug(`7 Last processed index: ${lastProcessedIndex}`); + logger.debug(`8 To be processed index: ${toBeProcessedIndex}`); + const kvDiff = await datalayer.getRootDiff( organization.registryId, root1.root_hash, From 3f16e10673467d6a3eed821e2fb517f7f742970a Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Thu, 15 Feb 2024 13:04:55 -0500 Subject: [PATCH 6/6] test: add logging to sync --- src/tasks/sync-registries.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/tasks/sync-registries.js b/src/tasks/sync-registries.js index f017dda3..c67af761 100644 --- a/src/tasks/sync-registries.js +++ b/src/tasks/sync-registries.js @@ -143,9 +143,7 @@ const syncOrganizationAudit = async (organization) => { let afterCommitCallbacks = []; const homeOrg = await Organization.getHomeOrg(); - const rootHistory = ( - await datalayer.getRootHistory(organization.registryId) - ).sort((a, b) => a.timestamp - b.timestamp); + const rootHistory = await datalayer.getRootHistory(organization.registryId); if (!rootHistory.length) { logger.info(`No root history found for ${organization.name}`);