Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: sync remaining bug #1005

Merged
merged 6 commits into from
Feb 15, 2024
Merged

fix: sync remaining bug #1005

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
fc4ccb5
fix: sync remaining bug
MichaelTaylor3D Feb 9, 2024
192a9e5
fix: sync remaining bug
MichaelTaylor3D Feb 9, 2024
64dacdf
fix: sync remaining bug
MichaelTaylor3D Feb 12, 2024
dd4c289
fix: sync remaining bug
MichaelTaylor3D Feb 12, 2024
8528fb3
test: add logging to sync
MichaelTaylor3D Feb 15, 2024
3f16e10
test: add logging to sync
MichaelTaylor3D Feb 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions src/models/governance/governance.model.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,9 +126,8 @@ class Governance extends Model {
return;
}

const governanceData = await datalayer.getSubscribedStoreData(
GOVERNANCE_BODY_ID,
);
const governanceData =
await datalayer.getSubscribedStoreData(GOVERNANCE_BODY_ID);

// Check if there is v1, v2, v3 ..... and if not, then we assume this is a legacy governance table that isnt versioned
const shouldSyncLegacy = !Object.keys(governanceData).some((key) =>
Expand Down
35 changes: 23 additions & 12 deletions src/tasks/sync-registries.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,9 +143,7 @@ const syncOrganizationAudit = async (organization) => {
let afterCommitCallbacks = [];

const homeOrg = await Organization.getHomeOrg();
const rootHistory = (
await datalayer.getRootHistory(organization.registryId)
).sort((a, b) => a.timestamp - b.timestamp);
const rootHistory = await datalayer.getRootHistory(organization.registryId);

if (!rootHistory.length) {
logger.info(`No root history found for ${organization.name}`);
Expand Down Expand Up @@ -211,16 +209,17 @@ const syncOrganizationAudit = async (organization) => {
currentGeneration = lastRootSaved;
}

const historyIndex = currentGeneration.generation;
const lastProcessedIndex = currentGeneration.generation;
logger.debug(`1 Last processed index: ${lastProcessedIndex}`);

if (historyIndex > rootHistory.length) {
if (lastProcessedIndex > rootHistory.length) {
logger.error(
`Could not find root history for ${organization.name} with timestamp ${currentGeneration.timestamp}, something is wrong and the sync for this organization will be paused until this is resolved.`,
);
}

const rootHistoryCount = rootHistory.length - 1;
const syncRemaining = rootHistoryCount - historyIndex;
const rootHistoryZeroBasedCount = rootHistory.length - 1;
const syncRemaining = rootHistoryZeroBasedCount - lastProcessedIndex;
const isSynced = syncRemaining === 0;

await Organization.update(
Expand All @@ -232,12 +231,19 @@ const syncOrganizationAudit = async (organization) => {
);

if (process.env.NODE_ENV !== 'test' && isSynced) {
logger.debug(`3 Last processed index: ${lastProcessedIndex}`);
return;
}

const toBeProcessedIndex = lastProcessedIndex + 1;
logger.debug(`3 Last processed index: ${lastProcessedIndex}`);
logger.debug(`4 To be processed index: ${toBeProcessedIndex}`);

// Organization not synced, sync it
logger.info(' ');
logger.info(`Syncing ${organization.name} generation ${historyIndex}`);
logger.info(
`Syncing ${organization.name} generation ${toBeProcessedIndex}`,
);
logger.info(
`${organization.name} is ${syncRemaining} DataLayer generations away from being fully synced.`,
);
Expand All @@ -246,8 +252,10 @@ const syncOrganizationAudit = async (organization) => {
await new Promise((resolve) => setTimeout(resolve, 30000));
}

const root1 = _.get(rootHistory, `[${historyIndex}]`);
const root2 = _.get(rootHistory, `[${historyIndex + 1}]`);
logger.debug(`5 Last processed index: ${lastProcessedIndex}`);
const root1 = _.get(rootHistory, `[${lastProcessedIndex}]`);
logger.debug(`6 To be processed index: ${toBeProcessedIndex}`);
const root2 = _.get(rootHistory, `[${toBeProcessedIndex}]`);

logger.info(`ROOT 1 ${JSON.stringify(root1)}`);
logger.info(`ROOT 2', ${JSON.stringify(root2)}`);
Expand All @@ -259,6 +267,9 @@ const syncOrganizationAudit = async (organization) => {
return;
}

logger.debug(`7 Last processed index: ${lastProcessedIndex}`);
logger.debug(`8 To be processed index: ${toBeProcessedIndex}`);

const kvDiff = await datalayer.getRootDiff(
organization.registryId,
root1.root_hash,
Expand Down Expand Up @@ -299,7 +310,7 @@ const syncOrganizationAudit = async (organization) => {

const updateTransaction = async (transaction, mirrorTransaction) => {
logger.info(
`Syncing ${organization.name} generation ${historyIndex + 1}`,
`Syncing ${organization.name} generation ${toBeProcessedIndex}`,
);
for (const diff of optimizedKvDiff) {
const key = decodeHex(diff.key);
Expand All @@ -313,7 +324,7 @@ const syncOrganizationAudit = async (organization) => {
table: modelKey,
change: decodeHex(diff.value),
onchainConfirmationTimeStamp: root2.timestamp,
generation: historyIndex + 1,
generation: toBeProcessedIndex,
comment: _.get(
tryParseJSON(
decodeHex(_.get(comment, '[0].value', encodeHex('{}'))),
Expand Down
Loading