diff --git a/package-lock.json b/package-lock.json index acd1a2c33..b54ba5cc7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,8 +10,7 @@ "dependencies": { "@babel/eslint-parser": "^7.22.9", "@chia-carbon/core-registry-config": "^1.0.2", - "@chia-carbon/core-registry-logger": "^1.0.11", - "async-waterfall": "^0.1.5", + "@chia-carbon/core-registry-logger": "^1.0.12", "body-parser": "^1.20.2", "cli-spinner": "^0.2.10", "cors": "^2.8.5", @@ -3105,11 +3104,6 @@ "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" }, - "node_modules/async-waterfall": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/async-waterfall/-/async-waterfall-0.1.5.tgz", - "integrity": "sha512-aJXVeDPYaXZUw/lfkeEK13OLSxHO4AZWA2so1Q0n4ey3ix26rWzSCzl4hnkJvolA12MvmUF8covY8bZvKjbtmA==" - }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", diff --git a/package.json b/package.json index 629ce1f89..8f051a1c5 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "dependencies": { "@babel/eslint-parser": "^7.22.9", "@chia-carbon/core-registry-config": "^1.0.2", - "@chia-carbon/core-registry-logger": "^1.0.11", + "@chia-carbon/core-registry-logger": "^1.0.12", "body-parser": "^1.20.2", "cli-spinner": "^0.2.10", "cors": "^2.8.5", @@ -74,7 +74,6 @@ "eslint-plugin-es": "^4.1.0", "eslint-plugin-mocha": "^10.2.0", "husky": "^8.0.3", - "mocha": "^10.2.0", "semver": "^7.5.4", "sinon": "^16.1.0", diff --git a/src/datalayer/syncService.js b/src/datalayer/syncService.js index e4a553221..1a02e1872 100644 --- a/src/datalayer/syncService.js +++ b/src/datalayer/syncService.js @@ -297,7 +297,7 @@ const getRootHistory = (storeId) => { }; const getRootDiff = (storeId, root1, root2) => { - if (!USE_SIMULATOR) { + if (USE_SIMULATOR) { return Simulator.getMockedKvDiffFromStagingTable(); } else { return dataLayer.getRootDiff(storeId, root1, root2); diff --git a/src/tasks/sync-audit-table.js b/src/tasks/sync-audit-table.js index fe9c08d6b..9576cedc0 100644 --- a/src/tasks/sync-audit-table.js +++ b/src/tasks/sync-audit-table.js @@ -5,16 +5,17 @@ import { Organization, Audit, ModelKeys, Staging } from '../models'; import datalayer from '../datalayer'; import { decodeHex } from '../utils/datalayer-utils'; import dotenv from 'dotenv'; -import { logger } from '../logger.js'; +import { logger } from '../config/logger.cjs'; import { sequelize, sequelizeMirror } from '../database'; - +import { getConfig } from '../utils/config-loader'; import { assertDataLayerAvailable, assertWalletIsSynced, } from '../utils/data-assertions'; dotenv.config(); -import { CONFIG } from '../user-config'; + +const CONFIG = getConfig().APP; let taskIsRunning = false; @@ -30,7 +31,7 @@ const task = new Task('sync-audit', async () => { // Log additional information if present in the error object if (error.response && error.response.body) { logger.error( - `Additional error details: ${JSON.stringify(error.response.body)}`, + `Additional error details: ${JSON.stringify(error.response.body)}`, ); } } finally { @@ -39,37 +40,38 @@ const task = new Task('sync-audit', async () => { }); const job = new SimpleIntervalJob( - { - seconds: CONFIG().CADT.TASKS?.AUDIT_SYNC_TASK_INTERVAL || 30, - runImmediately: true, - }, - task, - 'sync-audit', + { + seconds: CONFIG?.TASKS?.AUDIT_SYNC_TASK_INTERVAL || 30, + runImmediately: true, + }, + task, + 'sync-audit', ); const processJob = async () => { await assertDataLayerAvailable(); await assertWalletIsSynced(); - logger.task('Syncing Audit Information'); + logger.info('Syncing Audit Information'); const organizations = await Organization.findAll({ where: { subscribed: true }, raw: true, }); for (const organization of organizations) { + console.log(`Syncing ${organization.name}`); await syncOrganizationAudit(organization); - if (!CONFIG().CADT.USE_SIMULATOR) { + if (!CONFIG.USE_SIMULATOR) { await new Promise((resolve) => - setTimeout( - resolve, - (CONFIG().CADT.TASKS?.AUDIT_SYNC_TASK_INTERVAL || 30) * 1000, - ), + setTimeout( + resolve, + (CONFIG.TASKS?.AUDIT_SYNC_TASK_INTERVAL || 30) * 1000, + ), ); } } - if (!CONFIG().CADT.USE_SIMULATOR) { + if (!CONFIG.USE_SIMULATOR) { await new Promise((resolve) => setTimeout(resolve, 5000)); } }; @@ -81,13 +83,7 @@ async function createTransaction(callback, afterCommitCallbacks) { let mirrorTransaction; try { - // Check if the database is locked and wait until it's unlocked - /* while (await isDatabaseLocked()) { - logger.debug('Database is locked. Waiting...'); - await waitFor(retryDelay); - }*/ - - logger.trace('Starting transaction'); + logger.info('Starting transaction'); // Start a transaction transaction = await sequelize.transaction(); mirrorTransaction = await sequelizeMirror.transaction(); @@ -103,7 +99,7 @@ async function createTransaction(callback, afterCommitCallbacks) { await afterCommitCallback(); } - logger.trace('Commited transaction'); + logger.info('Commited transaction'); return result; } catch (error) { @@ -118,13 +114,14 @@ async function createTransaction(callback, afterCommitCallbacks) { const syncOrganizationAudit = async (organization) => { try { - logger.task(`Syncing Audit: ${_.get(organization, 'name')}`); + logger.info(`Syncing Audit: ${_.get(organization, 'name')}`); let afterCommitCallbacks = []; const rootHistory = await datalayer.getRootHistory(organization.registryId); let lastRootSaved; - if (CONFIG().CADT.USE_SIMULATOR) { + if (CONFIG.USE_SIMULATOR) { + console.log('USING MOCK ROOT HISTORY'); lastRootSaved = rootHistory[0]; lastRootSaved.rootHash = lastRootSaved.root_hash; } else { @@ -146,7 +143,7 @@ const syncOrganizationAudit = async (organization) => { } const historyIndex = rootHistory.findIndex( - (root) => root.root_hash === rootHash, + (root) => root.root_hash === rootHash, ); if (!lastRootSaved) { @@ -165,13 +162,13 @@ const syncOrganizationAudit = async (organization) => { // is reset this will ensure that this organizations regsitry data is // cleaned up on both the local db and mirror db and ready to resync await Promise.all( - Object.keys(ModelKeys).map(async (modelKey) => { - ModelKeys[modelKey].destroy({ - where: { - orgUid: organization.orgUid, - }, - }); - }), + Object.keys(ModelKeys).map(async (modelKey) => { + ModelKeys[modelKey].destroy({ + where: { + orgUid: organization.orgUid, + }, + }); + }), ); return; @@ -189,9 +186,9 @@ const syncOrganizationAudit = async (organization) => { } const kvDiff = await datalayer.getRootDiff( - organization.registryId, - root1.root_hash, - root2.root_hash, + organization.registryId, + root1.root_hash, + root2.root_hash, ); if (_.isEmpty(kvDiff)) { @@ -200,16 +197,16 @@ const syncOrganizationAudit = async (organization) => { // 0x636f6d6d656e74 is hex for 'comment' const comment = kvDiff.filter( - (diff) => - (diff.key === '636f6d6d656e74' || diff.key === '0x636f6d6d656e74') && - diff.type === 'INSERT', + (diff) => + (diff.key === '636f6d6d656e74' || diff.key === '0x636f6d6d656e74') && + diff.type === 'INSERT', ); // 0x617574686F72 is hex for 'author' const author = kvDiff.filter( - (diff) => - (diff.key === '617574686f72' || diff.key === '0x617574686F72') && - diff.type === 'INSERT', + (diff) => + (diff.key === '617574686f72' || diff.key === '0x617574686F72') && + diff.type === 'INSERT', ); // Process any deletes in the kv diff first to ensure correct processing order @@ -219,7 +216,6 @@ const syncOrganizationAudit = async (organization) => { }); const homeOrg = await Organization.getHomeOrg(); - // console.log(kvDiff); const updateTransaction = async (transaction, mirrorTransaction) => { for (const diff of kvDiff) { @@ -236,34 +232,34 @@ const syncOrganizationAudit = async (organization) => { change: decodeHex(diff.value), onchainConfirmationTimeStamp: root2.timestamp, comment: _.get( - JSON.parse(decodeHex(_.get(comment, '[0].value', '7b7d'))), - 'comment', - '', + JSON.parse(decodeHex(_.get(comment, '[0].value', '7b7d'))), + 'comment', + '', ), author: _.get( - JSON.parse(decodeHex(_.get(author, '[0].value', '7b7d'))), - 'author', - '', + JSON.parse(decodeHex(_.get(author, '[0].value', '7b7d'))), + 'author', + '', ), }; if (modelKey) { const record = JSON.parse(decodeHex(diff.value)); const primaryKeyValue = - record[ModelKeys[modelKey].primaryKeyAttributes[0]]; + record[ModelKeys[modelKey].primaryKeyAttributes[0]]; if (diff.type === 'INSERT') { - logger.trace(`INSERTING: ${modelKey} - ${primaryKeyValue}`); + logger.info(`INSERTING: ${modelKey} - ${primaryKeyValue}`); await ModelKeys[modelKey].upsert(record, { transaction, mirrorTransaction, }); } else if (diff.type === 'DELETE') { - logger.trace(`DELETING: ${modelKey} - ${primaryKeyValue}`); + logger.info(`DELETING: ${modelKey} - ${primaryKeyValue}`); await ModelKeys[modelKey].destroy({ where: { [ModelKeys[modelKey].primaryKeyAttributes[0]]: - primaryKeyValue, + primaryKeyValue, }, transaction, mirrorTransaction, @@ -277,12 +273,12 @@ const syncOrganizationAudit = async (organization) => { 'units', 'projects', ].includes(modelKey) - ? primaryKeyValue - : undefined; + ? primaryKeyValue + : undefined; if (stagingUuid) { afterCommitCallbacks.push(async () => { - logger.trace(`DELETING STAGING: ${stagingUuid}`); + logger.info(`DELETING STAGING: ${stagingUuid}`); await Staging.destroy({ where: { uuid: stagingUuid }, }); @@ -294,18 +290,18 @@ const syncOrganizationAudit = async (organization) => { // Create the Audit record await Audit.create(auditData, { transaction, mirrorTransaction }); await Organization.update( - { registryHash: root2.root_hash }, - { - where: { orgUid: organization.orgUid }, - transaction, - mirrorTransaction, - }, + { registryHash: root2.root_hash }, + { + where: { orgUid: organization.orgUid }, + transaction, + mirrorTransaction, + }, ); } } }; - return createTransaction(updateTransaction, afterCommitCallbacks); + return await createTransaction(updateTransaction, afterCommitCallbacks); } catch (error) { logger.error('Error syncing org audit', error); } diff --git a/src/utils/config-loader.js b/src/utils/config-loader.js index 94fcf63cb..ed4f031c6 100644 --- a/src/utils/config-loader.js +++ b/src/utils/config-loader.js @@ -29,6 +29,7 @@ export const getConfig = _.memoize(() => { if (process.env.USE_SIMULATOR) { defaultConfig.APP.USE_SIMULATOR = true; defaultConfig.APP.CHIA_NETWORK = 'testnet'; + defaultConfig.APP.TASKS.AUDIT_SYNC_TASK_INTERVAL = 30; logger.info(`ENV FILE OVERRIDE: RUNNING IN SIMULATOR MODE`); } diff --git a/tests/integration/project.spec.js b/tests/integration/project.spec.js index cc8ff64a8..02bd5d50f 100644 --- a/tests/integration/project.spec.js +++ b/tests/integration/project.spec.js @@ -57,6 +57,7 @@ describe('Project Resource Integration Tests', function () { // Now push the staging table live await testFixtures.commitStagingRecords(); await testFixtures.waitForDataLayerSync(); + await testFixtures.waitForDataLayerSync(); // The staging table should be empty after committing expect(await testFixtures.getLastCreatedStagingRecord()).to.equal( diff --git a/tests/test-fixtures/staging-fixtures.js b/tests/test-fixtures/staging-fixtures.js index 0d9f2ebd0..b7d9fc615 100644 --- a/tests/test-fixtures/staging-fixtures.js +++ b/tests/test-fixtures/staging-fixtures.js @@ -14,6 +14,7 @@ export const resetStagingTable = async () => { export const getLastCreatedStagingRecord = async () => { const result = await supertest(app).get('/v1/staging'); + console.log('!!!!!!!!!!!!!!!!!!', result.body); expect(result.body).to.be.an('array'); return _.last(result.body); };