From b2899508443c8a7812f5e5bb58321f5bb3d6cb66 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 31 Oct 2024 22:23:40 +0000 Subject: [PATCH 1/6] build(deps): bump actions/upload-artifact from 3 to 4 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3 to 4. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6c9e7d67..ffe60550 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -166,13 +166,13 @@ jobs: - name: Upload Mac Installer if: matrix.runs-on == 'macos-latest' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: cadt-mac-installer path: ${{ github.workspace }}/build-scripts/macos/target/ready-to-upload - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ matrix.artifact-name }} path: ${{ github.workspace }}/dist @@ -213,7 +213,7 @@ jobs: sudo cp ./node_modules/sqlite3/build/Release/node_sqlite3.node ./dist/ - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: cadt-linux-arm64 path: ${{ github.workspace }}/dist @@ -265,7 +265,7 @@ jobs: dpkg-deb --build --root-owner-group "deb/$CLI_DEB_BASE" - name: Upload deb - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ matrix.name }}-deb path: ${{ github.workspace }}/deb/*.deb From 9ff548843849f73633c913592568ad076931d36a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 31 Oct 2024 22:27:04 +0000 Subject: [PATCH 2/6] build(deps): bump actions/download-artifact from 3 to 4 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 3 to 4. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a5d44b00..07ce2539 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -236,7 +236,7 @@ jobs: uses: actions/checkout@v4 - name: Download Linux artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ matrix.name }} path: ${{ matrix.name }} @@ -277,31 +277,31 @@ jobs: - debs steps: - name: Download Windows artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: cadt-windows-x64 path: cadt-windows-x64 - name: Download MacOS artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: cadt-mac-installer path: cadt-mac-installer - name: Download Linux artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: cadt-linux-x64 path: cadt-linux-x64 - name: Download Linux x64 deb - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: cadt-linux-x64-deb path: cadt-linux-x64-deb - name: Download Linux arm64 deb - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: cadt-linux-arm64-deb path: cadt-linux-arm64-deb From dcab9e9a6e33c9c15895ea4cebd1fce03beac15e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Nov 2024 14:16:11 +0000 Subject: [PATCH 3/6] build(deps): bump softprops/action-gh-release from 2.0.9 to 2.1.0 Bumps [softprops/action-gh-release](https://github.com/softprops/action-gh-release) from 2.0.9 to 2.1.0. - [Release notes](https://github.com/softprops/action-gh-release/releases) - [Changelog](https://github.com/softprops/action-gh-release/blob/master/CHANGELOG.md) - [Commits](https://github.com/softprops/action-gh-release/compare/v2.0.9...v2.1.0) --- updated-dependencies: - dependency-name: softprops/action-gh-release dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c165f550..653d9a68 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -318,7 +318,7 @@ jobs: zip -r cadt-linux-x64-${{ steps.tag-name.outputs.TAGNAME }}.zip cadt-linux-x64 - name: Release - uses: softprops/action-gh-release@v2.0.9 + uses: softprops/action-gh-release@v2.1.0 with: files: | cadt-windows-x64-${{ steps.tag-name.outputs.TAGNAME }}.zip From b48985967d911e7d1f0923808b374ecac3d75b19 Mon Sep 17 00:00:00 2001 From: William Wills Date: Mon, 18 Nov 2024 14:35:37 -0500 Subject: [PATCH 4/6] feat: sqlite db locking mitigation via organizations.model.js and audit.model.js accessing mutex --- src/models/audit/audit.model.js | 5 ++++ .../organizations/organizations.model.js | 5 ++++ src/tasks/sync-registries.js | 21 ++++++++++------ src/utils/model-utils.js | 25 +++++++++++++++++++ 4 files changed, 49 insertions(+), 7 deletions(-) diff --git a/src/models/audit/audit.model.js b/src/models/audit/audit.model.js index e10057df..31a6610c 100644 --- a/src/models/audit/audit.model.js +++ b/src/models/audit/audit.model.js @@ -7,6 +7,7 @@ import { AuditMirror } from './audit.model.mirror'; import ModelTypes from './audit.modeltypes.cjs'; import findDuplicateIssuancesSql from './sql/find-duplicate-issuances.sql.js'; import { Organization } from '../organizations/index.js'; +import { waitForSyncRegistries } from '../../utils/model-utils.js'; class Audit extends Model { static async create(values, options) { @@ -107,4 +108,8 @@ Audit.init(ModelTypes, { updatedAt: true, }); +Audit.addHook('beforeFind', async () => { + await waitForSyncRegistries(); +}); + export { Audit }; diff --git a/src/models/organizations/organizations.model.js b/src/models/organizations/organizations.model.js index 2e62da8f..5c4cc850 100644 --- a/src/models/organizations/organizations.model.js +++ b/src/models/organizations/organizations.model.js @@ -19,6 +19,7 @@ import { getConfig } from '../../utils/config-loader'; const { USE_SIMULATOR, AUTO_SUBSCRIBE_FILESTORE } = getConfig().APP; import ModelTypes from './organizations.modeltypes.cjs'; +import { waitForSyncRegistries } from '../../utils/model-utils.js'; class Organization extends Model { static async getHomeOrg(includeAddress = true) { @@ -469,4 +470,8 @@ Organization.init(ModelTypes, { timestamps: true, }); +Organization.addHook('beforeFind', async () => { + await waitForSyncRegistries(); +}); + export { Organization }; diff --git a/src/tasks/sync-registries.js b/src/tasks/sync-registries.js index 095bd440..2e164e72 100644 --- a/src/tasks/sync-registries.js +++ b/src/tasks/sync-registries.js @@ -1,7 +1,6 @@ import _ from 'lodash'; import { SimpleIntervalJob, Task } from 'toad-scheduler'; -import { Mutex } from 'async-mutex'; import { Organization, Audit, ModelKeys, Staging, Meta } from '../models'; import datalayer from '../datalayer'; import { @@ -22,15 +21,18 @@ import { migrateToNewSync, generateGenerationIndex, } from '../utils/sync-migration-utils'; +import { + processingUpdateTransactionMutex, + syncRegistriesTaskMutex, +} from '../utils/model-utils.js'; dotenv.config(); -const mutex = new Mutex(); const CONFIG = getConfig().APP; const task = new Task('sync-registries', async () => { - if (!mutex.isLocked()) { - logger.debug('running sync registries task'); - const releaseMutex = await mutex.acquire(); + logger.debug('sync registries task invoked'); + if (!syncRegistriesTaskMutex.isLocked()) { + const releaseSyncTaskMutex = await syncRegistriesTaskMutex.acquire(); try { const hasMigratedToNewSyncMethod = await Meta.findOne({ where: { metaKey: 'migratedToNewSync' }, @@ -61,7 +63,7 @@ const task = new Task('sync-registries', async () => { ); } } finally { - releaseMutex(); + releaseSyncTaskMutex(); } } }); @@ -97,8 +99,11 @@ async function createTransaction(callback, afterCommitCallbacks) { let transaction; let mirrorTransaction; + logger.info('Starting sequelize transaction and acquiring transaction mutex'); + const releaseTransactionMutex = + await processingUpdateTransactionMutex.acquire(); + try { - logger.info('Starting sequelize transaction'); // Start a transaction transaction = await sequelize.transaction(); @@ -130,6 +135,8 @@ async function createTransaction(callback, afterCommitCallbacks) { console.error(error); await transaction.rollback(); } + } finally { + releaseTransactionMutex(); } } diff --git a/src/utils/model-utils.js b/src/utils/model-utils.js index f0fbca82..02decbd0 100644 --- a/src/utils/model-utils.js +++ b/src/utils/model-utils.js @@ -1,6 +1,31 @@ import { columnsToInclude } from './helpers.js'; import Sequelize from 'sequelize'; +import { Mutex } from 'async-mutex'; + +export async function waitForSyncRegistries() { + if (processingUpdateTransactionMutex.isLocked()) { + // when the mutex is acquired, the current sync transaction has completed + const releaseMutex = await processingUpdateTransactionMutex.acquire(); + await releaseMutex(); + } +} + +/** + * mutex which must be acquired to run the sync-registries task job. + * this mutex exists to prevent multiple registry sync tasks from running at the same time and overloading the chia + * RPC's or causing a SQLite locking error due to multiple task instances trying to commit large update transactions + * @type {Mutex} + */ +export const syncRegistriesTaskMutex = new Mutex(); + +/** + * mutex which must be acquired when writing registry update information until the transaction has been committed + * audit model update transactions are large and lock the DB for long periods. + * @type {Mutex} + */ +export const processingUpdateTransactionMutex = new Mutex(); + export function formatModelAssociationName(model) { if (model == null || model.model == null) return ''; From a3f33f791e470aed6fa707df6cf2ca483b4fab85 Mon Sep 17 00:00:00 2001 From: William Wills Date: Fri, 22 Nov 2024 12:39:14 -0500 Subject: [PATCH 5/6] feat: updated org model and sync registries to prevent locking organization table during audit sync --- src/models/audit/audit.model.js | 4 +- .../organizations/organizations.model.js | 5 - src/tasks/sync-registries.js | 98 +++++++++++++++---- src/utils/model-utils.js | 9 +- 4 files changed, 86 insertions(+), 30 deletions(-) diff --git a/src/models/audit/audit.model.js b/src/models/audit/audit.model.js index 31a6610c..5e758530 100644 --- a/src/models/audit/audit.model.js +++ b/src/models/audit/audit.model.js @@ -7,7 +7,7 @@ import { AuditMirror } from './audit.model.mirror'; import ModelTypes from './audit.modeltypes.cjs'; import findDuplicateIssuancesSql from './sql/find-duplicate-issuances.sql.js'; import { Organization } from '../organizations/index.js'; -import { waitForSyncRegistries } from '../../utils/model-utils.js'; +import { waitForSyncRegistriesTransaction } from '../../utils/model-utils.js'; class Audit extends Model { static async create(values, options) { @@ -109,7 +109,7 @@ Audit.init(ModelTypes, { }); Audit.addHook('beforeFind', async () => { - await waitForSyncRegistries(); + await waitForSyncRegistriesTransaction(); }); export { Audit }; diff --git a/src/models/organizations/organizations.model.js b/src/models/organizations/organizations.model.js index 5c4cc850..2e62da8f 100644 --- a/src/models/organizations/organizations.model.js +++ b/src/models/organizations/organizations.model.js @@ -19,7 +19,6 @@ import { getConfig } from '../../utils/config-loader'; const { USE_SIMULATOR, AUTO_SUBSCRIBE_FILESTORE } = getConfig().APP; import ModelTypes from './organizations.modeltypes.cjs'; -import { waitForSyncRegistries } from '../../utils/model-utils.js'; class Organization extends Model { static async getHomeOrg(includeAddress = true) { @@ -470,8 +469,4 @@ Organization.init(ModelTypes, { timestamps: true, }); -Organization.addHook('beforeFind', async () => { - await waitForSyncRegistries(); -}); - export { Organization }; diff --git a/src/tasks/sync-registries.js b/src/tasks/sync-registries.js index 2e164e72..86c53fea 100644 --- a/src/tasks/sync-registries.js +++ b/src/tasks/sync-registries.js @@ -22,7 +22,7 @@ import { generateGenerationIndex, } from '../utils/sync-migration-utils'; import { - processingUpdateTransactionMutex, + processingSyncRegistriesTransactionMutex, syncRegistriesTaskMutex, } from '../utils/model-utils.js'; @@ -89,11 +89,57 @@ const processJob = async () => { }); for (const organization of organizations) { - await syncOrganizationAudit(organization); + if (CONFIG.USE_SIMULATOR || process.env.NODE_ENV === 'test') { + await syncOrganizationAudit(organization); + } else { + const mostRecentOrgAuditRecord = await Audit.findOne({ + where: { + orgUid: organization.orgUid, + }, + order: [['createdAt', 'DESC']], + limit: 1, + raw: true, + }); + + // verify that the latest organization root hash is up to date with the audit records. attempt correction. + if (mostRecentOrgAuditRecord.rootHash !== organization.registryHash) { + logger.warn( + `latest root hash in org table for organization ${organization.name} (orgUid ${organization.orgUid}) does not match the audit records. attempting to correct`, + ); + try { + const result = await Organization.update( + { registryHash: mostRecentOrgAuditRecord.rootHash }, + { + where: { orgUid: organization.orgUid }, + }, + ); + + if (result?.length) { + logger.info( + `registry hash record corrected for ${organization.name} (orgUid ${organization.orgUid}). proceeding with audit sync`, + ); + const correctedOrganizationRecord = await Organization.findOne({ + where: { orgUid: organization.orgUid }, + }); + + await syncOrganizationAudit(correctedOrganizationRecord); + } else { + throw new Error('organizations update query affected 0 records'); + } + } catch (error) { + logger.error( + `failed to update organization table record for ${organization.name} (orgUid ${organization.orgUid}) with correct root hash. Something is wrong. Skipping audit sync and trying again shortly. Error: ${error}`, + ); + } + } else { + // normal state, proceed with audit sync + await syncOrganizationAudit(organization); + } + } } }; -async function createTransaction(callback, afterCommitCallbacks) { +async function updateAuditTransaction(callback, afterCommitCallbacks) { let result = null; let transaction; @@ -101,7 +147,7 @@ async function createTransaction(callback, afterCommitCallbacks) { logger.info('Starting sequelize transaction and acquiring transaction mutex'); const releaseTransactionMutex = - await processingUpdateTransactionMutex.acquire(); + await processingSyncRegistriesTransactionMutex.acquire(); try { // Start a transaction @@ -131,8 +177,9 @@ async function createTransaction(callback, afterCommitCallbacks) { } catch (error) { // Roll back the transaction if an error occurs if (transaction) { - logger.error('Rolling back transaction'); - console.error(error); + logger.error( + `encountered error syncing organization audit. Rolling back transaction. Error: ${error}`, + ); await transaction.rollback(); } } finally { @@ -439,7 +486,10 @@ const syncOrganizationAudit = async (organization) => { // by not processing the DELETE for that record. const optimizedKvDiff = optimizeAndSortKvDiff(kvDiff); - const updateTransaction = async (transaction, mirrorTransaction) => { + const createAndProcessTransaction = async ( + transaction, + mirrorTransaction, + ) => { logger.info( `Syncing ${organization.name} generation ${toBeProcessedDatalayerGenerationIndex} (orgUid ${organization.orgUid}, registryId ${organization.registryId})`, ); @@ -536,18 +586,8 @@ const syncOrganizationAudit = async (organization) => { } // Create the Audit record - logger.debug( - `creating audit model entry for ${organization.name} transacton`, - ); + logger.debug(`creating audit model transaction entry`); await Audit.create(auditData, { transaction, mirrorTransaction }); - await Organization.update( - { registryHash: rootToBeProcessed.root_hash }, - { - where: { orgUid: organization.orgUid }, - transaction, - mirrorTransaction, - }, - ); } } }; @@ -556,7 +596,27 @@ const syncOrganizationAudit = async (organization) => { afterCommitCallbacks.push(truncateStaging); } - await createTransaction(updateTransaction, afterCommitCallbacks); + const transactionSucceeded = await createAndProcessTransaction( + updateAuditTransaction, + afterCommitCallbacks, + ); + + if (transactionSucceeded) { + logger.debug( + `updateAuditTransaction successfully completed and committed audit updates for ${organization.name} (orgUid: ${organization.orgUid}, registryId: ${organization.registryId}) generation index ${toBeProcessedDatalayerGenerationIndex}. updating registry hash to ${rootToBeProcessed.root_hash}`, + ); + + await Organization.update( + { registryHash: rootToBeProcessed.root_hash }, + { + where: { orgUid: organization.orgUid }, + }, + ); + } else { + logger.debug( + `updateAuditTransaction failed to complete and commit audit updates for ${organization.name} (orgUid: ${organization.orgUid}, registryId: ${organization.registryId}) generation index ${toBeProcessedDatalayerGenerationIndex}`, + ); + } } catch (error) { logger.error('Error syncing org audit', error); } diff --git a/src/utils/model-utils.js b/src/utils/model-utils.js index 02decbd0..0d77887a 100644 --- a/src/utils/model-utils.js +++ b/src/utils/model-utils.js @@ -3,10 +3,11 @@ import Sequelize from 'sequelize'; import { Mutex } from 'async-mutex'; -export async function waitForSyncRegistries() { - if (processingUpdateTransactionMutex.isLocked()) { +export async function waitForSyncRegistriesTransaction() { + if (processingSyncRegistriesTransactionMutex.isLocked()) { // when the mutex is acquired, the current sync transaction has completed - const releaseMutex = await processingUpdateTransactionMutex.acquire(); + const releaseMutex = + await processingSyncRegistriesTransactionMutex.acquire(); await releaseMutex(); } } @@ -24,7 +25,7 @@ export const syncRegistriesTaskMutex = new Mutex(); * audit model update transactions are large and lock the DB for long periods. * @type {Mutex} */ -export const processingUpdateTransactionMutex = new Mutex(); +export const processingSyncRegistriesTransactionMutex = new Mutex(); export function formatModelAssociationName(model) { if (model == null || model.model == null) return ''; From debad53293b5c79cf592451144d89c031c4d4b90 Mon Sep 17 00:00:00 2001 From: William Wills Date: Fri, 22 Nov 2024 13:42:28 -0500 Subject: [PATCH 6/6] fix: failing tests --- src/tasks/sync-registries.js | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/tasks/sync-registries.js b/src/tasks/sync-registries.js index 86c53fea..b5d9db9c 100644 --- a/src/tasks/sync-registries.js +++ b/src/tasks/sync-registries.js @@ -139,9 +139,7 @@ const processJob = async () => { } }; -async function updateAuditTransaction(callback, afterCommitCallbacks) { - let result = null; - +async function createAndProcessTransaction(callback, afterCommitCallbacks) { let transaction; let mirrorTransaction; @@ -158,7 +156,7 @@ async function updateAuditTransaction(callback, afterCommitCallbacks) { } // Execute the provided callback with the transaction - result = await callback(transaction, mirrorTransaction); + await callback(transaction, mirrorTransaction); // Commit the transaction if the callback completes without errors await transaction.commit(); @@ -173,7 +171,7 @@ async function updateAuditTransaction(callback, afterCommitCallbacks) { logger.info('Commited sequelize transaction'); - return result; + return true; } catch (error) { // Roll back the transaction if an error occurs if (transaction) { @@ -181,6 +179,7 @@ async function updateAuditTransaction(callback, afterCommitCallbacks) { `encountered error syncing organization audit. Rolling back transaction. Error: ${error}`, ); await transaction.rollback(); + return false; } } finally { releaseTransactionMutex(); @@ -486,10 +485,7 @@ const syncOrganizationAudit = async (organization) => { // by not processing the DELETE for that record. const optimizedKvDiff = optimizeAndSortKvDiff(kvDiff); - const createAndProcessTransaction = async ( - transaction, - mirrorTransaction, - ) => { + const updateAuditTransaction = async (transaction, mirrorTransaction) => { logger.info( `Syncing ${organization.name} generation ${toBeProcessedDatalayerGenerationIndex} (orgUid ${organization.orgUid}, registryId ${organization.registryId})`, );