feat: sync remaining count for organizations

src/controllers/organization.controller.js

@@ -1,3 +1,4 @@
+import { Sequelize } from 'sequelize';
 import { sequelize } from '../database';
 import { Organization } from '../models/organizations';
 
@@ -135,7 +136,11 @@ export const resetHomeOrg = async (req, res) => {
     await Promise.all([
       Organization.destroy({ where: { isHome: true } }),
       Staging.destroy({
-        where: {},
+        where: {
+          id: {
+            [Sequelize.Op.ne]: null,
+          },
+        },
         truncate: true,
       }),
     ]);

src/controllers/staging.controller.js

@@ -1,5 +1,6 @@
 import _ from 'lodash';
 
+import { Sequelize } from 'sequelize';
 import { Staging } from '../models';
 
 import {
@@ -140,7 +141,11 @@ export const clean = async (req, res) => {
     await assertIfReadOnlyMode();
     await assertHomeOrgExists();
     await Staging.destroy({
-      where: {},
+      where: {
+        id: {
+          [Sequelize.Op.ne]: null,
+        },
+      },
       truncate: true,
     });
     res.json({

src/database/index.js

@@ -18,8 +18,6 @@ const mirrorConfig =
   (process.env.NODE_ENV || 'local') === 'local' ? 'mirror' : 'mirrorTest';
 export const sequelizeMirror = new Sequelize(config[mirrorConfig]);
 
-logger.info('CADT:mirror-database');
-
 const logDebounce = _.debounce(() => {
   console.log('Mirror DB not connected');
   logger.info('Mirror DB not connected');

src/database/migrations/20231020201652-OrgSyncStatus.js

@@ -6,7 +6,7 @@ export default {
       ['organizations'].map((table) => {
         queryInterface.addColumn(table, 'synced', {
           type: Sequelize.BOOLEAN,
-          allowNull: true,
+          allowNull: false,
           defaultValue: false,
         });
       }),

src/database/migrations/20231020214357-OrgSyncRemainingCount.js

@@ -0,0 +1,23 @@
+'use strict';
+
+export default {
+  async up(queryInterface, Sequelize) {
+    await Promise.all(
+      ['organizations'].map((table) => {
+        queryInterface.addColumn(table, 'sync_remaining', {
+          type: Sequelize.INTEGER,
+          allowNull: true,
+          defaultValue: false,
+        });
+      }),
+    );
+  },
+
+  async down(queryInterface) {
+    await Promise.all(
+      ['organizations'].map((table) => {
+        queryInterface.removeColumn(table, 'sync_remaining');
+      }),
+    );
+  },
+};

src/database/migrations/index.js

@@ -31,6 +31,7 @@ import ResetDBForNewSingletons from './20220816155101-reset-db-for-new-singleton
 import AddIsTransferColumn from './20220825124702-add-isTransfer-column';
 import AddOrgMetadata from './20220831023546-add-org-metadata';
 import OrgSyncStatus from './20231020201652-OrgSyncStatus';
+import OrgSyncRemaining from './20231020214357-OrgSyncRemainingCount';
 
 export const migrations = [
   {
@@ -169,4 +170,8 @@ export const migrations = [
     migration: OrgSyncStatus,
     name: '20231020201652-OrgSyncStatus',
   },
+  {
+    migration: OrgSyncRemaining,
+    name: '20231020214357-OrgSyncRemainingCount',
+  },
 ];

src/datalayer/persistance.js

@@ -10,7 +10,6 @@ import { Organization } from '../models';
 import { logger } from '../config/logger.cjs';
 import { getChiaRoot } from '../utils/chia-root.js';
 
-logger.info('CADT:datalayer:persistance');
 process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;
 
 const CONFIG = getConfig().APP;

src/datalayer/syncService.js

@@ -230,7 +230,7 @@ const getSubscribedStoreData = async (storeId, retry = 0) => {
   if (!USE_SIMULATOR) {
     logger.info(`Getting confirmation for ${storeId}.`);
     const storeExistAndIsConfirmed = await dataLayer.getRoot(storeId, true);
-    logger.info(`Store exists and is found ${storeId}.`);
+    logger.info(`Store found in DataLayer: ${storeId}.`);
     if (!storeExistAndIsConfirmed) {
       logger.info(
         `Retrying subscribe to ${storeId}, store not yet confirmed.`,
@@ -242,9 +242,7 @@ const getSubscribedStoreData = async (storeId, retry = 0) => {
       );
       return getSubscribedStoreData(storeId, retry + 1);
     } else {
-      logger.debug(
-        `Store Exists and is confirmed, proceeding to get data ${storeId}`,
-      );
+      logger.debug(`Store is confirmed, proceeding to get data ${storeId}`);
     }
   }
 

src/datalayer/writeService.js

@@ -10,8 +10,6 @@ import { logger } from '../config/logger.cjs';
 import { Organization } from '../models';
 import { publicIpv4 } from '../utils/ip-tools';
 
-logger.info('CADT:datalayer:writeService');
-
 const { USE_SIMULATOR, DATALAYER_FILE_SERVER_URL } = getConfig().APP;
 
 const createDataLayerStore = async () => {

src/middleware.js

@@ -26,6 +26,7 @@ const headerKeys = Object.freeze({
   WALLET_SYNCED: 'x-wallet-synced',
   HOME_ORGANIZATION_SYNCED: 'x-home-org-synced',
   ALL_DATA_SYNCED: 'x-data-synced',
+  SYNC_REMAINING: 'x-sync-remaining',
 });
 
 const app = express();
@@ -123,6 +124,12 @@ app.use(async function (req, res, next) {
   const notSynced = Object.keys(orgMap).find((key) => !orgMap[key].synced);
 
   res.setHeader(headerKeys.ALL_DATA_SYNCED, !notSynced);
+
+  const syncRemaining = Object.keys(orgMap).reduce((agg, key) => {
+    return agg + orgMap[key].sync_remaining;
+  }, 0);
+
+  res.setHeader(headerKeys.SYNC_REMAINING, syncRemaining);
   next();
 });
 

src/models/governance/governance.model.js

@@ -194,7 +194,11 @@ class Governance extends Model {
     const rollbackChangesIfFailed = async () => {
       logger.info('Reverting Goverance Records');
       await Governance.destroy({
-        where: {},
+        where: {
+          id: {
+            [Sequelize.Op.ne]: null,
+          },
+        },
         truncate: true,
       });
 

src/models/organizations/organizations.model.js

@@ -18,8 +18,6 @@ import { getDataModelVersion } from '../../utils/helpers';
 import { getConfig } from '../../utils/config-loader';
 const { USE_SIMULATOR, AUTO_SUBSCRIBE_FILESTORE } = getConfig().APP;
 
-logger.info('CADT:organizations');
-
 import ModelTypes from './organizations.modeltypes.cjs';
 
 class Organization extends Model {
@@ -74,6 +72,7 @@ class Organization extends Model {
         'subscribed',
         'synced',
         'fileStoreSubscribed',
+        'sync_remaining',
       ],
     });
 
@@ -333,7 +332,7 @@ class Organization extends Model {
   static async syncOrganizationMeta() {
     try {
       const allSubscribedOrganizations = await Organization.findAll({
-        subscribed: true,
+        where: { subscribed: true },
       });
 
       await Promise.all(

src/models/organizations/organizations.modeltypes.cjs

@@ -35,9 +35,12 @@ module.exports = {
   },
   synced: {
     type: Sequelize.BOOLEAN,
-    allowNull: true,
     defaultValue: false,
   },
+  sync_remaining: {
+    type: Sequelize.INTEGER,
+    defaultValue: 0,
+  },
   createdAt: Sequelize.DATE,
   updatedAt: Sequelize.DATE,
 };

src/server.js

@@ -12,8 +12,6 @@ import dotenv from 'dotenv';
 
 dotenv.config();
 
-logger.info('CADT:server');
-
 const port = getConfig().APP.CW_PORT || 3030;
 const bindAddress = getConfig().APP.BIND_ADDRESS || 'localhost';
 const server = http.createServer(rootRouter);