Fix/reorg negative generations

package.json

@@ -73,8 +73,8 @@
     "@babel/register": "^7.24.6",
     "@commitlint/cli": "^19.4.0",
     "@commitlint/config-conventional": "^19.2.2",
-    "@eslint/eslintrc": "^3.1.0",
     "@eslint/js": "^9.9.0",
+    "@eslint/eslintrc": "^3.1.0",
     "@yao-pkg/pkg": "^5.12.1",
     "babel-plugin-module-resolver": "^5.0.2",
     "chai": "^5.1.1",

src/config/config.js

@@ -1,16 +1,33 @@
 import { getConfig } from '../utils/config-loader';
 import { getDataModelVersion } from '../utils/helpers';
 import { getChiaRoot } from '../utils/chia-root.js';
+import { logger } from './logger.js';
+import { createHash } from 'crypto';
 
 const chiaRoot = getChiaRoot();
-
 const persistanceFolder = `${chiaRoot}/cadt/${getDataModelVersion()}`;
 
+const localQueryLogger = (query) => {
+  const queryString = query.split(/:\s(.+)/)[1];
+  const queryHash = createHash('md5').update(queryString).digest('hex');
+  logger.silly(`SQLite Sequelize [query hash: ${queryHash}]\n\t${query}`);
+};
+
+const mirrorQueryLogger = (query) => {
+  const queryString = query.split(/:\s(.+)/)[1];
+  const queryHash = createHash('md5').update(queryString).digest('hex');
+  logger.silly(`Mirror DB Sequelize [query hash: ${queryHash}]\n\t${query}`);
+};
+
+const appLogLevel = getConfig().APP.LOG_LEVEL;
+const localLogging = appLogLevel === 'silly' ? localQueryLogger : false;
+const mirrorLogging = appLogLevel === 'silly' ? mirrorQueryLogger : false;
+
 export default {
   local: {
     dialect: 'sqlite',
     storage: `${persistanceFolder}/data.sqlite3`,
-    logging: false,
+    logging: localLogging,
     dialectOptions: {
       busyTimeout: 10000,
     },
@@ -36,6 +53,6 @@ export default {
     database: getConfig().MIRROR_DB.DB_NAME || '',
     host: getConfig().MIRROR_DB.DB_HOST || '',
     dialect: 'mysql',
-    logging: false,
+    logging: mirrorLogging,
   },
 };

src/config/logger.cjs → src/config/logger.js

@@ -1,16 +1,17 @@
-const winston = require('winston');
-const { format, transports, createLogger } = winston;
-const DailyRotateFile = require('winston-daily-rotate-file');
+import winston from 'winston';
+import DailyRotateFile from 'winston-daily-rotate-file';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import packageJson from '../../package.json' assert { type: 'json' };
+import { getConfig } from '../utils/config-loader.js';
 
-const fs = require('fs');
-const os = require('os');
-const path = require('path');
-const packageJson = require('../../package.json');
+const { format, transports, createLogger } = winston;
 
 const getChiaRoot = () => {
   let chiaRoot;
 
-  if(process.env.CHIA_ROOT) {
+  if (process.env.CHIA_ROOT) {
     chiaRoot = path.resolve(process.env.CHIA_ROOT);
   } else {
     const homeDir = os.homedir();
@@ -33,6 +34,7 @@ const logDir = `${chiaRoot}/cadt/${getDataModelVersion()}/logs`;
 if (!fs.existsSync(logDir)) {
   fs.mkdirSync(logDir, { recursive: true });
 }
+
 const logFormat = format.printf(
   (info) =>
     `${info.timestamp} [${packageJson.version}] [${info.level}]: ${info.message} ${
@@ -43,26 +45,22 @@ const logFormat = format.printf(
 );
 
 const logger = createLogger({
-  level: 'info',
+  level: getConfig().APP.LOG_LEVEL || 'info',
   format: format.combine(
     logFormat,
-    // format.label({ label: path.basename(process.main.filename) }),
     format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
     format.metadata({ fillExcept: ['message', 'level', 'timestamp'] }),
   ),
   transports: [
-    // Write all logs with importance level of `error` or less to `error.log`
     new transports.File({
       filename: `${logDir}/error.log`,
       level: 'error',
       format: format.combine(format.json()),
     }),
-    // Write all logs with importance level of `info` or less to `combined.log`
     new transports.File({
       filename: `${logDir}/combined.log`,
       format: format.combine(format.json()),
     }),
-    // Rotate logs to `application-%DATE%.log`
     new DailyRotateFile({
       filename: `${logDir}/application-%DATE%.log`,
       datePattern: 'YYYY-MM-DD',
@@ -71,6 +69,15 @@ const logger = createLogger({
       utc: true,
       format: format.combine(format.json()),
     }),
+    new DailyRotateFile({
+      filename: `${logDir}/debug-%DATE%.log`,
+      datePattern: 'YYYY-MM-DD',
+      level: 'debug',
+      zippedArchive: true,
+      maxSize: '20m',
+      utc: true,
+      format: format.combine(format.json()),
+    }),
   ],
   exceptionHandlers: [
     new transports.File({
@@ -99,9 +106,6 @@ const logger = createLogger({
   exitOnError: false,
 });
 
-//
-// If not in production then log to the `console`
-//
 if (process.env.NODE_ENV !== 'production') {
   logger.add(
     new transports.Console({
@@ -114,6 +118,4 @@ if (process.env.NODE_ENV !== 'production') {
   );
 }
 
-module.exports = {
-  logger,
-};
+export { logger };

src/controllers/project.controller.js

@@ -6,7 +6,7 @@ import { uuid as uuidv4 } from 'uuidv4';
 
 import { Staging, Project, Organization, ModelKeys } from '../models';
 
-import { logger } from '../config/logger.cjs';
+import { logger } from '../config/logger.js';
 
 import {
   genericFilterRegex,

src/controllers/units.controller.js

@@ -40,7 +40,7 @@ import {
 } from '../utils/string-utils';
 
 import { formatModelAssociationName } from '../utils/model-utils.js';
-import { logger } from '../config/logger.cjs';
+import { logger } from '../config/logger.js';
 
 export const create = async (req, res) => {
   try {

src/database/index.js

@@ -1,7 +1,7 @@
 import _ from 'lodash';
 import { Sequelize, QueryTypes } from 'sequelize';
 import config from '../config/config.js';
-import { logger } from '../config/logger.cjs';
+import { logger } from '../config/logger.js';
 import mysql from 'mysql2/promise';
 import { getConfig } from '../utils/config-loader';
 

src/datalayer/persistance.js

@@ -5,7 +5,7 @@ import superagent from 'superagent';
 import { getConfig } from '../utils/config-loader';
 import wallet from './wallet';
 import { Organization } from '../models';
-import { logger } from '../config/logger.cjs';
+import { logger } from '../config/logger.js';
 import { getChiaRoot } from '../utils/chia-root.js';
 import { getMirrorUrl } from '../utils/datalayer-utils';
 
@@ -733,6 +733,10 @@ const getSyncStatus = async (storeId) => {
     // We just care that we got some response, not what the response is
     if (Object.keys(data).includes('success')) {
       return data;
+    } else {
+      logger.warn(
+        `datalayer '/get_sync_status' RPC failed to get sync status for ${storeId}`,
+      );
     }
 
     return false;

src/datalayer/syncService.js

@@ -3,7 +3,7 @@ import _ from 'lodash';
 import { decodeDataLayerResponse } from '../utils/datalayer-utils';
 import { Simulator } from '../models';
 import { getConfig } from '../utils/config-loader';
-import { logger } from '../config/logger.cjs';
+import { logger } from '../config/logger.js';
 
 import * as dataLayer from './persistance';
 import * as simulator from './simulator';

src/datalayer/wallet.js

@@ -3,7 +3,7 @@ import path from 'path';
 import superagent from 'superagent';
 import { getConfig } from '../utils/config-loader';
 import { getChiaRoot } from '../utils/chia-root.js';
-import { logger } from '../config/logger.cjs';
+import { logger } from '../config/logger.js';
 
 process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;
 

src/datalayer/writeService.js

@@ -5,7 +5,7 @@ import wallet from './wallet';
 import * as simulator from './simulator';
 import { encodeHex, getMirrorUrl } from '../utils/datalayer-utils';
 import { getConfig } from '../utils/config-loader';
-import { logger } from '../config/logger.cjs';
+import { logger } from '../config/logger.js';
 import { Organization } from '../models';
 
 const { USE_SIMULATOR, AUTO_MIRROR_EXTERNAL_STORES } = getConfig().APP;

src/middleware.js

@@ -14,6 +14,7 @@ import {
 import packageJson from '../package.json' assert { type: 'json' };
 import datalayer from './datalayer';
 import { Organization } from './models';
+import { logger } from './config/logger.js';
 
 const { CADT_API_KEY, READ_ONLY, IS_GOVERNANCE_BODY, USE_SIMULATOR } =
   getConfig().APP;
@@ -40,6 +41,34 @@ app.use(
 app.use(express.json());
 app.use(bodyParser.urlencoded({ extended: false }));
 
+// Request logger middleware
+app.use((req, res, next) => {
+  logger.verbose(`Received request: ${req.method} ${req.originalUrl}`, {
+    method: req.method,
+    url: req.originalUrl,
+    headers: req.headers,
+    timestamp: new Date().toISOString(),
+  });
+
+  const startTime = Date.now();
+
+  res.on('finish', () => {
+    const duration = Date.now() - startTime;
+    logger.verbose(
+      `Processed request: ${req.method} ${req.originalUrl}, status: ${res.statusCode}, duration: ${duration}ms`,
+      {
+        method: req.method,
+        url: req.originalUrl,
+        status: res.statusCode,
+        responseTime: duration,
+        timestamp: new Date().toISOString(),
+      },
+    );
+  });
+
+  next();
+});
+
 // Common assertions on every endpoint
 app.use(async function (req, res, next) {
   try {

src/models/governance/governance.model.js

@@ -7,7 +7,7 @@ import { Meta } from '../../models';
 import datalayer from '../../datalayer';
 import { keyValueToChangeList } from '../../utils/datalayer-utils';
 import { getConfig } from '../../utils/config-loader';
-import { logger } from '../../config/logger.cjs';
+import { logger } from '../../config/logger.js';
 import { getDataModelVersion } from '../../utils/helpers';
 import PickListStub from './governance.stub.js';
 

src/models/organizations/organizations.model.js

@@ -8,7 +8,7 @@ const { Model } = Sequelize;
 import { sequelize } from '../../database';
 
 import datalayer from '../../datalayer';
-import { logger } from '../../config/logger.cjs';
+import { logger } from '../../config/logger.js';
 import { FileStore, Staging } from '../';
 
 import { getDefaultOrganizationList } from '../../utils/data-loaders';
@@ -212,7 +212,6 @@ class Organization extends Model {
     }
   }
 
-  // eslint-disable-next-line
   static async appendNewRegistry(registryId, dataVersion) {
     const registryVersionId = await datalayer.createDataLayerStore();
     await datalayer.syncDataLayer(registryId, {

src/routes/index.js

@@ -3,7 +3,7 @@
 import { prepareDb } from '../database';
 import scheduler from '../tasks';
 import { sequelize } from '../database';
-import { logger } from '../config/logger.cjs';
+import { logger } from '../config/logger.js';
 import { pullPickListValues } from '../utils/data-loaders';
 
 import app from '../middleware';

src/server.js

@@ -6,7 +6,7 @@ import http from 'http';
 import { Server } from 'socket.io';
 import { connection } from './websocket';
 import { getConfig } from './utils/config-loader';
-import { logger } from './config/logger.cjs';
+import { logger } from './config/logger.js';
 
 import dotenv from 'dotenv';
 

src/tasks/mirror-check.js

@@ -4,7 +4,7 @@ import {
   assertDataLayerAvailable,
   assertWalletIsSynced,
 } from '../utils/data-assertions';
-import { logger } from '../config/logger.cjs';
+import { logger } from '../config/logger.js';
 import { getConfig } from '../utils/config-loader';
 import { getMirrorUrl } from '../utils/datalayer-utils';
 import dotenv from 'dotenv';

src/tasks/reset-audit-table.js

@@ -1,6 +1,6 @@
 import { SimpleIntervalJob, Task } from 'toad-scheduler';
 import { Audit, Meta } from '../models';
-import { logger } from '../config/logger.cjs';
+import { logger } from '../config/logger.js';
 import dotenv from 'dotenv';
 import _ from 'lodash';
 dotenv.config();

src/tasks/sync-default-organizations.js

@@ -4,7 +4,7 @@ import {
   assertDataLayerAvailable,
   assertWalletIsSynced,
 } from '../utils/data-assertions';
-import { logger } from '../config/logger.cjs';
+import { logger } from '../config/logger.js';
 import { getConfig } from '../utils/config-loader';
 const CONFIG = getConfig().APP;
 

src/tasks/sync-governance-body.js

@@ -6,7 +6,7 @@ import {
   assertWalletIsSynced,
 } from '../utils/data-assertions';
 import { getConfig } from '../utils/config-loader';
-import { logger } from '../config/logger.cjs';
+import { logger } from '../config/logger.js';
 import { Organization } from '../models';
 
 const CONFIG = getConfig();

src/tasks/sync-organization-meta.js

@@ -5,7 +5,7 @@ import {
   assertDataLayerAvailable,
   assertWalletIsSynced,
 } from '../utils/data-assertions';
-import { logger } from '../config/logger.cjs';
+import { logger } from '../config/logger.js';
 
 const CONFIG = getConfig().APP;
 

src/tasks/sync-picklists.js

@@ -1,6 +1,6 @@
 import { SimpleIntervalJob, Task } from 'toad-scheduler';
 import { pullPickListValues } from '../utils/data-loaders';
-import { logger } from '../config/logger.cjs';
+import { logger } from '../config/logger.js';
 import {
   assertDataLayerAvailable,
   assertWalletIsSynced,