chore: Refresh develop branch with hotfixes from main

.github/dependabot.yml

@@ -4,4 +4,5 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+      day: "sunday"
     target-branch: "develop"

.gitignore

@@ -1,6 +1,7 @@
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
 data.sqlite3
 test.sqlite3
+test.sqlite3*
 testMirror.sqlite3
 
 # dependencies

CHANGELOG.md

@@ -1,3 +1,25 @@
+## [1.7.12](https://github.com/Chia-Network/cadt/compare/1.7.11...1.7.12) (2024-03-08)
+
+
+### Bug Fixes
+
+* allow other params when using projectids ([ddf21be](https://github.com/Chia-Network/cadt/commit/ddf21be38cc04860029d8a5e587ef0f0da0e960d))
+* joi ([f437359](https://github.com/Chia-Network/cadt/commit/f437359380bb2f24241c8611a22e5f5e68f730b1))
+* projectIds ([2c0a083](https://github.com/Chia-Network/cadt/commit/2c0a083a7dbeac26c077f72b36f03d58911bf065))
+* refactor validation ([4c742af](https://github.com/Chia-Network/cadt/commit/4c742af78b324d2745fa844bbb5bc57bd7cc9c25))
+* refactor validation ([b023d65](https://github.com/Chia-Network/cadt/commit/b023d653b1f0d4b1968bc6b2448e6a21a1f95711))
+
+
+
+## [1.7.11](https://github.com/Chia-Network/cadt/compare/1.7.10...1.7.11) (2024-03-08)
+
+
+### Bug Fixes
+
+* improve mirror logic ([af205a5](https://github.com/Chia-Network/cadt/commit/af205a58fbff2812a4e2cc37121e8c9a3cb724b9))
+
+
+
 ## [1.7.10](https://github.com/Chia-Network/cadt/compare/1.7.9...1.7.10) (2024-03-06)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cadt",
-  "version": "1.7.10",
+  "version": "1.7.12",
   "_comment": "DONT CHANGE MAJOR UNLESS DATAMODEL CHANGES: The major version corresponds to the datamodel version your using, so 2.0.0 means it'll use datamodel v2",
   "private": true,
   "bin": "build/server.js",
@@ -60,18 +60,18 @@
   },
   "devDependencies": {
     "@babel/cli": "^7.23.9",
-    "@babel/core": "^7.23.9",
+    "@babel/core": "^7.24.0",
     "@babel/plugin-syntax-import-attributes": "^7.23.3",
-    "@babel/preset-env": "^7.23.9",
+    "@babel/preset-env": "^7.24.0",
     "@babel/register": "^7.23.7",
-    "@commitlint/cli": "^18.6.1",
-    "@commitlint/config-conventional": "^18.6.2",
+    "@commitlint/cli": "^19.0.3",
+    "@commitlint/config-conventional": "^19.0.3",
     "babel-plugin-module-resolver": "^5.0.0",
     "chai": "^5.1.0",
     "chai-http": "^4.4.0",
-    "eslint": "^8.56.0",
+    "eslint": "^8.57.0",
     "eslint-plugin-es": "^4.1.0",
-    "eslint-plugin-mocha": "^10.2.0",
+    "eslint-plugin-mocha": "^10.3.0",
     "husky": "^9.0.11",
     "mocha": "^10.3.0",
     "semver": "^7.6.0",

src/datalayer/persistance.js

@@ -3,12 +3,11 @@ import fs from 'fs';
 import path from 'path';
 import superagent from 'superagent';
 import { getConfig } from '../utils/config-loader';
-import fullNode from './fullNode';
-import { publicIpv4 } from '../utils/ip-tools';
 import wallet from './wallet';
 import { Organization } from '../models';
 import { logger } from '../config/logger.cjs';
 import { getChiaRoot } from '../utils/chia-root.js';
+import { getMirrorUrl } from '../utils/datalayer-utils';
 
 process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;
 
@@ -117,6 +116,17 @@ const addMirror = async (storeId, url, forceAddMirror = false) => {
   await wallet.waitForAllTransactionsToConfirm();
   const homeOrg = await Organization.getHomeOrg();
 
+  logger.info(
+    `Checking mirrors for storeID is ${storeId} with mirror URL ${url}`,
+  );
+
+  if (!url) {
+    logger.info(
+      `No DATALAYER_FILE_SERVER_URL specified so skipping mirror for ${storeId}`,
+    );
+    return false;
+  }
+
   if (!homeOrg && !forceAddMirror) {
     logger.info(`No home org detected so skipping mirror for ${storeId}`);
     return false;
@@ -130,7 +140,7 @@ const addMirror = async (storeId, url, forceAddMirror = false) => {
   );
 
   if (mirror) {
-    logger.info(`Mirror already available for ${storeId}`);
+    logger.info(`Mirror already available for ${storeId} at ${url}`);
     return true;
   }
 
@@ -553,12 +563,9 @@ const subscribeToStoreOnDataLayer = async (storeId) => {
     if (Object.keys(data).includes('success') && data.success) {
       logger.info(`Successfully Subscribed: ${storeId}`);
 
-      const chiaConfig = fullNode.getChiaConfig();
+      const mirrorUrl = await getMirrorUrl();
 
-      await addMirror(
-        storeId,
-        `http://${await publicIpv4()}:${chiaConfig.data_layer.host_port}`,
-      );
+      await addMirror(storeId, mirrorUrl, true);
 
       return data;
     }

src/datalayer/writeService.js

@@ -38,8 +38,8 @@ const createDataLayerStore = async () => {
   return storeId;
 };
 
-const addMirror = async (storeId, url) => {
-  return dataLayer.addMirror(storeId, url);
+const addMirror = async (storeId, url, force = false) => {
+  return dataLayer.addMirror(storeId, url, force);
 };
 
 const waitForStoreToBeConfirmed = async (storeId, retry = 0) => {

src/models/organizations/organizations.model.js

@@ -222,8 +222,8 @@ class Organization extends Model {
     return registryVersionId;
   }
 
-  static async addMirror(storeId, url) {
-    await datalayer.addMirror(storeId, url);
+  static async addMirror(storeId, url, force = false) {
+    await datalayer.addMirror(storeId, url, force);
   }
 
   static async importHomeOrg(orgUid) {

src/tasks/mirror-check.js

@@ -43,18 +43,19 @@ const job = new SimpleIntervalJob(
 );
 
 const runMirrorCheck = async () => {
-  const homeOrg = Organization.getHomeOrg();
-
-  if (homeOrg) {
-    const organizations = Organization.getOrgsMap();
-    const orgs = Object.keys(organizations);
-    for (const org of orgs) {
-      const orgData = organizations[org];
-      const mirrorUrl = await getMirrorUrl();
-
+  const organizations = await Organization.getOrgsMap();
+  const orgs = Object.keys(organizations);
+  for (const org of orgs) {
+    const orgData = organizations[org];
+    const mirrorUrl = await getMirrorUrl();
+    if (mirrorUrl) {
       // There is logic within the addMirror function to check if the mirror already exists
-      await Organization.addMirror(orgData.orgUid, mirrorUrl);
-      await Organization.addMirror(orgData.registryId, mirrorUrl);
+      await Organization.addMirror(orgData.orgUid, mirrorUrl, true);
+      await Organization.addMirror(orgData.registryId, mirrorUrl, true);
+    } else {
+      logger.error(
+        'DATALAYER_FILE_SERVER_URL not set, skipping mirror announcement',
+      );
     }
   }
 };

src/utils/datalayer-utils.js

@@ -1,6 +1,5 @@
 import { getConfig } from './config-loader';
-import fullNode from '../datalayer/fullNode';
-import { publicIpv4 } from './ip-tools';
+import { logger } from '../config/logger.cjs';
 
 export const encodeHex = (str) => {
   return Buffer.from(str).toString('hex');
@@ -113,10 +112,12 @@ export const optimizeAndSortKvDiff = (kvDiff) => {
 };
 
 export const getMirrorUrl = async () => {
-  const { DATALAYER_FILE_SERVER_URL } = getConfig().APP;
-  const chiaConfig = fullNode.getChiaConfig();
-  return (
-    DATALAYER_FILE_SERVER_URL ||
-    `http://${await publicIpv4()}:${chiaConfig.data_layer.host_port}`
-  );
+  try {
+    const { DATALAYER_FILE_SERVER_URL } = getConfig().APP;
+    logger.info(`Resolved Mirror Url: ${DATALAYER_FILE_SERVER_URL}`);
+    return DATALAYER_FILE_SERVER_URL;
+  } catch (error) {
+    logger.error('Error getting DATALAYER_FILE_SERVER_URL: ${error}');
+    return null;
+  }
 };

src/validations/projects.validations.js

@@ -61,43 +61,33 @@ export const baseSchema = {
 };
 
 export const projectsGetQuerySchema = Joi.object({
-  page: Joi.number(),
-  limit: Joi.number(),
-  search: Joi.string(),
-  columns: Joi.array().items(Joi.string()).single(),
-  orgUid: Joi.string(),
-  warehouseProjectId: Joi.string(),
-  xls: Joi.boolean(),
-  projectIds: Joi.array().items(Joi.string()).single(),
-  order: Joi.string().regex(genericSortColumnRegex).max(100).min(1),
-  filter: Joi.string().regex(genericFilterRegex).max(100).min(1),
-  onlyMarketplaceProjects: Joi.boolean(),
-})
-  .when(
-    Joi.object({
-      warehouseProjectId: Joi.string().min(1),
-    }).or('warehouseProjectId'),
-    {
-      then: Joi.object(),
-      otherwise: Joi.object({
-        page: Joi.number().required(),
-        limit: Joi.number().required(),
-      }),
-    },
-  )
-  .when(
-    Joi.object({
-      projectIds: Joi.array().items(Joi.string()).single().min(1),
+  page: Joi.number().optional(),
+  limit: Joi.number().max(100).min(1).optional(),
+  search: Joi.string().optional(),
+  columns: Joi.array().items(Joi.string()).single().optional(),
+  orgUid: Joi.string().optional(),
+  warehouseProjectId: Joi.string().optional(),
+  xls: Joi.boolean().optional(),
+  projectIds: Joi.array().items(Joi.string()).single().optional(),
+  order: Joi.string().regex(genericSortColumnRegex).optional(),
+  filter: Joi.string().regex(genericFilterRegex).optional(),
+  onlyMarketplaceProjects: Joi.boolean().optional(),
+}).when(
+  Joi.alternatives([
+    Joi.object({ projectIds: Joi.string().required() }).unknown(),
+    Joi.object({ warehouseProjectId: Joi.string().required() }).unknown(),
+  ]),
+  {
+    then: Joi.object({
+      page: Joi.number().optional(),
+      limit: Joi.number().optional(),
     }),
-    {
-      then: Joi.object(),
-      otherwise: Joi.object({
-        page: Joi.number().required(),
-        limit: Joi.number().required(),
-      }),
-    },
-  )
-  .and('page', 'limit');
+    otherwise: Joi.object({
+      page: Joi.number().required(),
+      limit: Joi.number().required(),
+    }),
+  },
+);
 
 export const projectsPostSchema = Joi.object({
   ...baseSchema,