Merge pull request #117 from zelcash/development

v0.66.0

ZelBack/config/default.js

@@ -47,6 +47,9 @@ module.exports = {
     porttestnet: 26225,
     rpcporttestnet: 26224,
   },
+  zelcash: {
+    chainValidHeight: 640000,
+  },
   zelTeamZelId: '132hG26CFTNhLM3MRsLEJhp9DpBrK6vg5N',
   zelapps: {
     // in zel per month
@@ -56,7 +59,7 @@ module.exports = {
       hdd: 0.5 * 5, // per 1gb,
     },
     address: 't1...', // apps registration address
-    epochstart: 1000000, // zelapps epoch blockheight start
+    epochstart: 690000, // zelapps epoch blockheight start
     portMin: 30001, // originally should have been from 30000 but we got temporary folding there
     portMax: 39999,
     maxImageSize: 300000000, // 300mb

ZelBack/server.js

@@ -6,7 +6,7 @@ const config = require('config');
 // const path = require('path');
 const app = require('./src/lib/server.js');
 const log = require('./src/lib/log');
-const communication = require('./src/services/zelfluxCommunication');
+const serviceManager = require('./src/services/serviceManager');
 
 // const key = fs.readFileSync(path.join(__dirname, '../certs/selfsigned.key'), 'utf8');
 // const cert = fs.readFileSync(path.join(__dirname, '../certs/selfsigned.crt'), 'utf8');
@@ -19,5 +19,5 @@ const communication = require('./src/services/zelfluxCommunication');
 
 app.listen(config.server.apiport, () => {
   log.info(`ZelBack listening on port ${config.server.apiport}!`);
-  communication.startFluxFunctions();
+  serviceManager.startFluxFunctions();
 });

ZelBack/src/lib/log.js

@@ -8,7 +8,8 @@ function getFilesizeInBytes(filename) {
     const stats = fs.statSync(filename);
     const fileSizeInBytes = stats.size;
     return fileSizeInBytes;
-  } catch {
+  } catch (e) {
+    console.log(e);
     return 0;
   }
 }

ZelBack/src/routes.js

@@ -251,6 +251,9 @@ module.exports = (app, expressWs) => {
   app.get('/zelapps/globalspecifications', (req, res) => {
     zelappsService.getGlobalZelAppsSpecifications(req, res);
   });
+  app.get('/zelapps/hashes', (req, res) => {
+    zelappsService.getZelAppHashes(req, res);
+  });
 
   // app.get('/explorer/allutxos', (req, res) => {
   //   explorerService.getAllUtxos(req, res);

ZelBack/src/services/explorerService.js

@@ -325,9 +325,9 @@ async function processBlock(blockHeight) {
           await serviceHelper.updateOneInDatabase(database, addressTransactionIndexCollection, query, update, options);
         }));
         // MAY contain ZelApp transaction. Store it.
-        if (isZelAppMessageValue > 0 && message.length === 64) {
+        if (isZelAppMessageValue >= 10 && message.length === 64 && blockDataVerbose.height >= config.zelapps.epochstart) { // min of 10 zel had to be paid for us bothering checking
           const zelappTxRecord = {
-            txid: tx.txid, height: blockDataVerbose.height, hash: message, value: isZelAppMessageValue,
+            txid: tx.txid, height: blockDataVerbose.height, hash: message, value: isZelAppMessageValue, message: false, // message is boolean saying if we already have it stored as permanent message
           };
           await serviceHelper.insertOneToDatabase(database, zelappsHashesCollection, zelappTxRecord);
           zelappsService.checkAndRequestZelApp(message, tx.txid, blockDataVerbose.height, isZelAppMessageValue);
@@ -435,7 +435,7 @@ async function restoreDatabaseToBlockheightState(height, rescanGlobalApps = fals
     throw error;
   });
   if (rescanGlobalApps === true) {
-    const databaseGlobal = database.db(config.database.zelappsglobal.database);
+    const databaseGlobal = dbopen.db(config.database.zelappsglobal.database);
     log.info('Rescanning Apps!');
     await serviceHelper.removeDocumentsFromCollection(databaseGlobal, config.database.zelappsglobal.collections.zelappsMessages, query).catch((error) => {
       log.error(error);
@@ -460,17 +460,17 @@ async function initiateBlockProcessor(restoreDatabase, deepRestore, reindexOrRes
     isInInitiationOfBP = true;
     const db = serviceHelper.databaseConnection();
     const database = db.db(config.database.zelcash.database);
-    const query = {};
+    const query = { generalScannedHeight: { $gte: 0 } };
     const projection = {
       projection: {
         _id: 0,
         generalScannedHeight: 1,
       },
     };
     let scannedBlockHeight = 0;
-    const scannedBlockHeightsResult = await serviceHelper.findInDatabase(database, scannedHeightCollection, query, projection);
-    if (scannedBlockHeightsResult[0]) {
-      scannedBlockHeight = scannedBlockHeightsResult[0].generalScannedHeight;
+    const currentHeight = await serviceHelper.findOneInDatabase(database, scannedHeightCollection, query, projection);
+    if (currentHeight && currentHeight.generalScannedHeight) {
+      scannedBlockHeight = currentHeight.generalScannedHeight;
     }
     const zelcashGetInfo = await zelcashService.getInfo();
     let zelcashHeight = 0;
@@ -519,6 +519,7 @@ async function initiateBlockProcessor(restoreDatabase, deepRestore, reindexOrRes
       await database.collection(zelappsHashesCollection).createIndex({ txid: 1 }, { name: 'query for getting txid' });
       await database.collection(zelappsHashesCollection).createIndex({ height: 1 }, { name: 'query for getting height' });
       await database.collection(zelappsHashesCollection).createIndex({ hash: 1 }, { name: 'query for getting app hash' });
+      await database.collection(zelappsHashesCollection).createIndex({ message: 1 }, { name: 'query for getting app hashes depending if we have message' });
 
       const databaseGlobal = db.db(config.database.zelappsglobal.database);
       log.info('Preparing apps collections');
@@ -535,17 +536,17 @@ async function initiateBlockProcessor(restoreDatabase, deepRestore, reindexOrRes
         });
         log.info(resultE, resultF);
       }
-      await databaseGlobal.collection(config.database.zelappsglobal.collections.zelappsMessages).createIndex({ hash: 1 }, { name: 'query for getting zelapp message based on hash' });
+      await databaseGlobal.collection(config.database.zelappsglobal.collections.zelappsMessages).createIndex({ hash: 1 }, { name: 'query for getting zelapp message based on hash' }); // , unique: true
       await databaseGlobal.collection(config.database.zelappsglobal.collections.zelappsMessages).createIndex({ txid: 1 }, { name: 'query for getting zelapp message based on txid' });
       await databaseGlobal.collection(config.database.zelappsglobal.collections.zelappsMessages).createIndex({ height: 1 }, { name: 'query for getting zelapp message based on height' });
-      await databaseGlobal.collection(config.database.zelappsglobal.collections.zelappsMessages).createIndex({ 'zelAppSpecifications.name': 1 }, { name: 'query for getting zelapp message based on zelapp specs name' });
+      await databaseGlobal.collection(config.database.zelappsglobal.collections.zelappsMessages).createIndex({ 'zelAppSpecifications.name': 1 }, { name: 'query for getting zelapp message based on zelapp specs name' }); // , unique: true
       await databaseGlobal.collection(config.database.zelappsglobal.collections.zelappsMessages).createIndex({ 'zelAppSpecifications.owner': 1 }, { name: 'query for getting zelapp message based on zelapp specs owner' });
       await databaseGlobal.collection(config.database.zelappsglobal.collections.zelappsMessages).createIndex({ 'zelAppSpecifications.repotag': 1 }, { name: 'query for getting zelapp message based on image' });
-      await databaseGlobal.collection(config.database.zelappsglobal.collections.zelappsInformation).createIndex({ name: 1 }, { name: 'query for getting zelapp based on zelapp specs name' });
+      await databaseGlobal.collection(config.database.zelappsglobal.collections.zelappsInformation).createIndex({ name: 1 }, { name: 'query for getting zelapp based on zelapp specs name' }); // , unique: true
       await databaseGlobal.collection(config.database.zelappsglobal.collections.zelappsInformation).createIndex({ owner: 1 }, { name: 'query for getting zelapp based on zelapp specs owner' });
       await databaseGlobal.collection(config.database.zelappsglobal.collections.zelappsInformation).createIndex({ repotag: 1 }, { name: 'query for getting zelapp based on image' });
       await databaseGlobal.collection(config.database.zelappsglobal.collections.zelappsInformation).createIndex({ height: 1 }, { name: 'query for getting zelapp based on last height update' }); // we need to know the height of app adjustment
-      await databaseGlobal.collection(config.database.zelappsglobal.collections.zelappsInformation).createIndex({ hash: 1 }, { name: 'query for getting zelapp based on last hash' }); // we need to know the hash of the last message update which is the true identifier
+      await databaseGlobal.collection(config.database.zelappsglobal.collections.zelappsInformation).createIndex({ hash: 1 }, { name: 'query for getting zelapp based on last hash' }); // , unique: true // we need to know the hash of the last message update which is the true identifier
       // what if 2 app adjustment come in the same block?
       // log.info(resultE, resultF);
       log.info('Preparation done');
@@ -574,7 +575,7 @@ async function initiateBlockProcessor(restoreDatabase, deepRestore, reindexOrRes
           log.error('Error restoring database!');
           throw e;
         }
-      } else {
+      } else if (scannedBlockHeight > config.zelcash.chainValidHeight) {
         const zelcashGetChainTips = await zelcashService.getChainTips();
         if (zelcashGetChainTips.status !== 'success') {
           throw new Error(zelcashGetChainTips.data);
@@ -831,7 +832,7 @@ async function getScannedHeight(req, res) {
   const projection = {
     projection: {
       _id: 0,
-      generalScannedHeight: 1, //
+      generalScannedHeight: 1,
     },
   };
   const result = await serviceHelper.findOneInDatabase(database, scannedHeightCollection, query, projection).catch((error) => {

ZelBack/src/services/serviceManager.js

@@ -0,0 +1,60 @@
+const config = require('config');
+const log = require('../lib/log');
+
+const serviceHelper = require('./serviceHelper');
+const explorerService = require('./explorerService');
+const zelfluxCommunication = require('./zelfluxCommunication');
+const zelappsService = require('./zelappsService');
+
+async function startFluxFunctions() {
+  try {
+    log.info('Initiating MongoDB connection');
+    await serviceHelper.initiateDB(); // either true or throws error
+    log.info('DB connected');
+    log.info('Preparing local database...');
+    const db = serviceHelper.databaseConnection();
+    const database = db.db(config.database.local.database);
+    await serviceHelper.dropCollection(database, config.database.local.collections.activeLoginPhrases).catch((error) => {
+      if (error.message !== 'ns not found') {
+        log.error(error);
+      }
+    });
+    await serviceHelper.dropCollection(database, config.database.local.collections.activeSignatures).catch((error) => {
+      if (error.message !== 'ns not found') {
+        log.error(error);
+      }
+    });
+    await database.collection(config.database.local.collections.activeLoginPhrases).createIndex({ createdAt: 1 }, { expireAfterSeconds: 900 });
+    await database.collection(config.database.local.collections.activeSignatures).createIndex({ createdAt: 1 }, { expireAfterSeconds: 900 });
+    log.info('Local database prepared');
+    log.info('Preparing temporary database...');
+    // no need to drop temporary messages
+    const databaseTemp = db.db(config.database.zelappsglobal.database);
+    await databaseTemp.collection(config.database.zelappsglobal.collections.zelappsTemporaryMessages).createIndex({ receivedAt: 1 }, { expireAfterSeconds: 3600 });
+    log.info('Temporary database prepared');
+    zelfluxCommunication.adjustFirewall();
+    zelfluxCommunication.fluxDisovery();
+    log.info('Flux Discovery started');
+    zelfluxCommunication.keepConnectionsAlive();
+    zelfluxCommunication.keepIncomingConnectionsAlive();
+    zelfluxCommunication.checkDeterministicNodesCollisions();
+    setInterval(() => {
+      zelfluxCommunication.checkDeterministicNodesCollisions();
+    }, 60000);
+    log.info('Flux checks operational');
+    explorerService.initiateBlockProcessor(true, true);
+    setInterval(() => { // every 8 mins (4 blocks)
+      zelappsService.continuousZelAppHashesCheck();
+    }, 8 * 60 * 1000);
+    log.info('Flux Block Processing Service started');
+  } catch (e) {
+    log.error(e);
+    setTimeout(() => {
+      startFluxFunctions();
+    }, 15000);
+  }
+}
+
+module.exports = {
+  startFluxFunctions,
+};

ZelBack/src/services/zelappsService.js

@@ -28,6 +28,8 @@ const globalZelAppsMessages = config.database.zelappsglobal.collections.zelappsM
 const globalZelAppsInformation = config.database.zelappsglobal.collections.zelappsInformation;
 const globalZelAppsTempMessages = config.database.zelappsglobal.collections.zelappsTemporaryMessages;
 
+const zelappsHashesCollection = config.database.zelcash.collections.zelappsHashes;
+
 function getZelAppIdentifier(zelappName) {
   // this id is used for volumes, docker names so we know it reall belongs to zelflux
   if (zelappName.startsWith('zel')) {
@@ -2501,7 +2503,7 @@ async function storeZelAppPermanentMessage(message) {
   return true;
 }
 
-async function updateZelAppSpecifications(zelAppSpecs, i = 0) {
+async function updateZelAppSpecifications(zelAppSpecs) {
   try {
     // zelAppSpecs: {
     //   version: 1,
@@ -2552,11 +2554,10 @@ async function updateZelAppSpecifications(zelAppSpecs, i = 0) {
       await serviceHelper.updateOneInDatabase(database, globalZelAppsInformation, query, update, options);
     }
   } catch (error) {
+    // retry
     log.error(error);
-    if (i < 60) {
-      await serviceHelper.delay(60 * 1000);
-      updateZelAppSpecifications(zelAppSpecs, i + 1);
-    }
+    await serviceHelper.delay(60 * 1000);
+    updateZelAppSpecifications(zelAppSpecs);
   }
 }
 
@@ -2669,9 +2670,21 @@ async function checkZelAppTemporaryMessageExistence(hash) {
   }
 }
 
+async function zelappHashHasMessage(hash) {
+  const db = serviceHelper.databaseConnection();
+  const database = db.db(config.database.zelcash.database);
+  const query = { hash };
+  const update = { $set: { message: true } };
+  const options = {};
+  await serviceHelper.updateOneInDatabase(database, zelappsHashesCollection, query, update, options);
+  return true;
+}
+
 // hash of zelapp information, txid it was in, height of blockchain containing the txid
 async function checkAndRequestZelApp(hash, txid, height, valueSat, i = 0) {
   try {
+    const randomDelay = Math.floor((Math.random() * 1280)) + 420;
+    await serviceHelper.delay(randomDelay);
     const appMessageExists = await checkZelAppMessageExistence(hash);
     if (appMessageExists === false) { // otherwise do nothing
       // we surely do not have that message in permanent storaage.
@@ -2695,6 +2708,8 @@ async function checkAndRequestZelApp(hash, txid, height, valueSat, i = 0) {
             valueSat: serviceHelper.ensureNumber(valueSat),
           };
           await storeZelAppPermanentMessage(permanentZelAppMessage);
+          // await update zelapphashes that we already have it stored
+          await zelappHashHasMessage(hash);
           const updateForSpecifications = permanentZelAppMessage.zelAppSpecifications;
           updateForSpecifications.hash = permanentZelAppMessage.hash;
           updateForSpecifications.height = permanentZelAppMessage.height;
@@ -2706,13 +2721,16 @@ async function checkAndRequestZelApp(hash, txid, height, valueSat, i = 0) {
         // request the message and broadcast the message further to our connected peers.
         requestZelAppMessage(hash);
         // rerun this after 1 min delay
-        // stop this loop after 1 hour, as it might be a scammy message or simply this message is nowhere on the network
-        if (i < 60) {
+        // stop this loop after 7 mins, as it might be a scammy message or simply this message is nowhere on the network, we dont have connections etc. We also have continous checkup for it every 8 min
+        if (i < 7) {
           await serviceHelper.delay(60 * 1000);
           checkAndRequestZelApp(hash, txid, height, valueSat, i + 1);
         }
         // additional requesting of missing zelapp messages is done on rescans
       }
+    } else {
+      // update zelapphashes that we already have it stored
+      await zelappHashHasMessage(hash);
     }
   } catch (error) {
     log.error(error);
@@ -2835,6 +2853,59 @@ async function rescanGlobalAppsInformation(height = 0, removeLastInformation = f
   }
 }
 
+async function continuousZelAppHashesCheck() {
+  try {
+    log.info('Requesting missing ZelApp messages');
+    // get zelapp hashes that do not have a message;
+    const dbopen = serviceHelper.databaseConnection();
+    const database = dbopen.db(config.database.zelcash.database);
+    const query = { message: false };
+    const projection = {
+      projection: {
+        _id: 0,
+        txid: 1,
+        hash: 1,
+        height: 1,
+        value: 1,
+        message: 1,
+      },
+    };
+    const results = await serviceHelper.findInDatabase(database, zelappsHashesCollection, query, projection);
+    // eslint-disable-next-line no-restricted-syntax
+    for (const result of results) {
+      checkAndRequestZelApp(result.hash, result.txid, result.height, result.value);
+      // eslint-disable-next-line no-await-in-loop
+      await serviceHelper.delay(1234);
+    }
+  } catch (error) {
+    log.error(error);
+  }
+}
+
+async function getZelAppHashes(req, res) {
+  const dbopen = serviceHelper.databaseConnection();
+  const database = dbopen.db(config.database.zelcash.database);
+  const query = {};
+  const projection = {
+    projection: {
+      _id: 0,
+      txid: 1,
+      hash: 1,
+      height: 1,
+      value: 1,
+      message: 1,
+    },
+  };
+  const results = await serviceHelper.findInDatabase(database, zelappsHashesCollection, query, projection).catch((error) => {
+    const errMessage = serviceHelper.createErrorMessage(error.message, error.name, error.code);
+    res.json(errMessage);
+    log.error(error);
+    throw error;
+  });
+  const resultsResponse = serviceHelper.createDataMessage(results);
+  res.json(resultsResponse);
+}
+
 module.exports = {
   dockerListContainers,
   zelAppPull,
@@ -2881,4 +2952,6 @@ module.exports = {
   verifyZelAppMessageSignature,
   reindexGlobalAppsInformation,
   rescanGlobalAppsInformation,
+  continuousZelAppHashesCheck,
+  getZelAppHashes,
 };