From 985ca1ac390d78da9790553240c3e4a519a7b36a Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Fri, 30 May 2014 14:10:32 -0300 Subject: [PATCH 1/2] add ignore Cache --- README.md | 10 +++++++--- app/controllers/addresses.js | 14 +++++++------- app/models/Address.js | 11 +++++++++-- config/config.js | 9 ++++++--- lib/TransactionDb.js | 14 ++++++-------- test/integration/addrCache.js | 11 +++++++++++ 6 files changed, 46 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index ca29c6c85..7e7c2e50c 100644 --- a/README.md +++ b/README.md @@ -26,15 +26,17 @@ Alternatively, a total resync can be made, running `$ util/sync.js -D` ## IMPORTANT: v0.2 Caching schema In v0.2 a new cache schema has been introduced. Only information from transactions with -SAFE_CONFIRMATIONS+ settings will be cached (by default SAFE_CONFIRMATIONS=6). There +INSIGHT_SAFE_CONFIRMATIONS+ settings will be cached (by default SAFE_CONFIRMATIONS=6). There are 3 different caches: * nr. of confirmations * transaction spent information * scriptPubKey for unspent transactions -Cache data is only completed on request, i.e., only after accessing the required data for +Cache data is only filled on request, i.e., only after accessing the required data for the first time, the information is cached, there is not pre-caching procedure. +To ignore cache, use INSIGHT_IGNORE_CACHE; + ## Prerequisites * **bitcoind** - Download and Install [Bitcoin](http://bitcoin.org/en/download) @@ -91,7 +93,9 @@ BITCOIND_PASS # RPC password BITCOIND_DATADIR # bitcoind datadir for livenet, or datadir/testnet3 for testnet INSIGHT_NETWORK [= 'livenet' | 'testnet'] INSIGHT_DB # Path where to store insight's internal DB. (defaults to $HOME/.insight) -SAFE_CONFIRMATIONS=6 # Nr. of confirmation needed to start caching transaction information +INSIGHT_SAFE_CONFIRMATIONS=6 # Nr. of confirmation needed to start caching transaction information +INSIGHT_IGNORE_CACHE # True to ignore cache of spents in transaction, with more than INSIGHT_SAFE_CONFIRMATIONS confirmations. This is useful for tracking double spents for old transactions. + ``` Make sure that bitcoind is configured to [accept incoming connections using 'rpcallowip'](https://en.bitcoin.it/wiki/Running_Bitcoin). diff --git a/app/controllers/addresses.js b/app/controllers/addresses.js index 48b5f34a4..3f405858d 100644 --- a/app/controllers/addresses.js +++ b/app/controllers/addresses.js @@ -53,7 +53,7 @@ exports.show = function(req, res, next) { } else { return res.jsonp(a.getObj()); } - }, {txLimit: req.query.noTxList?0:-1}); + }, {txLimit: req.query.noTxList?0:-1, ignoreCache: req.param('nocache')}); } }; @@ -68,7 +68,7 @@ exports.utxo = function(req, res, next) { else { return res.jsonp(a.unspent); } - }, {onlyUnspent: 1}); + }, {onlyUnspent:1, ignoreCache: req.param('nocache')}); } }; @@ -81,7 +81,7 @@ exports.multiutxo = function(req, res, next) { if (err) callback(err); utxos = utxos.concat(a.unspent); callback(); - }, {onlyUnspent:1}); + }, {onlyUnspent:1, ignoreCache: req.param('nocache')}); }, function(err) { // finished callback if (err) return common.handleErrors(err, res); res.jsonp(utxos); @@ -99,7 +99,7 @@ exports.balance = function(req, res, next) { } else { return res.jsonp(a.balanceSat); } - }); + }, {ignoreCache: req.param('nocache')}); }; exports.totalReceived = function(req, res, next) { @@ -111,7 +111,7 @@ exports.totalReceived = function(req, res, next) { } else { return res.jsonp(a.totalReceivedSat); } - }); + }, {ignoreCache: req.param('nocache')}); }; exports.totalSent = function(req, res, next) { @@ -123,7 +123,7 @@ exports.totalSent = function(req, res, next) { } else { return res.jsonp(a.totalSentSat); } - }); + }, {ignoreCache: req.param('nocache')}); }; exports.unconfirmedBalance = function(req, res, next) { @@ -135,5 +135,5 @@ exports.unconfirmedBalance = function(req, res, next) { } else { return res.jsonp(a.unconfirmedBalanceSat); } - }); + }, {ignoreCache: req.param('nocache')}); }; diff --git a/app/models/Address.js b/app/models/Address.js index 193bfe5f9..0ad6d25f4 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -149,10 +149,15 @@ Address.prototype.update = function(next, opts) { if (!self.addrStr) return next(); opts = opts || {}; + if (! ('ignoreCache' in opts) ) + opts.ignoreCache = config.ignoreCache; + + // should collect txList from address? var txList = opts.txLimit === 0 ? null: []; + var tDb = TransactionDb; var bDb = BlockDb; - tDb.fromAddr(self.addrStr, function(err,txOut){ + tDb.fromAddr(self.addrStr, opts, function(err,txOut){ if (err) return next(err); bDb.fillConfirmations(txOut, function(err) { @@ -184,7 +189,9 @@ Address.prototype.update = function(next, opts) { txOut.forEach(function(txItem){ self._addTxItem(txItem, txList); }); - if (txList) self.transactions = txList; + if (txList) + self.transactions = txList; + return next(); } }); diff --git a/config/config.js b/config/config.js index 17f8134c8..a92cbf55e 100644 --- a/config/config.js +++ b/config/config.js @@ -60,7 +60,8 @@ if (!dataDir) { } dataDir += network === 'testnet' ? 'testnet3' : ''; -var safeConfirmations = process.env.SAFE_CONFIRMATIONS || 6; +var safeConfirmations = process.env.INSIGHT_SAFE_CONFIRMATIONS || 6; +var ignoreCache = process.env.INSIGHT_IGNORE_CACHE || 0; var bitcoindConf = { @@ -88,7 +89,8 @@ console.log( # Configuration:\n\ \t\tNetwork: %s\tINSIGHT_NETWORK\n\ \t\tDatabase Path: %s\tINSIGHT_DB\n\ -\t\tSafe Confirmations: %s\tSAFE_CONFIRMATIONS\n\ +\t\tSafe Confirmations: %s\tINSIGHT_SAFE_CONFIRMATIONS\n\ +\t\tIgnore Cache: %s\tINSIGHT_IGNORE_CACHE\n\ # Bicoind Connection configuration:\n\ \t\tRPC Username: %s\tBITCOIND_USER\n\ \t\tRPC Password: %s\tBITCOIND_PASS\n\ @@ -102,7 +104,7 @@ console.log( $ INSIGHT_NETWORK="testnet" BITCOIND_HOST="123.123.123.123" ./insight.js\ \n\n', version, -network, home, safeConfirmations, +network, home, safeConfirmations, ignoreCache?'yes':'no', bitcoindConf.user, bitcoindConf.pass?'Yes(hidden)':'No', bitcoindConf.protocol, @@ -139,4 +141,5 @@ module.exports = { segmentio: process.env.INSIGHT_SEGMENTIO_KEY }, safeConfirmations: safeConfirmations, // PLEASE NOTE THAT *FULL RESYNC* IS NEEDED TO CHANGE safeConfirmations + ignoreCache: ignoreCache, }; diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index afb8153d6..7d6c2d8d6 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -201,7 +201,6 @@ TransactionDb.prototype._fillOutpoints = function(txInfo, cb) { i.value = ret.valueSat / util.COIN; valueIn += i.valueSat; -console.log('[TransactionDb.js.204:ret:]',ret); //TODO if (ret.multipleSpentAttempt || !ret.spentTxId || (ret.spentTxId && ret.spentTxId !== txInfo.txid) ) { @@ -209,7 +208,6 @@ console.log('[TransactionDb.js.204:ret:]',ret); //TODO ret.multipleSpentAttempts.forEach(function(mul) { if (mul.spentTxId !== txInfo.txid) { -console.log('[TransactionDb.js.210]'); //TODO i.doubleSpentTxID = ret.spentTxId; i.doubleSpentIndex = ret.spentIndex; } @@ -218,7 +216,6 @@ console.log('[TransactionDb.js.210]'); //TODO i.dbError = 'Input spent not registered'; } else { -console.log('[TransactionDb.js.219]'); //TODO i.doubleSpentTxID = ret.spentTxId; i.doubleSpentIndex = ret.spentIndex; } @@ -409,7 +406,7 @@ TransactionDb.prototype.cacheScriptPubKey = function(txouts,cb) { -TransactionDb.prototype._parseAddrData = function(data) { +TransactionDb.prototype._parseAddrData = function(data, ignoreCache) { var k = data.key.split('-'); var v = data.value.split(':'); // console.log('[TransactionDb.js.375]',data.key,data.value); //TODO @@ -425,7 +422,7 @@ TransactionDb.prototype._parseAddrData = function(data) { // v[1]== isConfirmedCached // v[2]=== '1' -> is SpendCached -> [4]=spendTxId [5]=spentIndex [6]=spendTs // v[3]!== '1' -> is ScriptPubkey -> [[3] = scriptPubkey - if (v[1]){ + if (v[1] && !ignoreCache){ item.isConfirmed = 1; item.isConfirmedCached = 1; // console.log('[TransactionDb.js.356] CACHE HIT CONF:', item.key); //TODO @@ -448,7 +445,8 @@ TransactionDb.prototype._parseAddrData = function(data) { return item; }; -TransactionDb.prototype.fromAddr = function(addr, cb, txLimit) { +TransactionDb.prototype.fromAddr = function(addr, opts, cb) { + opts = opts || {}; var self = this; var k = ADDR_PREFIX + addr + '-'; var ret = []; @@ -456,10 +454,10 @@ TransactionDb.prototype.fromAddr = function(addr, cb, txLimit) { db.createReadStream({ start: k, end: k + '~', - limit: txLimit>0 ? txLimit: -1, // -1 means not limit + limit: opts.txLimit>0 ? opts.txLimit: -1, // -1 means not limit }) .on('data', function(data) { - ret.push(self._parseAddrData(data)); + ret.push(self._parseAddrData(data, opts.ignoreCache)); }) .on('error', cb) .on('end', function() { diff --git a/test/integration/addrCache.js b/test/integration/addrCache.js index bdc9252ec..4b847fc6e 100644 --- a/test/integration/addrCache.js +++ b/test/integration/addrCache.js @@ -93,6 +93,17 @@ describe('Address cache ', function() { return done(); },{txLimit:0}); }); + it('cache case 2 w ignore cache', function(done) { + var a = new Address('mt2AzeCorSf7yFckj19HFiXJgh9aNyc4h3', txDb); + a.update(function(err) { + if (err) done(err); + a.balance.should.equal(0, 'balance'); + a.totalReceived.should.equal(1376000, 'totalReceived'); + a.txApperances.should.equal(8003, 'txApperances'); + return done(); + },{txLimit:0, ignoreCache:1}); + }); + }); From 7becad01aafbce5e991eff03b94feebf654347ab Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Fri, 30 May 2014 14:19:13 -0300 Subject: [PATCH 2/2] add per call cache disable options --- README.md | 16 ++++++++-------- app/controllers/addresses.js | 14 +++++++------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 7e7c2e50c..5f2420cd0 100644 --- a/README.md +++ b/README.md @@ -32,10 +32,10 @@ are 3 different caches: * transaction spent information * scriptPubKey for unspent transactions -Cache data is only filled on request, i.e., only after accessing the required data for -the first time, the information is cached, there is not pre-caching procedure. - -To ignore cache, use INSIGHT_IGNORE_CACHE; +Cache data is only populated on request, i.e., only after accessing the required data for +the first time, the information is cached, there is not pre-caching procedure. To ignore +cache by default, use INSIGHT_IGNORE_CACHE. Also, address related calls support `?noCache=1` +to ignore the cache in a particular API request. ## Prerequisites @@ -167,12 +167,12 @@ The end-points are: ``` ### Address ``` - /api/addr/[:addr][?noTxList=1] + /api/addr/[:addr][?noTxList=1&noCache=1] /api/addr/mmvP3mTe53qxHdPqXEvdu8WdC7GfQ2vmx5?noTxList=1 ``` ### Unspent Outputs ``` - /api/addr/[:addr]/utxo + /api/addr/[:addr]/utxo[?noCache=1] ``` Sample return: ``` json @@ -197,8 +197,8 @@ Sample return: } ] ``` -Please not that in case confirmations are cached and are more that SAFE_CONFIRMATIONS setting, the -return can be a string of the form 'SAFE_CONFIRMATIONS+' +Please not that in case confirmations are cached and are more that INSIGHT_SAFE_CONFIRMATIONS setting, the +return can be a string of the form `SAFE_CONFIRMATIONS+`, e.g.: the string `6+` ### Unspent Outputs for multiple addresses diff --git a/app/controllers/addresses.js b/app/controllers/addresses.js index 3f405858d..7678d4f0f 100644 --- a/app/controllers/addresses.js +++ b/app/controllers/addresses.js @@ -53,7 +53,7 @@ exports.show = function(req, res, next) { } else { return res.jsonp(a.getObj()); } - }, {txLimit: req.query.noTxList?0:-1, ignoreCache: req.param('nocache')}); + }, {txLimit: req.query.noTxList?0:-1, ignoreCache: req.param('noCache')}); } }; @@ -68,7 +68,7 @@ exports.utxo = function(req, res, next) { else { return res.jsonp(a.unspent); } - }, {onlyUnspent:1, ignoreCache: req.param('nocache')}); + }, {onlyUnspent:1, ignoreCache: req.param('noCache')}); } }; @@ -81,7 +81,7 @@ exports.multiutxo = function(req, res, next) { if (err) callback(err); utxos = utxos.concat(a.unspent); callback(); - }, {onlyUnspent:1, ignoreCache: req.param('nocache')}); + }, {onlyUnspent:1, ignoreCache: req.param('noCache')}); }, function(err) { // finished callback if (err) return common.handleErrors(err, res); res.jsonp(utxos); @@ -99,7 +99,7 @@ exports.balance = function(req, res, next) { } else { return res.jsonp(a.balanceSat); } - }, {ignoreCache: req.param('nocache')}); + }, {ignoreCache: req.param('noCache')}); }; exports.totalReceived = function(req, res, next) { @@ -111,7 +111,7 @@ exports.totalReceived = function(req, res, next) { } else { return res.jsonp(a.totalReceivedSat); } - }, {ignoreCache: req.param('nocache')}); + }, {ignoreCache: req.param('noCache')}); }; exports.totalSent = function(req, res, next) { @@ -123,7 +123,7 @@ exports.totalSent = function(req, res, next) { } else { return res.jsonp(a.totalSentSat); } - }, {ignoreCache: req.param('nocache')}); + }, {ignoreCache: req.param('noCache')}); }; exports.unconfirmedBalance = function(req, res, next) { @@ -135,5 +135,5 @@ exports.unconfirmedBalance = function(req, res, next) { } else { return res.jsonp(a.unconfirmedBalanceSat); } - }, {ignoreCache: req.param('nocache')}); + }, {ignoreCache: req.param('noCache')}); };