diff --git a/lib/mempool/mempool.js b/lib/mempool/mempool.js index e6eea0f8f..28babb9cc 100644 --- a/lib/mempool/mempool.js +++ b/lib/mempool/mempool.js @@ -78,20 +78,37 @@ class Mempool extends EventEmitter { this.tip = this.network.genesis.hash; this.nextState = this.chain.state; - this.waiting = new BufferMap(); - this.orphans = new BufferMap(); - this.map = new BufferMap(); - this.spents = new BufferMap(); - this.claims = new BufferMap(); - this.airdrops = new BufferMap(); - this.airdropIndex = new Map(); - this.claimNames = new BufferMap(); + // "The" mempool + this.map = new BufferMap(); // hash -> MempoolEntry + this.claims = new BufferMap(); // hash -> ClaimEntry + this.airdrops = new BufferMap(); // hash -> AirdropEntry + + // Orphans and missing parents + this.waiting = new BufferMap(); // parent hash -> BufferSet[spender hashes] + this.orphans = new BufferMap(); // orphan tx hash -> Orphan + + // Prevent double-spends + this.spents = new BufferMap(); // prevout key -> MempoolEntry of spender + this.claimNames = new BufferMap(); // namehash -> ClaimEntry + this.airdropIndex = new Map(); // airdrop position -> AirdropEntry + + // Track namestates to validate incoming covenants + this.contracts = new ContractState(this.network); + + // Recently rejected txs by hash this.rejects = new RollingFilter(120000, 0.000001); + // TXs removed from the mempool because their + // parent TX was removed from a block and their + // covenants prevent them from joining in a block or mempool. + // Will be valid again once the parent is re-confirmed. + this.disconnected = + new BufferMap(); // parent hash -> BufferSet[child hashes] + this.children = new BufferMap(); // child tx hash -> TX (not MempoolEntry) + + // Extensions of blockchain indexes by tx hash for API this.coinIndex = new CoinIndex(); this.txIndex = new TXIndex(); - - this.contracts = new ContractState(this.network); } /** @@ -193,13 +210,6 @@ class Mempool extends EventEmitter { */ async _addBlock(block, txs, view) { - if (this.map.size === 0 - && this.claims.size === 0 - && this.airdrops.size === 0) { - this.tip = block.hash; - return; - } - const entries = []; const cb = txs[0]; @@ -209,11 +219,16 @@ class Mempool extends EventEmitter { const entry = this.getEntry(hash); if (!entry) { + // Confirmed TX was not in mempool + // maybe clear other references and conflicts this.removeOrphan(hash); this.removeDoubleSpends(tx); + this.removeDoubleOpens(tx); + + // Confirmed TX resolves an orphan if (this.waiting.has(hash)) await this.handleOrphans(tx); - this.removeDoubleOpens(tx); + continue; } @@ -221,6 +236,10 @@ class Mempool extends EventEmitter { this.emit('confirmed', tx, block); + // Confirmed TX reconnects a child + if (this.disconnected.has(hash)) + await this.reconnectSpenders(tx); + entries.push(entry); } @@ -382,6 +401,18 @@ class Mempool extends EventEmitter { if (this.hasEntry(hash)) continue; + // Some covenants can only be used once per name per block. + // If the TX we want to re-insert into the mempool conflicts + // with another TX already in the mempool because of this rule, + // the solution is to evict the child TX (the TX already in the + // mempool) and then insert the parent TX (from the disconnected block). + // Since the child TX spends the output of the parent TX, evicting the + // parent TX but keeping the child TX would leave the mempool in an + // invalid state, and the miner would produce invalid blocks. + // We can hang on to those evicted child TXs until they are valid again. + if (this.contracts.hasNames(tx)) + this.disconnectSpenders(tx); + try { await this.insertTX(tx, -1); total += 1; @@ -557,6 +588,9 @@ class Mempool extends EventEmitter { this.claimNames.clear(); this.spents.clear(); this.contracts.clear(); + this.disconnected.clear(); + this.children.clear(); + this.coinIndex.reset(); this.txIndex.reset(); @@ -1851,6 +1885,75 @@ class Mempool extends EventEmitter { } } + /** + * Recursively disconnect spenders of a transaction. + * @private + * @param {TX} parent + */ + + disconnectSpenders(parent) { + const parentHash = parent.hash(); + const children = new BufferSet(); + const names = new BufferSet(); + rules.addNames(parent, names); + + for (let i = 0; i < parent.outputs.length; i++) { + const spender = this.getSpent(parentHash, i); // MempoolEntry + + // No child spending this output + if (!spender) + continue; + + const {tx} = spender; + + // Child is not linked by name. + // Covenant rules don't prevent it from + // staying in the mempool. + if (!rules.hasNames(tx, names)) + continue; + + // This child has to be removed from the mempool + // but it will be valid again once its parent confirms. + this.disconnectSpenders(tx); + children.add(tx.hash()); + this.children.set(tx.hash(), tx); + this.removeEntry(spender); + } + + if (children.size) + this.disconnected.set(parentHash, children); + } + + /** + * Recursively reconnect spenders of a transaction. + * @private + * @param {TX} parent + */ + + async reconnectSpenders(parent) { + const hashes = this.disconnected.get(parent.hash()); + + if (!hashes) + return; + + for (const hash of hashes) { + const tx = this.children.get(hash); + assert(tx); + + try { + const missing = await this.insertTX(tx, -1); + if (missing) + throw new Error('spender is orphan.'); + // Recurse only on success + await this.reconnectSpenders(tx); + } catch (err) { + this.logger.debug( + 'Could not reconnect spender %x: %s.', + tx.hash(), err.message); + } + } + } + /** * Count the highest number of * ancestors a transaction may have. @@ -3098,7 +3201,7 @@ class Orphan { * Create an orphan. * @constructor * @param {TX} tx - * @param {Hash[]} missing + * @param {Number} missing * @param {Number} id */ diff --git a/test/mempool-reorg-test.js b/test/mempool-reorg-test.js new file mode 100644 index 000000000..7b59ce12a --- /dev/null +++ b/test/mempool-reorg-test.js @@ -0,0 +1,168 @@ +'use strict'; + +const assert = require('bsert'); +const Network = require('../lib/protocol/network'); +const {Resource} = require('../lib/dns/resource'); +const FullNode = require('../lib/node/fullnode'); +const plugin = require('../lib/wallet/plugin'); + +const network = Network.get('regtest'); +const { + treeInterval, + biddingPeriod, + revealPeriod +} = network.names; + +describe('Mempool Covenant Reorg', function () { + const node = new FullNode({ + network: 'regtest' + }); + node.use(plugin); + + let wallet, name; + + before(async () => { + await node.open(); + wallet = node.get('walletdb').wdb.primary; + }); + + after(async () => { + await node.close(); + }); + + let counter = 0; + function makeResource() { + return Resource.fromJSON({ + records: [{type: 'TXT', txt: [`${counter++}`]}] + }); + } + + async function getConfirmedResource() { + const res = await node.rpc.getNameResource([name]); + return res.records[0].txt[0]; + } + + function getMempoolResource() { + if (!node.mempool.map.size) + return null; + + assert.strictEqual(node.mempool.map.size, 1); + const {tx} = node.mempool.map.toValues()[0]; + const res = Resource.decode(tx.outputs[0].covenant.items[2]); + return res.records[0].txt[0]; + } + + it('should fund wallet and win name', async () => { + await node.rpc.generate([10]); + name = await node.rpc.grindName([3]); + await wallet.sendOpen(name, true); + await node.rpc.generate([treeInterval + 1]); + await wallet.sendBid(name, 10000, 20000); + await node.rpc.generate([biddingPeriod]); + await wallet.sendReveal(name); + await node.rpc.generate([revealPeriod]); + await node.rpc.generate([1]); + await wallet.sendUpdate(name, makeResource()); + await node.rpc.generate([1]); + + assert.strictEqual(await getConfirmedResource(), '0'); + }); + + it('should generate UPDATE chain', async () => { + for (let i = 0; i < 10; i++) { + await wallet.sendUpdate( + name, + makeResource(), + {selection: 'age'} // avoid spending coinbase early + ); + await node.rpc.generate([1]); + } + }); + + it('should shallow reorg chain', async () => { + // Initial state + assert.strictEqual(await getConfirmedResource(), '10'); + + // Mempool is empty + assert.strictEqual(getMempoolResource(), null); + + // Do not reorg beyond tree interval + assert(node.chain.height % treeInterval === 3); + + // Reorganize + const waiter = new Promise((resolve) => { + node.once('reorganize', () => { + resolve(); + }); + }); + + const depth = 3; + let entry = await node.chain.getEntryByHeight(node.chain.height - depth); + for (let i = 0; i <= depth; i++) { + const block = await node.miner.cpu.mineBlock(entry); + entry = await node.chain.add(block); + } + await waiter; + + // State after reorg + assert.strictEqual(await getConfirmedResource(), '7'); + + // Mempool is NOT empty, "next" tx is waiting + assert.strictEqual(getMempoolResource(), '8'); + + // This next block would be invalid in our own chain + // if mempool was corrupted with the wrong tx from the reorg. + await node.rpc.generate([1]); + + // State after new block + assert.strictEqual(await getConfirmedResource(), '8'); + + // Mempool is again NOT empty, "next NEXT" tx is waiting + assert.strictEqual(getMempoolResource(), '9'); + + // One more + await node.rpc.generate([1]); + assert.strictEqual(await getConfirmedResource(), '9'); + assert.strictEqual(getMempoolResource(), '10'); + + // Finally + await node.rpc.generate([1]); + assert.strictEqual(await getConfirmedResource(), '10'); + assert.strictEqual(getMempoolResource(), null); + }); + + it('should deep reorg chain', async () => { + // Initial state + assert.strictEqual(await getConfirmedResource(), '10'); + + // Mempool is empty + assert.strictEqual(getMempoolResource(), null); + + // Reorganize beyond tree interval + const waiter = new Promise((resolve) => { + node.once('reorganize', () => { + resolve(); + }); + }); + + const depth = 12; + let entry = await node.chain.getEntryByHeight(node.chain.height - depth); + // Intentionally forking from historical tree interval requires dirty hack + const {treeRoot} = await node.chain.getEntryByHeight(node.chain.height - depth + 1); + await node.chain.db.tree.inject(treeRoot); + for (let i = 0; i <= depth; i++) { + const block = await node.miner.cpu.mineBlock(entry); + entry = await node.chain.add(block); + } + await waiter; + + // State after reorg + assert.strictEqual(await getConfirmedResource(), '2'); + assert.strictEqual(getMempoolResource(), '3'); + + // Confirm entire update chain one by one + await node.rpc.generate([8]); + assert.strictEqual(await getConfirmedResource(), '10'); + assert.strictEqual(getMempoolResource(), null); + }); +}); diff --git a/test/mempool-test.js b/test/mempool-test.js index 419022119..178056356 100644 --- a/test/mempool-test.js +++ b/test/mempool-test.js @@ -581,18 +581,18 @@ describe('Mempool', function() { } const cb = mtx.toTX(); - const [block] = await getMockBlock(chain, [cb], false); + const [block, view] = await getMockBlock(chain, [cb], false); const entry = await chain.add(block, VERIFY_BODY); - await mempool._addBlock(entry, block.txs); + await mempool.addBlock(entry, block.txs, view); // Add 100 blocks so we don't get // premature spend of coinbase. for (let i = 0; i < 100; i++) { - const [block] = await getMockBlock(chain); + const [block, view] = await getMockBlock(chain); const entry = await chain.add(block, VERIFY_BODY); - await mempool._addBlock(entry, block.txs); + await mempool.addBlock(entry, block.txs, view); } chaincoins.addTX(cb); @@ -653,8 +653,66 @@ describe('Mempool', function() { // Ensure mempool contents are valid in next block const [newBlock, newView] = await getMockBlock(chain, [tx1, tx2]); const newEntry = await chain.add(newBlock, VERIFY_BODY); + await mempool.addBlock(newEntry, newBlock.txs, newView); + assert.strictEqual(mempool.map.size, 0); + }); + + it('should insert resolved orphan tx after parent confirmed', async () => { + await mempool.reset(); + // Mempool is empty + assert.strictEqual(mempool.map.size, 0); + // No orphans either + assert.strictEqual(mempool.waiting.size, 0); + assert.strictEqual(mempool.orphans.size, 0); + + // Create first TX + const coin1 = chaincoins.getCoins()[0]; + const addr = wallet.createReceive().getAddress(); + const mtx1 = new MTX(); + mtx1.addCoin(coin1); + mtx1.addOutput(addr, 90000); + chaincoins.sign(mtx1); + const tx1 = mtx1.toTX(); + chaincoins.addTX(tx1); + wallet.addTX(tx1); + + // Create second TX, spending output of first + const mtx2 = new MTX(); + mtx2.addTX(tx1, 0); + mtx2.addOutput(addr, 80000); + wallet.sign(mtx2); + const tx2 = mtx2.toTX(); + chaincoins.addTX(tx2); + wallet.addTX(tx2); + + // Attempt to add second TX to mempool + await mempool.addTX(tx2); + + // tx2 is orphan waiting on tx1 + assert.strictEqual(mempool.map.size, 0); + assert.strictEqual(mempool.waiting.size, 1); + assert.strictEqual(mempool.orphans.size, 1); + assert(mempool.waiting.has(tx1.hash())); + assert(mempool.orphans.has(tx2.hash())); + + // Confirm tx1 in a block + const [block, view] = await getMockBlock(chain, [tx1], true); + const entry = await chain.add(block, VERIFY_BODY); + await mempool.addBlock(entry, block.txs, view); + + // tx2 has been resolved back in to mempool + assert.strictEqual(mempool.map.size, 1); + assert.strictEqual(mempool.waiting.size, 0); + assert.strictEqual(mempool.orphans.size, 0); + assert(mempool.map.has(tx2.hash())); + + // Ensure mempool contents are valid in next block + const [newBlock, newView] = await getMockBlock(chain, [tx2]); + const newEntry = await chain.add(newBlock, VERIFY_BODY); await mempool._addBlock(newEntry, newBlock.txs, newView); assert.strictEqual(mempool.map.size, 0); + assert.strictEqual(mempool.waiting.size, 0); + assert.strictEqual(mempool.orphans.size, 0); }); it('should handle reorg: coinbase spends', async () => { @@ -673,7 +731,7 @@ describe('Mempool', function() { // Add it to block and mempool const [block1, view1] = await getMockBlock(chain, [cb], false); const entry1 = await chain.add(block1, VERIFY_BODY); - await mempool._addBlock(entry1, block1.txs, view1); + await mempool.addBlock(entry1, block1.txs, view1); // The coinbase output is a valid UTXO in the chain assert(await chain.getCoin(cb.hash(), 0)); @@ -704,7 +762,7 @@ describe('Mempool', function() { [block2, view2] = await getMockBlock(chain); entry2 = await chain.add(block2, VERIFY_BODY); - await mempool._addBlock(entry2, block2.txs, view2); + await mempool.addBlock(entry2, block2.txs, view2); } // Try again @@ -717,7 +775,7 @@ describe('Mempool', function() { // Confirm coinbase spend in a block const [block3, view3] = await getMockBlock(chain, [spend]); const entry3 = await chain.add(block3, VERIFY_BODY); - await mempool._addBlock(entry3, block3.txs, view3); + await mempool.addBlock(entry3, block3.txs, view3); // Coinbase spend has been removed from the mempool assert.strictEqual(mempool.map.size, 0); @@ -767,7 +825,7 @@ describe('Mempool', function() { // Add it to block and mempool const [block1, view1] = await getMockBlock(chain, [fund]); const entry1 = await chain.add(block1, VERIFY_BODY); - await mempool._addBlock(entry1, block1.txs, view1); + await mempool.addBlock(entry1, block1.txs, view1); // The fund TX output is a valid UTXO in the chain const spendCoin = await chain.getCoin(fund.hash(), 0); @@ -794,7 +852,7 @@ describe('Mempool', function() { // Confirm spend into block const [block2, view2] = await getMockBlock(chain, [spend]); const entry2 = await chain.add(block2, VERIFY_BODY); - await mempool._addBlock(entry2, block2.txs, view2); + await mempool.addBlock(entry2, block2.txs, view2); // Spend has been removed from the mempool assert.strictEqual(mempool.map.size, 0); @@ -832,7 +890,7 @@ describe('Mempool', function() { // Ensure mempool contents are valid in next block const [newBlock, newView] = await getMockBlock(chain, [fund]); const newEntry = await chain.add(newBlock, VERIFY_BODY); - await mempool._addBlock(newEntry, newBlock.txs, newView); + await mempool.addBlock(newEntry, newBlock.txs, newView); assert.strictEqual(mempool.map.size, 0); }); @@ -862,7 +920,7 @@ describe('Mempool', function() { // Add it to block and mempool const [block1, view1] = await getMockBlock(chain, [open]); const entry1 = await chain.add(block1, VERIFY_BODY); - await mempool._addBlock(entry1, block1.txs, view1); + await mempool.addBlock(entry1, block1.txs, view1); // The open TX output is a valid UTXO in the chain assert(await chain.getCoin(open.hash(), 0)); @@ -909,7 +967,7 @@ describe('Mempool', function() { [block2, view2] = await getMockBlock(chain); entry2 = await chain.add(block2, VERIFY_BODY); - await mempool._addBlock(entry2, block2.txs, view2); + await mempool.addBlock(entry2, block2.txs, view2); } // BIDDING is activated in the next block @@ -930,7 +988,7 @@ describe('Mempool', function() { // Confirm bid into block const [block3, view3] = await getMockBlock(chain, [bid]); const entry3 = await chain.add(block3, VERIFY_BODY); - await mempool._addBlock(entry3, block3.txs, view3); + await mempool.addBlock(entry3, block3.txs, view3); // Bid has been removed from the mempool assert.strictEqual(mempool.map.size, 0); @@ -1028,7 +1086,7 @@ describe('Mempool', function() { try { ownership.ignore = true; entry2 = await chain.add(block2, VERIFY_BODY); - await mempool._addBlock(entry2, block2.txs, view2); + await mempool.addBlock(entry2, block2.txs, view2); } finally { ownership.ignore = false; } @@ -1095,7 +1153,7 @@ describe('Mempool', function() { [block2, view2] = await getMockBlock(chain); entry2 = await chain.add(block2, VERIFY_BODY); - await mempool._addBlock(entry2, block2.txs, view2); + await mempool.addBlock(entry2, block2.txs, view2); } // Update the claim with a *very recent* block commitment @@ -1136,7 +1194,7 @@ describe('Mempool', function() { try { ownership.ignore = true; entry3 = await chain.add(block3, VERIFY_BODY); - await mempool._addBlock(entry3, block3.txs, view3); + await mempool.addBlock(entry3, block3.txs, view3); } finally { ownership.ignore = false; } @@ -1271,18 +1329,18 @@ describe('Mempool', function() { } const cb = mtx.toTX(); - const [block] = await getMockBlock(chain, [cb], false); + const [block, view] = await getMockBlock(chain, [cb], false); const entry = await chain.add(block, VERIFY_BODY); - await mempool._addBlock(entry, block.txs); + await mempool.addBlock(entry, block.txs, view); // Add 100 blocks so we don't get // premature spend of coinbase. for (let i = 0; i < 100; i++) { - const [block] = await getMockBlock(chain); + const [block, view] = await getMockBlock(chain); const entry = await chain.add(block, VERIFY_BODY); - await mempool._addBlock(entry, block.txs); + await mempool.addBlock(entry, block.txs, view); } chaincoins.addTX(cb);