From 0d8469427af33a052db68a2bca3494b46ca77dc3 Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Thu, 20 Jul 2023 23:32:44 +0530 Subject: [PATCH] feat: Headers-Sync --- bin/bcoin | 5 +- bin/neutrino | 43 +++++ lib/bcoin-browser.js | 1 + lib/bcoin.js | 1 + lib/blockchain/chain.js | 37 ++++- lib/blockchain/chaindb.js | 63 +++++++- lib/blockchain/layout.js | 3 + lib/net/peer.js | 12 ++ lib/net/pool.js | 111 +++++++++++-- lib/node/neutrino.js | 319 ++++++++++++++++++++++++++++++++++++++ lib/protocol/networks.js | 2 +- 11 files changed, 577 insertions(+), 20 deletions(-) create mode 100755 bin/neutrino create mode 100644 lib/node/neutrino.js diff --git a/bin/bcoin b/bin/bcoin index 0df74c54f..990d72f19 100755 --- a/bin/bcoin +++ b/bin/bcoin @@ -43,7 +43,10 @@ for arg in "$@"; do --daemon) daemon=1 ;; - --spv) + --neutrino) + cmd='neutrino' + ;; + --spv) cmd='spvnode' ;; esac diff --git a/bin/neutrino b/bin/neutrino new file mode 100755 index 000000000..7872de324 --- /dev/null +++ b/bin/neutrino @@ -0,0 +1,43 @@ +#!/usr/bin/env node + +'use strict'; + +console.log('Starting bcoin'); +process.title = 'bcoin'; +const Neutrino = require('../lib/node/neutrino'); + +const node = new Neutrino({ + file: true, + argv: true, + env: true, + logFile: true, + logConsole: true, // todo: remove + logLevel: 'debug', // todo: remove + db: 'leveldb', + memory: false, + workers: true, + loader: require +}); + +if (!node.config.bool('no-wallet') && !node.has('walletdb')) { + const plugin = require('../lib/wallet/plugin'); + node.use(plugin); +} + +(async () => { + await node.ensure(); + await node.open(); + await node.connect(); + node.startSync(); +})().catch((err) => { + console.error(err.stack); + process.exit(1); +}); + +process.on('unhandledRejection', (err, promise) => { + throw err; +}); + +process.on('SIGINT', async () => { + await node.close(); +}); diff --git a/lib/bcoin-browser.js b/lib/bcoin-browser.js index 1f7254be8..8b2d46cb5 100644 --- a/lib/bcoin-browser.js +++ b/lib/bcoin-browser.js @@ -89,6 +89,7 @@ bcoin.node = require('./node'); bcoin.Node = require('./node/node'); bcoin.FullNode = require('./node/fullnode'); bcoin.SPVNode = require('./node/spvnode'); +bcoin.Neutrino = require('./node/neutrino'); // Primitives bcoin.primitives = require('./primitives'); diff --git a/lib/bcoin.js b/lib/bcoin.js index a2edf9e78..4cc519e74 100644 --- a/lib/bcoin.js +++ b/lib/bcoin.js @@ -123,6 +123,7 @@ bcoin.define('node', './node'); bcoin.define('Node', './node/node'); bcoin.define('FullNode', './node/fullnode'); bcoin.define('SPVNode', './node/spvnode'); +bcoin.define('Neutrino', './node/neutrino'); // Primitives bcoin.define('primitives', './primitives'); diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 9cd0a312f..6537e72ef 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -1791,6 +1791,24 @@ class Chain extends AsyncEmitter { return this.hasEntry(hash); } + async getCFHeaderHeight() { + return await this.db.getCFHeaderHeight(); + } + + async saveCFHeaderHeight(height) { + this.db.neutrinoState.headerHeight = height; + await this.db.saveNeutrinoState(); + } + + async getCFilterHeight() { + return await this.db.getCFilterHeight(); + } + + async saveCFilterHeight(height) { + this.db.neutrinoState.filterHeight = height; + await this.db.saveNeutrinoState(); + } + /** * Find the corresponding block entry by hash or height. * @param {Hash|Number} hash/height @@ -2003,19 +2021,22 @@ class Chain extends AsyncEmitter { if (this.synced) return; + if (this.options.neutrino && this.getProgress() < 1) + return; if (this.options.checkpoints) { if (this.height < this.network.lastCheckpoint) return; - } - - if (this.tip.time < util.now() - this.network.block.maxTipAge) + } else if (!this.options.neutrino && + this.tip.time < util.now() - this.network.block.maxTipAge) return; if (!this.hasChainwork()) return; - this.synced = true; - this.emit('full'); + if (this.options.neutrino) + this.emit('headersFull'); + else + this.emit('full'); } /** @@ -2616,6 +2637,7 @@ class ChainOptions { this.compression = true; this.spv = false; + this.neutrino = false; this.bip91 = false; this.bip148 = false; this.prune = false; @@ -2662,6 +2684,11 @@ class ChainOptions { this.spv = options.spv; } + if (options.neutrino != null) { + assert(typeof options.neutrino === 'boolean'); + this.neutrino = options.neutrino; + } + if (options.prefix != null) { assert(typeof options.prefix === 'string'); this.prefix = options.prefix; diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index cb91accaa..c4d2e0671 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -46,6 +46,7 @@ class ChainDB { this.state = new ChainState(); this.pending = null; this.current = null; + this.neutrinoState = null; this.cacheHash = new LRU(this.options.entryCache, null, BufferMap); this.cacheHeight = new LRU(this.options.entryCache); @@ -90,6 +91,11 @@ class ChainDB { this.logger.info('ChainDB successfully initialized.'); } + if (this.options.neutrino) { + if (!this.neutrinoState) + this.neutrinoState = await this.getNeutrinoState(); + } + this.logger.info( 'Chain State: hash=%h tx=%d coin=%d value=%s.', this.state.tip, @@ -1001,7 +1007,7 @@ class ChainDB { */ async getRawBlock(block) { - if (this.options.spv) + if (this.options.spv && !this.options.neutrino) return null; const hash = await this.getHash(block); @@ -1670,6 +1676,39 @@ class ChainDB { b.put(layout.O.encode(), flags.toRaw()); return b.write(); } + + /** + * Get Neutrino State + * @returns {Promise} - Returns neutrino state + */ + + async getNeutrinoState() { + const data = await this.db.get(layout.N.encode()); + if (!data) + return new NeutrinoState(); + return NeutrinoState.fromRaw(data); + } + + async getCFHeaderHeight() { + const state = await this.getNeutrinoState(); + return state.headerHeight; + } + + async getCFilterHeight() { + const state = await this.getNeutrinoState(); + return state.filterHeight; + } + + /** + * Save Neutrino State + * @returns {void} + */ + async saveNeutrinoState() { + const state = this.neutrinoState.toRaw(); + const b = this.db.batch(); + b.put(layout.N.encode(), state); + return b.write(); + } } /** @@ -1952,6 +1991,28 @@ function fromU32(num) { return data; } +class NeutrinoState { + constructor() { // TODO: do we add support for multiple filters? + this.headerHeight = 0; + this.filterHeight = 0; + } + + toRaw() { + const bw = bio.write(8); + bw.writeU32(this.headerHeight); + bw.writeU32(this.filterHeight); + return bw.render(); + } + + static fromRaw(data) { + const state = new NeutrinoState(); + const br = bio.read(data); + state.headerHeight = br.readU32(); + state.filterHeight = br.readU32(); + return state; + } +} + /* * Expose */ diff --git a/lib/blockchain/layout.js b/lib/blockchain/layout.js index 337f95900..532ccb050 100644 --- a/lib/blockchain/layout.js +++ b/lib/blockchain/layout.js @@ -14,6 +14,7 @@ const bdb = require('bdb'); * O -> chain options * R -> tip hash * D -> versionbits deployments + * N -> neutrino state * e[hash] -> entry * h[hash] -> height * H[height] -> hash @@ -33,6 +34,8 @@ const layout = { O: bdb.key('O'), R: bdb.key('R'), D: bdb.key('D'), + N: bdb.key('N'), + F: bdb.key('H', ['hash256']), e: bdb.key('e', ['hash256']), h: bdb.key('h', ['hash256']), H: bdb.key('H', ['uint32']), diff --git a/lib/net/peer.js b/lib/net/peer.js index 2271e7896..dac2e265d 100644 --- a/lib/net/peer.js +++ b/lib/net/peer.js @@ -1449,6 +1449,12 @@ class Peer extends EventEmitter { if (!(this.services & services.NETWORK)) throw new Error('Peer does not support network services.'); + if (this.options.neutrino) { + if (!(this.services & services.NODE_COMPACT_FILTERS)) { + throw new Error('Peer does not support Compact Filters.'); + } + } + if (this.options.headers) { if (this.version < common.HEADERS_VERSION) throw new Error('Peer does not support getheaders.'); @@ -2080,6 +2086,7 @@ class PeerOptions { this.agent = common.USER_AGENT; this.noRelay = false; this.spv = false; + this.neutrino = false; this.compact = false; this.headers = false; this.banScore = common.BAN_SCORE; @@ -2143,6 +2150,11 @@ class PeerOptions { this.spv = options.spv; } + if (options.neutrino != null) { + assert(typeof options.neutrino === 'boolean'); + this.neutrino = options.neutrino; + } + if (options.compact != null) { assert(typeof options.compact === 'boolean'); this.compact = options.compact; diff --git a/lib/net/pool.js b/lib/net/pool.js index 234b23bc2..fcf11a3b8 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -204,16 +204,22 @@ class Pool extends EventEmitter { */ resetChain() { - if (!this.options.checkpoints) + if (!this.options.checkpoints && !this.options.neutrino) return; - this.checkpoints = false; + if (!this.options.neutrino) + this.checkpoints = false; this.headerTip = null; this.headerChain.reset(); this.headerNext = null; const tip = this.chain.tip; - + if (this.options.neutrino) { + this.headerChain.push(new HeaderEntry(tip.hash, tip.height)); + this.cfHeaderChain = new List(); + this.cfHeaderChain.push(new CFHeaderEntry(consensus.ZERO_HASH, 0)); + return; + } if (tip.height < this.network.lastCheckpoint) { this.checkpoints = true; this.headerTip = this.getNextTip(tip.height); @@ -650,7 +656,10 @@ class Pool extends EventEmitter { return; this.syncing = true; - this.resync(false); + if (this.options.neutrino) { + this.startHeadersSync(); + } else + this.resync(false); } /** @@ -704,6 +713,32 @@ class Pool extends EventEmitter { this.compactBlocks.clear(); } + /** + * Start the headers sync using getHeaders messages. + * @private + * @return {Promise} + */ + + async startHeadersSync() { + if (!this.syncing) + return; + let locator; + try { + locator = await this.chain.getLocator(); + } catch (e) { + this.emit('error', e); + return; + } + + const peer = this.peers.load; + if (!peer) { + this.logger.info('No loader peer.'); + return; + } + this.chain.synced = false; + peer.sendGetHeaders(locator); + } + /** * Send a sync to each peer. * @private @@ -814,7 +849,12 @@ class Pool extends EventEmitter { peer.syncing = true; peer.blockTime = Date.now(); - + if (this.options.neutrino) { + peer.sendGetHeaders(locator); + if (!this.syncing) + this.startFilterHeadersSync(); + return true; + } if (this.checkpoints) { peer.sendGetHeaders(locator, this.headerTip.hash); return true; @@ -1634,6 +1674,13 @@ class Pool extends EventEmitter { if (this.options.hasWitness() && !peer.hasWitness()) return; + if (this.options.neutrino) { + const filterHeight = await this.chain.getCFilterHeight(); + if (filterHeight === this.chain.height) + this.startSync(); + return; + } + // Request headers instead. if (this.checkpoints) return; @@ -1698,6 +1745,8 @@ class Pool extends EventEmitter { */ async handleTXInv(peer, hashes) { + if (this.options.neutrino) + return; assert(hashes.length > 0); if (this.syncing && !this.chain.synced) @@ -2027,6 +2076,9 @@ class Pool extends EventEmitter { if (this.options.selfish) return; + if (this.options.neutrino) + return; + if (this.chain.options.spv) return; @@ -2139,7 +2191,8 @@ class Pool extends EventEmitter { async _handleHeaders(peer, packet) { const headers = packet.items; - if (!this.checkpoints) + if (!this.checkpoints && !this.options.neutrino) + // todo add support for checkpoints return; if (!this.syncing) @@ -2179,13 +2232,14 @@ class Pool extends EventEmitter { this.logger.warning( 'Peer sent a bad header chain (%s).', peer.hostname()); - peer.destroy(); + peer.increaseBan(10); return; } node = new HeaderEntry(hash, height); - if (node.height === this.headerTip.height) { + if (!this.options.neutrino && node.height === this.headerTip.height) { + // todo add support for checkpoints if (!node.hash.equals(this.headerTip.hash)) { this.logger.warning( 'Peer sent an invalid checkpoint (%s).', @@ -2200,6 +2254,8 @@ class Pool extends EventEmitter { this.headerNext = node; this.headerChain.push(node); + if (this.options.neutrino) + await this._addBlock(peer, header, chainCommon.flags.VERIFY_POW); } this.logger.debug( @@ -2212,14 +2268,19 @@ class Pool extends EventEmitter { peer.blockTime = Date.now(); // Request the blocks we just added. - if (checkpoint) { + if (checkpoint && !this.options.neutrino) { this.headerChain.shift(); this.resolveHeaders(peer); return; } // Request more headers. - peer.sendGetHeaders([node.hash], this.headerTip.hash); + if (this.chain.synced) + return; + if (this.options.neutrino) + peer.sendGetHeaders([node.hash]); + else + peer.sendGetHeaders([node.hash], this.headerTip.hash); } /** @@ -2293,7 +2354,7 @@ class Pool extends EventEmitter { const hash = block.hash(); - if (!this.resolveBlock(peer, hash)) { + if (!this.options.neutrino && !this.resolveBlock(peer, hash)) { this.logger.warning( 'Received unrequested block: %h (%s).', block.hash(), peer.hostname()); @@ -3690,6 +3751,7 @@ class PoolOptions { this.prefix = null; this.checkpoints = true; this.spv = false; + this.neutrino = false; this.bip37 = false; this.bip157 = false; this.listen = false; @@ -3772,12 +3834,17 @@ class PoolOptions { if (options.spv != null) { assert(typeof options.spv === 'boolean'); - assert(options.spv === this.chain.options.spv); this.spv = options.spv; } else { this.spv = this.chain.options.spv; } + if (options.neutrino != null) { + assert(options.compact !== true); + assert(typeof options.neutrino === 'boolean'); + this.neutrino = options.neutrino; + } + if (options.bip37 != null) { assert(typeof options.bip37 === 'boolean'); this.bip37 = options.bip37; @@ -3953,6 +4020,12 @@ class PoolOptions { this.listen = false; } + if (this.neutrino) { + this.requiredServices |= common.services.NODE_COMPACT_FILTERS; + this.checkpoints = true; + this.compact = false; + } + if (this.selfish) { this.services &= ~common.services.NETWORK; this.bip37 = false; @@ -4494,6 +4567,20 @@ class HeaderEntry { } } +class CFHeaderEntry { + /** + * Create cfheader entry. + * @constructor + */ + + constructor(hash, height) { + this.hash = hash; + this.height = height; + this.prev = null; + this.next = null; + } +} + /* * Expose */ diff --git a/lib/node/neutrino.js b/lib/node/neutrino.js new file mode 100644 index 000000000..8077943f2 --- /dev/null +++ b/lib/node/neutrino.js @@ -0,0 +1,319 @@ +/*! + * neutrino.js - spv node for bcoin + * Copyright (c) 2023, Manav Desai (MIT License) + * Copyright (c) 2023, Shaswat Gupta (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +const assert = require('bsert'); +const Chain = require('../blockchain/chain'); +const Pool = require('../net/pool'); +const Node = require('./node'); +const HTTP = require('./http'); +const RPC = require('./rpc'); +const blockstore = require('../blockstore'); +const FilterIndexer = require('../indexer/filterindexer'); + +/** + * Neutrino Node + * Create a neutrino node which only maintains + * a chain, a pool, and an http server. + * @alias module:node.Neutrino + * @extends Node + */ + +class Neutrino extends Node { + /** + * Create Neutrino node. + * @constructor + * @param {Object?} options + * @param {Buffer?} options.sslKey + * @param {Buffer?} options.sslCert + * @param {Number?} options.httpPort + * @param {String?} options.httpHost + */ + + constructor(options) { + super('bcoin', 'bcoin.conf', 'debug.log', options); + + this.opened = false; + + // SPV flag. + this.spv = false; + this.neutrino = true; + + // Instantiate block storage. + this.blocks = blockstore.create({ + network: this.network, + logger: this.logger, + prefix: this.config.prefix, + cacheSize: this.config.mb('block-cache-size'), + memory: this.memory, + spv: this.spv, + neutrino: this.neutrino + }); + + this.chain = new Chain({ + blocks: this.blocks, + network: this.network, + logger: this.logger, + prefix: this.config.prefix, + memory: this.memory, + maxFiles: this.config.uint('max-files'), + cacheSize: this.config.mb('cache-size'), + entryCache: this.config.uint('entry-cache'), + forceFlags: this.config.bool('force-flags'), + checkpoints: this.config.bool('checkpoints'), + bip91: this.config.bool('bip91'), + bip148: this.config.bool('bip148'), + spv: true, + neutrino: this.neutrino + }); + + this.filterIndexers.set( + 'BASIC', + new FilterIndexer({ + network: this.network, + logger: this.logger, + blocks: this.blocks, + chain: this.chain, + memory: this.config.bool('memory'), + prefix: this.config.str('index-prefix', this.config.prefix), + filterType: 'BASIC', + neutrino: true + }) + ); + + this.pool = new Pool({ + network: this.network, + logger: this.logger, + chain: this.chain, + prefix: this.config.prefix, + checkpoints: true, + filterIndexers: this.filterIndexers, + proxy: this.config.str('proxy'), + onion: this.config.bool('onion'), + upnp: this.config.bool('upnp'), + seeds: this.config.array('seeds'), + nodes: this.config.array('nodes'), + only: this.config.array('only'), + maxOutbound: this.config.uint('max-outbound'), + createSocket: this.config.func('create-socket'), + memory: this.memory, + selfish: true, + listen: false, + neutrino: this.neutrino, + spv: this.spv + }); + + this.rpc = new RPC(this); + + this.http = new HTTP({ + network: this.network, + logger: this.logger, + node: this, + prefix: this.config.prefix, + ssl: this.config.bool('ssl'), + keyFile: this.config.path('ssl-key'), + certFile: this.config.path('ssl-cert'), + host: this.config.str('http-host'), + port: this.config.uint('http-port'), + apiKey: this.config.str('api-key'), + noAuth: this.config.bool('no-auth'), + cors: this.config.bool('cors') + }); + + this.init(); + } + + /** + * Initialize the node. + * @private + */ + + init() { + // Bind to errors + this.chain.on('error', err => this.error(err)); + this.pool.on('error', err => this.error(err)); + + if (this.http) + this.http.on('error', err => this.error(err)); + + this.chain.on('block', (block) => { + this.emit('block', block); + }); + + this.chain.on('connect', async (entry, block) => { + this.emit('connect', entry, block); + }); + + this.chain.on('disconnect', (entry, block) => { + this.emit('disconnect', entry, block); + }); + + this.chain.on('reorganize', (tip, competitor) => { + this.emit('reorganize', tip, competitor); + }); + + this.chain.on('reset', (tip) => { + this.emit('reset', tip); + }); + + this.chain.on('headersFull', async () => { + if (this.chain.height === 0) + return; + this.logger.info('Block Headers are fully synced'); + await this.pool.startFilterHeadersSync(); + }); + + this.pool.on('cfheaders', async () => { + if (this.chain.height === 0) + return; + this.logger.info('Filter Headers are fully synced'); + await this.pool.startFilterSync(); + }); + + this.loadPlugins(); + } + + /** + * Open the node and all its child objects, + * wait for the database to load. + * @returns {Promise} + */ + + async open() { + assert(!this.opened, 'Neutrino Node is already open.'); + this.opened = true; + + await this.handlePreopen(); + await this.blocks.open(); + await this.chain.open(); + await this.pool.open(); + + await this.openPlugins(); + + await this.http.open(); + await this.handleOpen(); + + for (const filterindex of this.filterIndexers.values()) { + await filterindex.open(); + } + + this.logger.info('Node is loaded.'); + } + + /** + * Close the node, wait for the database to close. + * @returns {Promise} + */ + + async close() { + assert(this.opened, 'Neutrino Node is not open.'); + this.opened = false; + + await this.handlePreclose(); + await this.http.close(); + + await this.closePlugins(); + + await this.pool.close(); + await this.chain.close(); + await this.handleClose(); + + for (const filterindex of this.filterIndexers.values()) { + await filterindex.close(); + } + } + + /** + * Scan for any missed transactions. + * Note that this will replay the blockchain sync. + * @param {Number|Hash} start - Start block. + * @returns {Promise} + */ + + async scan(start) { + throw new Error('Not implemented.'); + } + + /** + * Broadcast a transaction (note that this will _not_ be verified + * by the mempool - use with care, lest you get banned from + * bitcoind nodes). + * @param {TX|Block} item + * @returns {Promise} + */ + + async broadcast(item) { + try { + await this.pool.broadcast(item); + } catch (e) { + this.emit('error', e); + } + } + + /** + * Broadcast a transaction (note that this will _not_ be verified + * by the mempool - use with care, lest you get banned from + * bitcoind nodes). + * @param {TX} tx + * @returns {Promise} + */ + + sendTX(tx) { + return this.broadcast(tx); + } + + /** + * Broadcast a transaction. Silence errors. + * @param {TX} tx + * @returns {Promise} + */ + + relay(tx) { + return this.broadcast(tx); + } + + /** + * Connect to the network. + * @returns {Promise} + */ + + connect() { + return this.pool.connect(); + } + + /** + * Disconnect from the network. + * @returns {Promise} + */ + + disconnect() { + return this.pool.disconnect(); + } + + /** + * Start the blockchain sync. + */ + + startSync() { + return this.pool.startSync(); + } + + /** + * Stop syncing the blockchain. + */ + + stopSync() { + return this.pool.stopSync(); + } +} + +/* + * Expose + */ + +module.exports = Neutrino; diff --git a/lib/protocol/networks.js b/lib/protocol/networks.js index 16e6bedf7..8c2db9e8e 100644 --- a/lib/protocol/networks.js +++ b/lib/protocol/networks.js @@ -792,7 +792,7 @@ regtest.block = { bip66hash: null, pruneAfterHeight: 1000, keepBlocks: 10000, - maxTipAge: 0xffffffff, + maxTipAge: 24 * 60 * 60, slowHeight: 0 };