diff --git a/bin/bcoin-cli b/bin/bcoin-cli index b9136ab43..9ca39cdfe 100755 --- a/bin/bcoin-cli +++ b/bin/bcoin-cli @@ -129,6 +129,22 @@ class CLI { this.log(filter); } + async getFilterHeader() { + let hash = this.config.str(0, ''); + + if (hash.length !== 64) + hash = parseInt(hash, 10); + + const filterHeader = await this.client.getFilterHeader(hash); + + if (!filterHeader) { + this.log('Filter header not found.'); + return; + } + + this.log(filterHeader); + } + async estimateFee() { const blocks = this.config.uint(0, 1); @@ -246,6 +262,9 @@ class CLI { case 'filter': await this.getFilter(); break; + case 'filterheader': + await this.getFilterHeader(); + break; case 'fee': await this.estimateFee(); break; @@ -263,6 +282,7 @@ class CLI { this.log(' $ coin [hash+index/address]: View coins.'); this.log(' $ fee [target]: Estimate smart fee.'); this.log(' $ filter [hash/height]: View filter.'); + this.log(' $ filterheader [hash/height]: View filter header.'); this.log(' $ header [hash/height]: View block header.'); this.log(' $ info: Get server info.'); this.log(' $ mempool: Get mempool snapshot.'); diff --git a/lib/client/node.js b/lib/client/node.js index 50800cac1..5df661dfa 100644 --- a/lib/client/node.js +++ b/lib/client/node.js @@ -164,9 +164,18 @@ class NodeClient extends Client { * @returns {Promise} */ - getFilter(filter) { - assert(typeof filter === 'string' || typeof filter === 'number'); - return this.get(`/filter/${filter}`); + getFilter(block) { + assert(typeof block === 'string' || typeof block === 'number'); + return this.get(`/filter/${block}`); + } + + getFilterHeader(block) { + assert(typeof block === 'string' || typeof block === 'number'); + return this.get(`/filterheader/${block}`); + } + + getBlockPeer(hash) { + return this.call('get block peer', hash); } /** diff --git a/lib/indexer/filterindexer.js b/lib/indexer/filterindexer.js index 97265253b..ae88af139 100644 --- a/lib/indexer/filterindexer.js +++ b/lib/indexer/filterindexer.js @@ -85,6 +85,49 @@ class FilterIndexer extends Indexer { this.put(layout.f.encode(hash), gcsFilter.hash()); } + /** + * save filter header + * @param {Hash} blockHash + * @param {Hash} filterHeader + * @param {Hash} filterHash + * @returns {Promise} + */ + + async saveFilterHeader(blockHash, filterHeader, filterHash) { + assert(blockHash); + assert(filterHeader); + assert(filterHash); + + const filter = new Filter(); + filter.header = filterHeader; + + await this.blocks.writeFilter(blockHash, filter.toRaw(), this.filterType); + // console.log(layout.f.encode(blockHash)); + this.put(layout.f.encode(blockHash), filterHash); + } + + /** + * Save filter + * @param {Hash} blockHash + * @param {BasicFilter} basicFilter + * @param {Hash} filterHeader + * @returns {Promise} + */ + + async saveFilter(blockHash, basicFilter, filterHeader) { + assert(blockHash); + assert(basicFilter); + assert(filterHeader); + + const filter = new Filter(); + filter.filter = basicFilter.toRaw(); + filter.header = filterHeader; + + await this.blocks.writeFilter(blockHash, filter.toRaw(), this.filterType); + // console.log(layout.f.encode(blockHash)); + this.put(layout.f.encode(blockHash), basicFilter.hash()); + } + /** * Prune compact filters. * @private diff --git a/lib/indexer/indexer.js b/lib/indexer/indexer.js index 97d85f76b..b052d6a97 100644 --- a/lib/indexer/indexer.js +++ b/lib/indexer/indexer.js @@ -50,6 +50,8 @@ class Indexer extends EventEmitter { this.blocks = this.options.blocks; this.chain = this.options.chain; + this.neutrino = this.options.neutrino; + this.closing = false; this.db = null; this.batch = null; @@ -292,6 +294,11 @@ class Indexer extends EventEmitter { */ async _syncBlock(meta, block, view) { + if (this.neutrino) { + if (!this.batch) + this.start(); + return true; + } // In the case that the next block is being // connected or the current block disconnected // use the block and view being passed directly, @@ -636,6 +643,8 @@ class IndexOptions { this.cacheSize = 16 << 20; this.compression = true; + this.neutrino = false; + if (options) this.fromOptions(options); } @@ -697,6 +706,11 @@ class IndexOptions { this.compression = options.compression; } + if (options.neutrino != null) { + assert(typeof options.neutrino === 'boolean'); + this.neutrino = options.neutrino; + } + return this; } diff --git a/lib/net/peer.js b/lib/net/peer.js index dac2e265d..154c83e52 100644 --- a/lib/net/peer.js +++ b/lib/net/peer.js @@ -1009,6 +1009,12 @@ class Peer extends EventEmitter { case packetTypes.GETHEADERS: this.request(packetTypes.HEADERS, timeout * 2); break; + case packetTypes.GETCFHEADERS: + this.request(packetTypes.CFHEADERS, timeout); + break; + case packetTypes.GETCFILTERS: + this.request(packetTypes.CFILTER, timeout); + break; case packetTypes.GETDATA: this.request(packetTypes.DATA, timeout * 2); break; @@ -1751,6 +1757,26 @@ class Peer extends EventEmitter { this.send(packet); } + /** + * @param {Number} filterType - `0` = basic + * @param {Number} startHeight - Height to start at. + * @param {Hash} stopHash - Hash to stop at. + * @returns {void} + * @description Send `getcfilters` to peer. + */ + sendGetCFilters(filterType, startHeight, stopHash) { + const packet = new packets.GetCFiltersPacket( + filterType, + startHeight, + stopHash); + + this.logger.debug( + 'Sending getcfilters (type=%d, startHeight=%d, stopHash=%h).', + filterType, startHeight, stopHash); + + this.send(packet); + } + /** * Send `cfheaders` to peer. * @param {Number} filterType @@ -1773,6 +1799,27 @@ class Peer extends EventEmitter { this.send(packet); } + /** + * @param {Number} filterType + * @param {Number} startHeight + * @param {Hash} stopHash + * @returns {void} + * @description Send `getcfheaders` to peer. + */ + + sendGetCFHeaders(filterType, startHeight, stopHash) { + const packet = new packets.GetCFHeadersPacket( + filterType, + startHeight, + stopHash); + + this.logger.debug( + 'Sending getcfheaders (type=%d, start=%h, stop=%h).', + filterType, startHeight, stopHash); + + this.send(packet); + } + /** * send `cfcheckpt` to peer. * @param {Number} filterType @@ -1793,6 +1840,25 @@ class Peer extends EventEmitter { this.send(packet); } + /** + * Send `getcfcheckpt` to peer. + * @param {Number} filterType + * @param {Hash} stopHash + * @returns {void} + */ + + sendGetCFCheckpt(filterType, stopHash) { + const packet = new packets.GetCFCheckptPacket( + filterType, + stopHash); + + this.logger.debug( + 'Sending getcfcheckpt (type=%d, stop=%h).', + filterType, stopHash); + + this.send(packet); + } + /** * Send `mempool` to peer. */ diff --git a/lib/net/pool.js b/lib/net/pool.js index fcf11a3b8..ae8ae859b 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -35,6 +35,7 @@ const packetTypes = packets.types; const scores = HostList.scores; const {inspectSymbol} = require('../utils'); const {consensus} = require('../protocol'); +const BasicFilter = require('../golomb/basicFilter'); /** * Pool @@ -87,6 +88,10 @@ class Pool extends EventEmitter { this.hosts = new HostList(this.options); this.id = 0; + this.requestedFilterType = null; + this.getcfiltersStartHeight = null; + this.requestedStopHash = null; + if (this.options.spv) { this.spvFilter = BloomFilter.fromRate( 20000, 0.001, BloomFilter.flags.ALL); @@ -713,6 +718,57 @@ class Pool extends EventEmitter { this.compactBlocks.clear(); } + /** + * Start the filters headers sync. + */ + + async startFilterHeadersSync() { + this.logger.info('Starting filter headers sync (%s).', + this.chain.options.network); + if (!this.opened || !this.connected) + return; + + const cFHeaderHeight = await this.chain.getCFHeaderHeight(); + const startHeight = cFHeaderHeight + ? cFHeaderHeight + 1 : 1; + const chainHeight = await this.chain.height; + const stopHeight = chainHeight - startHeight + 1 > 2000 + ? 2000 : chainHeight; + const stopHash = await this.chain.getHash(stopHeight); + this.requestedFilterType = common.FILTERS.BASIC; + this.requestedStopHash = stopHash; + await this.peers.load.sendGetCFHeaders( + common.FILTERS.BASIC, + startHeight, + stopHash); + } + + /** + * Start the filters sync. + */ + + async startFilterSync() { + this.logger.info('Starting filter sync (%s).', + this.chain.options.network); + if (!this.opened || !this.connected) + return; + + const cFilterHeight = await this.chain.getCFilterHeight(); + const startHeight = cFilterHeight + ? cFilterHeight + 1 : 1; + const chainHeight = await this.chain.height; + const stopHeight = chainHeight - startHeight + 1 > 1000 + ? 1000 : chainHeight; + const stopHash = await this.chain.getHash(stopHeight); + this.requestedFilterType = common.FILTERS.BASIC; + this.getcfiltersStartHeight = startHeight; + this.requestedStopHash = stopHash; + await this.peers.load.sendGetCFilters( + common.FILTERS.BASIC, + startHeight, + stopHash); + } + /** * Start the headers sync using getHeaders messages. * @private @@ -1234,6 +1290,12 @@ class Pool extends EventEmitter { case packetTypes.GETCFCHECKPT: await this.handleGetCFCheckpt(peer, packet); break; + case packetTypes.CFCHECKPT: + await this.handleCFCheckpt(peer, packet); + break; + case packetTypes.CFHEADERS: + await this.handleCFHeaders(peer, packet); + break; case packetTypes.GETBLOCKS: await this.handleGetBlocks(peer, packet); break; @@ -1286,8 +1348,8 @@ class Pool extends EventEmitter { await this.handleBlockTxn(peer, packet); break; case packetTypes.CFILTER: - case packetTypes.CFHEADERS: - case packetTypes.CFCHECKPT: + await this.handleCFilters(peer, packet); + break; case packetTypes.UNKNOWN: await this.handleUnknown(peer, packet); break; @@ -1965,7 +2027,7 @@ class Pool extends EventEmitter { if (!stopHeight) return; - if (stopHeight - packet.startHeight >= common.MAX_CFILTERS) + if (stopHeight - packet.startHeight > common.MAX_CFILTERS) return; const indexer = this.getFilterIndexer(filtersByVal[packet.filterType]); @@ -2061,6 +2123,160 @@ class Pool extends EventEmitter { peer.sendCFCheckpt(packet.filterType, packet.stopHash, filterHeaders); } + /** + * Handle peer `CFCheckpt` packet. + * @method + * @private + * @param {Peer} peer + * @param {CFCheckptPacket} packet + */ + + async handleCFCheckpt(peer, packet) { + if (!this.options.neutrino) { + peer.ban(); + peer.destroy(); + } + } + + /** + * Handle peer `CFHeaders` packet. + * @method + * @private + * @param {Peer} peer - Sender. + * @param {CFHeadersPacket} packet - Packet to handle. + * @returns {void} + */ + async handleCFHeaders(peer, packet) { + this.logger.info('Received CFHeaders packet from %s', peer.hostname()); + if (!this.options.neutrino) { + peer.ban(); + peer.destroy(); + return; + } + + const filterType = packet.filterType; + + if (filterType !== this.requestedFilterType) { + this.logger.warning('Received CFHeaders packet with wrong filterType'); + peer.ban(); + peer.destroy(); + return; + } + + const stopHash = packet.stopHash; + if (!stopHash.equals(this.requestedStopHash)) { + this.logger.warning('Received CFHeaders packet with wrong stopHash'); + peer.ban(); + return; + } + let previousFilterHeader = packet.previousFilterHeader; + const filterHashes = packet.filterHashes; + let blockHeight = await this.chain.getHeight(stopHash) + - filterHashes.length + 1; + const stopHeight = await this.chain.getHeight(stopHash); + for (const filterHash of filterHashes) { + if (blockHeight > stopHeight) { + peer.ban(); + return; + } + const basicFilter = new BasicFilter(); + basicFilter._hash = filterHash; + const filterHeader = basicFilter.header(previousFilterHeader); + const lastFilterHeader = this.cfHeaderChain.tail; + const cfHeaderEntry = new CFHeaderEntry( + filterHash, lastFilterHeader.height + 1); + this.cfHeaderChain.push(cfHeaderEntry); + const blockHash = await this.chain.getHash(blockHeight); + const indexer = this.getFilterIndexer(filtersByVal[filterType]); + await indexer.saveFilterHeader(blockHash, filterHeader, filterHash); + previousFilterHeader = filterHeader; + await this.chain.saveCFHeaderHeight(blockHeight); + blockHeight++; + const cFHeaderHeight = await this.chain.getCFHeaderHeight(); + this.logger.info('CFHeaderHeight: %d', cFHeaderHeight); + } + if (this.headerChain.tail.height <= stopHeight) + this.emit('cfheaders'); + else { + const nextStopHeight = stopHeight + 2000 < this.chain.height + ? stopHeight + 2000 : this.chain.height; + const nextStopHash = await this.chain.getHash(nextStopHeight); + this.requestedStopHash = nextStopHash; + peer.sendGetCFHeaders(filterType, stopHeight + 1, nextStopHash); + } + } + + async handleCFilters(peer, packet) { + this.logger.info('Received CFilter packet from %s', peer.hostname()); + if (!this.options.neutrino) { + peer.ban(); + peer.destroy(); + return; + } + + const blockHash = packet.blockHash; + const filterType = packet.filterType; + const filter = packet.filterBytes; + + if (filterType !== this.requestedFilterType) { + this.logger.warning('Received CFilter packet with wrong filterType'); + peer.ban(); + peer.destroy(); + return; + } + + const blockHeight = await this.chain.getHeight(blockHash); + const stopHeight = await this.chain.getHeight(this.requestedStopHash); + + if (!(blockHeight >= this.getcfiltersStartHeight + && blockHeight <= stopHeight)) { + this.logger.warning('Received CFilter packet with wrong blockHeight'); + peer.ban(); + return; + } + + const basicFilter = new BasicFilter(); + const gcsFilter = basicFilter.fromNBytes(filter); + + const indexer = this.getFilterIndexer(filtersByVal[filterType]); + const filterHeader = await indexer.getFilterHeader(blockHash); + await indexer.saveFilter(blockHash, gcsFilter, filterHeader); + + await this.chain.saveCFilterHeight(blockHeight); + const cFilterHeight = await this.chain.getCFilterHeight(); + this.logger.info('CFilter height: %d', cFilterHeight); + this.emit('cfilter', blockHash, gcsFilter); + const startHeight = stopHeight + 1; + let nextStopHeight; + if (cFilterHeight === stopHeight + && stopHeight < this.chain.height) { + if (startHeight + 1000 < this.chain.height) { + nextStopHeight = stopHeight + 1000; + const stopHash = await this.chain.getHash(nextStopHeight); + this.getcfiltersStartHeight = startHeight; + this.requestedStopHash = stopHash; + this.peers.load.sendGetCFilters( + common.FILTERS.BASIC, + startHeight, + stopHash + ); + } else { + nextStopHeight = this.chain.height; + const stopHash = await this.chain.getHash(nextStopHeight); + this.getcfiltersStartHeight = startHeight; + this.requestedStopHash = stopHash; + this.peers.load.sendGetCFilters( + common.FILTERS.BASIC, + startHeight, + stopHash + ); + return; + } + } else if (cFilterHeight === this.chain.height) { + this.chain.emit('full'); + } + } + /** * Handle `getblocks` packet. * @method diff --git a/lib/node/fullnode.js b/lib/node/fullnode.js index cd373d3b9..928a3756d 100644 --- a/lib/node/fullnode.js +++ b/lib/node/fullnode.js @@ -645,28 +645,6 @@ class FullNode extends Node { return false; } - - /** - * Retrieve compact filter by hash. - * @param {Hash | Number} hash - * @param {Number} type - * @returns {Promise} - Returns {@link Buffer}. - */ - - async getBlockFilter(hash, filterType) { - const Indexer = this.filterIndexers.get(filterType); - - if (!Indexer) - return null; - - if (typeof hash === 'number') - hash = await this.chain.getHash(hash); - - if (!hash) - return null; - - return Indexer.getFilter(hash); - } } /* diff --git a/lib/node/http.js b/lib/node/http.js index 8448ec015..36ea3eb92 100644 --- a/lib/node/http.js +++ b/lib/node/http.js @@ -291,7 +291,8 @@ class HTTP extends Server { enforce(hash != null, 'Hash or height required.'); - const filter = await this.node.getBlockFilter(hash); + const filterName = valid.str(1, 'BASIC').toUpperCase(); + const filter = await this.node.getBlockFilter(hash, filterName); if (!filter) { res.json(404); @@ -301,6 +302,24 @@ class HTTP extends Server { res.json(200, filter.toJSON()); }); + this.get('/filterheader/:block', async (req, res) => { + const valid = Validator.fromRequest(req); + const hash = valid.uintbrhash('block'); + + enforce(hash != null, 'Hash or height required.'); + + const filterName = valid.str(1, 'BASIC').toUpperCase(); + const filterHeader = await this.node. + getBlockFilterHeader(hash, filterName); + + if (!filterHeader) { + res.json(404); + return; + } + + res.json(200, filterHeader.toJSON()); + }); + // Mempool snapshot this.get('/mempool', async (req, res) => { enforce(this.mempool, 'No mempool available.'); diff --git a/lib/node/node.js b/lib/node/node.js index 6ef9803fe..e2e20f453 100644 --- a/lib/node/node.js +++ b/lib/node/node.js @@ -413,6 +413,50 @@ class Node extends EventEmitter { await plugin.close(); } } + + /** + * Retrieve compact filter by hash/height. + * @param {Hash | Number} hash + * @param {Number} type + * @returns {Promise} - Returns {@link Buffer}. + */ + + async getBlockFilter(hash, filterType) { + const Indexer = this.filterIndexers.get(filterType); + + if (!Indexer) + return null; + + if (typeof hash === 'number') + hash = await this.chain.getHash(hash); + + if (!hash) + return null; + + return Indexer.getFilter(hash); + } + + /** + * Retrieve compact filter header by hash/height. + * @param {Hash | Number} hash + * @param {Number} type + * @returns {Promise} - Returns {@link Buffer}. + */ + + async getBlockFilterHeader(hash, filterType) { + const Indexer = this.filterIndexers.get(filterType); + + if (!Indexer) + return null; + + if (typeof hash === 'number') + hash = await this.chain.getHash(hash); + + if (!hash) + return null; + + return Indexer.getFilterHeader(hash); + } } /* diff --git a/lib/node/rpc.js b/lib/node/rpc.js index ab67affb9..bc1e93d44 100644 --- a/lib/node/rpc.js +++ b/lib/node/rpc.js @@ -155,11 +155,14 @@ class RPC extends RPCBase { this.add('getblockchaininfo', this.getBlockchainInfo); this.add('getbestblockhash', this.getBestBlockHash); this.add('getblockcount', this.getBlockCount); + this.add('getfiltercount', this.getFilterCount); + this.add('getfilterheadercount', this.getFilterHeaderCount); this.add('getblock', this.getBlock); this.add('getblockbyheight', this.getBlockByHeight); this.add('getblockhash', this.getBlockHash); this.add('getblockheader', this.getBlockHeader); this.add('getblockfilter', this.getBlockFilter); + this.add('getblockfilterheader', this.getBlockFilterHeader); this.add('getchaintips', this.getChainTips); this.add('getdifficulty', this.getDifficulty); this.add('getmempoolancestors', this.getMempoolAncestors); @@ -629,6 +632,22 @@ class RPC extends RPCBase { return this.chain.tip.height; } + async getFilterCount(args, help) { + if (help || args.length !== 0) + throw new RPCError(errs.MISC_ERROR, 'getfiltercount'); + + const height = await this.chain.getCFilterHeight(); + return height; + } + + async getFilterHeaderCount(args, help) { + if (help || args.length !== 0) + throw new RPCError(errs.MISC_ERROR, 'getfilterheadercount'); + + const height = await this.chain.getCFHeaderHeight(); + return height; + } + async getBlock(args, help) { if (help || args.length < 1 || args.length > 3) throw new RPCError(errs.MISC_ERROR, 'getblock "hash" ( verbose )'); @@ -769,6 +788,32 @@ class RPC extends RPCBase { return filter.toJSON(); } + async getBlockFilterHeader(args, help) { + if (help || args.length < 1 || args.length > 2) { + throw new RPCError(errs.MISC_ERROR, + 'getblockfilterheader "hash" ( "type" )'); + } + + const valid = new Validator(args); + const hash = valid.brhash(0); + const filterName = valid.str(1, 'BASIC').toUpperCase(); + + const filterType = filters[filterName]; + + if (!hash) + throw new RPCError(errs.MISC_ERROR, 'Invalid block hash.'); + + if (!filterType) + throw new RPCError(errs.MISC_ERROR, 'Filter type not supported'); + + const filterHeader = await this.node.getBlockFilterHeader(hash, filterName); + + if (!filterHeader) + throw new RPCError(errs.MISC_ERROR, 'Block filter header not found.'); + + return filterHeader; + } + async getChainTips(args, help) { if (help || args.length !== 0) throw new RPCError(errs.MISC_ERROR, 'getchaintips'); diff --git a/lib/wallet/nodeclient.js b/lib/wallet/nodeclient.js index 9f6c43600..9a9b1ee6b 100644 --- a/lib/wallet/nodeclient.js +++ b/lib/wallet/nodeclient.js @@ -37,6 +37,13 @@ class NodeClient extends AsyncEmitter { init() { this.node.chain.on('connect', async (entry, block) => { + if (!this.opened || this.node.neutrino) + return; + + await this.emitAsync('block connect', entry, block.txs); + }); + + this.node.chain.on('getblockpeer', async (entry, block) => { if (!this.opened) return; @@ -50,6 +57,13 @@ class NodeClient extends AsyncEmitter { await this.emitAsync('block disconnect', entry); }); + this.node.pool.on('cfilter', async (blockHeight, filter) => { + if (!this.opened) + return; + + await this.emitAsync('cfilter', blockHeight, filter); + }); + this.node.on('tx', (tx) => { if (!this.opened) return;