diff --git a/blockchain/chainio.go b/blockchain/chainio.go index fdcbb4c3..3a86fcd6 100644 --- a/blockchain/chainio.go +++ b/blockchain/chainio.go @@ -1080,6 +1080,32 @@ func SerializeUtreexoRoots(numLeaves uint64, roots []utreexo.Hash) ([]byte, erro return w.Bytes(), nil } +// SerializeUtreexoRootsHash serializes the numLeaves and the roots into a byte slice. +// it takes in a slice of chainhash.Hash instead of utreexo.Hash. chainhash.Hash is the hashed +// value of the utreexo.Hash. +func SerializeUtreexoRootsHash(numLeaves uint64, roots []*chainhash.Hash) ([]byte, error) { + // 8 byte NumLeaves + (32 byte roots * len(roots)) + w := bytes.NewBuffer(make([]byte, 0, 8+(len(roots)*chainhash.HashSize))) + + // Write the NumLeaves first. + var buf [8]byte + byteOrder.PutUint64(buf[:], numLeaves) + _, err := w.Write(buf[:]) + if err != nil { + return nil, err + } + + // Then write the roots. + for _, root := range roots { + _, err = w.Write(root[:]) + if err != nil { + return nil, err + } + } + + return w.Bytes(), nil +} + // DeserializeUtreexoRoots deserializes the provided byte slice into numLeaves and roots. func DeserializeUtreexoRoots(serializedUView []byte) (uint64, []utreexo.Hash, error) { totalLen := len(serializedUView) diff --git a/blockchain/indexers/utreexocfindex.go b/blockchain/indexers/utreexocfindex.go new file mode 100644 index 00000000..8a2846d6 --- /dev/null +++ b/blockchain/indexers/utreexocfindex.go @@ -0,0 +1,394 @@ +package indexers + +import ( + "errors" + + "github.com/utreexo/utreexod/blockchain" + "github.com/utreexo/utreexod/btcutil" + "github.com/utreexo/utreexod/btcutil/gcs/builder" + "github.com/utreexo/utreexod/chaincfg" + "github.com/utreexo/utreexod/chaincfg/chainhash" + "github.com/utreexo/utreexod/database" + "github.com/utreexo/utreexod/wire" +) + +// utreexoCFIndexName is the human-readable name for the index. +const ( + utreexoCFIndexName = "utreexo custom commited filter index" +) + +// utreexocfilter is a custom commited filter which serves utreexo roots +// these roots are already present, so they need not be created/stored, their +// headers could be stored though +var ( + // utreexoCFIndexParentBucketKey is the name of the parent bucket used to + // house the index. The rest of the buckets live below this bucket. + utreexoCFIndexParentBucketKey = []byte("utreexocfindexparentbucket") + + // utreexoCfHeaderKeys is an array of db bucket names used to house indexes of + // block hashes to cf headers. + utreexoCfHeaderKeys = [][]byte{ + []byte("utreexocfheaderbyhashidx"), + } +) + +// dbFetchFilterIdxEntry retrieves a data blob from the filter index database. +// An entry's absence is not considered an error. +// Right now, the value of 'key' will always be the key to the utreexocfheader key +// as we don't need to fetch filters or filter hashes from the main bucket, as those +// are not stored +func dbFetchUtreexoCFilterIdxEntry(dbTx database.Tx, key []byte, h *chainhash.Hash) ([]byte, error) { + idx := dbTx.Metadata().Bucket(utreexoCFIndexParentBucketKey).Bucket(key) + return idx.Get(h[:]), nil +} + +// dbStoreFilterIdxEntry stores a data blob in the filter index database. +func dbStoreUtreexoCFilterIdxEntry(dbTx database.Tx, key []byte, h *chainhash.Hash, f []byte) error { + idx := dbTx.Metadata().Bucket(utreexoCFIndexParentBucketKey).Bucket(key) + return idx.Put(h[:], f) +} + +// dbDeleteFilterIdxEntry deletes a data blob from the filter index database. +func dbDeleteUtreexoCFilterIdxEntry(dbTx database.Tx, key []byte, h *chainhash.Hash) error { + idx := dbTx.Metadata().Bucket(utreexoCFIndexParentBucketKey).Bucket(key) + return idx.Delete(h[:]) +} + +var _ Indexer = (*UtreexoCFIndex)(nil) + +var _ NeedsInputser = (*UtreexoCFIndex)(nil) + +type UtreexoCFIndex struct { + db database.DB + chainParams *chaincfg.Params + + noUtreexo bool + + chain *blockchain.BlockChain + + utreexoProofIndex *UtreexoProofIndex + + flatUtreexoProofIndex *FlatUtreexoProofIndex +} + +func (idx *UtreexoCFIndex) NeedsInputs() bool { + return true +} + +// Init initializes the utreexo cf index. This is part of the Indexer +// interface. +func (idx *UtreexoCFIndex) Init(chain *blockchain.BlockChain) error { + idx.chain = chain + return nil // Nothing to do. +} + +// Key returns the database key to use for the index as a byte slice. This is +// part of the Indexer interface. +func (idx *UtreexoCFIndex) Key() []byte { + return utreexoCFIndexParentBucketKey +} + +// Name returns the human-readable name of the index. This is part of the +// Indexer interface. +func (idx *UtreexoCFIndex) Name() string { + return utreexoCFIndexName +} + +// Create is invoked when the index manager determines the index needs to +// be created for the first time. It creates buckets for the custom utreexo +// filter index. +func (idx *UtreexoCFIndex) Create(dbTx database.Tx) error { + meta := dbTx.Metadata() + + utreexoCfIndexParentBucket, err := meta.CreateBucket(utreexoCFIndexParentBucketKey) + if err != nil { + return err + } + + for _, bucketName := range utreexoCfHeaderKeys { + _, err = utreexoCfIndexParentBucket.CreateBucket(bucketName) + if err != nil { + return err + } + } + + return nil +} + +// storeUtreexoCFilter stores a given utreexocfilter header +func (idx *UtreexoCFIndex) storeUtreexoCFHeader(dbTx database.Tx, block *btcutil.Block, filterData []byte, + filterType wire.FilterType) error { + if filterType != wire.UtreexoCFilter { + return errors.New("invalid filter type") + } + + // Figure out which header bucket to use. + hkey := utreexoCfHeaderKeys[0] + h := block.Hash() + + // fetch the previous block's filter header. + var prevHeader *chainhash.Hash + ph := &block.MsgBlock().Header.PrevBlock + if ph.IsEqual(&zeroHash) { + prevHeader = &zeroHash + } else { + var pfh []byte + err := idx.db.View(func(dbTx database.Tx) error { + var err error + pfh, err = dbFetchUtreexoCFilterIdxEntry(dbTx, hkey, ph) + return err + }) + if err != nil { + return err + } + + // Construct the new block's filter header, and store it. + prevHeader, err = chainhash.NewHash(pfh) + if err != nil { + return err + } + } + + fh, err := builder.MakeHeaderForUtreexoCFilter(filterData, *prevHeader) + if err != nil { + return err + } + return dbStoreUtreexoCFilterIdxEntry(dbTx, hkey, h, fh[:]) +} + +// ConnectBlock is invoked by the index manager when a new block has been +// connected to the main chain. +// This is part of the Indexer interface. +func (idx *UtreexoCFIndex) ConnectBlock(dbTx database.Tx, block *btcutil.Block, + stxos []blockchain.SpentTxOut) error { + + blockHash := block.Hash() + roots, leaves, err := idx.fetchUtreexoRoots(dbTx, blockHash, block) + + if err != nil { + return err + } + + // serialize the hashes of the utreexo roots hash + serializedUtreexo, err := blockchain.SerializeUtreexoRootsHash(leaves, roots) + if err != nil { + return err + } + + return idx.storeUtreexoCFHeader(dbTx, block, serializedUtreexo, wire.UtreexoCFilter) +} + +// fetches the utreexo roots for a given block hash +func (idx *UtreexoCFIndex) fetchUtreexoRoots(dbTx database.Tx, blockHash *chainhash.Hash, + block *btcutil.Block) ([]*chainhash.Hash, uint64, error) { + + // For compact state nodes + if !idx.noUtreexo { + viewPoint, err := idx.chain.FetchUtreexoViewpoint(blockHash) + if err != nil { + return nil, 0, err + } + roots := viewPoint.GetRoots() + leaves := viewPoint.NumLeaves() + return roots, leaves, nil + } else if idx.utreexoProofIndex != nil { + var roots []*chainhash.Hash + var numLeaves uint64 + var err error + if block.Height() < idx.chain.BestSnapshot().Height { + log.Infof("block height is less than best snapshot") + roots, numLeaves, err = idx.utreexoProofIndex.FetchUtreexoState(dbTx, block.Hash()) + } else { + roots, numLeaves = idx.utreexoProofIndex.FetchCurrentUtreexoState() + } + if err != nil { + log.Errorf("Error fetching utreexo state at block %s: %v", block.Hash(), err) + return nil, 0, err + } + return roots, numLeaves, nil + + } else if idx.flatUtreexoProofIndex != nil { + var roots []*chainhash.Hash + var numLeaves uint64 + var err error + if block.Height() < idx.chain.BestSnapshot().Height { + log.Infof("block height is less than best snapshot") + roots, numLeaves, err = idx.flatUtreexoProofIndex.FetchUtreexoState(block.Height()) + } else { + roots, numLeaves = idx.flatUtreexoProofIndex.FetchCurrentUtreexoState() + } + if err != nil { + log.Errorf("Error fetching utreexo state at block %s: %v", block.Hash(), err) + return nil, 0, err + } + return roots, numLeaves, nil + } + + return nil, 0, errors.New("unsupported filter type") +} + +// DisconnectBlock is invoked by the index manager when a block has been +// disconnected from the main chain. This indexer removes the hash-to-cf +// mapping for every passed block. This is part of the Indexer interface. +func (idx *UtreexoCFIndex) DisconnectBlock(dbTx database.Tx, block *btcutil.Block, + _ []blockchain.SpentTxOut) error { + + for _, key := range utreexoCfHeaderKeys { + err := dbDeleteUtreexoCFilterIdxEntry(dbTx, key, block.Hash()) + if err != nil { + return err + } + } + + return nil +} + +// PruneBlock is invoked when an older block is deleted after it's been +// processed. +// TODO (kcalvinalvin): Consider keeping the filters at a later date to help with +// reindexing as a pruned node. +// +// This is part of the Indexer interface. +func (idx *UtreexoCFIndex) PruneBlock(dbTx database.Tx, blockHash *chainhash.Hash) error { + + for _, key := range utreexoCfHeaderKeys { + err := dbDeleteUtreexoCFilterIdxEntry(dbTx, key, blockHash) + if err != nil { + return err + } + } + + return nil +} + +// entryByBlockHash fetches a filter index entry of a particular type +// (eg. filter, filter header, etc) for a filter type and block hash. +func (idx *UtreexoCFIndex) entryByBlockHash(filterTypeKeys [][]byte, + filterType wire.FilterType, h *chainhash.Hash) ([]byte, error) { + + if uint8(filterType) != uint8(wire.UtreexoCFilter) { + return nil, errors.New("unsupported filter type") + } + + // if the filtertype keys is empty, then we are fetching the filter data itself + // if not, we are fetching the filter header + if len(filterTypeKeys) == 0 { + var serializedUtreexo []byte + err := idx.db.View(func(dbTx database.Tx) error { + var err error + block, errheight := idx.chain.BlockByHash(h) + if errheight != nil { + return errheight + } + roots, leaves, err := idx.fetchUtreexoRoots(dbTx, h, block) + + if err != nil { + return err + } + // serialize the hashes of the utreexo roots hash + serializedUtreexo, err = blockchain.SerializeUtreexoRootsHash(leaves, roots) + return err + }) + + // serialize the hashes of the utreexo roots hash + // serializedUtreexo, err := blockchain.SerializeUtreexoRootsHash(leaves, roots) + if err != nil { + return nil, err + } + + return serializedUtreexo, err + } else { + // using filtertypekeys[0] as there is only one entry in the filterTypeKeys (utreexoCfHeaderKeys) + key := filterTypeKeys[0] + + var entry []byte + err := idx.db.View(func(dbTx database.Tx) error { + var err error + entry, err = dbFetchUtreexoCFilterIdxEntry(dbTx, key, h) + return err + }) + return entry, err + } +} + +// entriesByBlockHashes batch fetches a filter index entry of a particular type +// (eg. filter, filter header, etc) for a filter type and slice of block hashes. +func (idx *UtreexoCFIndex) entriesByBlockHashes(filterTypeKeys [][]byte, + filterType wire.FilterType, blockHashes []*chainhash.Hash) ([][]byte, error) { + + if uint8(filterType) != uint8(wire.UtreexoCFilter) { + return nil, errors.New("unsupported filter type") + } + + // we use filterTypeKeys[0] as the key for the utreexo cfilter since for now, + // there is only one type of utreexo cfilter and the filterTypeKeys always + // returns the filterheaderkeys, which has just the one value + key := filterTypeKeys[0] + + entries := make([][]byte, 0, len(blockHashes)) + err := idx.db.View(func(dbTx database.Tx) error { + for _, blockHash := range blockHashes { + entry, err := dbFetchUtreexoCFilterIdxEntry(dbTx, key, blockHash) + if err != nil { + return err + } + entries = append(entries, entry) + } + return nil + }) + return entries, err +} + +// FilterByBlockHash returns the serialized contents of a block's utreexo +// cfilter. +func (idx *UtreexoCFIndex) FilterByBlockHash(h *chainhash.Hash, + filterType wire.FilterType) ([]byte, error) { + // we create an ampty variable of type [][]byte to pass to the entryByBlockHash + // in order to fetch the filter data itself + utreexoCfIndexKeys := [][]byte{} + return idx.entryByBlockHash(utreexoCfIndexKeys, filterType, h) +} + +// FilterHeaderByBlockHash returns the serialized contents of a block's utreexo +// committed filter header. +func (idx *UtreexoCFIndex) FilterHeaderByBlockHash(h *chainhash.Hash, + filterType wire.FilterType) ([]byte, error) { + return idx.entryByBlockHash(utreexoCfHeaderKeys, filterType, h) +} + +// FilterHeadersByBlockHashes returns the serialized contents of a block's +// utreexo commited filter header for a set of blocks by hash. +func (idx *UtreexoCFIndex) FilterHeadersByBlockHashes(blockHashes []*chainhash.Hash, + filterType wire.FilterType) ([][]byte, error) { + return idx.entriesByBlockHashes(utreexoCfHeaderKeys, filterType, blockHashes) +} + +// NewUtreexoCfIndex returns a new instance of an indexer that is used to create a +// mapping of the hashes of all blocks in the blockchain to their respective +// utreexo committed filters. +// +// It implements the Indexer interface which plugs into the IndexManager that +// in turn is used by the blockchain package. This allows the index to be +// seamlessly maintained along with the chain. +func NewUtreexoCfIndex(db database.DB, chainParams *chaincfg.Params, utreexoProofIndex *UtreexoProofIndex, + flatUtreexoProofIndex *FlatUtreexoProofIndex, noUtreexo bool) *UtreexoCFIndex { + return &UtreexoCFIndex{db: db, chainParams: chainParams, utreexoProofIndex: utreexoProofIndex, + flatUtreexoProofIndex: flatUtreexoProofIndex, noUtreexo: noUtreexo} +} + +// DropUtreexoCfIndex drops the utreexo CF index from the provided database if exists. +func DropUtreexoCfIndex(db database.DB, interrupt <-chan struct{}) error { + return dropIndex(db, utreexoCFIndexParentBucketKey, utreexoCFIndexName, interrupt) +} + +// UtreexoCfIndexInitialized returns true if the utreexocfindex has been created previously. +func UtreexoCfIndexInitialized(db database.DB) bool { + var exists bool + db.View(func(dbTx database.Tx) error { + bucket := dbTx.Metadata().Bucket(utreexoCFIndexParentBucketKey) + exists = bucket != nil + return nil + }) + + return exists +} diff --git a/blockchain/indexers/utreexoproofindex.go b/blockchain/indexers/utreexoproofindex.go index 27224899..7362b41b 100644 --- a/blockchain/indexers/utreexoproofindex.go +++ b/blockchain/indexers/utreexoproofindex.go @@ -648,6 +648,9 @@ func dbStoreUtreexoState(dbTx database.Tx, hash *chainhash.Hash, p utreexo.Utree func dbFetchUtreexoState(dbTx database.Tx, hash *chainhash.Hash) (utreexo.Stump, error) { stateBucket := dbTx.Metadata().Bucket(utreexoParentBucketKey).Bucket(utreexoStateKey) serialized := stateBucket.Get(hash[:]) + if serialized == nil { + return utreexo.Stump{}, nil + } numLeaves, roots, err := blockchain.DeserializeUtreexoRoots(serialized) if err != nil { diff --git a/btcutil/gcs/builder/builder.go b/btcutil/gcs/builder/builder.go index fe11b80d..33bbd3e8 100644 --- a/btcutil/gcs/builder/builder.go +++ b/btcutil/gcs/builder/builder.go @@ -369,3 +369,19 @@ func MakeHeaderForFilter(filter *gcs.Filter, prevHeader chainhash.Hash) (chainha // above. return chainhash.DoubleHashH(filterTip), nil } + +// MakeHeaderForUtreexoCFilter makes a filter chain header for a utreexoc filter, given the +// filter data and the previous filter chain header. +func MakeHeaderForUtreexoCFilter(filterData []byte, prevHeader chainhash.Hash) (chainhash.Hash, error) { + filterTip := make([]byte, 2*chainhash.HashSize) + filterHash := chainhash.DoubleHashH(filterData) + + // In the buffer we created above we'll compute hash || prevHash as an + // intermediate value. + copy(filterTip, filterHash[:]) + copy(filterTip[chainhash.HashSize:], prevHeader[:]) + + // The final filter hash is the double-sha256 of the hash computed + // above. + return chainhash.DoubleHashH(filterTip), nil +} diff --git a/config.go b/config.go index 59f21754..854a1614 100644 --- a/config.go +++ b/config.go @@ -204,9 +204,11 @@ type config struct { FlatUtreexoProofIndex bool `long:"flatutreexoproofindex" description:"Maintain a utreexo proof for all blocks in flat files"` UtreexoProofIndexMaxMemory int64 `long:"utreexoproofindexmaxmemory" description:"The maxmimum memory in mebibytes (MiB) that the utreexo proof indexes will use up. Passing in 0 will make the entire proof index stay on disk. Passing in a negative value will make the entire proof index stay in memory. Default of 250MiB."` CFilters bool `long:"cfilters" description:"Enable committed filtering (CF) support"` + UtreexoCFilters bool `long:"utreexocfilters" description:"Enable committed filtering (CF) support serving utreexo roots."` NoPeerBloomFilters bool `long:"nopeerbloomfilters" description:"Disable bloom filtering support"` DropAddrIndex bool `long:"dropaddrindex" description:"Deletes the address-based transaction index from the database on start up and then exits."` DropCfIndex bool `long:"dropcfindex" description:"Deletes the index used for committed filtering (CF) support from the database on start up and then exits."` + DropUtreexoCfIndex bool `long:"droputreexocfindex" description:"Deletes the index used for custom utreexo commited filter indexing support serving utreexo roots from the database on start up and then exits."` DropTxIndex bool `long:"droptxindex" description:"Deletes the hash-based transaction index from the database on start up and then exits."` DropTTLIndex bool `long:"dropttlindex" description:"Deletes the time to live index from the database on start up and then exits."` DropUtreexoProofIndex bool `long:"droputreexoproofindex" description:"Deletes the utreexo proof index from the database on start up and then exits."` @@ -1206,6 +1208,14 @@ func loadConfig() (*config, []string, error) { cfg.NoAssumeUtreexo = true } + if cfg.NoUtreexo && cfg.UtreexoCFilters && (!cfg.UtreexoProofIndex && !cfg.FlatUtreexoProofIndex) { + err := fmt.Errorf("%s: the --utreexocfilters can not be called with --noutreexo option when "+ + "neither of utreexoproofindex or flatutreexoproofindex options are enabled", funcName) + fmt.Fprintln(os.Stderr, err) + fmt.Fprintln(os.Stderr, usageMessage) + return nil, nil, err + } + // Specifying --noonion means the onion address dial function results in // an error. if cfg.NoOnion { diff --git a/rpcserver.go b/rpcserver.go index 14b0cea3..c3fa83e0 100644 --- a/rpcserver.go +++ b/rpcserver.go @@ -158,6 +158,8 @@ var rpcHandlersBeforeInit = map[string]commandHandler{ "getchaintips": handleGetChainTips, "getcfilter": handleGetCFilter, "getcfilterheader": handleGetCFilterHeader, + "getutreexocfilter": handleGetUtreexoCFilter, + "getutreexocfilterheader": handleGetUtreexoCFilterHeader, "getconnectioncount": handleGetConnectionCount, "getcurrentnet": handleGetCurrentNet, "getdifficulty": handleGetDifficulty, @@ -294,6 +296,8 @@ var rpcLimited = map[string]struct{}{ "getchaintips": {}, "getcfilter": {}, "getcfilterheader": {}, + "getutreexocfilter": {}, + "getutreexocfilterheader": {}, "getcurrentnet": {}, "getdifficulty": {}, "getheaders": {}, @@ -2395,6 +2399,7 @@ func handleGetCFilter(s *rpcServer, cmd interface{}, closeChan <-chan struct{}) } filterBytes, err := s.cfg.CfIndex.FilterByBlockHash(hash, c.FilterType) + if err != nil { rpcsLog.Debugf("Could not find committed filter for %v: %v", hash, err) @@ -2424,6 +2429,7 @@ func handleGetCFilterHeader(s *rpcServer, cmd interface{}, closeChan <-chan stru } headerBytes, err := s.cfg.CfIndex.FilterHeaderByBlockHash(hash, c.FilterType) + if len(headerBytes) > 0 { rpcsLog.Debugf("Found header of committed filter for %v", hash) } else { @@ -2439,6 +2445,68 @@ func handleGetCFilterHeader(s *rpcServer, cmd interface{}, closeChan <-chan stru return hash.String(), nil } +// handleGetCFilter implements the getcfilter command. +func handleGetUtreexoCFilter(s *rpcServer, cmd interface{}, closeChan <-chan struct{}) (interface{}, error) { + if s.cfg.UtreexoCfIndex == nil { + return nil, &btcjson.RPCError{ + Code: btcjson.ErrRPCNoCFIndex, + Message: "The utreexo CF index must be enabled for this command", + } + } + + c := cmd.(*btcjson.GetCFilterCmd) + hash, err := chainhash.NewHashFromStr(c.Hash) + if err != nil { + return nil, rpcDecodeHexError(c.Hash) + } + + filterBytes, err := s.cfg.UtreexoCfIndex.FilterByBlockHash(hash, c.FilterType) + + if err != nil { + rpcsLog.Debugf("Could not find utreexo committed filter for %v: %v", + hash, err) + return nil, &btcjson.RPCError{ + Code: btcjson.ErrRPCBlockNotFound, + Message: "Block not found", + } + } + + rpcsLog.Debugf("Found utreexo committed filter for %v", hash) + return hex.EncodeToString(filterBytes), nil +} + +// handleGetCFilterHeader implements the getcfilterheader command. +func handleGetUtreexoCFilterHeader(s *rpcServer, cmd interface{}, closeChan <-chan struct{}) (interface{}, error) { + if s.cfg.UtreexoCfIndex == nil { + return nil, &btcjson.RPCError{ + Code: btcjson.ErrRPCNoCFIndex, + Message: "Utreexo CF index must be enabled for this command", + } + } + + c := cmd.(*btcjson.GetCFilterHeaderCmd) + hash, err := chainhash.NewHashFromStr(c.Hash) + if err != nil { + return nil, rpcDecodeHexError(c.Hash) + } + + headerBytes, err := s.cfg.UtreexoCfIndex.FilterHeaderByBlockHash(hash, c.FilterType) + + if len(headerBytes) > 0 { + rpcsLog.Debugf("Found header of utreexo committed filter for %v", hash) + } else { + rpcsLog.Debugf("Could not find header of utreexo committed filter for %v: %v", + hash, err) + return nil, &btcjson.RPCError{ + Code: btcjson.ErrRPCBlockNotFound, + Message: "Block not found", + } + } + + hash.SetBytes(headerBytes) + return hash.String(), nil +} + // handleGetConnectionCount implements the getconnectioncount command. func handleGetConnectionCount(s *rpcServer, cmd interface{}, closeChan <-chan struct{}) (interface{}, error) { return s.cfg.ConnMgr.ConnectedCount(), nil @@ -5649,6 +5717,7 @@ type rpcserverConfig struct { TxIndex *indexers.TxIndex AddrIndex *indexers.AddrIndex CfIndex *indexers.CfIndex + UtreexoCfIndex *indexers.UtreexoCFIndex TTLIndex *indexers.TTLIndex UtreexoProofIndex *indexers.UtreexoProofIndex FlatUtreexoProofIndex *indexers.FlatUtreexoProofIndex diff --git a/server.go b/server.go index 74fcc96f..7edf4d02 100644 --- a/server.go +++ b/server.go @@ -252,6 +252,7 @@ type server struct { txIndex *indexers.TxIndex addrIndex *indexers.AddrIndex cfIndex *indexers.CfIndex + utreexoCfIndex *indexers.UtreexoCFIndex ttlIndex *indexers.TTLIndex utreexoProofIndex *indexers.UtreexoProofIndex flatUtreexoProofIndex *indexers.FlatUtreexoProofIndex @@ -863,170 +864,345 @@ func (sp *serverPeer) OnGetCFilters(_ *peer.Peer, msg *wire.MsgGetCFilters) { return } + var hashes []chainhash.Hash + var hashPtrs []*chainhash.Hash + // if the filter type is supported, we initialize variables to avoid duplicate code + if msg.FilterType == wire.GCSFilterRegular || msg.FilterType == wire.UtreexoCFilter { + var err error + // get the block hashes included in the getcfilters message + hashes, err = sp.server.chain.HeightToHashRange( + int32(msg.StartHeight), &msg.StopHash, wire.MaxGetCFiltersReqRange, + ) + if err != nil { + peerLog.Debugf("Invalid getcfilters request: %v", err) + return + } + + // Create []*chainhash.Hash from []chainhash.Hash to pass to + // FiltersByBlockHashes. + hashPtrs = make([]*chainhash.Hash, len(hashes)) + for i := range hashes { + hashPtrs[i] = &hashes[i] + } + } + // We'll also ensure that the remote party is requesting a set of // filters that we actually currently maintain. switch msg.FilterType { case wire.GCSFilterRegular: - break + filters, err := sp.server.cfIndex.FiltersByBlockHashes( + hashPtrs, msg.FilterType, + ) + if err != nil { + peerLog.Errorf("Error retrieving cfilters: %v", err) + return + } + + for i, filterBytes := range filters { + if len(filterBytes) == 0 { + peerLog.Warnf("Could not obtain cfilter for %v", + hashes[i]) + return + } + + filterMsg := wire.NewMsgCFilter( + msg.FilterType, &hashes[i], filterBytes, + ) + sp.QueueMessage(filterMsg, nil) + } + + case wire.UtreexoCFilter: + for i, blockHash := range hashPtrs { + var serializedUtreexo []byte + + leaves, roots, err := sp.getUtreexoRoots(blockHash) + if err != nil { + return + } + + // serialize the hashes of the utreexo roots hash + serializedUtreexo, err = blockchain.SerializeUtreexoRootsHash(leaves, roots) + if err != nil { + peerLog.Errorf("error serializing utreexoc filter: %v", err) + return + } + + if len(serializedUtreexo) == 0 { + peerLog.Warnf("Could not obtain utreexocfilter for %v", + hashes[i]) + return + } + + filterMsg := wire.NewMsgCFilter( + msg.FilterType, &hashes[i], serializedUtreexo, + ) + sp.QueueMessage(filterMsg, nil) + } default: peerLog.Debug("Filter request for unknown filter: %v", msg.FilterType) return } +} - hashes, err := sp.server.chain.HeightToHashRange( - int32(msg.StartHeight), &msg.StopHash, wire.MaxGetCFiltersReqRange, - ) - if err != nil { - peerLog.Debugf("Invalid getcfilters request: %v", err) +// OnGetCFHeaders is invoked when a peer receives a getcfheader bitcoin message. +func (sp *serverPeer) OnGetCFHeaders(_ *peer.Peer, msg *wire.MsgGetCFHeaders) { + // Ignore getcfilterheader requests if not in sync. + if !sp.server.syncManager.IsCurrent() { return } - // Create []*chainhash.Hash from []chainhash.Hash to pass to - // FiltersByBlockHashes. - hashPtrs := make([]*chainhash.Hash, len(hashes)) - for i := range hashes { - hashPtrs[i] = &hashes[i] - } + var startHeight int32 + var maxResults int + var hashList []chainhash.Hash + var hashPtrs []*chainhash.Hash + // if the filter type is supported, we initialize variables to avoid duplicate code + if msg.FilterType == wire.GCSFilterRegular || msg.FilterType == wire.UtreexoCFilter { - filters, err := sp.server.cfIndex.FiltersByBlockHashes( - hashPtrs, msg.FilterType, - ) - if err != nil { - peerLog.Errorf("Error retrieving cfilters: %v", err) - return - } + startHeight = int32(msg.StartHeight) + maxResults = wire.MaxCFHeadersPerMsg - for i, filterBytes := range filters { - if len(filterBytes) == 0 { - peerLog.Warnf("Could not obtain cfilter for %v", - hashes[i]) - return + // If StartHeight is positive, fetch the predecessor block hash so we + // can populate the PrevFilterHeader field. + if msg.StartHeight > 0 { + startHeight-- + maxResults++ } - filterMsg := wire.NewMsgCFilter( - msg.FilterType, &hashes[i], filterBytes, + // Fetch the hashes from the block index. + var err error + hashList, err = sp.server.chain.HeightToHashRange( + startHeight, &msg.StopHash, maxResults, ) - sp.QueueMessage(filterMsg, nil) - } -} + if err != nil { + peerLog.Debugf("Invalid getcfheaders request: %v", err) + } -// OnGetCFHeaders is invoked when a peer receives a getcfheader bitcoin message. -func (sp *serverPeer) OnGetCFHeaders(_ *peer.Peer, msg *wire.MsgGetCFHeaders) { - // Ignore getcfilterheader requests if not in sync. - if !sp.server.syncManager.IsCurrent() { - return + // This is possible if StartHeight is one greater that the height of + // StopHash, and we pull a valid range of hashes including the previous + // filter header. + if len(hashList) == 0 || (msg.StartHeight > 0 && len(hashList) == 1) { + peerLog.Debug("No results for getcfheaders request") + return + } + + // Create []*chainhash.Hash from []chainhash.Hash to pass to + // FilterHeadersByBlockHashes. + hashPtrs = make([]*chainhash.Hash, len(hashList)) + for i := range hashList { + hashPtrs[i] = &hashList[i] + } } // We'll also ensure that the remote party is requesting a set of // headers for filters that we actually currently maintain. switch msg.FilterType { case wire.GCSFilterRegular: - break - default: - peerLog.Debug("Filter request for unknown headers for "+ - "filter: %v", msg.FilterType) - return - } + // Fetch the raw filter hash bytes from the database for all blocks. + filterHashes, err := sp.server.cfIndex.FilterHashesByBlockHashes( + hashPtrs, msg.FilterType, + ) + if err != nil { + peerLog.Errorf("Error retrieving cfilter hashes: %v", err) + return + } - startHeight := int32(msg.StartHeight) - maxResults := wire.MaxCFHeadersPerMsg + // Generate cfheaders message and send it. + headersMsg := wire.NewMsgCFHeaders() - // If StartHeight is positive, fetch the predecessor block hash so we - // can populate the PrevFilterHeader field. - if msg.StartHeight > 0 { - startHeight-- - maxResults++ - } + // Populate the PrevFilterHeader field. + if msg.StartHeight > 0 { + prevBlockHash := &hashList[0] - // Fetch the hashes from the block index. - hashList, err := sp.server.chain.HeightToHashRange( - startHeight, &msg.StopHash, maxResults, - ) - if err != nil { - peerLog.Debugf("Invalid getcfheaders request: %v", err) - } - - // This is possible if StartHeight is one greater that the height of - // StopHash, and we pull a valid range of hashes including the previous - // filter header. - if len(hashList) == 0 || (msg.StartHeight > 0 && len(hashList) == 1) { - peerLog.Debug("No results for getcfheaders request") - return - } + // Fetch the raw committed filter header bytes from the + // database. + headerBytes, err := sp.server.cfIndex.FilterHeaderByBlockHash( + prevBlockHash, msg.FilterType) + if err != nil { + peerLog.Errorf("Error retrieving CF header: %v", err) + return + } + if len(headerBytes) == 0 { + peerLog.Warnf("Could not obtain CF header for %v", prevBlockHash) + return + } - // Create []*chainhash.Hash from []chainhash.Hash to pass to - // FilterHeadersByBlockHashes. - hashPtrs := make([]*chainhash.Hash, len(hashList)) - for i := range hashList { - hashPtrs[i] = &hashList[i] - } + // Deserialize the hash into PrevFilterHeader. + err = headersMsg.PrevFilterHeader.SetBytes(headerBytes) + if err != nil { + peerLog.Warnf("Committed filter header deserialize "+ + "failed: %v", err) + return + } - // Fetch the raw filter hash bytes from the database for all blocks. - filterHashes, err := sp.server.cfIndex.FilterHashesByBlockHashes( - hashPtrs, msg.FilterType, - ) - if err != nil { - peerLog.Errorf("Error retrieving cfilter hashes: %v", err) - return - } + hashList = hashList[1:] + filterHashes = filterHashes[1:] + } - // Generate cfheaders message and send it. - headersMsg := wire.NewMsgCFHeaders() + // Populate HeaderHashes. + for i, hashBytes := range filterHashes { + if len(hashBytes) == 0 { + peerLog.Warnf("Could not obtain CF hash for %v", hashList[i]) + return + } - // Populate the PrevFilterHeader field. - if msg.StartHeight > 0 { - prevBlockHash := &hashList[0] + // Deserialize the hash. + filterHash, err := chainhash.NewHash(hashBytes) + if err != nil { + peerLog.Warnf("Committed filter hash deserialize "+ + "failed: %v", err) + return + } - // Fetch the raw committed filter header bytes from the - // database. - headerBytes, err := sp.server.cfIndex.FilterHeaderByBlockHash( - prevBlockHash, msg.FilterType) - if err != nil { - peerLog.Errorf("Error retrieving CF header: %v", err) - return + headersMsg.AddCFHash(filterHash) } - if len(headerBytes) == 0 { - peerLog.Warnf("Could not obtain CF header for %v", prevBlockHash) - return + + headersMsg.FilterType = msg.FilterType + headersMsg.StopHash = msg.StopHash + + sp.QueueMessage(headersMsg, nil) + + // handle custom utreexocfilter message + case wire.UtreexoCFilter: + + // Generate cfheaders message and send it. + headersMsg := wire.NewMsgCFHeaders() + + // Populate the PrevFilterHeader field. + if msg.StartHeight > 0 { + prevBlockHash := &hashList[0] + + // Fetch the raw committed filter header bytes from the + // database. + headerBytes, err := sp.server.utreexoCfIndex.FilterHeaderByBlockHash( + prevBlockHash, msg.FilterType) + if err != nil { + peerLog.Errorf("Error retrieving CF header: %v", err) + return + } + if len(headerBytes) == 0 { + peerLog.Warnf("Could not obtain CF header for %v", prevBlockHash) + return + } + + // Deserialize the hash into PrevFilterHeader. + err = headersMsg.PrevFilterHeader.SetBytes(headerBytes) + if err != nil { + peerLog.Warnf("Committed filter header deserialize "+ + "failed: %v", err) + return + } } - // Deserialize the hash into PrevFilterHeader. - err = headersMsg.PrevFilterHeader.SetBytes(headerBytes) - if err != nil { - peerLog.Warnf("Committed filter header deserialize "+ - "failed: %v", err) - return + // fetch filter hashes and add to cf hashes field + for i, blockHash := range hashPtrs { + var serializedUtreexo []byte + // skip the first index as this index was added so as to enable us + // to get the previous filter's header + if i == 0 { + continue + } + + leaves, roots, err := sp.getUtreexoRoots(blockHash) + if err != nil { + return + } + + // serialize the hashes of the utreexo roots hash + serializedUtreexo, err = blockchain.SerializeUtreexoRootsHash(leaves, roots) + if err != nil { + peerLog.Errorf("error serializing utreexoc filter: %v", err) + return + } + + if len(serializedUtreexo) == 0 { + peerLog.Warnf("Could not obtain utreexocfilter for %v", + hashList[i]) + return + } + hashBytes := chainhash.DoubleHashB(serializedUtreexo) + + if len(hashBytes) == 0 { + peerLog.Warnf("Could not obtain CF hash for %v", hashList[i]) + return + } + + // Deserialize the hash. + filterHash, err := chainhash.NewHash(hashBytes) + if err != nil { + peerLog.Warnf("Committed filter hash deserialize "+ + "failed: %v", err) + return + } + + headersMsg.AddCFHash(filterHash) } + headersMsg.FilterType = msg.FilterType + headersMsg.StopHash = msg.StopHash + + sp.QueueMessage(headersMsg, nil) - hashList = hashList[1:] - filterHashes = filterHashes[1:] + default: + peerLog.Debug("Filter request for unknown headers for "+ + "filter: %v", msg.FilterType) + return } +} - // Populate HeaderHashes. - for i, hashBytes := range filterHashes { - if len(hashBytes) == 0 { - peerLog.Warnf("Could not obtain CF hash for %v", hashList[i]) - return - } +// getUtreexoRoots fetches utreexo roots from the appropriate locations, i.e fetches +// roots for CSN from a different location from utreexoviewpoint and flatfile +func (sp *serverPeer) getUtreexoRoots(blockHash *chainhash.Hash) (uint64, []*chainhash.Hash, error) { + + var leaves uint64 + var roots []*chainhash.Hash - // Deserialize the hash. - filterHash, err := chainhash.NewHash(hashBytes) + // For compact state nodes + if !cfg.NoUtreexo { + viewPoint, err := sp.server.chain.FetchUtreexoViewpoint(blockHash) if err != nil { - peerLog.Warnf("Committed filter hash deserialize "+ - "failed: %v", err) - return + peerLog.Errorf("could not obtain utreexo view: %v", err) + return 0, nil, err } - - headersMsg.AddCFHash(filterHash) + roots = viewPoint.GetRoots() + leaves = viewPoint.NumLeaves() } + // for bridge nodes + if sp.server.utreexoProofIndex != nil { + var uleaves uint64 + var uroots []*chainhash.Hash + var err error + err = sp.server.db.View(func(dbTx database.Tx) error { + uroots, uleaves, err = sp.server.utreexoProofIndex.FetchUtreexoState(dbTx, blockHash) + if err != nil { + return err + } - headersMsg.FilterType = msg.FilterType - headersMsg.StopHash = msg.StopHash - - sp.QueueMessage(headersMsg, nil) + return nil + }) + if err != nil { + peerLog.Errorf("error fetching utreexo view for blockhash %s: error: %v", blockHash, err) + return 0, nil, err + } + roots = uroots + leaves = uleaves + } else if sp.server.flatUtreexoProofIndex != nil { + height, err := sp.server.chain.BlockHeightByHash(blockHash) + if err != nil { + peerLog.Errorf("couldn't fetch the block height for blockhash %s from "+ + "the blockindex. Error: %v", blockHash, err) + return 0, nil, err + } + uroots, uleaves, err := sp.server.flatUtreexoProofIndex.FetchUtreexoState(height) + if err != nil { + peerLog.Errorf("error fetching utreexo view for blockhash: %s: error: %v", err) + return 0, nil, err + } + roots = uroots + leaves = uleaves + } + return leaves, roots, nil } // OnGetCFCheckpt is invoked when a peer receives a getcfcheckpt bitcoin message. @@ -1040,6 +1216,7 @@ func (sp *serverPeer) OnGetCFCheckpt(_ *peer.Peer, msg *wire.MsgGetCFCheckpt) { // checkpoints for filters that we actually currently maintain. switch msg.FilterType { case wire.GCSFilterRegular: + case wire.UtreexoCFilter: break default: @@ -1135,11 +1312,21 @@ func (sp *serverPeer) OnGetCFCheckpt(_ *peer.Peer, msg *wire.MsgGetCFCheckpt) { for i := forkIdx; i < len(blockHashes); i++ { blockHashPtrs = append(blockHashPtrs, &blockHashes[i]) } - filterHeaders, err := sp.server.cfIndex.FilterHeadersByBlockHashes( - blockHashPtrs, msg.FilterType, - ) - if err != nil { - peerLog.Errorf("Error retrieving cfilter headers: %v", err) + var filterHeaders [][]byte + var filterHeaderErr error + + if msg.FilterType == wire.GCSFilterRegular { + filterHeaders, filterHeaderErr = sp.server.cfIndex.FilterHeadersByBlockHashes( + blockHashPtrs, msg.FilterType, + ) + } else if msg.FilterType == wire.UtreexoCFilter { + filterHeaders, filterHeaderErr = sp.server.utreexoCfIndex.FilterHeadersByBlockHashes( + blockHashPtrs, msg.FilterType, + ) + } + + if filterHeaderErr != nil { + peerLog.Errorf("Error retrieving cfilter headers: %v", filterHeaderErr) return } @@ -3152,6 +3339,10 @@ func newServer(listenAddrs, agentBlacklist, agentWhitelist []string, services |= wire.SFNodeUtreexo } + if cfg.UtreexoCFilters { + services |= wire.SFNodeUtreexoCF + } + amgr := addrmgr.New(cfg.DataDir, btcdLookup) var listeners []net.Listener @@ -3262,6 +3453,12 @@ func newServer(listenAddrs, agentBlacklist, agentWhitelist []string, } indexes = append(indexes, s.flatUtreexoProofIndex) } + if cfg.UtreexoCFilters { + indxLog.Info("Utreexo C filter index enabled") + s.utreexoCfIndex = indexers.NewUtreexoCfIndex(db, chainParams, + s.utreexoProofIndex, s.flatUtreexoProofIndex, cfg.NoUtreexo) + indexes = append(indexes, s.utreexoCfIndex) + } // Create an index manager if any of the optional indexes are enabled. var indexManager blockchain.IndexManager @@ -3608,6 +3805,7 @@ func newServer(listenAddrs, agentBlacklist, agentWhitelist []string, TxIndex: s.txIndex, AddrIndex: s.addrIndex, CfIndex: s.cfIndex, + UtreexoCfIndex: s.utreexoCfIndex, TTLIndex: s.ttlIndex, UtreexoProofIndex: s.utreexoProofIndex, FlatUtreexoProofIndex: s.flatUtreexoProofIndex, diff --git a/utreexod.go b/utreexod.go index 427e408a..8c49d2db 100644 --- a/utreexod.go +++ b/utreexod.go @@ -94,6 +94,14 @@ func pruneChecks(db database.DB) error { "and sync from the beginning to enable the desired index. You may "+ "start the node up without the --cfilters flag", cfg.DataDir) } + // If we've previously been pruned and the utreexocfindex isn't present, it means that the + // user wants to enable the utreexocfindex after the node has already synced up while being pruned. + if beenPruned && !indexers.UtreexoCfIndexInitialized(db) && cfg.UtreexoCFilters { + return fmt.Errorf("utreexo cfilters cannot be enabled as the node has been "+ + "previously pruned. You must delete the files in the datadir: \"%s\" "+ + "and sync from the beginning to enable the desired index. You may "+ + "start the node up without the --utreexocfilters flag", cfg.DataDir) + } // If the user wants to disable the cfindex and is pruned or has enabled pruning, force // the user to either drop the cfindex manually or restart the node without the --cfilters @@ -114,6 +122,25 @@ func pruneChecks(db database.DB) error { "To keep the compact filters, restart the node with the --cfilters "+ "flag", prunedStr) } + // If the user wants to disable the utreexocfindex and is pruned or has enabled pruning, force + // the user to either drop the utreexocfindex manually or restart the node without the + // --utreexocfilters flag. + if (beenPruned || cfg.Prune != 0) && indexers.UtreexoCfIndexInitialized(db) && !cfg.UtreexoCFilters { + var prunedStr string + if beenPruned { + prunedStr = "has been previously pruned" + } else { + prunedStr = fmt.Sprintf("was started with prune flag (--prune=%d)", cfg.Prune) + } + return fmt.Errorf("--utreexocfilters flag was not given but the utreexo cfilters have "+ + "previously been enabled on this node and the index data currently "+ + "exists in the database. The node %s and "+ + "the database would be left in an inconsistent state if the utreexo c "+ + "filters don't get indexed now. To disable utreeco cfilters, please drop the "+ + "index completely with the --droputreexocfindex flag and restart the node. "+ + "To keep the compact filters, restart the node with the --utreexocfilters "+ + "flag", prunedStr) + } // If the user wants to disable the utreexo proof index and is pruned or has enabled pruning, // force the user to either drop the utreexo proof index manually or restart the node without // the --utreexoproofindex flag. @@ -287,6 +314,13 @@ func btcdMain(serverChan chan<- *server) error { return nil } + if cfg.DropUtreexoCfIndex { + if err := indexers.DropUtreexoCfIndex(db, interrupt); err != nil { + btcdLog.Errorf("%v", err) + return err + } + return nil + } if cfg.DropTTLIndex { if err := indexers.DropTTLIndex(db, interrupt); err != nil { btcdLog.Errorf("%v", err) diff --git a/wire/msgcfilter.go b/wire/msgcfilter.go index 682e9fd2..43a87b86 100644 --- a/wire/msgcfilter.go +++ b/wire/msgcfilter.go @@ -17,6 +17,9 @@ type FilterType uint8 const ( // GCSFilterRegular is the regular filter type. GCSFilterRegular FilterType = iota + // UtreexoCFilter is a custom filter type for serving Utreexo roots. + // creates a new filter type with FilterType value of 1 + UtreexoCFilter ) const ( diff --git a/wire/protocol.go b/wire/protocol.go index af73714a..9214a71c 100644 --- a/wire/protocol.go +++ b/wire/protocol.go @@ -87,6 +87,10 @@ const ( // filters (CFs). SFNodeCF + // SFNodeUtreexoCF is a flag used to indicate a peer supports utreexo + // committed filters (utreexo CFs). + SFNodeUtreexoCF + // SFNode2X is a flag used to indicate a peer is running the Segwit2X // software. SFNode2X @@ -115,6 +119,7 @@ var sfStrings = map[ServiceFlag]string{ SFNodeCF: "SFNodeCF", SFNode2X: "SFNode2X", SFNodeUtreexo: "SFNodeUtreexo", + SFNodeUtreexoCF: "SFNodeUtreexoCF", } // orderedSFStrings is an ordered list of service flags from highest to @@ -130,6 +135,7 @@ var orderedSFStrings = []ServiceFlag{ SFNodeCF, SFNode2X, SFNodeUtreexo, + SFNodeUtreexoCF, } // String returns the ServiceFlag in human-readable form.