Stopgap to re-fetch blocks with no children (#131)

* stopgap for handling blocks with no children in BFG

in BFG, we only increment block height to get each block in the chain; if there is a fork, we don't handle for it.  this introduces a check for blocks with no children, it will then "re-process" the block at that height from electrumx

* index for better query

* Update database/bfgd/postgres/postgres.go

Co-authored-by: Joshua Sing <joshua@bloq.com>

* Update database/bfgd/postgres/postgres.go

Co-authored-by: Joshua Sing <joshua@bloq.com>

* Update database/bfgd/postgres/postgres.go

Co-authored-by: Joshua Sing <joshua@bloq.com>

* removed extra newline

* Update database/bfgd/database_ext_test.go

Co-authored-by: Joshua Sing <joshua@bloq.com>

* handle for collision

* fix variable used

* correct function

* remove collision handling

---------

Co-authored-by: Joshua Sing <joshua@bloq.com>

database/bfgd/database.go

@@ -26,6 +26,7 @@ type Database interface {
 	BtcBlockInsert(ctx context.Context, bb *BtcBlock) error
 	BtcBlockByHash(ctx context.Context, hash [32]byte) (*BtcBlock, error)
 	BtcBlockHeightByHash(ctx context.Context, hash [32]byte) (uint64, error)
+	BtcBlocksHeightsWithNoChildren(ctx context.Context) ([]uint64, error)
 
 	// Pop data
 	PopBasisByL2KeystoneAbrevHash(ctx context.Context, aHash [32]byte, excludeUnconfirmed bool) ([]PopBasis, error)

database/bfgd/database_ext_test.go

@@ -1653,6 +1653,168 @@ func TestL2BtcFinalitiesByL2KeystoneNotPublishedHeight(t *testing.T) {
 	}
 }
 
+func TestBtcHeightsNoChildren(t *testing.T) {
+	type testTableItem struct {
+		name                         string
+		numberToCreateWithChildren   int
+		numberToCreateWithNoChildren int
+		overlapCount                 int
+	}
+
+	testTable := []testTableItem{
+		{
+			name:                         "0",
+			numberToCreateWithNoChildren: 0,
+			numberToCreateWithChildren:   43,
+		},
+		{
+			name:                         "less than 100",
+			numberToCreateWithNoChildren: 76,
+			numberToCreateWithChildren:   4,
+		},
+		{
+			name:                         "more than 100",
+			numberToCreateWithNoChildren: 126,
+			numberToCreateWithChildren:   333,
+		},
+		{
+			name:                         "more than 100 and overlap",
+			numberToCreateWithNoChildren: 126,
+			numberToCreateWithChildren:   333,
+			overlapCount:                 98,
+		},
+	}
+
+	createBlocksWithNoChildren := func(ctx context.Context, count int, db bfgd.Database) []int64 {
+		heights := make([]int64, count)
+		for i := range count {
+			height := mathrand.Int64()
+			hash := make([]byte, 32)
+			if _, err := rand.Read(hash); err != nil {
+				t.Fatal(err)
+			}
+			header := make([]byte, 80)
+			if _, err := rand.Read(header); err != nil {
+				t.Fatal(err)
+			}
+
+			btcBlock := bfgd.BtcBlock{
+				Height: uint64(height),
+				Hash:   hash,
+				Header: header,
+			}
+
+			if err := db.BtcBlockInsert(ctx, &btcBlock); err != nil {
+				t.Fatal(err)
+			}
+
+			heights[i] = height
+		}
+
+		return heights
+	}
+
+	createBlocksWithChildren := func(ctx context.Context, count int, db bfgd.Database, avoidHeights []int64, overlapHeights []int64) []int64 {
+		var prevHash []byte
+		overlapHeightI := 0
+		heights := make([]int64, count)
+		for i := range count {
+			var height int64
+			for {
+				if overlapHeightI < len(overlapHeights) {
+					height = overlapHeights[overlapHeightI]
+					overlapHeightI++
+					break
+				}
+
+				height = mathrand.Int64()
+				if !slices.Contains(avoidHeights, height) {
+					break
+				}
+			}
+			hash := make([]byte, 32)
+			if _, err := rand.Read(hash); err != nil {
+				t.Fatal(err)
+			}
+			header := make([]byte, 80)
+			if _, err := rand.Read(header); err != nil {
+				t.Fatal(err)
+			}
+
+			if len(prevHash) > 0 {
+				for k := range 32 {
+					header[k+4] = prevHash[k]
+				}
+			}
+
+			btcBlock := bfgd.BtcBlock{
+				Height: uint64(height),
+				Hash:   hash,
+				Header: header,
+			}
+
+			if err := db.BtcBlockInsert(ctx, &btcBlock); err != nil {
+				t.Fatal(err)
+			}
+			prevHash = hash
+			heights[i] = height
+		}
+		return heights
+	}
+
+	for _, tti := range testTable {
+		t.Run(tti.name, func(t *testing.T) {
+			ctx, cancel := defaultTestContext()
+			defer cancel()
+
+			db, sdb, cleanup := createTestDB(ctx, t)
+			defer func() {
+				db.Close()
+				sdb.Close()
+				cleanup()
+			}()
+
+			var overlapHeights []int64
+			noChildrenHeights := createBlocksWithNoChildren(ctx, tti.numberToCreateWithNoChildren, db)
+
+			childrenHeights := createBlocksWithChildren(ctx, tti.numberToCreateWithChildren, db, nil, overlapHeights)
+
+			if tti.overlapCount > 0 {
+				overlapHeights = noChildrenHeights[:tti.overlapCount]
+				oldChildrenHeights := childrenHeights
+				for _, o := range oldChildrenHeights {
+					if !slices.Contains(overlapHeights, o) {
+						childrenHeights = append(childrenHeights, o)
+					}
+				}
+			}
+
+			heights, err := db.BtcBlocksHeightsWithNoChildren(ctx)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			toCmp := make([]uint64, len(noChildrenHeights)+1)
+			for i, c := range noChildrenHeights {
+				toCmp[i] = uint64(c)
+			}
+			toCmp[len(toCmp)-1] = uint64(childrenHeights[len(childrenHeights)-1])
+
+			slices.Sort(heights)
+			slices.Sort(toCmp)
+
+			// we return a nil slice if emtpy, change that here for deep.Equal
+			if len(heights) == 0 {
+				heights = []uint64{}
+			}
+
+			if diff := deep.Equal(toCmp[:len(toCmp)-1], heights); len(diff) != 0 {
+				t.Fatalf("unexpected diff %s", diff)
+			}
+		})
+	}
+}
+
 func createBtcBlock(ctx context.Context, t *testing.T, db bfgd.Database, count int, chain bool, height int, lastHash []byte, l2BlockNumber uint32) bfgd.BtcBlock {
 	header := make([]byte, 80)
 	hash := make([]byte, 32)

database/bfgd/postgres/postgres.go

@@ -20,7 +20,7 @@ import (
 )
 
 const (
-	bfgdVersion = 7
+	bfgdVersion = 8
 
 	logLevel = "INFO"
 	verbose  = false
@@ -965,6 +965,57 @@ func (p *pgdb) AccessPublicKeyDelete(ctx context.Context, publicKey *bfgd.Access
 	return nil
 }
 
+// BtcBlocksHeightsWithNoChildren returns the heights of blocks stored in the
+// database that do not have any children, these represent possible forks that
+// have not been handled yet.
+func (p *pgdb) BtcBlocksHeightsWithNoChildren(ctx context.Context) ([]uint64, error) {
+	log.Tracef("BtcBlocksHeightsWithNoChildren")
+	defer log.Tracef("BtcBlocksHeightsWithNoChildren exit")
+
+	// Query all heights from btc_blocks where the block does not have any
+	// children and there are no other blocks at the same height with children.
+	// Excludes the tip because it will not have any children.
+	const q = `
+		SELECT height FROM btc_blocks bb1
+		WHERE NOT EXISTS (SELECT * FROM btc_blocks bb2 WHERE substr(bb2.header, 5, 32) = bb1.hash)
+		AND NOT EXISTS (
+			SELECT * FROM btc_blocks bb3 WHERE bb1.height = bb3.height 
+			AND EXISTS (
+				SELECT * FROM btc_blocks bb4 WHERE substr(bb4.header, 5, 32) = bb3.hash
+			)
+		)
+		ORDER BY height DESC
+		OFFSET $1 + 1
+		LIMIT 100
+	`
+
+	var heights []uint64
+	for offset := 0; ; offset += 100 {
+		rows, err := p.db.QueryContext(ctx, q, offset)
+		if err != nil {
+			return nil, err
+		}
+		defer rows.Close()
+
+		startingLength := len(heights)
+		for rows.Next() {
+			var v uint64
+			if err := rows.Scan(&v); err != nil {
+				return nil, err
+			}
+			heights = append(heights, v)
+		}
+
+		if startingLength == len(heights) {
+			return heights, nil
+		}
+
+		if rows.Err() != nil {
+			return nil, rows.Err()
+		}
+	}
+}
+
 // canonicalChainTipL2BlockNumber gets our best guess of the canonical tip
 // and returns it.  it finds the highest btc block with an associated
 // l2 keystone where only 1 btc block exists at that height

database/bfgd/scripts/0008.sql

@@ -0,0 +1,11 @@
+-- Copyright (c) 2024 Hemi Labs, Inc.
+-- Use of this source code is governed by the MIT License,
+-- which can be found in the LICENSE file.
+
+BEGIN;
+
+UPDATE version SET version = 8;
+
+CREATE INDEX btc_blocks_header_prev_hash_idx ON btc_blocks (substr(header, 5, 32));
+
+COMMIT;

service/bfg/bfg.go

@@ -124,6 +124,8 @@ type Server struct {
 	// record the last known canonical chain height,
 	// if this grows we need to notify subscribers
 	canonicalChainHeight uint64
+
+	checkForInvalidBlocks chan struct{}
 }
 
 func NewServer(cfg *Config) (*Server, error) {
@@ -145,7 +147,8 @@ func NewServer(cfg *Config) (*Server, error) {
 			Name:      "rpc_calls_total",
 			Help:      "The total number of succesful RPC commands",
 		}),
-		sessions: make(map[string]*bfgWs),
+		sessions:              make(map[string]*bfgWs),
+		checkForInvalidBlocks: make(chan struct{}),
 	}
 	for range requestLimit {
 		s.requestLimiter <- true
@@ -162,6 +165,36 @@ func NewServer(cfg *Config) (*Server, error) {
 	return s, nil
 }
 
+func (s *Server) queueCheckForInvalidBlocks() {
+	select {
+	case s.checkForInvalidBlocks <- struct{}{}:
+	default:
+	}
+}
+
+func (s *Server) invalidBlockChecker(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-s.checkForInvalidBlocks:
+			heights, err := s.db.BtcBlocksHeightsWithNoChildren(ctx)
+			if err != nil {
+				log.Errorf("error trying to get heights for btc blocks: %s", err)
+				return
+			}
+
+			log.Infof("received %d heights with no children, will re-check", len(heights))
+			for _, height := range heights {
+				log.Infof("reprocessing block at height %d", height)
+				if err := s.processBitcoinBlock(ctx, height); err != nil {
+					log.Errorf("error processing bitcoin block: %s", err)
+				}
+			}
+		}
+	}
+}
+
 // handleRequest is called as a go routine to handle a long-lived command.
 func (s *Server) handleRequest(parentCtx context.Context, bws *bfgWs, wsid string, requestType string, handler func(ctx context.Context) (any, error)) {
 	log.Tracef("handleRequest: %v", bws.addr)
@@ -546,6 +579,7 @@ func (s *Server) processBitcoinBlocks(ctx context.Context, start, end uint64) er
 		}
 		s.btcHeight = i
 	}
+	s.queueCheckForInvalidBlocks()
 	return nil
 }
 
@@ -1471,6 +1505,7 @@ func (s *Server) Run(pctx context.Context) error {
 
 	s.wg.Add(1)
 	go s.trackBitcoin(ctx)
+	go s.invalidBlockChecker(ctx)
 
 	select {
 	case <-ctx.Done():