remove fallback by height in canonical chain

remove fallback of finding any block at the next height if we cannot find a parent

redefine "tip" as: highest block with only 1 at its height with an existing parent, unless there are no parents to any blocks, then just the highest block with a count of 1 at that height

only refresh the canonical chain if we know have all heights' parents

database/bfgd/database_ext_test.go

@@ -1332,11 +1332,6 @@ func TestBtcBlockGetCanonicalChain(t *testing.T) {
 			onChainCount:  2,
 			offChainCount: 1,
 		},
-		{
-			name:          "1 on, 2 off",
-			onChainCount:  1,
-			offChainCount: 2,
-		},
 		{
 			name:          "100 on, 99 off",
 			onChainCount:  100,
@@ -1369,7 +1364,8 @@ func TestBtcBlockGetCanonicalChain(t *testing.T) {
 				)
 			}
 
-			height += 10000
+			height = 1
+
 			l2BlockNumber += 1000
 			// create on-chain blocks
 			onChainBlocks = createBtcBlocksAtStartingHeight(ctx, t, db, tti.onChainCount, true, height, []byte{}, l2BlockNumber)
@@ -1445,16 +1441,6 @@ func TestBtcBlockGetCanonicalChainWithForks(t *testing.T) {
 			chainPattern:       []int{2, 1, 1},
 			unconfirmedIndices: []bool{false, false, false, false},
 		},
-		{
-			name:               "fork in beginning with break",
-			chainPattern:       []int{2, 1, 1, 1},
-			unconfirmedIndices: []bool{false, false, true, false},
-		},
-		{
-			name:               "fork in beginning with multiple breaks",
-			chainPattern:       []int{2, 1, 1, 1, 1},
-			unconfirmedIndices: []bool{false, true, false, true, false},
-		},
 	}
 
 	for _, tti := range testTable {
@@ -1547,7 +1533,9 @@ func TestPublications(t *testing.T) {
 			lastHash := []byte{}
 			for _, height := range tti.heightPattern {
 				_onChainBlocks := createBtcBlocksAtStaticHeight(ctx, t, db, 1, true, height, lastHash, l2BlockNumber)
-				lastHash = _onChainBlocks[0].Hash
+				if len(_onChainBlocks[0].Hash) > 0 {
+					lastHash = _onChainBlocks[0].Hash
+				}
 				l2BlockNumber++
 			}
 

database/bfgd/postgres/postgres.go

@@ -20,7 +20,7 @@ import (
 )
 
 const (
-	bfgdVersion = 8
+	bfgdVersion = 9
 
 	logLevel = "INFO"
 	verbose  = false
@@ -976,16 +976,8 @@ func (p *pgdb) BtcBlocksHeightsWithNoChildren(ctx context.Context) ([]uint64, er
 	// children and there are no other blocks at the same height with children.
 	// Excludes the tip because it will not have any children.
 	const q = `
-		SELECT height FROM btc_blocks bb1
-		WHERE NOT EXISTS (SELECT * FROM btc_blocks bb2 WHERE substr(bb2.header, 5, 32) = bb1.hash)
-		AND NOT EXISTS (
-			SELECT * FROM btc_blocks bb3 WHERE bb1.height = bb3.height 
-			AND EXISTS (
-				SELECT * FROM btc_blocks bb4 WHERE substr(bb4.header, 5, 32) = bb3.hash
-			)
-		)
-		ORDER BY height DESC
-		OFFSET $1 + 1
+		SELECT height FROM heights_with_no_children
+		OFFSET $1
 		LIMIT 100
 	`
 

database/bfgd/scripts/0009.sql

@@ -0,0 +1,89 @@
+-- Copyright (c) 2024 Hemi Labs, Inc.
+-- Use of this source code is governed by the MIT License,
+-- which can be found in the LICENSE file.
+BEGIN;
+
+UPDATE version
+SET version = 9;
+
+DROP MATERIALIZED VIEW btc_blocks_can;
+-- this materialized view represents the canonical btc_blocks as we know it
+CREATE MATERIALIZED VIEW btc_blocks_can AS WITH RECURSIVE bb AS (
+	-- define the tip as the highest block in __highest, look below 
+	-- for definition of this result set
+	SELECT hash,
+		header,
+		height
+	FROM btc_blocks
+	WHERE height = (
+			-- give me the block at the max height with only 1 block at
+			-- that height
+			SELECT MAX(height) as height
+			FROM __highest
+			WHERE c = 1
+		)
+	UNION
+	SELECT btc_blocks.hash,
+		btc_blocks.header,
+		btc_blocks.height
+	FROM btc_blocks,
+		bb
+	WHERE -- find the parent block via header -> parent hash
+		(
+			substr(bb.header, 5, 32) = btc_blocks.hash
+		)
+),
+__highest AS (
+	-- use this to find the tip, creates "__highest" result set:
+	-- give me the count of blocks at each height
+	SELECT height,
+		count(*) AS c
+	FROM btc_blocks bbo -- where there exists a parent
+	WHERE EXISTS (
+			SELECT *
+			FROM btc_blocks bbi
+			WHERE substr(bbo.header, 5, 32) = bbi.hash
+		) -- unless there are no parents for ANY block
+		OR NOT EXISTS (
+			SELECT *
+			FROM btc_blocks bb1
+				INNER JOIN btc_blocks bb2 ON substr(bb1.header, 5, 32) = bb2.hash
+		)
+	GROUP BY height
+)
+SELECT *
+FROM bb;
+
+
+-- create view to return all heights that have no children
+CREATE VIEW heights_with_no_children AS
+SELECT height
+FROM btc_blocks bb1 
+-- for all blocks, check if there exists no children
+WHERE NOT EXISTS (
+		SELECT *
+		FROM btc_blocks bb2
+		WHERE substr(bb2.header, 5, 32) = bb1.hash
+	) -- then, check if there exist no other blocks at this height with children
+	AND NOT EXISTS (
+		SELECT *
+		FROM btc_blocks bb3
+		WHERE bb1.height = bb3.height
+			AND EXISTS (
+				SELECT *
+				FROM btc_blocks bb4
+				WHERE substr(bb4.header, 5, 32) = bb3.hash
+			)
+	) -- exclude the tip, as it will have no children by its nature
+ORDER BY height DESC OFFSET 1;
+
+-- only refresh materialized view if there are no heights without children
+CREATE OR REPLACE FUNCTION refresh_btc_blocks_can() RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ BEGIN IF NOT EXISTS(
+		SELECT *
+		FROM heights_with_no_children
+	) THEN REFRESH MATERIALIZED VIEW btc_blocks_can;
+END IF;
+RETURN NEW;
+END;
+$$;
+COMMIT;

e2e/docker-compose.yml

@@ -33,7 +33,7 @@ services:
       - "-rpcport=18443"
       - "-rpcconnect=bitcoind"
       - "generatetoaddress"
-      - "3000" # need to generate a lot for greater chance to not spend coinbase
+      - "1000" # need to generate a lot for greater chance to not spend coinbase
       - "$BTC_ADDRESS"
     restart: on-failure
 
@@ -97,7 +97,7 @@ services:
       BFG_POSTGRES_URI: "postgres://postgres@bfgd-postgres:5432/bfg?sslmode=disable"
       BFG_BTC_START_HEIGHT: "1"
       BFG_EXBTC_ADDRESS: "electrumx:50001"
-      BFG_LOG_LEVEL: "INFO"
+      BFG_LOG_LEVEL: "TRACE"
       BFG_PUBLIC_ADDRESS: ":8383"
       BFG_PRIVATE_ADDRESS: ":8080"
 

service/bfg/bfg.go

@@ -185,12 +185,24 @@ func (s *Server) invalidBlockChecker(ctx context.Context) {
 			}
 
 			log.Infof("received %d heights with no children, will re-check", len(heights))
+
+			// if all blocks have children, wait until next btc block to re-check
+			if len(heights) == 0 {
+				continue
+			}
+
+			// otherwise, reprocess blocks with missing children and re-check
+
 			for _, height := range heights {
 				log.Infof("reprocessing block at height %d", height)
 				if err := s.processBitcoinBlock(ctx, height); err != nil {
 					log.Errorf("error processing bitcoin block: %s", err)
 				}
 			}
+
+			go func() {
+				s.queueCheckForInvalidBlocks()
+			}()
 		}
 	}
 }
@@ -1507,6 +1519,8 @@ func (s *Server) Run(pctx context.Context) error {
 	go s.trackBitcoin(ctx)
 	go s.invalidBlockChecker(ctx)
 
+	s.queueCheckForInvalidBlocks()
+
 	select {
 	case <-ctx.Done():
 		err = ctx.Err()