chore: Add tests for unhappy path and alerting metrics for major reorg (

#129)

README.md

@@ -14,7 +14,8 @@ Staking protocol and serves as the ground truth for the Bitcoin Staking system.
    poller ensures that all the output blocks have at least `N` confirmations 
    where `N` is a configurable value, which should be large enough so that 
    the chance of the output blocks being forked is enormously low, e.g., 
-   greater than or equal to `6` in Bitcoin mainnet.
+   greater than or equal to `6` in Bitcoin mainnet. In case of major reorg,
+   the indexer will terminate and should manually bootstrap from a clean DB.
 2. Extracting transaction data for staking, unbonding, and withdrawal. These 
    transactions are verified and compared against the system parameters to 
    identify whether they are active, inactive due to staking cap overflow, 

btcscanner/block_handler.go

@@ -35,7 +35,7 @@ func (bs *BtcPoller) blockEventLoop(startHeight uint64) {
 
 	defer blockEventNotifier.Cancel()
 	if err != nil {
-		panic(fmt.Errorf("failed to register BTC notifier"))
+		panic("failed to register BTC notifier")
 	}
 
 	bs.logger.Info("BTC notifier registered")
@@ -51,7 +51,7 @@ func (bs *BtcPoller) blockEventLoop(startHeight uint64) {
 			bs.logger.Debug("received new best btc block",
 				zap.Int32("height", newBlock.Height))
 
-			err := bs.handleNewBlock(newBlock)
+			err := bs.HandleNewBlock(newBlock)
 			if err != nil {
 				bs.logger.Debug("failed to handle a new block, need bootstrapping",
 					zap.Int32("height", newBlock.Height),
@@ -77,11 +77,11 @@ func (bs *BtcPoller) blockEventLoop(startHeight uint64) {
 	}
 }
 
-// handleNewBlock handles a new block by adding it in the unconfirmed
+// HandleNewBlock handles a new block by adding it in the unconfirmed
 // block cache, and extracting confirmed blocks if there are any
 // error will be returned if the new block is not in the same branch
 // of the cache
-func (bs *BtcPoller) handleNewBlock(blockEpoch *notifier.BlockEpoch) error {
+func (bs *BtcPoller) HandleNewBlock(blockEpoch *notifier.BlockEpoch) error {
 	// get cache tip and check whether this block is expected
 	cacheTip := bs.unconfirmedBlockCache.Tip()
 	if cacheTip == nil {
@@ -135,7 +135,8 @@ func (bs *BtcPoller) commitChainUpdate(confirmedBlocks []*types.IndexedBlock) {
 			if !confirmedTipHash.IsEqual(&confirmedBlocks[0].Header.PrevBlock) {
 				// this indicates either programmatic error or the confirmation
 				// depth is not large enough to cover re-orgs
-				panic(fmt.Errorf("invalid canonical chain"))
+				majorReorgsCounter.Inc()
+				panic(fmt.Errorf("major reorgs happened at height %d", confirmedBlocks[0].Height))
 			}
 		}
 		bs.confirmedTipBlock = confirmedBlocks[len(confirmedBlocks)-1]

btcscanner/btc_scanner_test.go

@@ -7,16 +7,19 @@ import (
 
 	bbndatagen "github.com/babylonchain/babylon/testutil/datagen"
 	"github.com/golang/mock/gomock"
+	"github.com/lightningnetwork/lnd/chainntnfs"
 	"github.com/lightningnetwork/lnd/lntest/mock"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap"
 
 	"github.com/babylonchain/staking-indexer/btcscanner"
 	"github.com/babylonchain/staking-indexer/testutils/datagen"
 	"github.com/babylonchain/staking-indexer/testutils/mocks"
+	"github.com/babylonchain/staking-indexer/types"
 )
 
-func FuzzPoller(f *testing.F) {
+// FuzzBootstrap tests happy path of bootstrapping
+func FuzzBootstrap(f *testing.F) {
 	bbndatagen.AddRandomSeedsToFuzzer(f, 100)
 
 	f.Fuzz(func(t *testing.T, seed int64) {
@@ -64,3 +67,152 @@ func FuzzPoller(f *testing.F) {
 		wg.Wait()
 	})
 }
+
+// FuzzHandleNewBlock tests (1) happy path of handling an incoming block,
+// and (2) errors when the incoming block is not expected
+func FuzzHandleNewBlock(f *testing.F) {
+	bbndatagen.AddRandomSeedsToFuzzer(f, 100)
+
+	f.Fuzz(func(t *testing.T, seed int64) {
+		r := rand.New(rand.NewSource(seed))
+		versionedParams := datagen.GenerateGlobalParamsVersions(r, t)
+		k := uint64(versionedParams.Versions[0].ConfirmationDepth)
+		startHeight := versionedParams.Versions[0].ActivationHeight
+
+		// Generate a random number of blocks
+		numBlocks := bbndatagen.RandomIntOtherThan(r, 0, 50) + k // make sure we have at least k+1 entry
+		initialChain := datagen.GetRandomIndexedBlocks(r, startHeight, numBlocks)
+		bestHeight := initialChain[len(initialChain)-1].Height
+		bestBlockHash := initialChain[len(initialChain)-1].BlockHash()
+
+		numBlocks1 := bbndatagen.RandomIntOtherThan(r, 0, 50)
+		firstChainedIndexedBlocks := datagen.GetRandomIndexedBlocksFromHeight(r, numBlocks1, bestHeight, bestBlockHash)
+		firstChainedBlockEpochs := indexedBlocksToBlockEpochs(firstChainedIndexedBlocks)
+		canonicalChain := append(initialChain, firstChainedIndexedBlocks...)
+
+		ctl := gomock.NewController(t)
+		mockBtcClient := mocks.NewMockClient(ctl)
+		mockBtcClient.EXPECT().GetTipHeight().Return(uint64(bestHeight), nil).AnyTimes()
+		for _, b := range canonicalChain {
+			mockBtcClient.EXPECT().GetBlockByHeight(gomock.Eq(uint64(b.Height))).
+				Return(b, nil).AnyTimes()
+		}
+
+		numBlocks2 := bbndatagen.RandomIntOtherThan(r, 0, 50) + numBlocks1
+		secondChainedIndexedBlocks := datagen.GetRandomIndexedBlocksFromHeight(r, numBlocks2, bestHeight, bestBlockHash)
+		secondChainedBlockEpochs := indexedBlocksToBlockEpochs(secondChainedIndexedBlocks)
+
+		btcScanner, err := btcscanner.NewBTCScanner(uint16(k), zap.NewNop(), mockBtcClient, &mock.ChainNotifier{})
+		require.NoError(t, err)
+
+		// receive confirmed blocks
+		go func() {
+			for {
+				<-btcScanner.ChainUpdateInfoChan()
+			}
+		}()
+
+		err = btcScanner.Start(startHeight, startHeight)
+		require.NoError(t, err)
+		defer func() {
+			err := btcScanner.Stop()
+			require.NoError(t, err)
+		}()
+
+		for _, b := range firstChainedBlockEpochs {
+			err := btcScanner.HandleNewBlock(b)
+			require.NoError(t, err)
+		}
+
+		for i, b := range secondChainedBlockEpochs {
+			err := btcScanner.HandleNewBlock(b)
+			if i <= len(firstChainedBlockEpochs)-1 {
+				require.NoError(t, err)
+			} else if i == len(firstChainedBlockEpochs) {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), "the block's parent hash does not match the cache tip")
+			} else {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), "missing blocks")
+			}
+		}
+	})
+}
+
+// FuzzBootstrapMajorReorg tests the case when a major reorg is happening
+// this should cause panic
+func FuzzBootstrapMajorReorg(f *testing.F) {
+	bbndatagen.AddRandomSeedsToFuzzer(f, 100)
+
+	f.Fuzz(func(t *testing.T, seed int64) {
+		r := rand.New(rand.NewSource(seed))
+		versionedParams := datagen.GenerateGlobalParamsVersions(r, t)
+		k := uint64(10)
+		startHeight := versionedParams.Versions[0].ActivationHeight
+		// Generate a random number of blocks
+		numBlocks := bbndatagen.RandomIntOtherThan(r, 0, 50) + k // make sure we have at least k+1 entry
+		chainIndexedBlocks := datagen.GetRandomIndexedBlocks(r, startHeight, numBlocks)
+		bestHeight := chainIndexedBlocks[len(chainIndexedBlocks)-1].Height
+		confirmedBlocks := chainIndexedBlocks[:numBlocks-k+1]
+		lastConfirmedBlock := confirmedBlocks[len(confirmedBlocks)-1]
+
+		// major reorg chain is created from the last confirmed height but does not point
+		// to the last confirmed block
+		secondChain := datagen.GetRandomIndexedBlocks(r, uint64(lastConfirmedBlock.Height+1), numBlocks)
+		secondBestHeight := secondChain[len(secondChain)-1].Height
+
+		ctl := gomock.NewController(t)
+		mockBtcClient := mocks.NewMockClient(ctl)
+		tipHeightCall1 := mockBtcClient.EXPECT().GetTipHeight().Return(uint64(bestHeight), nil)
+		mockBtcClient.EXPECT().GetTipHeight().Return(uint64(secondBestHeight), nil).After(tipHeightCall1)
+		for i := chainIndexedBlocks[0].Height; i <= secondChain[len(secondChain)-1].Height; i++ {
+			if i < secondChain[0].Height {
+				firstBlock := chainIndexedBlocks[int(i-chainIndexedBlocks[0].Height)]
+				mockBtcClient.EXPECT().GetBlockByHeight(gomock.Eq(uint64(i))).
+					Return(firstBlock, nil)
+			} else if i <= bestHeight {
+				firstBlock := chainIndexedBlocks[int(i-chainIndexedBlocks[0].Height)]
+				c1 := mockBtcClient.EXPECT().GetBlockByHeight(gomock.Eq(uint64(i))).
+					Return(firstBlock, nil)
+				secondBlock := secondChain[int(i-secondChain[0].Height)]
+				mockBtcClient.EXPECT().GetBlockByHeight(gomock.Eq(uint64(i))).
+					Return(secondBlock, nil).After(c1)
+			} else {
+				secondBlock := secondChain[int(i-secondChain[0].Height)]
+				mockBtcClient.EXPECT().GetBlockByHeight(gomock.Eq(uint64(i))).
+					Return(secondBlock, nil)
+			}
+		}
+
+		btcScanner, err := btcscanner.NewBTCScanner(uint16(k), zap.NewNop(), mockBtcClient, &mock.ChainNotifier{})
+		require.NoError(t, err)
+
+		// receive confirmed blocks
+		go func() {
+			for {
+				<-btcScanner.ChainUpdateInfoChan()
+			}
+		}()
+
+		err = btcScanner.Bootstrap(startHeight)
+		require.NoError(t, err)
+		require.Panics(t, func() {
+			_ = btcScanner.Bootstrap(uint64(lastConfirmedBlock.Height) + 1)
+		})
+	})
+}
+
+func indexedBlocksToBlockEpochs(ibs []*types.IndexedBlock) []*chainntnfs.BlockEpoch {
+	blockEpochs := make([]*chainntnfs.BlockEpoch, 0)
+	for _, ib := range ibs {
+		blockHash := ib.BlockHash()
+		be := &chainntnfs.BlockEpoch{
+			Hash:        &blockHash,
+			Height:      ib.Height,
+			BlockHeader: ib.Header,
+		}
+		blockEpochs = append(blockEpochs, be)
+	}
+
+	return blockEpochs
+}

btcscanner/metrics.go

@@ -0,0 +1,15 @@
+package btcscanner
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	majorReorgsCounter = promauto.NewCounter(
+		prometheus.CounterOpts{
+			Name: "si_major_reorgs_counter",
+			Help: "Total number of major reorgs happened",
+		},
+	)
+)

doc/metrics.md

@@ -56,3 +56,5 @@ global parameters.
   failures when processing valid withdrawal transactions from unbonding
 
 * `invalidTransactionsCounter`: Total number of invalid transactions
+
+* `majorReorgsCounter`: Total number of major reorgs happened

itest/e2e_test.go

@@ -240,15 +240,17 @@ func TestWithdrawFromMultipleUnbondingTxs(t *testing.T) {
 		stakingTxHash1, storedStakingTx1.StakingOutputIdx, stakingInfo1, stakingTx1,
 		getCovenantPrivKeys(t), regtestParams,
 	)
-	unbondingTxHash1, err := tm.WalletClient.SendRawTransaction(unbondingTx1, true)
-	require.NoError(t, err)
+	unbondingTxHash1 := unbondingTx1.TxHash()
+	tm.SendTxWithNConfirmations(t, unbondingTx1, 1)
+
 	unbondingTx2, unbondingInfo2 := testutils.BuildUnbondingTx(
 		t, sysParams, tm.WalletPrivKey,
 		testStakingData2.FinalityProviderKey, testStakingData2.StakingAmount,
 		stakingTxHash2, storedStakingTx2.StakingOutputIdx, stakingInfo2, stakingTx2,
 		getCovenantPrivKeys(t), regtestParams,
 	)
-	unbondingTxHash2, err := tm.WalletClient.SendRawTransaction(unbondingTx2, true)
+	unbondingTxHash2 := unbondingTx2.TxHash()
+	tm.SendTxWithNConfirmations(t, unbondingTx2, 1)
 	require.NoError(t, err)
 
 	// wait for the unbonding tx expires
@@ -262,15 +264,15 @@ func TestWithdrawFromMultipleUnbondingTxs(t *testing.T) {
 
 	fundingInfo1 := &testutils.FundingInfo{
 		FundTxOutput:      unbondingTx1.TxOut[0],
-		FundTxHash:        *unbondingTxHash1,
+		FundTxHash:        unbondingTxHash1,
 		FundTxOutputIndex: 0,
 		FundTxSpendInfo:   withdrawSpendInfo1,
 		LockTime:          sysParams.UnbondingTime,
 		Value:             btcutil.Amount(unbondingTx1.TxOut[0].Value),
 	}
 	fundingInfo2 := &testutils.FundingInfo{
 		FundTxOutput:      unbondingTx2.TxOut[0],
-		FundTxHash:        *unbondingTxHash2,
+		FundTxHash:        unbondingTxHash2,
 		FundTxOutputIndex: 0,
 		FundTxSpendInfo:   withdrawSpendInfo2,
 		LockTime:          sysParams.UnbondingTime,
@@ -289,8 +291,8 @@ func TestWithdrawFromMultipleUnbondingTxs(t *testing.T) {
 	tm.CheckNextWithdrawEvent(t, *stakingTxHash1)
 	tm.CheckNextWithdrawEvent(t, *stakingTxHash2)
 	// consume unbonding events
-	tm.CheckNextUnbondingEvent(t, *unbondingTxHash1)
-	tm.CheckNextUnbondingEvent(t, *unbondingTxHash2)
+	tm.CheckNextUnbondingEvent(t, unbondingTxHash1)
+	tm.CheckNextUnbondingEvent(t, unbondingTxHash2)
 }
 
 // TestWithdrawStakingAndUnbondingTxs tests withdrawal from staking and unbonding tx in

testutils/utils.go

@@ -121,10 +121,6 @@ func CreateTxFromOutputsAndSign(
 		return nil, err
 	}
 
-	if err != nil {
-		return nil, err
-	}
-
 	fundedTx, signed, err := btcClient.SignRawTransactionWithWallet(tx)
 
 	if err != nil {