squashme

e2e/monitor/main_test.go

@@ -6,8 +6,10 @@ package main
 
 import (
 	"encoding/json"
+	"fmt"
 	"math/big"
 	"testing"
+	"time"
 
 	"github.com/hemilabs/heminetwork/hemi"
 )
@@ -26,36 +28,55 @@ func TestMonitor(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// each keystone is 25 seconds, so there are 4 keystones per 100 seconds,
-	// we expect the number of pop txs to be at least once every 25 seconds
-	// for the time we waited
-	// add 25 seconds for cushion
-	seconds := ms / 1000
-	popTxsPer100Seconds := 4
-	expectedPopTxs := popTxsPer100Seconds * (seconds / 100)
+	var lastErr error
+	const retryTimeoutSeconds = 25
+	const maxRetries = 5
 
-	t.Logf("expecting at least %d pop txs mined", expectedPopTxs)
+	for i := range maxRetries {
+		t.Logf("retry #%d, last error was %v", i, lastErr)
 
-	if jo.PopTxCount < uint64(expectedPopTxs) {
-		t.Fatalf("popTxCount %d < %d", jo.PopTxCount, expectedPopTxs)
-	}
+		lastErr = nil
 
-	// the expected balance should be at least 1 BaseHEMI per poptx - 8.  We say
-	// "- 8" because we lag 8 keystones behind a pop payout (200 L2 blocks at
-	// 1 block per second)
-	popMinerBalance := big.NewInt(0)
-	balance, ok := popMinerBalance.SetString(jo.PopMinerHemiBalance, 10)
-	if !ok {
-		t.Fatalf("could not parse balance from %s", jo.PopMinerHemiBalance)
-	}
+		// each keystone is 25 seconds, so there are 4 keystones per 100 seconds,
+		// we expect the number of pop txs to be at least once every 25 seconds
+		// for the time we waited
+		// add 25 seconds for cushion
+		seconds := ms / 1000
+		popTxsPer100Seconds := 4
+		expectedPopTxs := popTxsPer100Seconds * (seconds / 100)
+
+		t.Logf("expecting at least %d pop txs mined", expectedPopTxs)
+
+		if jo.PopTxCount < uint64(expectedPopTxs) {
+			lastErr = fmt.Errorf("popTxCount %d < %d", jo.PopTxCount, expectedPopTxs)
+			continue
+		}
 
-	expectedPayouts := expectedPopTxs - 8
-	expectedPayoutBalance := big.NewInt(hemi.HEMIBase)
-	expectedPayoutBalance = expectedPayoutBalance.Mul(big.NewInt(int64(expectedPayouts)), expectedPayoutBalance)
+		// the expected balance should be at least 1 BaseHEMI per poptx - 8.  We say
+		// "- 8" because we lag 8 keystones behind a pop payout (200 L2 blocks at
+		// 1 block per second)
+		popMinerBalance := big.NewInt(0)
+		balance, ok := popMinerBalance.SetString(jo.PopMinerHemiBalance, 10)
+		if !ok {
+			lastErr = fmt.Errorf("could not parse balance from %s", jo.PopMinerHemiBalance)
+			continue
+		}
 
-	t.Logf("expecting actual balance %d to be greater than %d", balance, expectedPayoutBalance)
+		expectedPayouts := expectedPopTxs - 8
+		expectedPayoutBalance := big.NewInt(hemi.HEMIBase)
+		expectedPayoutBalance = expectedPayoutBalance.Mul(big.NewInt(int64(expectedPayouts)), expectedPayoutBalance)
+
+		t.Logf("expecting actual balance %d to be greater than %d", balance, expectedPayoutBalance)
+
+		if expectedPayoutBalance.Cmp(balance) > 0 {
+			lastErr = fmt.Errorf("pop miner payout balance received %d, want at least %d", balance, expectedPayoutBalance)
+			continue
+		}
+
+		time.Sleep(retryTimeoutSeconds * time.Second)
+	}
 
-	if expectedPayoutBalance.Cmp(balance) > 0 {
-		t.Fatalf("pop miner payout balance received %d, want at least %d", balance, expectedPayoutBalance)
+	if lastErr != nil {
+		t.Fatal(lastErr)
 	}
 }