Transition to beacon API

.github/workflows/golangci-lint.yml

@@ -16,15 +16,15 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.22'
+          go-version: '1.23'
           cache: false
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v4
         with:
           # Require: The version of golangci-lint to use.
           # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version.
           # When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit.
-          version: v1.54
+          version: v1.60
 
           # XXX Workaround for https://github.com/golangci/golangci-lint-action/issues/135
           # Optional: if set to true, then the action won't cache or restore ~/go/pkg.

.github/workflows/main.yml

@@ -38,7 +38,7 @@ jobs:
     steps:
     - uses: actions/setup-go@v3
       with:
-        go-version: 1.21.x
+        go-version: 1.23.x
     - uses: actions/checkout@v3
     - name: Build
       run: |

README.md

@@ -7,7 +7,6 @@ Eth2 Monitor serves a few purposes:
 
 * monitors the attestation inclusion distance,
 * monitors and alerts slashing events,
-* exports rewards history and other data (TBD).
 
 ## Installation ##
 
@@ -39,32 +38,8 @@ It's recommended to use the latest Go version.
 
 ## Install Prysm beacon node ##
 
-`eth2-monitor` uses Prysm's GRPC API in order to query the Beacon Chain.
-You can re-use your existing Prysm beacon node or set up another one.
-
-Here is an example of how you can sync a Prysm beacon node:
-
-``` shell
-docker run                                                                              \
-  -v "${HOME}/prysm":/data                                                              \
-  -p 0.0.0.0:4000:4000/tcp                                                              \
-  -p 0.0.0.0:13000:13000                                                                \
-  -p 0.0.0.0:12000:12000/udp                                                            \
-  --rm                                                                                  \
-  --name prysm                                                                          \
-  gcr.io/prysmaticlabs/prysm/beacon-chain:stable                                        \
-    --accept-terms-of-use                                                               \
-    --mainnet                                                                           \
-    --datadir /data                                                                     \
-    --http-web3provider https://mainnet.infura.io/v3/YOUR_TOKEN                         \
-    --rpc-host 0.0.0.0
-```
-
-If you want to export a large amount of data involving all validators, add `--grpc-max-msg-size 419430400 --rpc-max-page-size 100000` to achieve adequate performance.
-
-Note, GRPC uses port 4000/tcp by default.
-
-Use `--beacon-node` to work with a remote Prysm. By default, the monitor connects to `localhost:4000`.
+`eth2-monitor` relies on [Beacon Node API](https://ethereum.github.io/beacon-APIs/#/) to query the Beacon
+Chain.  You can re-use your existing Prysm beacon node or set up another one.
 
 ## Usage ##
 
@@ -85,6 +60,12 @@ eth2-monitor cmd keys.txt
 
 All public keys are hex-encoded and case-insensitive. On the first run, the monitor will convert the keys into indexes: it takes some time if you have many keys. On the second run, the indexes are loaded from a cache.
 
+Here's a more involved example invocation:
+
+```
+./bin/eth2-monitor monitor --beacon-chain-api <PRYSM_NODE_IP_ADDRESS>:3500 --print-successful --log-level trace <VALIDATOR_PUBLIC_KEYS_FILE_PATH>
+```
+
 Don't hesitate to run commands with `--help` to learn more about CLI. 😉
 
 ### Monitor attestations ###
@@ -114,15 +95,3 @@ eth2-monitor slashings --slack-url https://hooks.slack.com/services/YOUR_TOKEN
 ```
 
 At stakefish, we use it for [our Twitter bot](https://twitter.com/Eth2SlashBot).
-
-### Export rewards history and other data (TBD) ###
-
-In case you want to see your rewards history, you can export it as a CSV file or into a PostgreSQL database.
-
-``` shell
-eth2-monitor export -o my-validators-rewards keys.txt
-```
-
-If no keys are specified, all validators will be exported.
-
-The format of the exported data is subject to modifications, and the schema may change in future releases.

beaconchain/service.go

@@ -2,10 +2,16 @@ package beaconchain
 
 import (
 	"context"
+	"encoding/hex"
+	"fmt"
+	"strings"
 	"time"
 
 	eth2client "github.com/attestantio/go-eth2-client"
+	"github.com/attestantio/go-eth2-client/api"
+	apiv1 "github.com/attestantio/go-eth2-client/api/v1"
 	eth2http "github.com/attestantio/go-eth2-client/http"
+	"github.com/attestantio/go-eth2-client/spec/phase0"
 )
 
 type BeaconChain struct {
@@ -32,6 +38,104 @@ func (beacon *BeaconChain) Service() eth2client.Service {
 	return beacon.service
 }
 
-func (beacon *BeaconChain) Timeout() time.Duration {
-	return beacon.timeout
+func NormalizedPublicKey(pubkey string) string {
+	if !strings.HasPrefix(pubkey, "0x") {
+		panic(fmt.Sprintf("Public key did not have the expected 0x prefix: %v", pubkey))
+	}
+	pubkey = strings.TrimPrefix(pubkey, "0x")
+	pubkey = strings.ToLower(pubkey)
+	return pubkey
+}
+
+func (beacon *BeaconChain) GetValidatorIndexes(ctx context.Context, pubkeys []string) (map[string]phase0.ValidatorIndex, error) {
+	provider := beacon.service.(eth2client.ValidatorsProvider)
+
+	blspubkeys := make([]phase0.BLSPubKey, len(pubkeys))
+	for i, strkey := range pubkeys {
+		binkey, err := hex.DecodeString(strkey)
+		if err != nil {
+			return nil, err
+		}
+		blspubkeys[i] = phase0.BLSPubKey(binkey)
+	}
+
+	resp, err := provider.Validators(ctx, &api.ValidatorsOpts{
+		State:   "justified",
+		PubKeys: blspubkeys,
+	})
+	if err != nil {
+		return nil, err
+	}
+	if len(resp.Data) > len(pubkeys) {
+		panic(fmt.Sprintf("Expected at most %v validator in Beacon API response, got %v", len(pubkeys), len(resp.Data)))
+	}
+
+	result := map[string]phase0.ValidatorIndex{}
+	for index, validator := range resp.Data {
+		// Includes the leading 0x
+		key := validator.Validator.PublicKey.String()
+		key = NormalizedPublicKey(key)
+		result[key] = index
+	}
+	return result, nil
+}
+
+func (beacon *BeaconChain) GetProposerDuties(ctx context.Context, epoch phase0.Epoch, indices []phase0.ValidatorIndex) ([]*apiv1.ProposerDuty, error) {
+	provider := beacon.service.(eth2client.ProposerDutiesProvider)
+	resp, err := provider.ProposerDuties(ctx, &api.ProposerDutiesOpts{
+		Epoch:   epoch,
+		Indices: indices,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return resp.Data, err
+}
+
+func (beacon *BeaconChain) GetBeaconCommitees(ctx context.Context, epoch phase0.Epoch) ([]*apiv1.BeaconCommittee, error) {
+	provider := beacon.service.(eth2client.BeaconCommitteesProvider)
+	resp, err := provider.BeaconCommittees(ctx, &api.BeaconCommitteesOpts{
+		State: "justified",
+		Epoch: &epoch,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return resp.Data, err
+}
+
+func (beacon *BeaconChain) getValidatorBalance(ctx context.Context, validator phase0.ValidatorIndex, slot phase0.Slot) (*phase0.Gwei, error) {
+	provider := beacon.service.(eth2client.ValidatorBalancesProvider)
+	resp, err := provider.ValidatorBalances(ctx, &api.ValidatorBalancesOpts{
+		State:   fmt.Sprintf("%d", slot),
+		Indices: []phase0.ValidatorIndex{validator},
+	})
+	if err != nil {
+		return nil, err
+	}
+	if len(resp.Data) == 0 {
+		return nil, nil
+	}
+	if len(resp.Data) > 1 {
+		panic(fmt.Sprintf("Expected at most 1 validator in Beacon API response, got %v", len(resp.Data)))
+	}
+	for _, balance := range resp.Data {
+		return &balance, nil
+	}
+	panic("unreachable")
+}
+
+func (beacon *BeaconChain) GetValidatorBalanceDiff(ctx context.Context, validator phase0.ValidatorIndex, earlierSlot phase0.Slot, laterSlot phase0.Slot) (*int64, error) {
+	earlierBalance, err := beacon.getValidatorBalance(ctx, validator, earlierSlot)
+	if err != nil {
+		return nil, err
+	}
+
+	laterBalance, err := beacon.getValidatorBalance(ctx, validator, laterSlot)
+	if err != nil {
+		return nil, err
+	}
+
+	balance := int64(*laterBalance) - int64(*earlierBalance)
+	return &balance, nil
 }

cmd/opts/opts.go

@@ -2,7 +2,6 @@ package opts
 
 var (
 	LogLevel       string
-	BeaconNode     string
 	BeaconChainAPI string
 	MetricsPort    string
 	SlackURL       string

cmd/root.go

@@ -10,9 +10,9 @@ import (
 	"eth2-monitor/beaconchain"
 	"eth2-monitor/cmd/opts"
 	"eth2-monitor/pkg"
-	"eth2-monitor/prysmgrpc"
 
 	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 
 	"net/http"
@@ -55,21 +55,18 @@ var (
 		Run: func(cmd *cobra.Command, args []string) {
 			ctx, cancel := context.WithCancel(context.Background())
 			defer cancel()
-			s, err := prysmgrpc.New(ctx,
-				prysmgrpc.WithAddress(opts.BeaconNode),
-				prysmgrpc.WithTimeout(time.Minute))
-			pkg.Must(err)
 
 			beacon, err := beaconchain.New(ctx, opts.BeaconChainAPI, time.Minute)
 			pkg.Must(err)
 
 			plainPubkeys, err := pkg.LoadKeys(args)
 			pkg.Must(err)
+			log.Info().Msgf("Validator keys loaded from file(s): %v", len(plainPubkeys))
 
 			var wg sync.WaitGroup
 			wg.Add(2)
 			go pkg.SubscribeToEpochs(ctx, beacon, &wg)
-			go pkg.MonitorAttestationsAndProposals(ctx, s, beacon, plainPubkeys, &wg)
+			go pkg.MonitorAttestationsAndProposals(ctx, beacon, plainPubkeys, &wg)
 
 			//Create Prometheus Metrics Client
 			http.Handle("/metrics", promhttp.Handler())
@@ -114,7 +111,6 @@ func Execute() error {
 
 func init() {
 	rootCmd.PersistentFlags().StringVarP(&opts.LogLevel, "log-level", "l", "info", "log level (error, warn, info, debug, trace)")
-	rootCmd.PersistentFlags().StringVar(&opts.BeaconNode, "beacon-node", "localhost:4000", "Prysm beacon node GRPC address")
 	rootCmd.PersistentFlags().StringVar(&opts.BeaconChainAPI, "beacon-chain-api", "localhost:3500", "Beacon Chain API HTTP address")
 	rootCmd.PersistentFlags().StringVar(&opts.MetricsPort, "metrics-port", "1337", "Metrics port to expose metrics for Prometheus")
 	rootCmd.PersistentFlags().StringVar(&opts.SlackURL, "slack-url", "", "Slack Webhook URL")

go.mod

@@ -1,8 +1,6 @@
 module eth2-monitor
 
-go 1.21
-
-toolchain go1.22.0
+go 1.23
 
 require (
 	github.com/attestantio/go-eth2-client v0.19.10
@@ -13,21 +11,21 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.16.0
 	github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7
-	github.com/prysmaticlabs/prysm/v5 v5.0.1
 	github.com/rs/zerolog v1.29.1
 	github.com/spf13/cobra v1.5.0
 	golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611
 	golang.org/x/text v0.14.0
-	google.golang.org/grpc v1.56.3
-	google.golang.org/protobuf v1.30.0
 )
 
 require (
+	github.com/DataDog/zstd v1.5.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.7.0 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
 	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cockroachdb/errors v1.9.1 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
 	github.com/consensys/bavard v0.1.13 // indirect
 	github.com/consensys/gnark-crypto v0.12.1 // indirect
 	github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect
@@ -36,31 +34,35 @@ require (
 	github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/ferranbt/fastssz v0.1.3 // indirect
+	github.com/getsentry/sentry-go v0.18.0 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-playground/validator/v10 v10.13.0 // indirect
 	github.com/go-stack/stack v1.8.1 // indirect
 	github.com/goccy/go-yaml v1.9.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.2 // indirect
 	github.com/holiman/uint256 v1.2.4 // indirect
 	github.com/huandu/go-clone v1.6.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/klauspost/compress v1.17.2 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mmcloughlin/addchain v0.4.0 // indirect
 	github.com/prometheus/client_model v0.4.0 // indirect
 	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect
-	github.com/prysmaticlabs/fastssz v0.0.0-20221107182844-78142813af44 // indirect
-	github.com/prysmaticlabs/gohashtree v0.0.4-beta // indirect
 	github.com/r3labs/sse/v2 v2.10.0 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/supranational/blst v0.3.11 // indirect
-	github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.opentelemetry.io/otel v1.16.0 // indirect
 	go.opentelemetry.io/otel/metric v1.16.0 // indirect
 	go.opentelemetry.io/otel/trace v1.16.0 // indirect
@@ -69,7 +71,7 @@ require (
 	golang.org/x/sync v0.5.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
-	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect