From 3e61a758e7040ae370b0e2b7168a2bb335758161 Mon Sep 17 00:00:00 2001 From: sin3point14 Date: Wed, 11 Dec 2024 22:48:12 +0530 Subject: [PATCH 1/6] Add validator_address to a few metrics labels --- cmd/run.go | 7 ++--- internal/pkg/daemon/daemon.go | 6 ++++- internal/pkg/metrics/metrics.go | 47 +++++++++++++++++++-------------- 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index 0fedf3b..edced85 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -34,10 +34,7 @@ var runCmd = &cobra.Command{ // setup metrics hostname := util.GetHostname() - metrics, err := metrics.NewMetrics(cfg.ComposeFile, hostname, BinVersion) - if err != nil { - return errors.Wrapf(err, "error creating metrics server") - } + metrics := metrics.NewMetrics(cfg.ComposeFile, hostname, BinVersion) // setup notifier notifier := notification.NewFallbackNotifier(cfg, metrics, lg, hostname) @@ -56,7 +53,7 @@ var runCmd = &cobra.Command{ ctx := notification.WithContextFallback(cmd.Context(), notifier) // initialize daemon (fetch initial state and run basic sanity checks) - if err := d.Init(ctx, cfg); err != nil { + if err := d.Init(ctx, cfg, BinVersion); err != nil { return errors.Wrapf(err, "failed to initialize daemon") } diff --git a/internal/pkg/daemon/daemon.go b/internal/pkg/daemon/daemon.go index 9ecdfde..fa4e381 100644 --- a/internal/pkg/daemon/daemon.go +++ b/internal/pkg/daemon/daemon.go @@ -14,6 +14,7 @@ import ( "blazar/internal/pkg/config" "blazar/internal/pkg/cosmos" "blazar/internal/pkg/daemon/checks" + "blazar/internal/pkg/daemon/util" "blazar/internal/pkg/docker" "blazar/internal/pkg/errors" "blazar/internal/pkg/log" @@ -107,7 +108,7 @@ func NewDaemon(ctx context.Context, cfg *config.Config, m *metrics.Metrics) (*Da }, nil } -func (d *Daemon) Init(ctx context.Context, cfg *config.Config) error { +func (d *Daemon) Init(ctx context.Context, cfg *config.Config, version string) error { logger := log.FromContext(ctx).With("package", "daemon") logger.Info("Starting up blazar daemon...") @@ -131,6 +132,9 @@ func (d *Daemon) Init(ctx context.Context, cfg *config.Config) error { return errors.Wrapf(err, "failed to get status response") } + hostname := util.GetHostname() + d.metrics.RegisterValidatorInfoMetrics(cfg.ComposeFile, hostname, version, status.ValidatorInfo.Address.String()) + // display information about the node d.nodeInfo, err = d.cosmosClient.NodeInfo(ctx) if err != nil { diff --git a/internal/pkg/metrics/metrics.go b/internal/pkg/metrics/metrics.go index 1525c73..77cc1e6 100644 --- a/internal/pkg/metrics/metrics.go +++ b/internal/pkg/metrics/metrics.go @@ -23,7 +23,7 @@ type Metrics struct { NotifErrs prometheus.Counter } -func NewMetrics(composeFile string, hostname string, version string) (*Metrics, error) { +func NewMetrics(composeFile, hostname, version string) *Metrics { labels := prometheus.Labels{"hostname": hostname, "compose_file": composeFile, "version": version} metrics := &Metrics{ @@ -35,24 +35,9 @@ func NewMetrics(composeFile string, hostname string, version string) (*Metrics, ConstLabels: labels, }, ), - Step: promauto.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespace, - Name: "upgrade_step", - Help: "ID of the current step of the upgrade process", - ConstLabels: labels, - }, - []string{"upgrade_height", "upgrade_name", "upgrade_status"}, - ), - BlocksToUpgrade: promauto.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespace, - Name: "blocks_to_upgrade_height", - Help: "Number of blocks to the upgrade height", - ConstLabels: labels, - }, - []string{"upgrade_height", "upgrade_name", "upgrade_status"}, - ), + // Filled by RegisterValidatorInfoMetrics + Step: nil, + BlocksToUpgrade: nil, UpwErrs: promauto.NewCounter( prometheus.CounterOpts{ Namespace: namespace, @@ -87,7 +72,29 @@ func NewMetrics(composeFile string, hostname string, version string) (*Metrics, ), } - return metrics, nil + return metrics +} + +func (m *Metrics) RegisterValidatorInfoMetrics(composeFile, hostname, version, validator_address string) { + labels := prometheus.Labels{"hostname": hostname, "compose_file": composeFile, "version": version, "validator_address": validator_address} + m.Step = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "upgrade_step", + Help: "ID of the current step of the upgrade process", + ConstLabels: labels, + }, + []string{"upgrade_height", "upgrade_name", "upgrade_status"}, + ) + m.BlocksToUpgrade = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "blocks_to_upgrade_height", + Help: "Number of blocks to the upgrade height", + ConstLabels: labels, + }, + []string{"upgrade_height", "upgrade_name", "upgrade_status"}, + ) } func RegisterHandler(mux *runtime.ServeMux) error { From 4f04eccdec67e20c622133e374525acf0d252ece Mon Sep 17 00:00:00 2001 From: sin3point14 Date: Wed, 11 Dec 2024 22:58:47 +0530 Subject: [PATCH 2/6] Adjust tests for metrics.NewMetrics changes --- internal/pkg/daemon/daemon_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/pkg/daemon/daemon_test.go b/internal/pkg/daemon/daemon_test.go index cd773e1..03817ab 100644 --- a/internal/pkg/daemon/daemon_test.go +++ b/internal/pkg/daemon/daemon_test.go @@ -75,8 +75,8 @@ func TestIntegrationDaemon(t *testing.T) { }() // we can't register 2 metrics, but this sharing this should probably cause no problems - metrics, err := metrics.NewMetrics("/path/to/docker-compose.yml", "dummy", "test") - require.NoError(t, err) + metrics := metrics.NewMetrics("/path/to/docker-compose.yml", "dummy", "test") + metrics.RegisterValidatorInfoMetrics("/path/to/docker-compose.yml", "dummy", "test", "AABBCCDD") ports := getFreePorts(t, 6) From abb6b4db4d82c7012c6abda2a5737831f6beba79 Mon Sep 17 00:00:00 2001 From: sin3point14 Date: Thu, 12 Dec 2024 13:22:30 +0530 Subject: [PATCH 3/6] Add chain id label to metrics --- internal/pkg/daemon/daemon.go | 2 +- internal/pkg/daemon/daemon_test.go | 2 +- internal/pkg/metrics/metrics.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/pkg/daemon/daemon.go b/internal/pkg/daemon/daemon.go index fa4e381..a48d03f 100644 --- a/internal/pkg/daemon/daemon.go +++ b/internal/pkg/daemon/daemon.go @@ -133,7 +133,7 @@ func (d *Daemon) Init(ctx context.Context, cfg *config.Config, version string) e } hostname := util.GetHostname() - d.metrics.RegisterValidatorInfoMetrics(cfg.ComposeFile, hostname, version, status.ValidatorInfo.Address.String()) + d.metrics.RegisterValidatorInfoMetrics(cfg.ComposeFile, hostname, version, status.NodeInfo.Network, status.ValidatorInfo.Address.String()) // display information about the node d.nodeInfo, err = d.cosmosClient.NodeInfo(ctx) diff --git a/internal/pkg/daemon/daemon_test.go b/internal/pkg/daemon/daemon_test.go index 03817ab..618f00e 100644 --- a/internal/pkg/daemon/daemon_test.go +++ b/internal/pkg/daemon/daemon_test.go @@ -76,7 +76,7 @@ func TestIntegrationDaemon(t *testing.T) { // we can't register 2 metrics, but this sharing this should probably cause no problems metrics := metrics.NewMetrics("/path/to/docker-compose.yml", "dummy", "test") - metrics.RegisterValidatorInfoMetrics("/path/to/docker-compose.yml", "dummy", "test", "AABBCCDD") + metrics.RegisterValidatorInfoMetrics("/path/to/docker-compose.yml", "dummy", "test", "test-chain-id", "AABBCCDD") ports := getFreePorts(t, 6) diff --git a/internal/pkg/metrics/metrics.go b/internal/pkg/metrics/metrics.go index 77cc1e6..af5846f 100644 --- a/internal/pkg/metrics/metrics.go +++ b/internal/pkg/metrics/metrics.go @@ -75,8 +75,8 @@ func NewMetrics(composeFile, hostname, version string) *Metrics { return metrics } -func (m *Metrics) RegisterValidatorInfoMetrics(composeFile, hostname, version, validator_address string) { - labels := prometheus.Labels{"hostname": hostname, "compose_file": composeFile, "version": version, "validator_address": validator_address} +func (m *Metrics) RegisterValidatorInfoMetrics(composeFile, hostname, version, chain_id, validator_address string) { + labels := prometheus.Labels{"hostname": hostname, "compose_file": composeFile, "version": version, "chain_id": chain_id, "validator_address": validator_address} m.Step = promauto.NewGaugeVec( prometheus.GaugeOpts{ Namespace: namespace, From 72a17e36107c7a5424ea99c545574c7a8b147237 Mon Sep 17 00:00:00 2001 From: sin3point14 Date: Thu, 12 Dec 2024 14:33:56 +0530 Subject: [PATCH 4/6] Remove blazar_upgrade_step and export step as label in blocks_to_upgrade_height --- internal/pkg/daemon/daemon.go | 11 ++++++---- internal/pkg/daemon/daemon_test.go | 1 - internal/pkg/daemon/metrics.go | 5 ++--- internal/pkg/metrics/metrics.go | 35 ++++++++---------------------- 4 files changed, 18 insertions(+), 34 deletions(-) diff --git a/internal/pkg/daemon/daemon.go b/internal/pkg/daemon/daemon.go index a48d03f..6dfe4a0 100644 --- a/internal/pkg/daemon/daemon.go +++ b/internal/pkg/daemon/daemon.go @@ -14,7 +14,6 @@ import ( "blazar/internal/pkg/config" "blazar/internal/pkg/cosmos" "blazar/internal/pkg/daemon/checks" - "blazar/internal/pkg/daemon/util" "blazar/internal/pkg/docker" "blazar/internal/pkg/errors" "blazar/internal/pkg/log" @@ -50,7 +49,11 @@ type Daemon struct { // initial counters startupHeight int64 nodeAddress bytes.HexBytes - nodeInfo *tmservice.GetNodeInfoResponse + + // node information + nodeInfo *tmservice.GetNodeInfoResponse + validatorAddress string + chainId string // tracking current height currHeight int64 @@ -132,8 +135,8 @@ func (d *Daemon) Init(ctx context.Context, cfg *config.Config, version string) e return errors.Wrapf(err, "failed to get status response") } - hostname := util.GetHostname() - d.metrics.RegisterValidatorInfoMetrics(cfg.ComposeFile, hostname, version, status.NodeInfo.Network, status.ValidatorInfo.Address.String()) + d.chainId = status.NodeInfo.Network + d.validatorAddress = status.ValidatorInfo.Address.String() // display information about the node d.nodeInfo, err = d.cosmosClient.NodeInfo(ctx) diff --git a/internal/pkg/daemon/daemon_test.go b/internal/pkg/daemon/daemon_test.go index 618f00e..c077987 100644 --- a/internal/pkg/daemon/daemon_test.go +++ b/internal/pkg/daemon/daemon_test.go @@ -76,7 +76,6 @@ func TestIntegrationDaemon(t *testing.T) { // we can't register 2 metrics, but this sharing this should probably cause no problems metrics := metrics.NewMetrics("/path/to/docker-compose.yml", "dummy", "test") - metrics.RegisterValidatorInfoMetrics("/path/to/docker-compose.yml", "dummy", "test", "test-chain-id", "AABBCCDD") ports := getFreePorts(t, 6) diff --git a/internal/pkg/daemon/metrics.go b/internal/pkg/daemon/metrics.go index c77f2bc..2d9ce66 100644 --- a/internal/pkg/daemon/metrics.go +++ b/internal/pkg/daemon/metrics.go @@ -6,7 +6,6 @@ import ( func (d *Daemon) updateMetrics() { // the upgrade state may change, we don't want to persist the metric with the old status - d.metrics.Step.Reset() d.metrics.BlocksToUpgrade.Reset() upcomingUpgrades := d.ur.GetUpcomingUpgradesWithCache(d.currHeight) @@ -14,7 +13,7 @@ func (d *Daemon) updateMetrics() { upgradeHeight := strconv.FormatInt(upgrade.Height, 10) status := d.stateMachine.GetStatus(upgrade.Height) - d.metrics.Step.WithLabelValues(upgradeHeight, upgrade.Name, status.String()).Set(float64(d.stateMachine.GetStep(upgrade.Height))) - d.metrics.BlocksToUpgrade.WithLabelValues(upgradeHeight, upgrade.Name, status.String()).Set(float64(upgrade.Height - d.currHeight)) + d.metrics.BlocksToUpgrade.WithLabelValues(upgradeHeight, upgrade.Name, status.String(), + d.stateMachine.GetStep(upgrade.Height).String(), d.chainId, d.validatorAddress).Set(float64(upgrade.Height - d.currHeight)) } } diff --git a/internal/pkg/metrics/metrics.go b/internal/pkg/metrics/metrics.go index af5846f..fe46ba1 100644 --- a/internal/pkg/metrics/metrics.go +++ b/internal/pkg/metrics/metrics.go @@ -15,7 +15,6 @@ const ( type Metrics struct { Up prometheus.Gauge - Step *prometheus.GaugeVec BlocksToUpgrade *prometheus.GaugeVec UpwErrs prometheus.Counter UiwErrs prometheus.Counter @@ -35,9 +34,15 @@ func NewMetrics(composeFile, hostname, version string) *Metrics { ConstLabels: labels, }, ), - // Filled by RegisterValidatorInfoMetrics - Step: nil, - BlocksToUpgrade: nil, + BlocksToUpgrade: promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "blocks_to_upgrade_height", + Help: "Number of blocks to the upgrade height", + ConstLabels: labels, + }, + []string{"upgrade_height", "upgrade_name", "upgrade_status", "upgrade_step", "chain_id", "validator_address"}, + ), UpwErrs: promauto.NewCounter( prometheus.CounterOpts{ Namespace: namespace, @@ -75,28 +80,6 @@ func NewMetrics(composeFile, hostname, version string) *Metrics { return metrics } -func (m *Metrics) RegisterValidatorInfoMetrics(composeFile, hostname, version, chain_id, validator_address string) { - labels := prometheus.Labels{"hostname": hostname, "compose_file": composeFile, "version": version, "chain_id": chain_id, "validator_address": validator_address} - m.Step = promauto.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespace, - Name: "upgrade_step", - Help: "ID of the current step of the upgrade process", - ConstLabels: labels, - }, - []string{"upgrade_height", "upgrade_name", "upgrade_status"}, - ) - m.BlocksToUpgrade = promauto.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespace, - Name: "blocks_to_upgrade_height", - Help: "Number of blocks to the upgrade height", - ConstLabels: labels, - }, - []string{"upgrade_height", "upgrade_name", "upgrade_status"}, - ) -} - func RegisterHandler(mux *runtime.ServeMux) error { handler := promhttp.Handler() return mux.HandlePath("GET", "/metrics", func(w http.ResponseWriter, r *http.Request, _ map[string]string) { From bb2e0b393e0184e27018ebde1fbf3c1924779fde Mon Sep 17 00:00:00 2001 From: sin3point14 Date: Thu, 12 Dec 2024 14:36:13 +0530 Subject: [PATCH 5/6] Remove unused parameter from daemon.Init --- cmd/run.go | 2 +- internal/pkg/daemon/daemon.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index edced85..60a1b49 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -53,7 +53,7 @@ var runCmd = &cobra.Command{ ctx := notification.WithContextFallback(cmd.Context(), notifier) // initialize daemon (fetch initial state and run basic sanity checks) - if err := d.Init(ctx, cfg, BinVersion); err != nil { + if err := d.Init(ctx, cfg); err != nil { return errors.Wrapf(err, "failed to initialize daemon") } diff --git a/internal/pkg/daemon/daemon.go b/internal/pkg/daemon/daemon.go index 6dfe4a0..a759e93 100644 --- a/internal/pkg/daemon/daemon.go +++ b/internal/pkg/daemon/daemon.go @@ -48,9 +48,9 @@ type Daemon struct { // initial counters startupHeight int64 - nodeAddress bytes.HexBytes // node information + nodeAddress bytes.HexBytes nodeInfo *tmservice.GetNodeInfoResponse validatorAddress string chainId string @@ -111,7 +111,7 @@ func NewDaemon(ctx context.Context, cfg *config.Config, m *metrics.Metrics) (*Da }, nil } -func (d *Daemon) Init(ctx context.Context, cfg *config.Config, version string) error { +func (d *Daemon) Init(ctx context.Context, cfg *config.Config) error { logger := log.FromContext(ctx).With("package", "daemon") logger.Info("Starting up blazar daemon...") From 96c80b162578caf02c41e3c4aa8c9624e1a9bf4f Mon Sep 17 00:00:00 2001 From: sin3point14 Date: Thu, 12 Dec 2024 15:20:53 +0530 Subject: [PATCH 6/6] Fix lint --- internal/pkg/daemon/daemon.go | 4 ++-- internal/pkg/daemon/metrics.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/pkg/daemon/daemon.go b/internal/pkg/daemon/daemon.go index a759e93..04cf58b 100644 --- a/internal/pkg/daemon/daemon.go +++ b/internal/pkg/daemon/daemon.go @@ -53,7 +53,7 @@ type Daemon struct { nodeAddress bytes.HexBytes nodeInfo *tmservice.GetNodeInfoResponse validatorAddress string - chainId string + chainID string // tracking current height currHeight int64 @@ -135,7 +135,7 @@ func (d *Daemon) Init(ctx context.Context, cfg *config.Config) error { return errors.Wrapf(err, "failed to get status response") } - d.chainId = status.NodeInfo.Network + d.chainID = status.NodeInfo.Network d.validatorAddress = status.ValidatorInfo.Address.String() // display information about the node diff --git a/internal/pkg/daemon/metrics.go b/internal/pkg/daemon/metrics.go index 2d9ce66..efdb615 100644 --- a/internal/pkg/daemon/metrics.go +++ b/internal/pkg/daemon/metrics.go @@ -14,6 +14,6 @@ func (d *Daemon) updateMetrics() { status := d.stateMachine.GetStatus(upgrade.Height) d.metrics.BlocksToUpgrade.WithLabelValues(upgradeHeight, upgrade.Name, status.String(), - d.stateMachine.GetStep(upgrade.Height).String(), d.chainId, d.validatorAddress).Set(float64(upgrade.Height - d.currHeight)) + d.stateMachine.GetStep(upgrade.Height).String(), d.chainID, d.validatorAddress).Set(float64(upgrade.Height - d.currHeight)) } }