From 9194f59edb6e70aaa30efe803630bbf3c4c88d33 Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Sat, 30 Nov 2024 14:39:11 -0600 Subject: [PATCH 1/5] Return from refreshing file sizes if we can't get the config --- internal/metrics/fullnode.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/metrics/fullnode.go b/internal/metrics/fullnode.go index 673b300..35f6caa 100644 --- a/internal/metrics/fullnode.go +++ b/internal/metrics/fullnode.go @@ -424,6 +424,7 @@ func (s *FullNodeServiceMetrics) RefreshFileSizes() { cfg, err := config.GetChiaConfig() if err != nil { log.Errorf("Error getting chia config: %s\n", err.Error()) + return } database := cfg.GetFullPath(cfg.FullNode.DatabasePath) databaseWal := fmt.Sprintf("%s-wal", database) From fd194a22bf9b7fcbe98a0370cf941a69ba9c245c Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Sat, 30 Nov 2024 15:24:05 -0600 Subject: [PATCH 2/5] if version isn't set when receiving responses, try to get it again --- internal/metrics/fullnode.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/internal/metrics/fullnode.go b/internal/metrics/fullnode.go index 35f6caa..f445421 100644 --- a/internal/metrics/fullnode.go +++ b/internal/metrics/fullnode.go @@ -36,7 +36,8 @@ type FullNodeServiceMetrics struct { metrics *Metrics // General Service Metrics - version *prometheus.GaugeVec + gotVersionResponse bool + version *prometheus.GaugeVec // GetBlockchainState Metrics difficulty *wrappedPrometheus.LazyGauge @@ -187,6 +188,7 @@ func (s *FullNodeServiceMetrics) SetupPollingMetrics(ctx context.Context) { // Disconnected clears/unregisters metrics when the connection drops func (s *FullNodeServiceMetrics) Disconnected() { s.version.Reset() + s.gotVersionResponse = false s.difficulty.Unregister() s.mempoolCost.Unregister() s.mempoolMinFee.Reset() @@ -228,9 +230,18 @@ func (s *FullNodeServiceMetrics) Reconnected() { // ReceiveResponse handles full node related responses that are returned over the websocket func (s *FullNodeServiceMetrics) ReceiveResponse(resp *types.WebsocketResponse) { + // Sometimes, when we reconnect, or start exporter before the full node is up + // the daemon is up before the full node, and the initial request for the version + // doesn't make it to the service, since it wasn't up yet + // daemon doesn't queue these messages for later, they just get dropped + if !s.gotVersionResponse { + utils.LogErr(s.metrics.client.FullNodeService.GetVersion(&rpc.GetVersionOptions{})) + } + switch resp.Command { case "get_version": versionHelper(resp, s.version) + s.gotVersionResponse = true case "get_blockchain_state": s.GetBlockchainState(resp) // Ask for connection info when we get updated blockchain state From d893f46b86bbf63c03d8f35333223bd2e444a7b4 Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Sat, 30 Nov 2024 15:28:31 -0600 Subject: [PATCH 3/5] Check whether or not we know the version once we start receiving messages from services Fixes a case where daemon is up but service is not for the initial "Get Version" messages --- internal/metrics/crawler.go | 11 +++++++++++ internal/metrics/farmer.go | 11 +++++++++++ internal/metrics/fullnode.go | 6 +++--- internal/metrics/harvester.go | 11 +++++++++++ internal/metrics/timelord.go | 11 +++++++++++ internal/metrics/wallet.go | 11 +++++++++++ 6 files changed, 58 insertions(+), 3 deletions(-) diff --git a/internal/metrics/crawler.go b/internal/metrics/crawler.go index 780046c..2b8816f 100644 --- a/internal/metrics/crawler.go +++ b/internal/metrics/crawler.go @@ -30,6 +30,7 @@ type CrawlerServiceMetrics struct { metrics *Metrics // General Service Metrics + gotVersionResponse bool version *prometheus.GaugeVec // Current network @@ -128,6 +129,7 @@ func (s *CrawlerServiceMetrics) SetupPollingMetrics(ctx context.Context) {} // Disconnected clears/unregisters metrics when the connection drops func (s *CrawlerServiceMetrics) Disconnected() { s.version.Reset() + s.gotVersionResponse = false s.totalNodes5Days.Unregister() s.reliableNodes.Unregister() s.ipv4Nodes5Days.Unregister() @@ -143,9 +145,18 @@ func (s *CrawlerServiceMetrics) Reconnected() { // ReceiveResponse handles crawler responses that are returned over the websocket func (s *CrawlerServiceMetrics) ReceiveResponse(resp *types.WebsocketResponse) { + // Sometimes, when we reconnect, or start exporter before chia is running + // the daemon is up before the service, and the initial request for the version + // doesn't make it to the service + // daemon doesn't queue these messages for later, they just get dropped + if !s.gotVersionResponse { + utils.LogErr(s.metrics.client.FullNodeService.GetVersion(&rpc.GetVersionOptions{})) + } + switch resp.Command { case "get_version": versionHelper(resp, s.version) + s.gotVersionResponse = true case "get_peer_counts": fallthrough case "loaded_initial_peers": diff --git a/internal/metrics/farmer.go b/internal/metrics/farmer.go index 17ee97b..ed3a1e9 100644 --- a/internal/metrics/farmer.go +++ b/internal/metrics/farmer.go @@ -24,6 +24,7 @@ type FarmerServiceMetrics struct { metrics *Metrics // General Service Metrics + gotVersionResponse bool version *prometheus.GaugeVec // Connection Metrics @@ -108,6 +109,7 @@ func (s *FarmerServiceMetrics) SetupPollingMetrics(ctx context.Context) {} // Disconnected clears/unregisters metrics when the connection drops func (s *FarmerServiceMetrics) Disconnected() { s.version.Reset() + s.gotVersionResponse = false s.connectionCount.Reset() s.plotFilesize.Reset() s.plotCount.Reset() @@ -123,9 +125,18 @@ func (s *FarmerServiceMetrics) Reconnected() { // ReceiveResponse handles crawler responses that are returned over the websocket func (s *FarmerServiceMetrics) ReceiveResponse(resp *types.WebsocketResponse) { + // Sometimes, when we reconnect, or start exporter before chia is running + // the daemon is up before the service, and the initial request for the version + // doesn't make it to the service + // daemon doesn't queue these messages for later, they just get dropped + if !s.gotVersionResponse { + utils.LogErr(s.metrics.client.FullNodeService.GetVersion(&rpc.GetVersionOptions{})) + } + switch resp.Command { case "get_version": versionHelper(resp, s.version) + s.gotVersionResponse = true case "get_connections": s.GetConnections(resp) case "new_farming_info": diff --git a/internal/metrics/fullnode.go b/internal/metrics/fullnode.go index f445421..5d65d21 100644 --- a/internal/metrics/fullnode.go +++ b/internal/metrics/fullnode.go @@ -230,9 +230,9 @@ func (s *FullNodeServiceMetrics) Reconnected() { // ReceiveResponse handles full node related responses that are returned over the websocket func (s *FullNodeServiceMetrics) ReceiveResponse(resp *types.WebsocketResponse) { - // Sometimes, when we reconnect, or start exporter before the full node is up - // the daemon is up before the full node, and the initial request for the version - // doesn't make it to the service, since it wasn't up yet + // Sometimes, when we reconnect, or start exporter before chia is running + // the daemon is up before the service, and the initial request for the version + // doesn't make it to the service // daemon doesn't queue these messages for later, they just get dropped if !s.gotVersionResponse { utils.LogErr(s.metrics.client.FullNodeService.GetVersion(&rpc.GetVersionOptions{})) diff --git a/internal/metrics/harvester.go b/internal/metrics/harvester.go index 6595a4c..2144a62 100644 --- a/internal/metrics/harvester.go +++ b/internal/metrics/harvester.go @@ -25,6 +25,7 @@ type HarvesterServiceMetrics struct { metrics *Metrics // General Service Metrics + gotVersionResponse bool version *prometheus.GaugeVec // Connection Metrics @@ -124,6 +125,7 @@ func (s *HarvesterServiceMetrics) httpGetPlots() { // Disconnected clears/unregisters metrics when the connection drops func (s *HarvesterServiceMetrics) Disconnected() { s.version.Reset() + s.gotVersionResponse = false s.connectionCount.Reset() s.totalPlots.Unregister() s.plotFilesize.Reset() @@ -140,9 +142,18 @@ func (s *HarvesterServiceMetrics) Reconnected() { // ReceiveResponse handles crawler responses that are returned over the websocket func (s *HarvesterServiceMetrics) ReceiveResponse(resp *types.WebsocketResponse) { + // Sometimes, when we reconnect, or start exporter before chia is running + // the daemon is up before the service, and the initial request for the version + // doesn't make it to the service + // daemon doesn't queue these messages for later, they just get dropped + if !s.gotVersionResponse { + utils.LogErr(s.metrics.client.FullNodeService.GetVersion(&rpc.GetVersionOptions{})) + } + switch resp.Command { case "get_version": versionHelper(resp, s.version) + s.gotVersionResponse = true case "get_connections": s.GetConnections(resp) case "farming_info": diff --git a/internal/metrics/timelord.go b/internal/metrics/timelord.go index bb1b48a..2983752 100644 --- a/internal/metrics/timelord.go +++ b/internal/metrics/timelord.go @@ -23,6 +23,7 @@ type TimelordServiceMetrics struct { metrics *Metrics // General Service Metrics + gotVersionResponse bool version *prometheus.GaugeVec // Timelord Metrics @@ -62,6 +63,7 @@ func (s *TimelordServiceMetrics) SetupPollingMetrics(ctx context.Context) {} // Disconnected clears/unregisters metrics when the connection drops func (s *TimelordServiceMetrics) Disconnected() { s.version.Reset() + s.gotVersionResponse = false s.fastestTimelord.Unregister() s.slowTimelord.Unregister() s.estimatedIPS.Unregister() @@ -75,10 +77,19 @@ func (s *TimelordServiceMetrics) Reconnected() { // ReceiveResponse handles crawler responses that are returned over the websocket func (s *TimelordServiceMetrics) ReceiveResponse(resp *types.WebsocketResponse) { + // Sometimes, when we reconnect, or start exporter before chia is running + // the daemon is up before the service, and the initial request for the version + // doesn't make it to the service + // daemon doesn't queue these messages for later, they just get dropped + if !s.gotVersionResponse { + utils.LogErr(s.metrics.client.FullNodeService.GetVersion(&rpc.GetVersionOptions{})) + } + //("finished_pot_challenge", "new_compact_proof", "skipping_peak", "new_peak") switch resp.Command { case "get_version": versionHelper(resp, s.version) + s.gotVersionResponse = true case "finished_pot": s.FinishedPoT(resp) case "new_compact_proof": diff --git a/internal/metrics/wallet.go b/internal/metrics/wallet.go index 4acaad3..9e82a1f 100644 --- a/internal/metrics/wallet.go +++ b/internal/metrics/wallet.go @@ -25,6 +25,7 @@ type WalletServiceMetrics struct { metrics *Metrics // General Service Metrics + gotVersionResponse bool version *prometheus.GaugeVec // Connection Metrics @@ -93,6 +94,7 @@ func (s *WalletServiceMetrics) SetupPollingMetrics(ctx context.Context) { // Disconnected clears/unregisters metrics when the connection drops func (s *WalletServiceMetrics) Disconnected() { s.version.Reset() + s.gotVersionResponse = false s.connectionCount.Reset() s.walletSynced.Unregister() s.confirmedBalance.Reset() @@ -109,9 +111,18 @@ func (s *WalletServiceMetrics) Reconnected() { // ReceiveResponse handles wallet responses that are returned over the websocket func (s *WalletServiceMetrics) ReceiveResponse(resp *types.WebsocketResponse) { + // Sometimes, when we reconnect, or start exporter before chia is running + // the daemon is up before the service, and the initial request for the version + // doesn't make it to the service + // daemon doesn't queue these messages for later, they just get dropped + if !s.gotVersionResponse { + utils.LogErr(s.metrics.client.FullNodeService.GetVersion(&rpc.GetVersionOptions{})) + } + switch resp.Command { case "get_version": versionHelper(resp, s.version) + s.gotVersionResponse = true case "get_connections": s.GetConnections(resp) case "coin_added": From f5555a9e372fb8ec2413127d307b05cb0534b252 Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Sat, 30 Nov 2024 21:03:08 -0600 Subject: [PATCH 4/5] Go chia libs 0.20.1 --- go.mod | 2 +- go.sum | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 87368ee..7ce5ba2 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/chia-network/chia-exporter go 1.21 require ( - github.com/chia-network/go-chia-libs v0.20.0 + github.com/chia-network/go-chia-libs v0.20.1 github.com/chia-network/go-modules v0.0.8 github.com/go-sql-driver/mysql v1.8.1 github.com/oschwald/maxminddb-golang v1.13.1 diff --git a/go.sum b/go.sum index a33bc98..736c1b4 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chia-network/go-chia-libs v0.20.0 h1:rdW1Ob/hb32Pl89w4/9SkoGWVhyC+tOkeRqvPr/xuvg= -github.com/chia-network/go-chia-libs v0.20.0/go.mod h1:npTqaFSjTdMxE7hc0LOmWJmWGqcs+IERarK5fDxXk/I= +github.com/chia-network/go-chia-libs v0.20.1 h1:uEPrjtxot0oipTuWT63Spw9t9kj56JdhdFBS4ZgYdjc= +github.com/chia-network/go-chia-libs v0.20.1/go.mod h1:+RMorskgxwYzPGf2gIyW0k7FGDdLrrH4X5ATrrMreb0= github.com/chia-network/go-modules v0.0.8 h1:VATMxehRISOhaRwPo/GL735IKWW0G7sUYH2OmBofsfE= github.com/chia-network/go-modules v0.0.8/go.mod h1:OdvlWftyJc3+i3QYv5cfQsiQASL7Em7fJnzdmPmj07M= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -94,8 +94,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= From ca125f91bf39102cc13e35b4fcebea19c9913c53 Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Sat, 30 Nov 2024 21:10:30 -0600 Subject: [PATCH 5/5] Get initial plots/wallets if those didn't work right away --- internal/metrics/harvester.go | 7 +++++++ internal/metrics/wallet.go | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/internal/metrics/harvester.go b/internal/metrics/harvester.go index 2144a62..788d0e0 100644 --- a/internal/metrics/harvester.go +++ b/internal/metrics/harvester.go @@ -28,6 +28,8 @@ type HarvesterServiceMetrics struct { gotVersionResponse bool version *prometheus.GaugeVec + gotPlotsResponse bool + // Connection Metrics connectionCount *prometheus.GaugeVec @@ -126,6 +128,7 @@ func (s *HarvesterServiceMetrics) httpGetPlots() { func (s *HarvesterServiceMetrics) Disconnected() { s.version.Reset() s.gotVersionResponse = false + s.gotPlotsResponse = false s.connectionCount.Reset() s.totalPlots.Unregister() s.plotFilesize.Reset() @@ -149,6 +152,9 @@ func (s *HarvesterServiceMetrics) ReceiveResponse(resp *types.WebsocketResponse) if !s.gotVersionResponse { utils.LogErr(s.metrics.client.FullNodeService.GetVersion(&rpc.GetVersionOptions{})) } + if !s.gotPlotsResponse { + s.httpGetPlots() + } switch resp.Command { case "get_version": @@ -160,6 +166,7 @@ func (s *HarvesterServiceMetrics) ReceiveResponse(resp *types.WebsocketResponse) s.FarmingInfo(resp) case "get_plots": s.GetPlots(resp) + s.gotPlotsResponse = true case "debug": debugHelper(resp, s.debug) } diff --git a/internal/metrics/wallet.go b/internal/metrics/wallet.go index 9e82a1f..76a8642 100644 --- a/internal/metrics/wallet.go +++ b/internal/metrics/wallet.go @@ -28,6 +28,8 @@ type WalletServiceMetrics struct { gotVersionResponse bool version *prometheus.GaugeVec + gotWalletsResponse bool + // Connection Metrics connectionCount *prometheus.GaugeVec @@ -95,6 +97,7 @@ func (s *WalletServiceMetrics) SetupPollingMetrics(ctx context.Context) { func (s *WalletServiceMetrics) Disconnected() { s.version.Reset() s.gotVersionResponse = false + s.gotWalletsResponse = false s.connectionCount.Reset() s.walletSynced.Unregister() s.confirmedBalance.Reset() @@ -118,6 +121,9 @@ func (s *WalletServiceMetrics) ReceiveResponse(resp *types.WebsocketResponse) { if !s.gotVersionResponse { utils.LogErr(s.metrics.client.FullNodeService.GetVersion(&rpc.GetVersionOptions{})) } + if !s.gotWalletsResponse { + utils.LogErr(s.metrics.client.WalletService.GetWallets(&rpc.GetWalletsOptions{})) + } switch resp.Command { case "get_version": @@ -135,6 +141,7 @@ func (s *WalletServiceMetrics) ReceiveResponse(resp *types.WebsocketResponse) { s.GetWalletBalance(resp) case "get_wallets": s.GetWallets(resp) + s.gotWalletsResponse = true case "debug": debugHelper(resp, s.debug) }