From c2329a403108920a581d5c6f46f00a017e4f61ad Mon Sep 17 00:00:00 2001 From: Azanul Date: Sat, 27 Jul 2024 11:54:43 +0530 Subject: [PATCH 1/3] fix: failing redis price calculation Signed-off-by: Azanul --- providers/gcp/redis/pricing.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/providers/gcp/redis/pricing.go b/providers/gcp/redis/pricing.go index 8d8cd99c4..7757dd731 100644 --- a/providers/gcp/redis/pricing.go +++ b/providers/gcp/redis/pricing.go @@ -5,6 +5,7 @@ import ( "fmt" "math" "net/http" + "strings" "time" "cloud.google.com/go/redis/apiv1/redispb" @@ -66,17 +67,18 @@ func calculateRedisCost(redis *redispb.Instance, pricing *GcpDatabasePricing) fl if redis.Tier == redispb.Instance_BASIC { priceMap = pricing.Gcp.Databases.CloudMemorystore.Redis.Basic - priceKey = fmt.Sprintf("Rediscapacitybasicm%ddefault", capacityTier) + priceKey = fmt.Sprintf("rediscapacitybasicm%ddefault", capacityTier) } else if redis.Tier == redispb.Instance_STANDARD_HA { priceMap = pricing.Gcp.Databases.CloudMemorystore.Redis.Standard if redis.ReadReplicasMode == redispb.Instance_READ_REPLICAS_DISABLED { - priceKey = fmt.Sprintf("Rediscapacitystandardm%ddefault", capacityTier) + priceKey = fmt.Sprintf("rediscapacitystandardm%ddefault", capacityTier) } else { - priceKey = fmt.Sprintf("Rediscapacitystandardnodem%d", capacityTier) + priceKey = fmt.Sprintf("rediscapacitystandardnodem%d", capacityTier) } } - pricePerHrPerGbInNanos := priceMap[priceKey].Regions[redis.LocationId].Price[0].Nanos + location := strings.Join(strings.Split(redis.LocationId, "-")[:2], "-") + pricePerHrPerGbInNanos := priceMap[priceKey].Regions[location].Price[0].Nanos pricePerHrPerGbInDollars := pricePerHrPerGbInNanos / math.Pow(10, 9) now := time.Now().UTC() From 1c87ef760b4b9a414534abadea6b4f722b7d62db Mon Sep 17 00:00:00 2001 From: Azanul Date: Tue, 30 Jul 2024 16:41:02 +0530 Subject: [PATCH 2/3] fix: gcp buckets size calculation Signed-off-by: Azanul --- providers/gcp/storage/buckets.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/providers/gcp/storage/buckets.go b/providers/gcp/storage/buckets.go index 6825a1081..43575c1f0 100644 --- a/providers/gcp/storage/buckets.go +++ b/providers/gcp/storage/buckets.go @@ -37,9 +37,11 @@ func GetBucketSize(ctx context.Context, client providers.ProviderClient, bucketN EndTime: ×tamp.Timestamp{Seconds: time.Now().Unix()}, }, Aggregation: &monitoringpb.Aggregation{ - AlignmentPeriod: &duration.Duration{Seconds: 60}, - PerSeriesAligner: monitoringpb.Aggregation_ALIGN_SUM, + AlignmentPeriod: &duration.Duration{Seconds: 86400}, + PerSeriesAligner: monitoringpb.Aggregation_ALIGN_MEAN, + GroupByFields: []string{"resource.label.bucket_name"}, }, + View: monitoringpb.ListTimeSeriesRequest_FULL, } res := monitoringClient.ListTimeSeries(ctx, req) From 92cffbd120c03732c853f5987b085f77ece76f56 Mon Sep 17 00:00:00 2001 From: Azanul Date: Tue, 30 Jul 2024 17:19:38 +0530 Subject: [PATCH 3/3] refac: show service disabled as warnings Signed-off-by: Azanul --- providers/gcp/alloydb/clusters.go | 10 ++++++++-- providers/gcp/alloydb/instances.go | 10 ++++++++-- providers/gcp/appengine/service.go | 10 ++++++++-- providers/gcp/artifactregistry/docker_images.go | 10 ++++++++-- providers/gcp/artifactregistry/packages.go | 10 ++++++++-- providers/gcp/artifactregistry/repositories.go | 10 ++++++++-- providers/gcp/bigquery/tables.go | 10 ++++++++-- providers/gcp/certificate/certificates.go | 10 ++++++++-- providers/gcp/compute/disks.go | 10 ++++++++-- providers/gcp/compute/instances.go | 10 ++++++++-- providers/gcp/compute/snapshots.go | 10 ++++++++-- providers/gcp/container/clusters.go | 10 ++++++++-- providers/gcp/firestore/documents.go | 10 ++++++++-- providers/gcp/function/functions.go | 10 ++++++++-- providers/gcp/gateway/gateways.go | 5 +++-- providers/gcp/iam/roles.go | 9 +++++++-- providers/gcp/iam/service_accounts.go | 10 ++++++++-- providers/gcp/kms/keys.go | 9 +++++++-- providers/gcp/redis/instances.go | 10 ++++++++-- providers/gcp/sql/instances.go | 10 ++++++++-- providers/gcp/storage/buckets.go | 9 +++++++-- 21 files changed, 160 insertions(+), 42 deletions(-) diff --git a/providers/gcp/alloydb/clusters.go b/providers/gcp/alloydb/clusters.go index 7a1acd7a4..8ac74101d 100644 --- a/providers/gcp/alloydb/clusters.go +++ b/providers/gcp/alloydb/clusters.go @@ -3,6 +3,7 @@ package alloydb import ( "context" "errors" + "strings" "time" "github.com/sirupsen/logrus" @@ -29,8 +30,13 @@ func Clusters(ctx context.Context, client providers.ProviderClient) ([]models.Re break } if err != nil { - logrus.WithError(err).Errorf("failed to get clusters") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to get clusters") + return resources, err + } } resources = append(resources, models.Resource{ Provider: "GCP", diff --git a/providers/gcp/alloydb/instances.go b/providers/gcp/alloydb/instances.go index ecdc29385..1747e3e98 100644 --- a/providers/gcp/alloydb/instances.go +++ b/providers/gcp/alloydb/instances.go @@ -3,6 +3,7 @@ package alloydb import ( "context" "errors" + "strings" "time" "github.com/sirupsen/logrus" @@ -29,8 +30,13 @@ func Instances(ctx context.Context, client providers.ProviderClient) ([]models.R break } if err != nil { - logrus.WithError(err).Errorf("failed to get instances") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to get instances") + return resources, err + } } resources = append(resources, models.Resource{ Provider: "GCP", diff --git a/providers/gcp/appengine/service.go b/providers/gcp/appengine/service.go index 89772f6fb..bb9a827e4 100644 --- a/providers/gcp/appengine/service.go +++ b/providers/gcp/appengine/service.go @@ -3,6 +3,7 @@ package appengine import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -29,8 +30,13 @@ func Services(ctx context.Context, client providers.ProviderClient) ([]models.Re for { svc, err := svcs.Next() if err != nil { - logrus.WithError(err).Errorf("failed to get app engine") - break + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to get app engine") + break + } } if svc == nil { break diff --git a/providers/gcp/artifactregistry/docker_images.go b/providers/gcp/artifactregistry/docker_images.go index cf97e5044..ab8acb5d2 100644 --- a/providers/gcp/artifactregistry/docker_images.go +++ b/providers/gcp/artifactregistry/docker_images.go @@ -3,6 +3,7 @@ package artifactregistry import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -29,8 +30,13 @@ func ArtifactregistryDockerImages(ctx context.Context, client providers.Provider for { image, err := imageItr.Next() if err != nil { - logrus.WithError(err).Errorf("failed to get nex image") - break + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to get nex image") + break + } } if image == nil { break diff --git a/providers/gcp/artifactregistry/packages.go b/providers/gcp/artifactregistry/packages.go index 4e381690c..526e91266 100644 --- a/providers/gcp/artifactregistry/packages.go +++ b/providers/gcp/artifactregistry/packages.go @@ -3,6 +3,7 @@ package artifactregistry import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -29,8 +30,13 @@ func ArtifactregistryPackages(ctx context.Context, client providers.ProviderClie for { pkg, err := pkgItr.Next() if err != nil { - logrus.WithError(err).Errorf("failed to get next package") - break + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to get next package") + break + } } if pkg == nil { break diff --git a/providers/gcp/artifactregistry/repositories.go b/providers/gcp/artifactregistry/repositories.go index 12b2cd3d7..1242d6e96 100644 --- a/providers/gcp/artifactregistry/repositories.go +++ b/providers/gcp/artifactregistry/repositories.go @@ -3,6 +3,7 @@ package artifactregistry import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -29,8 +30,13 @@ func ArtifactregistryRepositories(ctx context.Context, client providers.Provider for { repo, err := repoItr.Next() if err != nil { - logrus.WithError(err).Errorf("failed to get next repo") - break + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to get next repo") + break + } } if repo == nil { break diff --git a/providers/gcp/bigquery/tables.go b/providers/gcp/bigquery/tables.go index 08f669a65..f63ea06e5 100644 --- a/providers/gcp/bigquery/tables.go +++ b/providers/gcp/bigquery/tables.go @@ -3,6 +3,7 @@ package bigquery import ( "context" "fmt" + "strings" "time" "cloud.google.com/go/bigquery" @@ -50,8 +51,13 @@ func Tables(ctx context.Context, client providers.ProviderClient) ([]models.Reso break } if err != nil { - logrus.WithError(err).Errorf("failed to list dataset") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list dataset") + return resources, err + } } tablesIterator := dataset.Tables(ctx) diff --git a/providers/gcp/certificate/certificates.go b/providers/gcp/certificate/certificates.go index 0eb41c496..0c9372550 100644 --- a/providers/gcp/certificate/certificates.go +++ b/providers/gcp/certificate/certificates.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "regexp" + "strings" "time" "github.com/sirupsen/logrus" @@ -36,8 +37,13 @@ func Certificates(ctx context.Context, client providers.ProviderClient) ([]model break } if err != nil { - logrus.WithError(err).Errorf("failed to list certificates") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list certificates") + return resources, err + } } certificateNameWithoutProjectAndLocation := extractCertificateName(certificate.Name) diff --git a/providers/gcp/compute/disks.go b/providers/gcp/compute/disks.go index 7f00ae73d..d34c093ad 100644 --- a/providers/gcp/compute/disks.go +++ b/providers/gcp/compute/disks.go @@ -3,6 +3,7 @@ package compute import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -43,8 +44,13 @@ func Disks(ctx context.Context, client providers.ProviderClient) ([]models.Resou break } if err != nil { - logrus.WithError(err).Errorf("failed to list disks") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list disks") + return resources, err + } } if len(disksListPair.Value.Disks) == 0 { continue diff --git a/providers/gcp/compute/instances.go b/providers/gcp/compute/instances.go index ccfab2d47..d91422c76 100644 --- a/providers/gcp/compute/instances.go +++ b/providers/gcp/compute/instances.go @@ -3,6 +3,7 @@ package compute import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -43,8 +44,13 @@ func Instances(ctx context.Context, client providers.ProviderClient) ([]models.R break } if err != nil { - logrus.WithError(err).Errorf("failed to list instances") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list instances") + return resources, err + } } if len(instanceListPair.Value.Instances) == 0 { continue diff --git a/providers/gcp/compute/snapshots.go b/providers/gcp/compute/snapshots.go index 450344657..aea910137 100644 --- a/providers/gcp/compute/snapshots.go +++ b/providers/gcp/compute/snapshots.go @@ -3,6 +3,7 @@ package compute import ( "context" "fmt" + "strings" "time" compute "cloud.google.com/go/compute/apiv1" @@ -42,8 +43,13 @@ func Snapshots(ctx context.Context, client providers.ProviderClient) ([]models.R break } if err != nil { - logrus.WithError(err).Errorf("failed to list snapshots") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list snapshots") + return resources, err + } } tags := make([]models.Tag, 0) diff --git a/providers/gcp/container/clusters.go b/providers/gcp/container/clusters.go index 54fe772bb..10123115c 100644 --- a/providers/gcp/container/clusters.go +++ b/providers/gcp/container/clusters.go @@ -3,6 +3,7 @@ package container import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -36,8 +37,13 @@ func Clusters(ctx context.Context, client providers.ProviderClient) ([]models.Re ProjectId: client.GCPClient.Credentials.ProjectID, }) if err != nil { - logrus.WithError(err).Errorf("failed to collect clusters") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to collect clusters") + return resources, err + } } for _, cluster := range clusters.Clusters { diff --git a/providers/gcp/firestore/documents.go b/providers/gcp/firestore/documents.go index bdee76329..61904ef78 100644 --- a/providers/gcp/firestore/documents.go +++ b/providers/gcp/firestore/documents.go @@ -3,6 +3,7 @@ package firestore import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -26,8 +27,13 @@ func Documents(ctx context.Context, client providers.ProviderClient) ([]models.R list, err := itr.GetAll() if err != nil { - logrus.WithError(err).Errorf("failed to list all firestore collections") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list all firestore collections") + return resources, err + } } for _, collection := range list { diff --git a/providers/gcp/function/functions.go b/providers/gcp/function/functions.go index c27b9f6ce..4c01b9af2 100644 --- a/providers/gcp/function/functions.go +++ b/providers/gcp/function/functions.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "regexp" + "strings" "time" log "github.com/sirupsen/logrus" @@ -26,8 +27,13 @@ func Functions(ctx context.Context, client providers.ProviderClient) ([]models.R "projects/" + client.GCPClient.Credentials.ProjectID + "/locations/-", ).Do() if err != nil { - log.WithError(err).Errorf("failed to list Cloud Functions") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + log.Warn(err.Error()) + return resources, nil + } else { + log.WithError(err).Errorf("failed to list Cloud Functions") + return resources, err + } } for _, function := range functions.Functions { diff --git a/providers/gcp/gateway/gateways.go b/providers/gcp/gateway/gateways.go index d14461bc1..3f0903fc3 100644 --- a/providers/gcp/gateway/gateways.go +++ b/providers/gcp/gateway/gateways.go @@ -3,6 +3,7 @@ package gateway import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -34,12 +35,12 @@ RegionsLoop: "projects/" + client.GCPClient.Credentials.ProjectID + "/locations/" + regionName, ).Do() if err != nil { - if err.Error() == "googleapi: Error 403: Location "+regionName+" is not found or access is unauthorized., forbidden" { + if err.Error() == "googleapi: Error 403: Location "+regionName+" is not found or access is unauthorized., forbidden" || strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) continue RegionsLoop } else { logrus.WithError(err).Errorf("failed to list API Gateways") return resources, err - } } diff --git a/providers/gcp/iam/roles.go b/providers/gcp/iam/roles.go index b42501cc8..7026e8e0d 100644 --- a/providers/gcp/iam/roles.go +++ b/providers/gcp/iam/roles.go @@ -26,8 +26,13 @@ func Roles(ctx context.Context, client providers.ProviderClient) ([]models.Resou "projects/" + client.GCPClient.Credentials.ProjectID, ).Do() if err != nil { - logrus.WithError(err).Errorf("failed to list IAM roles") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list IAM roles") + return resources, err + } } for _, role := range roles.Roles { diff --git a/providers/gcp/iam/service_accounts.go b/providers/gcp/iam/service_accounts.go index 9751c57c9..68edf78c3 100644 --- a/providers/gcp/iam/service_accounts.go +++ b/providers/gcp/iam/service_accounts.go @@ -3,6 +3,7 @@ package iam import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -25,8 +26,13 @@ func ServiceAccounts(ctx context.Context, client providers.ProviderClient) ([]mo "projects/" + client.GCPClient.Credentials.ProjectID, ).Do() if err != nil { - logrus.WithError(err).Errorf("failed to list IAM Service Accounts") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list IAM Service Accounts") + return resources, err + } } for _, account := range serviceAccounts.Accounts { diff --git a/providers/gcp/kms/keys.go b/providers/gcp/kms/keys.go index 4cd4d5276..36607b03d 100644 --- a/providers/gcp/kms/keys.go +++ b/providers/gcp/kms/keys.go @@ -37,8 +37,13 @@ func Keys(ctx context.Context, client providers.ProviderClient) ([]models.Resour break } if err != nil { - logrus.WithError(err).Errorf("failed to list key rings") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list key rings") + return resources, err + } } if keyRing == nil { continue diff --git a/providers/gcp/redis/instances.go b/providers/gcp/redis/instances.go index 65b115c28..fd2a739bb 100644 --- a/providers/gcp/redis/instances.go +++ b/providers/gcp/redis/instances.go @@ -5,6 +5,7 @@ import ( "fmt" "regexp" + "strings" "time" redis "cloud.google.com/go/redis/apiv1" @@ -31,8 +32,13 @@ func Instances(ctx context.Context, client providers.ProviderClient) ([]models.R regions, err := utils.FetchGCPRegionsInRealtime(client.GCPClient.Credentials.ProjectID, option.WithCredentials(client.GCPClient.Credentials)) if err != nil { - logrus.WithError(err).Errorf("failed to list zones to fetch redis") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list zones to fetch redis") + return resources, err + } } redisClient, err := redis.NewCloudRedisRESTClient(ctx, option.WithCredentials(client.GCPClient.Credentials)) diff --git a/providers/gcp/sql/instances.go b/providers/gcp/sql/instances.go index 26c7e09f0..00794535a 100644 --- a/providers/gcp/sql/instances.go +++ b/providers/gcp/sql/instances.go @@ -3,6 +3,7 @@ package sql import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -23,8 +24,13 @@ func Instances(ctx context.Context, client providers.ProviderClient) ([]models.R instances, err := instancesClient.Instances.List(client.GCPClient.Credentials.ProjectID).Do() if err != nil { - logrus.WithError(err).Errorf("failed to lisrt sql servers") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list sql servers") + return resources, err + } } for _, sqlInstance := range instances.Items { diff --git a/providers/gcp/storage/buckets.go b/providers/gcp/storage/buckets.go index 43575c1f0..d19c75a13 100644 --- a/providers/gcp/storage/buckets.go +++ b/providers/gcp/storage/buckets.go @@ -88,8 +88,13 @@ func Buckets(ctx context.Context, client providers.ProviderClient) ([]models.Res break } if err != nil { - log.WithError(err).Errorf("failed to list buckets") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + log.Warn(err.Error()) + return resources, nil + } else { + log.WithError(err).Errorf("failed to list buckets") + return resources, err + } } tags := make([]models.Tag, 0)