From c2329a403108920a581d5c6f46f00a017e4f61ad Mon Sep 17 00:00:00 2001 From: Azanul Date: Sat, 27 Jul 2024 11:54:43 +0530 Subject: [PATCH 1/7] fix: failing redis price calculation Signed-off-by: Azanul --- providers/gcp/redis/pricing.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/providers/gcp/redis/pricing.go b/providers/gcp/redis/pricing.go index 8d8cd99c4..7757dd731 100644 --- a/providers/gcp/redis/pricing.go +++ b/providers/gcp/redis/pricing.go @@ -5,6 +5,7 @@ import ( "fmt" "math" "net/http" + "strings" "time" "cloud.google.com/go/redis/apiv1/redispb" @@ -66,17 +67,18 @@ func calculateRedisCost(redis *redispb.Instance, pricing *GcpDatabasePricing) fl if redis.Tier == redispb.Instance_BASIC { priceMap = pricing.Gcp.Databases.CloudMemorystore.Redis.Basic - priceKey = fmt.Sprintf("Rediscapacitybasicm%ddefault", capacityTier) + priceKey = fmt.Sprintf("rediscapacitybasicm%ddefault", capacityTier) } else if redis.Tier == redispb.Instance_STANDARD_HA { priceMap = pricing.Gcp.Databases.CloudMemorystore.Redis.Standard if redis.ReadReplicasMode == redispb.Instance_READ_REPLICAS_DISABLED { - priceKey = fmt.Sprintf("Rediscapacitystandardm%ddefault", capacityTier) + priceKey = fmt.Sprintf("rediscapacitystandardm%ddefault", capacityTier) } else { - priceKey = fmt.Sprintf("Rediscapacitystandardnodem%d", capacityTier) + priceKey = fmt.Sprintf("rediscapacitystandardnodem%d", capacityTier) } } - pricePerHrPerGbInNanos := priceMap[priceKey].Regions[redis.LocationId].Price[0].Nanos + location := strings.Join(strings.Split(redis.LocationId, "-")[:2], "-") + pricePerHrPerGbInNanos := priceMap[priceKey].Regions[location].Price[0].Nanos pricePerHrPerGbInDollars := pricePerHrPerGbInNanos / math.Pow(10, 9) now := time.Now().UTC() From 3ba592bf36989e5b214eb37884161b964bb56ee2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jul 2024 23:35:48 +0000 Subject: [PATCH 2/7] build(deps-dev): bump @storybook/addons in /dashboard Bumps [@storybook/addons](https://github.com/storybookjs/storybook/tree/HEAD/code/deprecated/addons) from 7.4.6 to 7.6.17. - [Release notes](https://github.com/storybookjs/storybook/releases) - [Changelog](https://github.com/storybookjs/storybook/blob/v7.6.17/CHANGELOG.md) - [Commits](https://github.com/storybookjs/storybook/commits/v7.6.17/code/deprecated/addons) --- updated-dependencies: - dependency-name: "@storybook/addons" dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- dashboard/package-lock.json | 199 +++++++++++++++++------------------- dashboard/package.json | 2 +- 2 files changed, 94 insertions(+), 107 deletions(-) diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json index 2e902e2d3..4065f462a 100644 --- a/dashboard/package-lock.json +++ b/dashboard/package-lock.json @@ -36,7 +36,7 @@ "@storybook/addon-essentials": "^8.0.5", "@storybook/addon-interactions": "^8.0.9", "@storybook/addon-links": "^8.2.4", - "@storybook/addons": "^7.4.6", + "@storybook/addons": "^7.6.17", "@storybook/blocks": "^7.6.4", "@storybook/nextjs": "^8.2.5", "@storybook/preview-api": "^8.1.6", @@ -6364,62 +6364,14 @@ } }, "node_modules/@storybook/addons": { - "version": "7.4.6", - "resolved": "https://registry.npmjs.org/@storybook/addons/-/addons-7.4.6.tgz", - "integrity": "sha512-c+4awrtwNlJayFdgLkEXa5H2Gj+KNlxuN+Z5oDAdZBLqXI8g0gn7eYO2F/eCSIDWdd/+zcU2uq57XPFKc8veHQ==", + "version": "7.6.17", + "resolved": "https://registry.npmjs.org/@storybook/addons/-/addons-7.6.17.tgz", + "integrity": "sha512-Ok18Y698Ccyg++MoUNJNHY0cXUvo8ETFIRLJk1g9ElJ70j6kPgNnzW2pAtZkBNmswHtofZ7pT156cj96k/LgfA==", "dev": true, "dependencies": { - "@storybook/manager-api": "7.4.6", - "@storybook/preview-api": "7.4.6", - "@storybook/types": "7.4.6" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/storybook" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0 || ^18.0.0", - "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0" - } - }, - "node_modules/@storybook/addons/node_modules/@storybook/channels": { - "version": "7.4.6", - "resolved": "https://registry.npmjs.org/@storybook/channels/-/channels-7.4.6.tgz", - "integrity": "sha512-yPv/sfo2c18fM3fvG0i1xse63vG8l33Al/OU0k/dtovltPu001/HVa1QgBgsb/QrEfZtvGjGhmtdVeYb39fv3A==", - "dev": true, - "dependencies": { - "@storybook/client-logger": "7.4.6", - "@storybook/core-events": "7.4.6", - "@storybook/global": "^5.0.0", - "qs": "^6.10.0", - "telejson": "^7.2.0", - "tiny-invariant": "^1.3.1" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/storybook" - } - }, - "node_modules/@storybook/addons/node_modules/@storybook/client-logger": { - "version": "7.4.6", - "resolved": "https://registry.npmjs.org/@storybook/client-logger/-/client-logger-7.4.6.tgz", - "integrity": "sha512-XDw31ZziU//86PKuMRnmc+L/G0VopaGKENQOGEpvAXCU9IZASwGKlKAtcyosjrpi+ZiUXlMgUXCpXM7x3b1Ehw==", - "dev": true, - "dependencies": { - "@storybook/global": "^5.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/storybook" - } - }, - "node_modules/@storybook/addons/node_modules/@storybook/core-events": { - "version": "7.4.6", - "resolved": "https://registry.npmjs.org/@storybook/core-events/-/core-events-7.4.6.tgz", - "integrity": "sha512-r5vrE+32lwrJh1NGFr1a0mWjvxo7q8FXYShylcwRWpacmL5NTtLkrXOoJSeGvJ4yKNYkvxQFtOPId4lzDxa32w==", - "dev": true, - "dependencies": { - "ts-dedent": "^2.0.0" + "@storybook/manager-api": "7.6.17", + "@storybook/preview-api": "7.6.17", + "@storybook/types": "7.6.17" }, "funding": { "type": "opencollective", @@ -6427,23 +6379,22 @@ } }, "node_modules/@storybook/addons/node_modules/@storybook/manager-api": { - "version": "7.4.6", - "resolved": "https://registry.npmjs.org/@storybook/manager-api/-/manager-api-7.4.6.tgz", - "integrity": "sha512-inrm3DIbCp8wjXSN/wK6e6i2ysQ/IEmtC7IN0OJ7vdrp+USCooPT448SQTUmVctUGCFmOU3fxXByq8g77oIi7w==", + "version": "7.6.17", + "resolved": "https://registry.npmjs.org/@storybook/manager-api/-/manager-api-7.6.17.tgz", + "integrity": "sha512-IJIV1Yc6yw1dhCY4tReHCfBnUKDqEBnMyHp3mbXpsaHxnxJZrXO45WjRAZIKlQKhl/Ge1CrnznmHRCmYgqmrWg==", "dev": true, "dependencies": { - "@storybook/channels": "7.4.6", - "@storybook/client-logger": "7.4.6", - "@storybook/core-events": "7.4.6", - "@storybook/csf": "^0.1.0", + "@storybook/channels": "7.6.17", + "@storybook/client-logger": "7.6.17", + "@storybook/core-events": "7.6.17", + "@storybook/csf": "^0.1.2", "@storybook/global": "^5.0.0", - "@storybook/router": "7.4.6", - "@storybook/theming": "7.4.6", - "@storybook/types": "7.4.6", + "@storybook/router": "7.6.17", + "@storybook/theming": "7.6.17", + "@storybook/types": "7.6.17", "dequal": "^2.0.2", "lodash": "^4.17.21", "memoizerific": "^1.11.3", - "semver": "^7.3.7", "store2": "^2.14.2", "telejson": "^7.2.0", "ts-dedent": "^2.0.0" @@ -6451,24 +6402,20 @@ "funding": { "type": "opencollective", "url": "https://opencollective.com/storybook" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0 || ^18.0.0", - "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0" } }, "node_modules/@storybook/addons/node_modules/@storybook/preview-api": { - "version": "7.4.6", - "resolved": "https://registry.npmjs.org/@storybook/preview-api/-/preview-api-7.4.6.tgz", - "integrity": "sha512-byUS/Opt3ytWD4cWz3sNEKw5Yks8MkQgRN+GDSyIomaEAQkLAM0rchPC0MYjwCeUSecV7IIQweNX5RbV4a34BA==", + "version": "7.6.17", + "resolved": "https://registry.npmjs.org/@storybook/preview-api/-/preview-api-7.6.17.tgz", + "integrity": "sha512-wLfDdI9RWo1f2zzFe54yRhg+2YWyxLZvqdZnSQ45mTs4/7xXV5Wfbv3QNTtcdw8tT3U5KRTrN1mTfTCiRJc0Kw==", "dev": true, "dependencies": { - "@storybook/channels": "7.4.6", - "@storybook/client-logger": "7.4.6", - "@storybook/core-events": "7.4.6", - "@storybook/csf": "^0.1.0", + "@storybook/channels": "7.6.17", + "@storybook/client-logger": "7.6.17", + "@storybook/core-events": "7.6.17", + "@storybook/csf": "^0.1.2", "@storybook/global": "^5.0.0", - "@storybook/types": "7.4.6", + "@storybook/types": "7.6.17", "@types/qs": "^6.9.5", "dequal": "^2.0.2", "lodash": "^4.17.21", @@ -6484,32 +6431,28 @@ } }, "node_modules/@storybook/addons/node_modules/@storybook/router": { - "version": "7.4.6", - "resolved": "https://registry.npmjs.org/@storybook/router/-/router-7.4.6.tgz", - "integrity": "sha512-Vl1esrHkcHxDKqc+HY7+6JQpBPW3zYvGk0cQ2rxVMhWdLZTAz1hss9DqzN9tFnPyfn0a1Q77EpMySkUrvWKKNQ==", + "version": "7.6.17", + "resolved": "https://registry.npmjs.org/@storybook/router/-/router-7.6.17.tgz", + "integrity": "sha512-GnyC0j6Wi5hT4qRhSyT8NPtJfGmf82uZw97LQRWeyYu5gWEshUdM7aj40XlNiScd5cZDp0owO1idduVF2k2l2A==", "dev": true, "dependencies": { - "@storybook/client-logger": "7.4.6", + "@storybook/client-logger": "7.6.17", "memoizerific": "^1.11.3", "qs": "^6.10.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/storybook" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0 || ^18.0.0", - "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0" } }, "node_modules/@storybook/addons/node_modules/@storybook/theming": { - "version": "7.4.6", - "resolved": "https://registry.npmjs.org/@storybook/theming/-/theming-7.4.6.tgz", - "integrity": "sha512-HW77iJ9ptCMqhoBOYFjRQw7VBap+38fkJGHP5KylEJCyYCgIAm2dEcQmtWpMVYFssSGcb6djfbtAMhYU4TL4Iw==", + "version": "7.6.17", + "resolved": "https://registry.npmjs.org/@storybook/theming/-/theming-7.6.17.tgz", + "integrity": "sha512-ZbaBt3KAbmBtfjNqgMY7wPMBshhSJlhodyMNQypv+95xLD/R+Az6aBYbpVAOygLaUQaQk4ar7H/Ww6lFIoiFbA==", "dev": true, "dependencies": { "@emotion/use-insertion-effect-with-fallbacks": "^1.0.0", - "@storybook/client-logger": "7.4.6", + "@storybook/client-logger": "7.6.17", "@storybook/global": "^5.0.0", "memoizerific": "^1.11.3" }, @@ -6522,22 +6465,6 @@ "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0" } }, - "node_modules/@storybook/addons/node_modules/@storybook/types": { - "version": "7.4.6", - "resolved": "https://registry.npmjs.org/@storybook/types/-/types-7.4.6.tgz", - "integrity": "sha512-6QLXtMVsFZFpzPkdGWsu/iuc8na9dnS67AMOBKm5qCLPwtUJOYkwhMdFRSSeJthLRpzV7JLAL8Kwvl7MFP3QSw==", - "dev": true, - "dependencies": { - "@storybook/channels": "7.4.6", - "@types/babel__core": "^7.0.0", - "@types/express": "^4.7.0", - "file-system-cache": "2.3.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/storybook" - } - }, "node_modules/@storybook/blocks": { "version": "7.6.4", "resolved": "https://registry.npmjs.org/@storybook/blocks/-/blocks-7.6.4.tgz", @@ -6880,6 +6807,37 @@ "undici-types": "~5.26.4" } }, + "node_modules/@storybook/channels": { + "version": "7.6.17", + "resolved": "https://registry.npmjs.org/@storybook/channels/-/channels-7.6.17.tgz", + "integrity": "sha512-GFG40pzaSxk1hUr/J/TMqW5AFDDPUSu+HkeE/oqSWJbOodBOLJzHN6CReJS6y1DjYSZLNFt1jftPWZZInG/XUA==", + "dev": true, + "dependencies": { + "@storybook/client-logger": "7.6.17", + "@storybook/core-events": "7.6.17", + "@storybook/global": "^5.0.0", + "qs": "^6.10.0", + "telejson": "^7.2.0", + "tiny-invariant": "^1.3.1" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/storybook" + } + }, + "node_modules/@storybook/client-logger": { + "version": "7.6.17", + "resolved": "https://registry.npmjs.org/@storybook/client-logger/-/client-logger-7.6.17.tgz", + "integrity": "sha512-6WBYqixAXNAXlSaBWwgljWpAu10tPRBJrcFvx2gPUne58EeMM20Gi/iHYBz2kMCY+JLAgeIH7ZxInqwO8vDwiQ==", + "dev": true, + "dependencies": { + "@storybook/global": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/storybook" + } + }, "node_modules/@storybook/codemod": { "version": "8.2.5", "resolved": "https://registry.npmjs.org/@storybook/codemod/-/codemod-8.2.5.tgz", @@ -7281,6 +7239,19 @@ "url": "https://opencollective.com/storybook" } }, + "node_modules/@storybook/core-events": { + "version": "7.6.17", + "resolved": "https://registry.npmjs.org/@storybook/core-events/-/core-events-7.6.17.tgz", + "integrity": "sha512-AriWMCm/k1cxlv10f+jZ1wavThTRpLaN3kY019kHWbYT9XgaSuLU67G7GPr3cGnJ6HuA6uhbzu8qtqVCd6OfXA==", + "dev": true, + "dependencies": { + "ts-dedent": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/storybook" + } + }, "node_modules/@storybook/core-webpack": { "version": "8.2.5", "resolved": "https://registry.npmjs.org/@storybook/core-webpack/-/core-webpack-8.2.5.tgz", @@ -8553,6 +8524,22 @@ "storybook": "^8.2.5" } }, + "node_modules/@storybook/types": { + "version": "7.6.17", + "resolved": "https://registry.npmjs.org/@storybook/types/-/types-7.6.17.tgz", + "integrity": "sha512-GRY0xEJQ0PrL7DY2qCNUdIfUOE0Gsue6N+GBJw9ku1IUDFLJRDOF+4Dx2BvYcVCPI5XPqdWKlEyZdMdKjiQN7Q==", + "dev": true, + "dependencies": { + "@storybook/channels": "7.6.17", + "@types/babel__core": "^7.0.0", + "@types/express": "^4.7.0", + "file-system-cache": "2.3.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/storybook" + } + }, "node_modules/@swc/core": { "version": "1.3.86", "resolved": "https://registry.npmjs.org/@swc/core/-/core-1.3.86.tgz", diff --git a/dashboard/package.json b/dashboard/package.json index de153fd7c..d6f24ec64 100644 --- a/dashboard/package.json +++ b/dashboard/package.json @@ -46,7 +46,7 @@ "@storybook/addon-essentials": "^8.0.5", "@storybook/addon-interactions": "^8.0.9", "@storybook/addon-links": "^8.2.4", - "@storybook/addons": "^7.4.6", + "@storybook/addons": "^7.6.17", "@storybook/blocks": "^7.6.4", "@storybook/nextjs": "^8.2.5", "@storybook/preview-api": "^8.1.6", From 1c87ef760b4b9a414534abadea6b4f722b7d62db Mon Sep 17 00:00:00 2001 From: Azanul Date: Tue, 30 Jul 2024 16:41:02 +0530 Subject: [PATCH 3/7] fix: gcp buckets size calculation Signed-off-by: Azanul --- providers/gcp/storage/buckets.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/providers/gcp/storage/buckets.go b/providers/gcp/storage/buckets.go index 6825a1081..43575c1f0 100644 --- a/providers/gcp/storage/buckets.go +++ b/providers/gcp/storage/buckets.go @@ -37,9 +37,11 @@ func GetBucketSize(ctx context.Context, client providers.ProviderClient, bucketN EndTime: ×tamp.Timestamp{Seconds: time.Now().Unix()}, }, Aggregation: &monitoringpb.Aggregation{ - AlignmentPeriod: &duration.Duration{Seconds: 60}, - PerSeriesAligner: monitoringpb.Aggregation_ALIGN_SUM, + AlignmentPeriod: &duration.Duration{Seconds: 86400}, + PerSeriesAligner: monitoringpb.Aggregation_ALIGN_MEAN, + GroupByFields: []string{"resource.label.bucket_name"}, }, + View: monitoringpb.ListTimeSeriesRequest_FULL, } res := monitoringClient.ListTimeSeries(ctx, req) From 92cffbd120c03732c853f5987b085f77ece76f56 Mon Sep 17 00:00:00 2001 From: Azanul Date: Tue, 30 Jul 2024 17:19:38 +0530 Subject: [PATCH 4/7] refac: show service disabled as warnings Signed-off-by: Azanul --- providers/gcp/alloydb/clusters.go | 10 ++++++++-- providers/gcp/alloydb/instances.go | 10 ++++++++-- providers/gcp/appengine/service.go | 10 ++++++++-- providers/gcp/artifactregistry/docker_images.go | 10 ++++++++-- providers/gcp/artifactregistry/packages.go | 10 ++++++++-- providers/gcp/artifactregistry/repositories.go | 10 ++++++++-- providers/gcp/bigquery/tables.go | 10 ++++++++-- providers/gcp/certificate/certificates.go | 10 ++++++++-- providers/gcp/compute/disks.go | 10 ++++++++-- providers/gcp/compute/instances.go | 10 ++++++++-- providers/gcp/compute/snapshots.go | 10 ++++++++-- providers/gcp/container/clusters.go | 10 ++++++++-- providers/gcp/firestore/documents.go | 10 ++++++++-- providers/gcp/function/functions.go | 10 ++++++++-- providers/gcp/gateway/gateways.go | 5 +++-- providers/gcp/iam/roles.go | 9 +++++++-- providers/gcp/iam/service_accounts.go | 10 ++++++++-- providers/gcp/kms/keys.go | 9 +++++++-- providers/gcp/redis/instances.go | 10 ++++++++-- providers/gcp/sql/instances.go | 10 ++++++++-- providers/gcp/storage/buckets.go | 9 +++++++-- 21 files changed, 160 insertions(+), 42 deletions(-) diff --git a/providers/gcp/alloydb/clusters.go b/providers/gcp/alloydb/clusters.go index 7a1acd7a4..8ac74101d 100644 --- a/providers/gcp/alloydb/clusters.go +++ b/providers/gcp/alloydb/clusters.go @@ -3,6 +3,7 @@ package alloydb import ( "context" "errors" + "strings" "time" "github.com/sirupsen/logrus" @@ -29,8 +30,13 @@ func Clusters(ctx context.Context, client providers.ProviderClient) ([]models.Re break } if err != nil { - logrus.WithError(err).Errorf("failed to get clusters") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to get clusters") + return resources, err + } } resources = append(resources, models.Resource{ Provider: "GCP", diff --git a/providers/gcp/alloydb/instances.go b/providers/gcp/alloydb/instances.go index ecdc29385..1747e3e98 100644 --- a/providers/gcp/alloydb/instances.go +++ b/providers/gcp/alloydb/instances.go @@ -3,6 +3,7 @@ package alloydb import ( "context" "errors" + "strings" "time" "github.com/sirupsen/logrus" @@ -29,8 +30,13 @@ func Instances(ctx context.Context, client providers.ProviderClient) ([]models.R break } if err != nil { - logrus.WithError(err).Errorf("failed to get instances") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to get instances") + return resources, err + } } resources = append(resources, models.Resource{ Provider: "GCP", diff --git a/providers/gcp/appengine/service.go b/providers/gcp/appengine/service.go index 89772f6fb..bb9a827e4 100644 --- a/providers/gcp/appengine/service.go +++ b/providers/gcp/appengine/service.go @@ -3,6 +3,7 @@ package appengine import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -29,8 +30,13 @@ func Services(ctx context.Context, client providers.ProviderClient) ([]models.Re for { svc, err := svcs.Next() if err != nil { - logrus.WithError(err).Errorf("failed to get app engine") - break + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to get app engine") + break + } } if svc == nil { break diff --git a/providers/gcp/artifactregistry/docker_images.go b/providers/gcp/artifactregistry/docker_images.go index cf97e5044..ab8acb5d2 100644 --- a/providers/gcp/artifactregistry/docker_images.go +++ b/providers/gcp/artifactregistry/docker_images.go @@ -3,6 +3,7 @@ package artifactregistry import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -29,8 +30,13 @@ func ArtifactregistryDockerImages(ctx context.Context, client providers.Provider for { image, err := imageItr.Next() if err != nil { - logrus.WithError(err).Errorf("failed to get nex image") - break + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to get nex image") + break + } } if image == nil { break diff --git a/providers/gcp/artifactregistry/packages.go b/providers/gcp/artifactregistry/packages.go index 4e381690c..526e91266 100644 --- a/providers/gcp/artifactregistry/packages.go +++ b/providers/gcp/artifactregistry/packages.go @@ -3,6 +3,7 @@ package artifactregistry import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -29,8 +30,13 @@ func ArtifactregistryPackages(ctx context.Context, client providers.ProviderClie for { pkg, err := pkgItr.Next() if err != nil { - logrus.WithError(err).Errorf("failed to get next package") - break + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to get next package") + break + } } if pkg == nil { break diff --git a/providers/gcp/artifactregistry/repositories.go b/providers/gcp/artifactregistry/repositories.go index 12b2cd3d7..1242d6e96 100644 --- a/providers/gcp/artifactregistry/repositories.go +++ b/providers/gcp/artifactregistry/repositories.go @@ -3,6 +3,7 @@ package artifactregistry import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -29,8 +30,13 @@ func ArtifactregistryRepositories(ctx context.Context, client providers.Provider for { repo, err := repoItr.Next() if err != nil { - logrus.WithError(err).Errorf("failed to get next repo") - break + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to get next repo") + break + } } if repo == nil { break diff --git a/providers/gcp/bigquery/tables.go b/providers/gcp/bigquery/tables.go index 08f669a65..f63ea06e5 100644 --- a/providers/gcp/bigquery/tables.go +++ b/providers/gcp/bigquery/tables.go @@ -3,6 +3,7 @@ package bigquery import ( "context" "fmt" + "strings" "time" "cloud.google.com/go/bigquery" @@ -50,8 +51,13 @@ func Tables(ctx context.Context, client providers.ProviderClient) ([]models.Reso break } if err != nil { - logrus.WithError(err).Errorf("failed to list dataset") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list dataset") + return resources, err + } } tablesIterator := dataset.Tables(ctx) diff --git a/providers/gcp/certificate/certificates.go b/providers/gcp/certificate/certificates.go index 0eb41c496..0c9372550 100644 --- a/providers/gcp/certificate/certificates.go +++ b/providers/gcp/certificate/certificates.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "regexp" + "strings" "time" "github.com/sirupsen/logrus" @@ -36,8 +37,13 @@ func Certificates(ctx context.Context, client providers.ProviderClient) ([]model break } if err != nil { - logrus.WithError(err).Errorf("failed to list certificates") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list certificates") + return resources, err + } } certificateNameWithoutProjectAndLocation := extractCertificateName(certificate.Name) diff --git a/providers/gcp/compute/disks.go b/providers/gcp/compute/disks.go index 7f00ae73d..d34c093ad 100644 --- a/providers/gcp/compute/disks.go +++ b/providers/gcp/compute/disks.go @@ -3,6 +3,7 @@ package compute import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -43,8 +44,13 @@ func Disks(ctx context.Context, client providers.ProviderClient) ([]models.Resou break } if err != nil { - logrus.WithError(err).Errorf("failed to list disks") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list disks") + return resources, err + } } if len(disksListPair.Value.Disks) == 0 { continue diff --git a/providers/gcp/compute/instances.go b/providers/gcp/compute/instances.go index ccfab2d47..d91422c76 100644 --- a/providers/gcp/compute/instances.go +++ b/providers/gcp/compute/instances.go @@ -3,6 +3,7 @@ package compute import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -43,8 +44,13 @@ func Instances(ctx context.Context, client providers.ProviderClient) ([]models.R break } if err != nil { - logrus.WithError(err).Errorf("failed to list instances") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list instances") + return resources, err + } } if len(instanceListPair.Value.Instances) == 0 { continue diff --git a/providers/gcp/compute/snapshots.go b/providers/gcp/compute/snapshots.go index 450344657..aea910137 100644 --- a/providers/gcp/compute/snapshots.go +++ b/providers/gcp/compute/snapshots.go @@ -3,6 +3,7 @@ package compute import ( "context" "fmt" + "strings" "time" compute "cloud.google.com/go/compute/apiv1" @@ -42,8 +43,13 @@ func Snapshots(ctx context.Context, client providers.ProviderClient) ([]models.R break } if err != nil { - logrus.WithError(err).Errorf("failed to list snapshots") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list snapshots") + return resources, err + } } tags := make([]models.Tag, 0) diff --git a/providers/gcp/container/clusters.go b/providers/gcp/container/clusters.go index 54fe772bb..10123115c 100644 --- a/providers/gcp/container/clusters.go +++ b/providers/gcp/container/clusters.go @@ -3,6 +3,7 @@ package container import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -36,8 +37,13 @@ func Clusters(ctx context.Context, client providers.ProviderClient) ([]models.Re ProjectId: client.GCPClient.Credentials.ProjectID, }) if err != nil { - logrus.WithError(err).Errorf("failed to collect clusters") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to collect clusters") + return resources, err + } } for _, cluster := range clusters.Clusters { diff --git a/providers/gcp/firestore/documents.go b/providers/gcp/firestore/documents.go index bdee76329..61904ef78 100644 --- a/providers/gcp/firestore/documents.go +++ b/providers/gcp/firestore/documents.go @@ -3,6 +3,7 @@ package firestore import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -26,8 +27,13 @@ func Documents(ctx context.Context, client providers.ProviderClient) ([]models.R list, err := itr.GetAll() if err != nil { - logrus.WithError(err).Errorf("failed to list all firestore collections") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list all firestore collections") + return resources, err + } } for _, collection := range list { diff --git a/providers/gcp/function/functions.go b/providers/gcp/function/functions.go index c27b9f6ce..4c01b9af2 100644 --- a/providers/gcp/function/functions.go +++ b/providers/gcp/function/functions.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "regexp" + "strings" "time" log "github.com/sirupsen/logrus" @@ -26,8 +27,13 @@ func Functions(ctx context.Context, client providers.ProviderClient) ([]models.R "projects/" + client.GCPClient.Credentials.ProjectID + "/locations/-", ).Do() if err != nil { - log.WithError(err).Errorf("failed to list Cloud Functions") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + log.Warn(err.Error()) + return resources, nil + } else { + log.WithError(err).Errorf("failed to list Cloud Functions") + return resources, err + } } for _, function := range functions.Functions { diff --git a/providers/gcp/gateway/gateways.go b/providers/gcp/gateway/gateways.go index d14461bc1..3f0903fc3 100644 --- a/providers/gcp/gateway/gateways.go +++ b/providers/gcp/gateway/gateways.go @@ -3,6 +3,7 @@ package gateway import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -34,12 +35,12 @@ RegionsLoop: "projects/" + client.GCPClient.Credentials.ProjectID + "/locations/" + regionName, ).Do() if err != nil { - if err.Error() == "googleapi: Error 403: Location "+regionName+" is not found or access is unauthorized., forbidden" { + if err.Error() == "googleapi: Error 403: Location "+regionName+" is not found or access is unauthorized., forbidden" || strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) continue RegionsLoop } else { logrus.WithError(err).Errorf("failed to list API Gateways") return resources, err - } } diff --git a/providers/gcp/iam/roles.go b/providers/gcp/iam/roles.go index b42501cc8..7026e8e0d 100644 --- a/providers/gcp/iam/roles.go +++ b/providers/gcp/iam/roles.go @@ -26,8 +26,13 @@ func Roles(ctx context.Context, client providers.ProviderClient) ([]models.Resou "projects/" + client.GCPClient.Credentials.ProjectID, ).Do() if err != nil { - logrus.WithError(err).Errorf("failed to list IAM roles") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list IAM roles") + return resources, err + } } for _, role := range roles.Roles { diff --git a/providers/gcp/iam/service_accounts.go b/providers/gcp/iam/service_accounts.go index 9751c57c9..68edf78c3 100644 --- a/providers/gcp/iam/service_accounts.go +++ b/providers/gcp/iam/service_accounts.go @@ -3,6 +3,7 @@ package iam import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -25,8 +26,13 @@ func ServiceAccounts(ctx context.Context, client providers.ProviderClient) ([]mo "projects/" + client.GCPClient.Credentials.ProjectID, ).Do() if err != nil { - logrus.WithError(err).Errorf("failed to list IAM Service Accounts") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list IAM Service Accounts") + return resources, err + } } for _, account := range serviceAccounts.Accounts { diff --git a/providers/gcp/kms/keys.go b/providers/gcp/kms/keys.go index 4cd4d5276..36607b03d 100644 --- a/providers/gcp/kms/keys.go +++ b/providers/gcp/kms/keys.go @@ -37,8 +37,13 @@ func Keys(ctx context.Context, client providers.ProviderClient) ([]models.Resour break } if err != nil { - logrus.WithError(err).Errorf("failed to list key rings") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list key rings") + return resources, err + } } if keyRing == nil { continue diff --git a/providers/gcp/redis/instances.go b/providers/gcp/redis/instances.go index 65b115c28..fd2a739bb 100644 --- a/providers/gcp/redis/instances.go +++ b/providers/gcp/redis/instances.go @@ -5,6 +5,7 @@ import ( "fmt" "regexp" + "strings" "time" redis "cloud.google.com/go/redis/apiv1" @@ -31,8 +32,13 @@ func Instances(ctx context.Context, client providers.ProviderClient) ([]models.R regions, err := utils.FetchGCPRegionsInRealtime(client.GCPClient.Credentials.ProjectID, option.WithCredentials(client.GCPClient.Credentials)) if err != nil { - logrus.WithError(err).Errorf("failed to list zones to fetch redis") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list zones to fetch redis") + return resources, err + } } redisClient, err := redis.NewCloudRedisRESTClient(ctx, option.WithCredentials(client.GCPClient.Credentials)) diff --git a/providers/gcp/sql/instances.go b/providers/gcp/sql/instances.go index 26c7e09f0..00794535a 100644 --- a/providers/gcp/sql/instances.go +++ b/providers/gcp/sql/instances.go @@ -3,6 +3,7 @@ package sql import ( "context" "fmt" + "strings" "time" "github.com/sirupsen/logrus" @@ -23,8 +24,13 @@ func Instances(ctx context.Context, client providers.ProviderClient) ([]models.R instances, err := instancesClient.Instances.List(client.GCPClient.Credentials.ProjectID).Do() if err != nil { - logrus.WithError(err).Errorf("failed to lisrt sql servers") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + logrus.Warn(err.Error()) + return resources, nil + } else { + logrus.WithError(err).Errorf("failed to list sql servers") + return resources, err + } } for _, sqlInstance := range instances.Items { diff --git a/providers/gcp/storage/buckets.go b/providers/gcp/storage/buckets.go index 43575c1f0..d19c75a13 100644 --- a/providers/gcp/storage/buckets.go +++ b/providers/gcp/storage/buckets.go @@ -88,8 +88,13 @@ func Buckets(ctx context.Context, client providers.ProviderClient) ([]models.Res break } if err != nil { - log.WithError(err).Errorf("failed to list buckets") - return resources, err + if strings.Contains(err.Error(), "SERVICE_DISABLED") { + log.Warn(err.Error()) + return resources, nil + } else { + log.WithError(err).Errorf("failed to list buckets") + return resources, err + } } tags := make([]models.Tag, 0) From c87c43d4ff1f4da106d6ea765369a2874e05edca Mon Sep 17 00:00:00 2001 From: Azanul Date: Thu, 1 Aug 2024 12:11:10 +0530 Subject: [PATCH 5/7] feat: support aws redshift cluster Signed-off-by: Azanul --- providers/aws/redshift/cluster.go | 90 +++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 providers/aws/redshift/cluster.go diff --git a/providers/aws/redshift/cluster.go b/providers/aws/redshift/cluster.go new file mode 100644 index 000000000..d77082561 --- /dev/null +++ b/providers/aws/redshift/cluster.go @@ -0,0 +1,90 @@ +package redshift + +import ( + "context" + "fmt" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/redshift" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/tailwarden/komiser/models" + "github.com/tailwarden/komiser/providers" + awsUtils "github.com/tailwarden/komiser/providers/aws/utils" +) + +func Resources(ctx context.Context, client providers.ProviderClient) ([]models.Resource, error) { + resources := make([]models.Resource, 0) + var config redshift.DescribeClustersInput + redshiftClient := redshift.NewFromConfig(*client.AWSClient) + + stsClient := sts.NewFromConfig(*client.AWSClient) + stsOutput, err := stsClient.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}) + if err != nil { + return resources, err + } + + accountId := stsOutput.Account + + serviceCost, err := awsUtils.GetCostAndUsage(ctx, client.AWSClient.Region, "Redshift") + if err != nil { + log.Warnln("Couldn't fetch Redshift cost and usage:", err) + } + + for { + output, err := redshiftClient.DescribeClusters(ctx, &config) + if err != nil { + return resources, err + } + + for _, cluster := range output.Clusters { + resourceArn := fmt.Sprintf("arn:aws:redshift:%s:%s:cluster/%s", client.AWSClient.Region, *accountId, *cluster.ClusterIdentifier) + outputTags := cluster.Tags + + tags := make([]models.Tag, 0) + + for _, tag := range outputTags { + tags = append(tags, models.Tag{ + Key: *tag.Key, + Value: *tag.Value, + }) + } + + monthlyCost := float64(0) + + resources = append(resources, models.Resource{ + Provider: "AWS", + Account: client.Name, + Service: "Redshift EventSubscription", + ResourceId: resourceArn, + Region: client.AWSClient.Region, + Name: *cluster.ClusterIdentifier, + Cost: monthlyCost, + Metadata: map[string]string{ + "serviceCost": fmt.Sprint(serviceCost), + }, + Tags: tags, + FetchedAt: time.Now(), + Link: fmt.Sprintf("https://%s.console.aws.amaxon.com/redshift/home?region=%s/clusters/%s", client.AWSClient.Region, client.AWSClient.Region, *cluster.ClusterIdentifier), + }) + + } + + if aws.ToString(output.Marker) == "" { + break + } + config.Marker = output.Marker + } + + log.WithFields(log.Fields{ + "provider": "AWS", + "account": client.Name, + "region": client.AWSClient.Region, + "service": "Redshift EventSubscription", + "resources": len(resources), + }).Info("Fetched resources") + return resources, nil + +} From 19d8c32481ce5a592d9798f701aba4dabd343537 Mon Sep 17 00:00:00 2001 From: Azanul Date: Thu, 1 Aug 2024 12:11:24 +0530 Subject: [PATCH 6/7] refac: redundent if and dot imports removed Signed-off-by: Azanul --- providers/aws/redshift/eventsubscription.go | 30 ++++++++++----------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/providers/aws/redshift/eventsubscription.go b/providers/aws/redshift/eventsubscription.go index b4b52317b..eea5996ef 100644 --- a/providers/aws/redshift/eventsubscription.go +++ b/providers/aws/redshift/eventsubscription.go @@ -10,13 +10,13 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/redshift" "github.com/aws/aws-sdk-go-v2/service/sts" - . "github.com/tailwarden/komiser/models" - . "github.com/tailwarden/komiser/providers" + "github.com/tailwarden/komiser/models" + "github.com/tailwarden/komiser/providers" awsUtils "github.com/tailwarden/komiser/providers/aws/utils" ) -func EventSubscriptions(ctx context.Context, client ProviderClient) ([]Resource, error) { - resources := make([]Resource, 0) +func EventSubscriptions(ctx context.Context, client providers.ProviderClient) ([]models.Resource, error) { + resources := make([]models.Resource, 0) var config redshift.DescribeEventSubscriptionsInput redshiftClient := redshift.NewFromConfig(*client.AWSClient) @@ -45,20 +45,18 @@ func EventSubscriptions(ctx context.Context, client ProviderClient) ([]Resource, resourceArn := fmt.Sprintf("arn:aws:redshift:%s:%s:eventsubscripion/%s", client.AWSClient.Region, *accountId, *eventSubscription.CustSubscriptionId) outputTags := eventSubscription.Tags - tags := make([]Tag, 0) + tags := make([]models.Tag, 0) - if err == nil { - for _, tag := range outputTags { - tags = append(tags, Tag{ - Key: *tag.Key, - Value: *tag.Value, - }) - } + for _, tag := range outputTags { + tags = append(tags, models.Tag{ + Key: *tag.Key, + Value: *tag.Value, + }) } monthlyCost := float64(0) - resources = append(resources, Resource{ + resources = append(resources, models.Resource{ Provider: "AWS", Account: client.Name, Service: "Redshift EventSubscription", @@ -69,9 +67,9 @@ func EventSubscriptions(ctx context.Context, client ProviderClient) ([]Resource, Metadata: map[string]string{ "serviceCost": fmt.Sprint(serviceCost), }, - Tags: tags, - FetchedAt: time.Now(), - Link: fmt.Sprintf("https://%s.console.aws.amaxon.com/redshift/home?region=%s/event-subscriptions/%s", client.AWSClient.Region, client.AWSClient.Region, *eventSubscription.CustSubscriptionId), + Tags: tags, + FetchedAt: time.Now(), + Link: fmt.Sprintf("https://%s.console.aws.amaxon.com/redshift/home?region=%s/event-subscriptions/%s", client.AWSClient.Region, client.AWSClient.Region, *eventSubscription.CustSubscriptionId), }) } } From cf2cac2b62a045da544f11a4cb3c14769c1fbe19 Mon Sep 17 00:00:00 2001 From: Azanul Date: Thu, 1 Aug 2024 12:17:10 +0530 Subject: [PATCH 7/7] refac: service name typo Signed-off-by: Azanul --- providers/aws/redshift/cluster.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/aws/redshift/cluster.go b/providers/aws/redshift/cluster.go index d77082561..ddedbe8ad 100644 --- a/providers/aws/redshift/cluster.go +++ b/providers/aws/redshift/cluster.go @@ -57,7 +57,7 @@ func Resources(ctx context.Context, client providers.ProviderClient) ([]models.R resources = append(resources, models.Resource{ Provider: "AWS", Account: client.Name, - Service: "Redshift EventSubscription", + Service: "Redshift Cluster", ResourceId: resourceArn, Region: client.AWSClient.Region, Name: *cluster.ClusterIdentifier, @@ -82,7 +82,7 @@ func Resources(ctx context.Context, client providers.ProviderClient) ([]models.R "provider": "AWS", "account": client.Name, "region": client.AWSClient.Region, - "service": "Redshift EventSubscription", + "service": "Redshift Cluster", "resources": len(resources), }).Info("Fetched resources") return resources, nil