From 284cdd6d45e5ca9c054ede43576f75bec1611e57 Mon Sep 17 00:00:00 2001 From: bishal7679 Date: Mon, 9 Oct 2023 12:04:44 +0530 Subject: [PATCH 1/4] cost of s3 showing 0 is fixed Signed-off-by: bishal7679 --- providers/aws/s3/buckets.go | 95 ++++++++++++++++++++++++++++++++----- 1 file changed, 82 insertions(+), 13 deletions(-) diff --git a/providers/aws/s3/buckets.go b/providers/aws/s3/buckets.go index 3401e99fd..eb42a941c 100644 --- a/providers/aws/s3/buckets.go +++ b/providers/aws/s3/buckets.go @@ -9,15 +9,18 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" - "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" + cloudwatchTypes "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" + "github.com/aws/aws-sdk-go-v2/service/pricing" + "github.com/aws/aws-sdk-go-v2/service/pricing/types" "github.com/aws/aws-sdk-go-v2/service/s3" . "github.com/tailwarden/komiser/models" . "github.com/tailwarden/komiser/providers" + awsUtils "github.com/tailwarden/komiser/providers/aws/utils" "github.com/tailwarden/komiser/utils" ) func ConvertBytesToTerabytes(bytes int64) float64 { - return float64(bytes) / 1000000000000 + return float64(bytes) / 1099511627776 } func Buckets(ctx context.Context, client ProviderClient) ([]Resource, error) { @@ -25,53 +28,119 @@ func Buckets(ctx context.Context, client ProviderClient) ([]Resource, error) { var config s3.ListBucketsInput s3Client := s3.NewFromConfig(*client.AWSClient) cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient) + pricingClient := pricing.NewFromConfig(*client.AWSClient) + + pricingOutput, err := pricingClient.GetProducts(ctx, &pricing.GetProductsInput{ + ServiceCode: aws.String("AmazonS3"), + Filters: []types.Filter{ + { + Field: aws.String("regionCode"), // Filter by region + Value: aws.String(client.AWSClient.Region), + Type: types.FilterTypeTermMatch, + }, + { + Field: aws.String("productFamily"), + Value: aws.String("Storage"), + Type: types.FilterTypeTermMatch, + }, + }, + }) + if err != nil { + log.Errorf("ERROR: Couldn't fetch pricing info for AWS S3: %v", err) + return resources, err + } + + priceMap, err := awsUtils.GetPriceMap(pricingOutput) + if err != nil { + log.Errorf("ERROR: Failed to calculate cost per month: %v", err) + return resources, err + } + + // --------------------------------------------------------------- output, err := s3Client.ListBuckets(context.Background(), &config) if err != nil { return resources, err } for _, bucket := range output.Buckets { + // metrics for bucket size metricsBucketSizebytesOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), EndTime: aws.Time(time.Now()), MetricName: aws.String("BucketSizeBytes"), Namespace: aws.String("AWS/S3"), - Dimensions: []types.Dimension{ - types.Dimension{ + Dimensions: []cloudwatchTypes.Dimension{ + { Name: aws.String("BucketName"), Value: bucket.Name, }, - types.Dimension{ + { Name: aws.String("StorageType"), Value: aws.String("StandardStorage"), }, }, - Unit: types.StandardUnitBytes, + Unit: cloudwatchTypes.StandardUnitBytes, Period: aws.Int32(3600), - Statistics: []types.Statistic{ - types.StatisticAverage, + Statistics: []cloudwatchTypes.Statistic{ + cloudwatchTypes.StatisticAverage, }, }) if err != nil { log.Warnf("Couldn't fetch invocations metric for %s", *bucket.Name) } - bucketSize := 0.0 if metricsBucketSizebytesOutput != nil && len(metricsBucketSizebytesOutput.Datapoints) > 0 { bucketSize = *metricsBucketSizebytesOutput.Datapoints[0].Average } sizeInTB := ConvertBytesToTerabytes(int64(bucketSize)) - monthlyCost := 0.0 + storageCostPerGB := 0.0 if sizeInTB <= 50 { - monthlyCost = (sizeInTB * 1000) * 0.023 + storageCostPerGB = (sizeInTB * 1024) * 0.023 } else if sizeInTB <= 450 { - monthlyCost = (sizeInTB * 1000) * 0.022 + storageCostPerGB = (sizeInTB * 1024) * 0.022 } else { - monthlyCost = (sizeInTB * 1000) * 0.021 + storageCostPerGB = (sizeInTB * 1024) * 0.021 + } + + // metrics for bucket usage + + metricsUsageOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ + StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), + EndTime: aws.Time(time.Now()), + MetricName: aws.String("AllRequests"), + Namespace: aws.String("AWS/S3"), + Dimensions: []cloudwatchTypes.Dimension{ + { + Name: aws.String("BucketName"), + Value: bucket.Name, + }, + { + Name: aws.String("StorageType"), + Value: aws.String("StandardStorage"), + }, + }, + Unit: cloudwatchTypes.StandardUnitCount, + Period: aws.Int32(3600), + Statistics: []cloudwatchTypes.Statistic{ + cloudwatchTypes.StatisticSum, + }, + }) + if err != nil { + log.Warnf("Couldn't fetch usage metric for %s", *bucket.Name) } + requestCount := 0.0 + if metricsUsageOutput != nil && len(metricsUsageOutput.Datapoints) > 0 { + requestCount = *metricsUsageOutput.Datapoints[0].Sum + } + // requestCost := (requestCount / 1000) * 0.0004 + + monthlyCost := 0.0 + + requestCharges := awsUtils.GetCost(priceMap["AWS-S3-Requests"], requestCount/1000) // charges per 1000 request + monthlyCost = storageCostPerGB + requestCharges tagsResp, err := s3Client.GetBucketTagging(context.Background(), &s3.GetBucketTaggingInput{ Bucket: bucket.Name, }) From eeef8dc834823ed721313a5ec94edec514dbca99 Mon Sep 17 00:00:00 2001 From: bishal7679 Date: Mon, 9 Oct 2023 19:44:21 +0530 Subject: [PATCH 2/4] modified for dynamic charges Signed-off-by: bishal7679 --- providers/aws/s3/buckets.go | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/providers/aws/s3/buckets.go b/providers/aws/s3/buckets.go index eb42a941c..8d5030d56 100644 --- a/providers/aws/s3/buckets.go +++ b/providers/aws/s3/buckets.go @@ -38,11 +38,6 @@ func Buckets(ctx context.Context, client ProviderClient) ([]Resource, error) { Value: aws.String(client.AWSClient.Region), Type: types.FilterTypeTermMatch, }, - { - Field: aws.String("productFamily"), - Value: aws.String("Storage"), - Type: types.FilterTypeTermMatch, - }, }, }) if err != nil { @@ -74,10 +69,6 @@ func Buckets(ctx context.Context, client ProviderClient) ([]Resource, error) { Name: aws.String("BucketName"), Value: bucket.Name, }, - { - Name: aws.String("StorageType"), - Value: aws.String("StandardStorage"), - }, }, Unit: cloudwatchTypes.StandardUnitBytes, Period: aws.Int32(3600), @@ -94,15 +85,6 @@ func Buckets(ctx context.Context, client ProviderClient) ([]Resource, error) { } sizeInTB := ConvertBytesToTerabytes(int64(bucketSize)) - storageCostPerGB := 0.0 - - if sizeInTB <= 50 { - storageCostPerGB = (sizeInTB * 1024) * 0.023 - } else if sizeInTB <= 450 { - storageCostPerGB = (sizeInTB * 1024) * 0.022 - } else { - storageCostPerGB = (sizeInTB * 1024) * 0.021 - } // metrics for bucket usage @@ -116,10 +98,6 @@ func Buckets(ctx context.Context, client ProviderClient) ([]Resource, error) { Name: aws.String("BucketName"), Value: bucket.Name, }, - { - Name: aws.String("StorageType"), - Value: aws.String("StandardStorage"), - }, }, Unit: cloudwatchTypes.StandardUnitCount, Period: aws.Int32(3600), @@ -139,8 +117,9 @@ func Buckets(ctx context.Context, client ProviderClient) ([]Resource, error) { monthlyCost := 0.0 + storageCharges := awsUtils.GetCost(priceMap["AWS-S3-Storage"], sizeInTB*1024) // charges per GB requestCharges := awsUtils.GetCost(priceMap["AWS-S3-Requests"], requestCount/1000) // charges per 1000 request - monthlyCost = storageCostPerGB + requestCharges + monthlyCost = storageCharges + requestCharges tagsResp, err := s3Client.GetBucketTagging(context.Background(), &s3.GetBucketTaggingInput{ Bucket: bucket.Name, }) From 66b6a6c180060397f66c1f6334a279de1f0e6b88 Mon Sep 17 00:00:00 2001 From: bishal7679 Date: Mon, 9 Oct 2023 23:13:41 +0530 Subject: [PATCH 3/4] removed unneccesary comments Signed-off-by: bishal7679 --- providers/aws/s3/buckets.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/providers/aws/s3/buckets.go b/providers/aws/s3/buckets.go index 8d5030d56..cc3678020 100644 --- a/providers/aws/s3/buckets.go +++ b/providers/aws/s3/buckets.go @@ -34,7 +34,7 @@ func Buckets(ctx context.Context, client ProviderClient) ([]Resource, error) { ServiceCode: aws.String("AmazonS3"), Filters: []types.Filter{ { - Field: aws.String("regionCode"), // Filter by region + Field: aws.String("regionCode"), Value: aws.String(client.AWSClient.Region), Type: types.FilterTypeTermMatch, }, @@ -51,14 +51,12 @@ func Buckets(ctx context.Context, client ProviderClient) ([]Resource, error) { return resources, err } - // --------------------------------------------------------------- output, err := s3Client.ListBuckets(context.Background(), &config) if err != nil { return resources, err } for _, bucket := range output.Buckets { - // metrics for bucket size metricsBucketSizebytesOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), EndTime: aws.Time(time.Now()), @@ -86,8 +84,6 @@ func Buckets(ctx context.Context, client ProviderClient) ([]Resource, error) { sizeInTB := ConvertBytesToTerabytes(int64(bucketSize)) - // metrics for bucket usage - metricsUsageOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), EndTime: aws.Time(time.Now()), @@ -113,12 +109,11 @@ func Buckets(ctx context.Context, client ProviderClient) ([]Resource, error) { if metricsUsageOutput != nil && len(metricsUsageOutput.Datapoints) > 0 { requestCount = *metricsUsageOutput.Datapoints[0].Sum } - // requestCost := (requestCount / 1000) * 0.0004 monthlyCost := 0.0 - storageCharges := awsUtils.GetCost(priceMap["AWS-S3-Storage"], sizeInTB*1024) // charges per GB - requestCharges := awsUtils.GetCost(priceMap["AWS-S3-Requests"], requestCount/1000) // charges per 1000 request + storageCharges := awsUtils.GetCost(priceMap["AWS-S3-Storage"], sizeInTB*1024) + requestCharges := awsUtils.GetCost(priceMap["AWS-S3-Requests"], requestCount/1000) monthlyCost = storageCharges + requestCharges tagsResp, err := s3Client.GetBucketTagging(context.Background(), &s3.GetBucketTaggingInput{ Bucket: bucket.Name, From 334e8049da55d4192b06c7756382a2c3f0b349d4 Mon Sep 17 00:00:00 2001 From: bishal7679 Date: Mon, 9 Oct 2023 23:15:22 +0530 Subject: [PATCH 4/4] added key for pricemap Signed-off-by: bishal7679 --- providers/aws/s3/buckets.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/aws/s3/buckets.go b/providers/aws/s3/buckets.go index cc3678020..de60e6ed6 100644 --- a/providers/aws/s3/buckets.go +++ b/providers/aws/s3/buckets.go @@ -113,7 +113,7 @@ func Buckets(ctx context.Context, client ProviderClient) ([]Resource, error) { monthlyCost := 0.0 storageCharges := awsUtils.GetCost(priceMap["AWS-S3-Storage"], sizeInTB*1024) - requestCharges := awsUtils.GetCost(priceMap["AWS-S3-Requests"], requestCount/1000) + requestCharges := awsUtils.GetCost(priceMap["S3-API-Tier1"], requestCount/1000) monthlyCost = storageCharges + requestCharges tagsResp, err := s3Client.GetBucketTagging(context.Background(), &s3.GetBucketTaggingInput{ Bucket: bucket.Name,