From 265b20969e722c64c6761b960bca72223f7b2af1 Mon Sep 17 00:00:00 2001 From: bishal7679 Date: Mon, 23 Oct 2023 22:40:14 +0530 Subject: [PATCH 1/6] fixed cost of cloudfront by distribution and function Signed-off-by: bishal7679 --- docs/configuration/cloud-providers/aws.mdx | 1 + policy.json | 1 + providers/aws/aws.go | 1 + providers/aws/cloudfront/distributions.go | 61 ++++++-- providers/aws/cloudfront/functions.go | 161 +++++++++++++++++++++ 5 files changed, 214 insertions(+), 11 deletions(-) create mode 100644 providers/aws/cloudfront/functions.go diff --git a/docs/configuration/cloud-providers/aws.mdx b/docs/configuration/cloud-providers/aws.mdx index fc1a827b9..55c6411eb 100644 --- a/docs/configuration/cloud-providers/aws.mdx +++ b/docs/configuration/cloud-providers/aws.mdx @@ -9,6 +9,7 @@ sidebar_label: Amazon Web Services - API Gateway - Access control lists - CloudFront distributions + - CloudFront functions - CloudWatch Dashboards - CloudWatch alarms - CloudWatch metrics diff --git a/policy.json b/policy.json index cd321e529..b5f6d1852 100644 --- a/policy.json +++ b/policy.json @@ -8,6 +8,7 @@ "apigateway:GET", "cloudwatch:GetMetricStatistics", "cloudfront:ListDistributions", + "cloudfront:Functions", "cloudfront:ListTagsForResource", "cloudwatch:DescribeAlarms", "cloudwatch:ListTagsForResource", diff --git a/providers/aws/aws.go b/providers/aws/aws.go index 045db0b86..aa0ba4338 100644 --- a/providers/aws/aws.go +++ b/providers/aws/aws.go @@ -58,6 +58,7 @@ func listOfSupportedServices() []providers.FetchDataFunction { ec2.Instances, eks.KubernetesClusters, cloudfront.Distributions, + cloudfront.Functions, dynamodb.Tables, ecs.Clusters, ecs.TaskDefinitions, diff --git a/providers/aws/cloudfront/distributions.go b/providers/aws/cloudfront/distributions.go index 92c56ce38..782cfe714 100644 --- a/providers/aws/cloudfront/distributions.go +++ b/providers/aws/cloudfront/distributions.go @@ -11,11 +11,19 @@ import ( "github.com/aws/aws-sdk-go-v2/service/cloudfront" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" + "github.com/aws/aws-sdk-go-v2/service/pricing" + pricingTypes "github.com/aws/aws-sdk-go-v2/service/pricing/types" . "github.com/tailwarden/komiser/models" . "github.com/tailwarden/komiser/providers" + awsUtils "github.com/tailwarden/komiser/providers/aws/utils" "github.com/tailwarden/komiser/utils" ) +const ( + freeTierRequests = 10000000 + freeTierUpload = 1099511627776 +) + func Distributions(ctx context.Context, client ProviderClient) ([]Resource, error) { resources := make([]Resource, 0) var config cloudfront.ListDistributionsInput @@ -25,6 +33,34 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro client.AWSClient.Region = "us-east-1" cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient) client.AWSClient.Region = tempRegion + pricingClient := pricing.NewFromConfig(*client.AWSClient) + + pricingOutput, err := pricingClient.GetProducts(ctx, &pricing.GetProductsInput{ + ServiceCode: aws.String("AmazonCloudFront"), + Filters: []pricingTypes.Filter{ + { + Field: aws.String("regionCode"), + Value: aws.String(client.AWSClient.Region), + Type: pricingTypes.FilterTypeTermMatch, + }, + }, + }) + if err != nil { + log.Errorf("ERROR: Couldn't fetch pricing info for AWS CloudFront: %v", err) + return resources, err + } + + priceMap, err := awsUtils.GetPriceMap(pricingOutput, "group") + if err != nil { + log.Errorf("ERROR: Failed to calculate cost per month: %v", err) + return resources, err + } + + priceMapForRequest, err := awsUtils.GetPriceMap(pricingOutput, "requestType") + if err != nil { + log.Errorf("ERROR: Failed to calculate cost per month: %v", err) + return resources, err + } for { output, err := cloudfrontClient.ListDistributions(ctx, &config) @@ -39,7 +75,7 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro MetricName: aws.String("BytesDownloaded"), Namespace: aws.String("AWS/CloudFront"), Dimensions: []types.Dimension{ - types.Dimension{ + { Name: aws.String("DistributionId"), Value: distribution.Id, }, @@ -65,7 +101,7 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro MetricName: aws.String("BytesUploaded"), Namespace: aws.String("AWS/CloudFront"), Dimensions: []types.Dimension{ - types.Dimension{ + { Name: aws.String("DistributionId"), Value: distribution.Id, }, @@ -84,6 +120,9 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro if metricsBytesUploadedOutput != nil && len(metricsBytesUploadedOutput.Datapoints) > 0 { bytesUploaded = *metricsBytesUploadedOutput.Datapoints[0].Sum } + if bytesUploaded > freeTierUpload { + bytesUploaded -= freeTierUpload + } metricsRequestsOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), @@ -91,7 +130,7 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro MetricName: aws.String("Requests"), Namespace: aws.String("AWS/CloudFront"), Dimensions: []types.Dimension{ - types.Dimension{ + { Name: aws.String("DistributionId"), Value: distribution.Id, }, @@ -110,17 +149,17 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro if metricsRequestsOutput != nil && len(metricsRequestsOutput.Datapoints) > 0 { requests = *metricsRequestsOutput.Datapoints[0].Sum } + if requests > freeTierRequests { + requests -= freeTierRequests + } - // calculate region data transfer out to internet - dataTransferToInternet := (bytesUploaded / 1000000000) * 0.085 + dataTransferToInternetCost := awsUtils.GetCost(priceMap["AWS-CloudFront-DataTransfer-In-Bytes"], (float64(bytesUploaded) / 1099511627776)*1024) - // calculate region data transfer out to origin - dataTransferToOrigin := (bytesDownloaded / 1000000000) * 0.02 + dataTransferToOriginCost := awsUtils.GetCost(priceMap["AWS-CloudFront-DataTransfer-Out-Bytes"], (float64(bytesDownloaded) / 1099511627776)*1024) - // calculate requests cost - requestsCost := requests * 0.000001 + requestsCost := awsUtils.GetCost(priceMapForRequest["CloudFront-Request-Origin-Shield"], requests/10000) - monthlyCost := dataTransferToInternet + dataTransferToOrigin + requestsCost + monthlyCost := dataTransferToInternetCost + dataTransferToOriginCost + requestsCost outputTags, err := cloudfrontClient.ListTagsForResource(ctx, &cloudfront.ListTagsForResourceInput{ Resource: distribution.ARN, @@ -164,4 +203,4 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro "resources": len(resources), }).Info("Fetched resources") return resources, nil -} +} \ No newline at end of file diff --git a/providers/aws/cloudfront/functions.go b/providers/aws/cloudfront/functions.go new file mode 100644 index 000000000..6e6a57f6a --- /dev/null +++ b/providers/aws/cloudfront/functions.go @@ -0,0 +1,161 @@ +package cloudfront + +import ( + "context" + "fmt" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/cloudfront" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" + "github.com/aws/aws-sdk-go-v2/service/pricing" + pricingTypes "github.com/aws/aws-sdk-go-v2/service/pricing/types" + . "github.com/tailwarden/komiser/models" + . "github.com/tailwarden/komiser/providers" + awsUtils "github.com/tailwarden/komiser/providers/aws/utils" + "github.com/tailwarden/komiser/utils" +) + +func Functions(ctx context.Context, client ProviderClient) ([]Resource, error) { + resources := make([]Resource, 0) + var config cloudfront.ListFunctionsInput + cloudfrontClient := cloudfront.NewFromConfig(*client.AWSClient) + + tempRegion := client.AWSClient.Region + client.AWSClient.Region = "us-east-1" + cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient) + client.AWSClient.Region = tempRegion + pricingClient := pricing.NewFromConfig(*client.AWSClient) + + pricingOutput, err := pricingClient.GetProducts(ctx, &pricing.GetProductsInput{ + ServiceCode: aws.String("AmazonCloudFront"), + Filters: []pricingTypes.Filter{ + { + Field: aws.String("regionCode"), + Value: aws.String(client.AWSClient.Region), + Type: pricingTypes.FilterTypeTermMatch, + }, + }, + }) + if err != nil { + log.Errorf("ERROR: Couldn't fetch pricing info for AWS CloudFront: %v", err) + return resources, err + } + + priceMap, err := awsUtils.GetPriceMap(pricingOutput, "group") + if err != nil { + log.Errorf("ERROR: Failed to calculate cost per month: %v", err) + return resources, err + } + + for { + output, err := cloudfrontClient.ListFunctions(ctx, &config) + if err != nil { + return resources, err + } + + for _, function := range output.FunctionList.Items { + metricsLambdaEdgeDurationOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ + StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), + EndTime: aws.Time(time.Now()), + MetricName: aws.String("Duration"), + Namespace: aws.String("AWS/LambdaEdge"), + Dimensions: []types.Dimension{ + { + Name: aws.String("FunctionName"), + Value: function.Name, + }, + }, + Period: aws.Int32(86400), + Statistics: []types.Statistic{ + types.StatisticAverage, + }, + }) + + if err != nil { + log.Warnf("Couldn't fetch Lambda@Edge Duration metric for %s", *function.Name) + } + + lambdaEdgeDuration := 0.0 + if metricsLambdaEdgeDurationOutput != nil && len(metricsLambdaEdgeDurationOutput.Datapoints) > 0 { + lambdaEdgeDuration = *metricsLambdaEdgeDurationOutput.Datapoints[0].Average + } + + metricsLambdaEdgeRequestsOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ + StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), + EndTime: aws.Time(time.Now()), + MetricName: aws.String("Requests"), + Namespace: aws.String("AWS/LambdaEdge"), + Dimensions: []types.Dimension{ + { + Name: aws.String("FunctionName"), + Value: function.Name, + }, + }, + Period: aws.Int32(86400), + Statistics: []types.Statistic{ + types.StatisticSum, + }, + }) + + if err != nil { + log.Warnf("Couldn't fetch Lambda@Edge Requests metric for %s", *function.Name) + } + + lambdaEdgeRequests := 0.0 + if metricsLambdaEdgeRequestsOutput != nil && len(metricsLambdaEdgeRequestsOutput.Datapoints) > 0 { + lambdaEdgeRequests = *metricsLambdaEdgeRequestsOutput.Datapoints[0].Sum + } + + lambdaEdgeDurationCost := awsUtils.GetCost(priceMap["AWS-Lambda-Edge-Duration"], lambdaEdgeDuration) + + lambdaEdgeRequestsCost := awsUtils.GetCost(priceMap["AWS-Lambda-Edge-Requests"], lambdaEdgeRequests/10000000) + + monthlyCost := lambdaEdgeDurationCost + lambdaEdgeRequestsCost + + outputTags, err := cloudfrontClient.ListTagsForResource(ctx, &cloudfront.ListTagsForResourceInput{ + Resource: function.FunctionMetadata.FunctionARN, + }) + + tags := make([]Tag, 0) + + if err == nil { + for _, tag := range outputTags.Tags.Items { + tags = append(tags, Tag{ + Key: *tag.Key, + Value: *tag.Value, + }) + } + } + + resources = append(resources, Resource{ + Provider: "AWS", + Account: client.Name, + Service: "CloudFront", + ResourceId: *function.FunctionMetadata.FunctionARN, + Region: client.AWSClient.Region, + Name: *function.Name, + Cost: monthlyCost, + Tags: tags, + FetchedAt: time.Now(), + Link: fmt.Sprintf("https://%s.console.aws.amazon.com/cloudfront/v3/home?region=%s#/distributions/%s", client.AWSClient.Region, client.AWSClient.Region, *function.Name), + }) + } + + if aws.ToString(output.FunctionList.NextMarker) == "" { + break + } + config.Marker = output.FunctionList.NextMarker + } + log.WithFields(log.Fields{ + "provider": "AWS", + "account": client.Name, + "region": client.AWSClient.Region, + "service": "CloudFront", + "resources": len(resources), + }).Info("Fetched resources") + return resources, nil +} \ No newline at end of file From ad9362d2958192501cbac66bb57ec3bb14868849 Mon Sep 17 00:00:00 2001 From: bishal7679 Date: Tue, 24 Oct 2023 19:34:46 +0530 Subject: [PATCH 2/6] modified pricemap key Signed-off-by: bishal7679 --- providers/aws/cloudfront/distributions.go | 52 +++-------------------- providers/aws/cloudfront/functions.go | 7 +-- 2 files changed, 10 insertions(+), 49 deletions(-) diff --git a/providers/aws/cloudfront/distributions.go b/providers/aws/cloudfront/distributions.go index 782cfe714..18c86a606 100644 --- a/providers/aws/cloudfront/distributions.go +++ b/providers/aws/cloudfront/distributions.go @@ -12,7 +12,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" "github.com/aws/aws-sdk-go-v2/service/pricing" - pricingTypes "github.com/aws/aws-sdk-go-v2/service/pricing/types" + // pricingTypes "github.com/aws/aws-sdk-go-v2/service/pricing/types" . "github.com/tailwarden/komiser/models" . "github.com/tailwarden/komiser/providers" awsUtils "github.com/tailwarden/komiser/providers/aws/utils" @@ -22,6 +22,7 @@ import ( const ( freeTierRequests = 10000000 freeTierUpload = 1099511627776 + per10kRequest = 10000 ) func Distributions(ctx context.Context, client ProviderClient) ([]Resource, error) { @@ -37,29 +38,19 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro pricingOutput, err := pricingClient.GetProducts(ctx, &pricing.GetProductsInput{ ServiceCode: aws.String("AmazonCloudFront"), - Filters: []pricingTypes.Filter{ - { - Field: aws.String("regionCode"), - Value: aws.String(client.AWSClient.Region), - Type: pricingTypes.FilterTypeTermMatch, - }, - }, }) if err != nil { log.Errorf("ERROR: Couldn't fetch pricing info for AWS CloudFront: %v", err) - return resources, err } - priceMap, err := awsUtils.GetPriceMap(pricingOutput, "group") + priceMapForDataTransfer, err := awsUtils.GetPriceMap(pricingOutput, "transferType") if err != nil { log.Errorf("ERROR: Failed to calculate cost per month: %v", err) - return resources, err } priceMapForRequest, err := awsUtils.GetPriceMap(pricingOutput, "requestType") if err != nil { log.Errorf("ERROR: Failed to calculate cost per month: %v", err) - return resources, err } for { @@ -95,35 +86,6 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro bytesDownloaded = *metricsBytesDownloadedOutput.Datapoints[0].Sum } - metricsBytesUploadedOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ - StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), - EndTime: aws.Time(time.Now()), - MetricName: aws.String("BytesUploaded"), - Namespace: aws.String("AWS/CloudFront"), - Dimensions: []types.Dimension{ - { - Name: aws.String("DistributionId"), - Value: distribution.Id, - }, - }, - Period: aws.Int32(86400), - Statistics: []types.Statistic{ - types.StatisticSum, - }, - }) - - if err != nil { - log.Warnf("Couldn't fetch invocations metric for %s", *distribution.Id) - } - - bytesUploaded := 0.0 - if metricsBytesUploadedOutput != nil && len(metricsBytesUploadedOutput.Datapoints) > 0 { - bytesUploaded = *metricsBytesUploadedOutput.Datapoints[0].Sum - } - if bytesUploaded > freeTierUpload { - bytesUploaded -= freeTierUpload - } - metricsRequestsOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), EndTime: aws.Time(time.Now()), @@ -153,13 +115,11 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro requests -= freeTierRequests } - dataTransferToInternetCost := awsUtils.GetCost(priceMap["AWS-CloudFront-DataTransfer-In-Bytes"], (float64(bytesUploaded) / 1099511627776)*1024) - - dataTransferToOriginCost := awsUtils.GetCost(priceMap["AWS-CloudFront-DataTransfer-Out-Bytes"], (float64(bytesDownloaded) / 1099511627776)*1024) + dataTransferToOriginCost := awsUtils.GetCost(priceMapForDataTransfer["CloudFront to Origin"], (float64(bytesDownloaded) / 1099511627776)*1024) - requestsCost := awsUtils.GetCost(priceMapForRequest["CloudFront-Request-Origin-Shield"], requests/10000) + requestsCost := awsUtils.GetCost(priceMapForRequest["CloudFront-Request-HTTP-Proxy"], requests/per10kRequest) - monthlyCost := dataTransferToInternetCost + dataTransferToOriginCost + requestsCost + monthlyCost := dataTransferToOriginCost + requestsCost outputTags, err := cloudfrontClient.ListTagsForResource(ctx, &cloudfront.ListTagsForResourceInput{ Resource: distribution.ARN, diff --git a/providers/aws/cloudfront/functions.go b/providers/aws/cloudfront/functions.go index 6e6a57f6a..371088c1f 100644 --- a/providers/aws/cloudfront/functions.go +++ b/providers/aws/cloudfront/functions.go @@ -19,6 +19,9 @@ import ( "github.com/tailwarden/komiser/utils" ) +const ( + perOneMillonRequest = 1000000 +) func Functions(ctx context.Context, client ProviderClient) ([]Resource, error) { resources := make([]Resource, 0) var config cloudfront.ListFunctionsInput @@ -42,13 +45,11 @@ func Functions(ctx context.Context, client ProviderClient) ([]Resource, error) { }) if err != nil { log.Errorf("ERROR: Couldn't fetch pricing info for AWS CloudFront: %v", err) - return resources, err } priceMap, err := awsUtils.GetPriceMap(pricingOutput, "group") if err != nil { log.Errorf("ERROR: Failed to calculate cost per month: %v", err) - return resources, err } for { @@ -112,7 +113,7 @@ func Functions(ctx context.Context, client ProviderClient) ([]Resource, error) { lambdaEdgeDurationCost := awsUtils.GetCost(priceMap["AWS-Lambda-Edge-Duration"], lambdaEdgeDuration) - lambdaEdgeRequestsCost := awsUtils.GetCost(priceMap["AWS-Lambda-Edge-Requests"], lambdaEdgeRequests/10000000) + lambdaEdgeRequestsCost := awsUtils.GetCost(priceMap["AWS-Lambda-Edge-Requests"], lambdaEdgeRequests/perOneMillonRequest) monthlyCost := lambdaEdgeDurationCost + lambdaEdgeRequestsCost From fe0a48e16d462a9e3c62e775fd63d7d4aeac21c9 Mon Sep 17 00:00:00 2001 From: bishal7679 Date: Sun, 29 Oct 2023 23:18:32 +0530 Subject: [PATCH 3/6] updated distribution with location function Signed-off-by: bishal7679 --- providers/aws/aws.go | 2 +- providers/aws/cloudfront/distributions.go | 26 ++++++++++++++++--- .../{functions.go => lambdaedge.go} | 6 ++--- 3 files changed, 27 insertions(+), 7 deletions(-) rename providers/aws/cloudfront/{functions.go => lambdaedge.go} (96%) diff --git a/providers/aws/aws.go b/providers/aws/aws.go index aa0ba4338..327054732 100644 --- a/providers/aws/aws.go +++ b/providers/aws/aws.go @@ -58,7 +58,7 @@ func listOfSupportedServices() []providers.FetchDataFunction { ec2.Instances, eks.KubernetesClusters, cloudfront.Distributions, - cloudfront.Functions, + cloudfront.LambdaEdgeFunctions, dynamodb.Tables, ecs.Clusters, ecs.TaskDefinitions, diff --git a/providers/aws/cloudfront/distributions.go b/providers/aws/cloudfront/distributions.go index 18c86a606..019daa411 100644 --- a/providers/aws/cloudfront/distributions.go +++ b/providers/aws/cloudfront/distributions.go @@ -12,6 +12,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" "github.com/aws/aws-sdk-go-v2/service/pricing" + // pricingTypes "github.com/aws/aws-sdk-go-v2/service/pricing/types" . "github.com/tailwarden/komiser/models" . "github.com/tailwarden/komiser/providers" @@ -22,7 +23,7 @@ import ( const ( freeTierRequests = 10000000 freeTierUpload = 1099511627776 - per10kRequest = 10000 + per10kRequest = 10000 ) func Distributions(ctx context.Context, client ProviderClient) ([]Resource, error) { @@ -115,7 +116,7 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro requests -= freeTierRequests } - dataTransferToOriginCost := awsUtils.GetCost(priceMapForDataTransfer["CloudFront to Origin"], (float64(bytesDownloaded) / 1099511627776)*1024) + dataTransferToOriginCost := awsUtils.GetCost(priceMapForDataTransfer["CloudFront to Origin"], (float64(bytesDownloaded)/1099511627776)*1024) requestsCost := awsUtils.GetCost(priceMapForRequest["CloudFront-Request-HTTP-Proxy"], requests/per10kRequest) @@ -163,4 +164,23 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro "resources": len(resources), }).Info("Fetched resources") return resources, nil -} \ No newline at end of file +} + +func getCloudFrontDistributionLocation(distributionID string,cloudfrontClient *cloudfront.Client) (string, error) { + + + // Specify the CloudFront distribution ID for which you want to fetch the regional location + input := &cloudfront.GetDistributionConfigInput{ + Id: &distributionID, + } + + // Fetch the distribution configuration + resp, err := cloudfrontClient.GetDistributionConfig(context.TODO(), input) + if err != nil { + return "", err + } + + // Extract the regional location from the distribution configuration + location := resp.DistributionConfig.Aliases.Items[0] + return location, nil +} diff --git a/providers/aws/cloudfront/functions.go b/providers/aws/cloudfront/lambdaedge.go similarity index 96% rename from providers/aws/cloudfront/functions.go rename to providers/aws/cloudfront/lambdaedge.go index 371088c1f..694f45403 100644 --- a/providers/aws/cloudfront/functions.go +++ b/providers/aws/cloudfront/lambdaedge.go @@ -22,7 +22,7 @@ import ( const ( perOneMillonRequest = 1000000 ) -func Functions(ctx context.Context, client ProviderClient) ([]Resource, error) { +func LambdaEdgeFunctions(ctx context.Context, client ProviderClient) ([]Resource, error) { resources := make([]Resource, 0) var config cloudfront.ListFunctionsInput cloudfrontClient := cloudfront.NewFromConfig(*client.AWSClient) @@ -63,7 +63,7 @@ func Functions(ctx context.Context, client ProviderClient) ([]Resource, error) { StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), EndTime: aws.Time(time.Now()), MetricName: aws.String("Duration"), - Namespace: aws.String("AWS/LambdaEdge"), + Namespace: aws.String("AWS/CloudFront"), Dimensions: []types.Dimension{ { Name: aws.String("FunctionName"), @@ -89,7 +89,7 @@ func Functions(ctx context.Context, client ProviderClient) ([]Resource, error) { StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), EndTime: aws.Time(time.Now()), MetricName: aws.String("Requests"), - Namespace: aws.String("AWS/LambdaEdge"), + Namespace: aws.String("AWS/CloudFront"), Dimensions: []types.Dimension{ { Name: aws.String("FunctionName"), From 4ddcb38f28f8ad3cf76ff627893a1237736e4f10 Mon Sep 17 00:00:00 2001 From: bishal7679 Date: Mon, 30 Oct 2023 23:18:08 +0530 Subject: [PATCH 4/6] updated distribution with location based cost Signed-off-by: bishal7679 --- docs/configuration/cloud-providers/aws.mdx | 1 - policy.json | 1 - providers/aws/cloudfront/distributions.go | 277 ++++++++++++--------- providers/aws/utils/utils.go | 12 +- 4 files changed, 170 insertions(+), 121 deletions(-) diff --git a/docs/configuration/cloud-providers/aws.mdx b/docs/configuration/cloud-providers/aws.mdx index 55c6411eb..fc1a827b9 100644 --- a/docs/configuration/cloud-providers/aws.mdx +++ b/docs/configuration/cloud-providers/aws.mdx @@ -9,7 +9,6 @@ sidebar_label: Amazon Web Services - API Gateway - Access control lists - CloudFront distributions - - CloudFront functions - CloudWatch Dashboards - CloudWatch alarms - CloudWatch metrics diff --git a/policy.json b/policy.json index b5f6d1852..cd321e529 100644 --- a/policy.json +++ b/policy.json @@ -8,7 +8,6 @@ "apigateway:GET", "cloudwatch:GetMetricStatistics", "cloudfront:ListDistributions", - "cloudfront:Functions", "cloudfront:ListTagsForResource", "cloudwatch:DescribeAlarms", "cloudwatch:ListTagsForResource", diff --git a/providers/aws/cloudfront/distributions.go b/providers/aws/cloudfront/distributions.go index 019daa411..c66140d95 100644 --- a/providers/aws/cloudfront/distributions.go +++ b/providers/aws/cloudfront/distributions.go @@ -2,6 +2,7 @@ package cloudfront import ( "context" + "encoding/json" "fmt" "time" @@ -26,15 +27,12 @@ const ( per10kRequest = 10000 ) +var EdgeLocation string + func Distributions(ctx context.Context, client ProviderClient) ([]Resource, error) { resources := make([]Resource, 0) var config cloudfront.ListDistributionsInput cloudfrontClient := cloudfront.NewFromConfig(*client.AWSClient) - - tempRegion := client.AWSClient.Region - client.AWSClient.Region = "us-east-1" - cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient) - client.AWSClient.Region = tempRegion pricingClient := pricing.NewFromConfig(*client.AWSClient) pricingOutput, err := pricingClient.GetProducts(ctx, &pricing.GetProductsInput{ @@ -44,143 +42,192 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro log.Errorf("ERROR: Couldn't fetch pricing info for AWS CloudFront: %v", err) } - priceMapForDataTransfer, err := awsUtils.GetPriceMap(pricingOutput, "transferType") + priceMapForDataTransfer, err := GetPriceMapCF(pricingOutput, "fromLocation") if err != nil { log.Errorf("ERROR: Failed to calculate cost per month: %v", err) } - priceMapForRequest, err := awsUtils.GetPriceMap(pricingOutput, "requestType") + priceMapForRequest, err := GetPriceMapCF(pricingOutput, "location") if err != nil { log.Errorf("ERROR: Failed to calculate cost per month: %v", err) } for { - output, err := cloudfrontClient.ListDistributions(ctx, &config) - if err != nil { - return resources, err - } - - for _, distribution := range output.DistributionList.Items { - metricsBytesDownloadedOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ - StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), - EndTime: aws.Time(time.Now()), - MetricName: aws.String("BytesDownloaded"), - Namespace: aws.String("AWS/CloudFront"), - Dimensions: []types.Dimension{ - { - Name: aws.String("DistributionId"), - Value: distribution.Id, - }, - }, - Period: aws.Int32(86400), - Statistics: []types.Statistic{ - types.StatisticSum, - }, - }) - - if err != nil { - log.Warnf("Couldn't fetch invocations metric for %s", *distribution.Id) - } - - bytesDownloaded := 0.0 - if metricsBytesDownloadedOutput != nil && len(metricsBytesDownloadedOutput.Datapoints) > 0 { - bytesDownloaded = *metricsBytesDownloadedOutput.Datapoints[0].Sum - } + getRegions := getRegionMapping() + for region, locations := range getRegions { + client.AWSClient.Region = region + for _, edgelocation := range locations { + output, err := cloudfrontClient.ListDistributions(ctx, &config) + if err != nil { + return resources, err + } - metricsRequestsOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ - StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), - EndTime: aws.Time(time.Now()), - MetricName: aws.String("Requests"), - Namespace: aws.String("AWS/CloudFront"), - Dimensions: []types.Dimension{ - { - Name: aws.String("DistributionId"), - Value: distribution.Id, - }, - }, - Period: aws.Int32(86400), - Statistics: []types.Statistic{ - types.StatisticSum, - }, - }) + cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient) + + for _, distribution := range output.DistributionList.Items { + metricsBytesDownloadedOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ + StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), + EndTime: aws.Time(time.Now()), + MetricName: aws.String("BytesDownloaded"), + Namespace: aws.String("AWS/CloudFront"), + Dimensions: []types.Dimension{ + { + Name: aws.String("DistributionId"), + Value: distribution.Id, + }, + }, + Period: aws.Int32(86400), + Statistics: []types.Statistic{ + types.StatisticSum, + }, + }) - if err != nil { - log.Warnf("Couldn't fetch invocations metric for %s", *distribution.Id) - } + if err != nil { + log.Warnf("Couldn't fetch invocations metric for %s", *distribution.Id) + } + + bytesDownloaded := 0.0 + if metricsBytesDownloadedOutput != nil && len(metricsBytesDownloadedOutput.Datapoints) > 0 { + bytesDownloaded = *metricsBytesDownloadedOutput.Datapoints[0].Sum + } + + metricsRequestsOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ + StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), + EndTime: aws.Time(time.Now()), + MetricName: aws.String("Requests"), + Namespace: aws.String("AWS/CloudFront"), + Dimensions: []types.Dimension{ + { + Name: aws.String("DistributionId"), + Value: distribution.Id, + }, + }, + Period: aws.Int32(86400), + Statistics: []types.Statistic{ + types.StatisticSum, + }, + }) - requests := 0.0 - if metricsRequestsOutput != nil && len(metricsRequestsOutput.Datapoints) > 0 { - requests = *metricsRequestsOutput.Datapoints[0].Sum - } - if requests > freeTierRequests { - requests -= freeTierRequests - } + if err != nil { + log.Warnf("Couldn't fetch invocations metric for %s", *distribution.Id) + } - dataTransferToOriginCost := awsUtils.GetCost(priceMapForDataTransfer["CloudFront to Origin"], (float64(bytesDownloaded)/1099511627776)*1024) + requests := 0.0 + if metricsRequestsOutput != nil && len(metricsRequestsOutput.Datapoints) > 0 { + requests = *metricsRequestsOutput.Datapoints[0].Sum + } + if requests > freeTierRequests { + requests -= freeTierRequests + } - requestsCost := awsUtils.GetCost(priceMapForRequest["CloudFront-Request-HTTP-Proxy"], requests/per10kRequest) + if priceMapForDataTransfer[edgelocation] != nil { + EdgeLocation = edgelocation + break + } + dataTransferToOriginCost := awsUtils.GetCost(priceMapForDataTransfer[EdgeLocation], (float64(bytesDownloaded)/1099511627776)*1024) - monthlyCost := dataTransferToOriginCost + requestsCost + requestsCost := awsUtils.GetCost(priceMapForRequest[EdgeLocation], requests/per10kRequest) - outputTags, err := cloudfrontClient.ListTagsForResource(ctx, &cloudfront.ListTagsForResourceInput{ - Resource: distribution.ARN, - }) + monthlyCost := dataTransferToOriginCost + requestsCost - tags := make([]Tag, 0) + outputTags, err := cloudfrontClient.ListTagsForResource(ctx, &cloudfront.ListTagsForResourceInput{ + Resource: distribution.ARN, + }) - if err == nil { - for _, tag := range outputTags.Tags.Items { - tags = append(tags, Tag{ - Key: *tag.Key, - Value: *tag.Value, + tags := make([]Tag, 0) + + if err == nil { + for _, tag := range outputTags.Tags.Items { + tags = append(tags, Tag{ + Key: *tag.Key, + Value: *tag.Value, + }) + } + } + + resources = append(resources, Resource{ + Provider: "AWS", + Account: client.Name, + Service: "CloudFront", + ResourceId: *distribution.ARN, + Region: client.AWSClient.Region, + Name: *distribution.DomainName, + Cost: monthlyCost, + Tags: tags, + FetchedAt: time.Now(), + Link: fmt.Sprintf("https://%s.console.aws.amazon.com/cloudfront/v3/home?region=%s#/distributions/%s", client.AWSClient.Region, client.AWSClient.Region, *distribution.Id), }) } - } - resources = append(resources, Resource{ - Provider: "AWS", - Account: client.Name, - Service: "CloudFront", - ResourceId: *distribution.ARN, - Region: client.AWSClient.Region, - Name: *distribution.DomainName, - Cost: monthlyCost, - Tags: tags, - FetchedAt: time.Now(), - Link: fmt.Sprintf("https://%s.console.aws.amazon.com/cloudfront/v3/home?region=%s#/distributions/%s", client.AWSClient.Region, client.AWSClient.Region, *distribution.Id), - }) + if aws.ToString(output.DistributionList.NextMarker) == "" { + break + } + config.Marker = output.DistributionList.Marker + } + log.WithFields(log.Fields{ + "provider": "AWS", + "account": client.Name, + "region": client.AWSClient.Region, + "service": "CloudFront", + "resources": len(resources), + }).Info("Fetched resources") + return resources, nil } - if aws.ToString(output.DistributionList.NextMarker) == "" { - break - } - config.Marker = output.DistributionList.Marker } - log.WithFields(log.Fields{ - "provider": "AWS", - "account": client.Name, - "region": client.AWSClient.Region, - "service": "CloudFront", - "resources": len(resources), - }).Info("Fetched resources") - return resources, nil } -func getCloudFrontDistributionLocation(distributionID string,cloudfrontClient *cloudfront.Client) (string, error) { - +func getRegionMapping() map[string][]string { + return map[string][]string{ + "us-east-1": {"United States", "Mexico", "Canada"}, + "eu-west-1": {"Europe", "Israel"}, + "ap-northeast-1": {"Australia", "New Zealand", "Taiwan"}, + "ap-northeast-2": {"South Korea"}, + "ap-southeast-1": {"Philippines", "Singapore", "Thailand", "Malaysia"}, + "ap-southeast-3": {"Indonesia"}, + "ap-south-1": {"India"}, + "sa-east-1": {"Japan", "South America"}, + "me-south-1": {"South Africa", "Kenya", "Middle East"}, + "ap-east-1": {"Hong Kong", "Vietnam"}, + "cn-north-1": {"China"}, + } +} + +// GetPriceMapCF is modified functions from awsUtils.GetPriceMap to get CF distribution unit price based on location +func GetPriceMapCF(pricingOutput *pricing.GetProductsOutput, field string) (map[string][]awsUtils.PriceDimensions, error) { + priceMap := make(map[string][]awsUtils.PriceDimensions) + + if pricingOutput != nil && len(pricingOutput.PriceList) > 0 { + for _, item := range pricingOutput.PriceList { + price := awsUtils.ProductEntry{} + err := json.Unmarshal([]byte(item), &price) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal JSON: %w", err) + } + + var key string + switch field { + case "fromLocation": + if price.Product.Attributes.TransferType == "CloudFront to Origin" { - // Specify the CloudFront distribution ID for which you want to fetch the regional location - input := &cloudfront.GetDistributionConfigInput{ - Id: &distributionID, - } + key = price.Product.Attributes.FromLocation + } + case "location": + if price.Product.Attributes.RequestType == "CloudFront-Request-HTTP-Proxy" { + key = price.Product.Attributes.RequestLocation + } + } - // Fetch the distribution configuration - resp, err := cloudfrontClient.GetDistributionConfig(context.TODO(), input) - if err != nil { - return "", err - } + unitPrices := []awsUtils.PriceDimensions{} + for _, pd := range price.Terms.OnDemand { + for _, p := range pd.PriceDimensions { + unitPrices = append(unitPrices, p) + } + } + + priceMap[key] = unitPrices + } + } - // Extract the regional location from the distribution configuration - location := resp.DistributionConfig.Aliases.Items[0] - return location, nil + return priceMap, nil } diff --git a/providers/aws/utils/utils.go b/providers/aws/utils/utils.go index 5ed566701..60f727069 100644 --- a/providers/aws/utils/utils.go +++ b/providers/aws/utils/utils.go @@ -15,8 +15,12 @@ type ProductEntry struct { Operation string `json:"operation"` GroupDescription string `json:"groupDescription"` RequestDescription string `json:"requestDescription"` + RequestType string `json:"requestType"` InstanceType string `json:"instanceType"` InstanceTypeFamily string `json:"instanceTypeFamily"` + TransferType string `json:"transferType"` + FromLocation string `json:"fromLocation"` + RequestLocation string `json:"location"` } `json:"attributes"` } `json:"product"` Terms struct { @@ -93,8 +97,8 @@ func GetPriceMap(pricingOutput *pricing.GetProductsOutput, field string) (map[st } func Int64PtrToFloat64(i *int64) float64 { - if i == nil { - return 0.0 // or any default value you prefer - } - return float64(*i) + if i == nil { + return 0.0 // or any default value you prefer + } + return float64(*i) } From 8fcfcc7a55fd704b91bca6b7ab8851a3e86cc814 Mon Sep 17 00:00:00 2001 From: bishal7679 Date: Tue, 31 Oct 2023 23:43:42 +0530 Subject: [PATCH 5/6] updated cloudfront distribution cost based on locations Signed-off-by: bishal7679 --- providers/aws/cloudfront/distributions.go | 248 +++++++++++----------- providers/aws/cloudfront/lambdaedge.go | 3 +- 2 files changed, 130 insertions(+), 121 deletions(-) diff --git a/providers/aws/cloudfront/distributions.go b/providers/aws/cloudfront/distributions.go index c66140d95..34d910703 100644 --- a/providers/aws/cloudfront/distributions.go +++ b/providers/aws/cloudfront/distributions.go @@ -33,7 +33,11 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro resources := make([]Resource, 0) var config cloudfront.ListDistributionsInput cloudfrontClient := cloudfront.NewFromConfig(*client.AWSClient) + tempRegion := client.AWSClient.Region + client.AWSClient.Region = "us-east-1" + cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient) pricingClient := pricing.NewFromConfig(*client.AWSClient) + client.AWSClient.Region = tempRegion pricingOutput, err := pricingClient.GetProducts(ctx, &pricing.GetProductsInput{ ServiceCode: aws.String("AmazonCloudFront"), @@ -52,144 +56,150 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro log.Errorf("ERROR: Failed to calculate cost per month: %v", err) } + getRegions := getRegionMapping() for { - getRegions := getRegionMapping() - for region, locations := range getRegions { - client.AWSClient.Region = region - for _, edgelocation := range locations { - output, err := cloudfrontClient.ListDistributions(ctx, &config) - if err != nil { - return resources, err + for region, edgelocation := range getRegions { + if client.AWSClient.Region == region { + log.Println("matched region--------------------------", region) + if priceMapForDataTransfer[edgelocation] != nil && priceMapForRequest[edgelocation] != nil { + EdgeLocation = edgelocation } - cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient) - - for _, distribution := range output.DistributionList.Items { - metricsBytesDownloadedOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ - StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), - EndTime: aws.Time(time.Now()), - MetricName: aws.String("BytesDownloaded"), - Namespace: aws.String("AWS/CloudFront"), - Dimensions: []types.Dimension{ - { - Name: aws.String("DistributionId"), - Value: distribution.Id, - }, - }, - Period: aws.Int32(86400), - Statistics: []types.Statistic{ - types.StatisticSum, - }, - }) + } - if err != nil { - log.Warnf("Couldn't fetch invocations metric for %s", *distribution.Id) - } - - bytesDownloaded := 0.0 - if metricsBytesDownloadedOutput != nil && len(metricsBytesDownloadedOutput.Datapoints) > 0 { - bytesDownloaded = *metricsBytesDownloadedOutput.Datapoints[0].Sum - } - - metricsRequestsOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ - StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), - EndTime: aws.Time(time.Now()), - MetricName: aws.String("Requests"), - Namespace: aws.String("AWS/CloudFront"), - Dimensions: []types.Dimension{ - { - Name: aws.String("DistributionId"), - Value: distribution.Id, - }, - }, - Period: aws.Int32(86400), - Statistics: []types.Statistic{ - types.StatisticSum, - }, - }) + } - if err != nil { - log.Warnf("Couldn't fetch invocations metric for %s", *distribution.Id) - } + output, err := cloudfrontClient.ListDistributions(ctx, &config) + if err != nil { + return resources, err + } - requests := 0.0 - if metricsRequestsOutput != nil && len(metricsRequestsOutput.Datapoints) > 0 { - requests = *metricsRequestsOutput.Datapoints[0].Sum - } - if requests > freeTierRequests { - requests -= freeTierRequests - } + for _, distribution := range output.DistributionList.Items { + metricsBytesDownloadedOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ + StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), + EndTime: aws.Time(time.Now()), + MetricName: aws.String("BytesDownloaded"), + Namespace: aws.String("AWS/CloudFront"), + Dimensions: []types.Dimension{ + { + Name: aws.String("DistributionId"), + Value: distribution.Id, + }, + }, + Period: aws.Int32(86400), + Statistics: []types.Statistic{ + types.StatisticSum, + }, + }) - if priceMapForDataTransfer[edgelocation] != nil { - EdgeLocation = edgelocation - break - } - dataTransferToOriginCost := awsUtils.GetCost(priceMapForDataTransfer[EdgeLocation], (float64(bytesDownloaded)/1099511627776)*1024) + if err != nil { + log.Warnf("Couldn't fetch invocations metric for %s", *distribution.Id) + } - requestsCost := awsUtils.GetCost(priceMapForRequest[EdgeLocation], requests/per10kRequest) + bytesDownloaded := 0.0 + if metricsBytesDownloadedOutput != nil && len(metricsBytesDownloadedOutput.Datapoints) > 0 { + bytesDownloaded = *metricsBytesDownloadedOutput.Datapoints[0].Sum + } - monthlyCost := dataTransferToOriginCost + requestsCost + metricsRequestsOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ + StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), + EndTime: aws.Time(time.Now()), + MetricName: aws.String("Requests"), + Namespace: aws.String("AWS/CloudFront"), + Dimensions: []types.Dimension{ + { + Name: aws.String("DistributionId"), + Value: distribution.Id, + }, + }, + Period: aws.Int32(86400), + Statistics: []types.Statistic{ + types.StatisticSum, + }, + }) - outputTags, err := cloudfrontClient.ListTagsForResource(ctx, &cloudfront.ListTagsForResourceInput{ - Resource: distribution.ARN, - }) + if err != nil { + log.Warnf("Couldn't fetch invocations metric for %s", *distribution.Id) + } - tags := make([]Tag, 0) - - if err == nil { - for _, tag := range outputTags.Tags.Items { - tags = append(tags, Tag{ - Key: *tag.Key, - Value: *tag.Value, - }) - } - } - - resources = append(resources, Resource{ - Provider: "AWS", - Account: client.Name, - Service: "CloudFront", - ResourceId: *distribution.ARN, - Region: client.AWSClient.Region, - Name: *distribution.DomainName, - Cost: monthlyCost, - Tags: tags, - FetchedAt: time.Now(), - Link: fmt.Sprintf("https://%s.console.aws.amazon.com/cloudfront/v3/home?region=%s#/distributions/%s", client.AWSClient.Region, client.AWSClient.Region, *distribution.Id), - }) - } + requests := 0.0 + if metricsRequestsOutput != nil && len(metricsRequestsOutput.Datapoints) > 0 { + requests = *metricsRequestsOutput.Datapoints[0].Sum + } + if requests > freeTierRequests { + requests -= freeTierRequests + } + + dataTransferToOriginCost := awsUtils.GetCost(priceMapForDataTransfer[EdgeLocation], (float64(bytesDownloaded)/1099511627776)*1024) + + requestsCost := awsUtils.GetCost(priceMapForRequest[EdgeLocation], requests/per10kRequest) + + monthlyCost := dataTransferToOriginCost + requestsCost - if aws.ToString(output.DistributionList.NextMarker) == "" { - break + outputTags, err := cloudfrontClient.ListTagsForResource(ctx, &cloudfront.ListTagsForResourceInput{ + Resource: distribution.ARN, + }) + + tags := make([]Tag, 0) + + if err == nil { + for _, tag := range outputTags.Tags.Items { + tags = append(tags, Tag{ + Key: *tag.Key, + Value: *tag.Value, + }) } - config.Marker = output.DistributionList.Marker } - log.WithFields(log.Fields{ - "provider": "AWS", - "account": client.Name, - "region": client.AWSClient.Region, - "service": "CloudFront", - "resources": len(resources), - }).Info("Fetched resources") - return resources, nil + + resources = append(resources, Resource{ + Provider: "AWS", + Account: client.Name, + Service: "CloudFront", + ResourceId: *distribution.ARN, + Region: client.AWSClient.Region, + Name: *distribution.DomainName, + Cost: monthlyCost, + Tags: tags, + FetchedAt: time.Now(), + Link: fmt.Sprintf("https://%s.console.aws.amazon.com/cloudfront/v3/home?region=%s#/distributions/%s", client.AWSClient.Region, client.AWSClient.Region, *distribution.Id), + }) } + if aws.ToString(output.DistributionList.NextMarker) == "" { + break + } + config.Marker = output.DistributionList.Marker } + log.WithFields(log.Fields{ + "provider": "AWS", + "account": client.Name, + "region": client.AWSClient.Region, + "service": "CloudFront", + "resources": len(resources), + }).Info("Fetched resources") + return resources, nil + } -func getRegionMapping() map[string][]string { - return map[string][]string{ - "us-east-1": {"United States", "Mexico", "Canada"}, - "eu-west-1": {"Europe", "Israel"}, - "ap-northeast-1": {"Australia", "New Zealand", "Taiwan"}, - "ap-northeast-2": {"South Korea"}, - "ap-southeast-1": {"Philippines", "Singapore", "Thailand", "Malaysia"}, - "ap-southeast-3": {"Indonesia"}, - "ap-south-1": {"India"}, - "sa-east-1": {"Japan", "South America"}, - "me-south-1": {"South Africa", "Kenya", "Middle East"}, - "ap-east-1": {"Hong Kong", "Vietnam"}, - "cn-north-1": {"China"}, +func getRegionMapping() map[string]string { + return map[string]string{ + "us-east-1": "United States", + "us-east-2": "United States", + "us-west-1": "United States", + "us-west-2": "United States", + "ca-central-1": "Canada", + "eu-north-1": "Europe", + "eu-west-1": "Europe", + "eu-west-2": "Europe", + "eu-west-3": "Europe", + "eu-central-1": "Europe", + "ap-northeast-1": "Japan", + "ap-northeast-2": "Asia Pacific", + "ap-northeast-3": "Australia", + "ap-southeast-1": "Asia Pacific", + "ap-southeast-2": "Australia", + "ap-south-1": "India", + "sa-east-1": "South America", } } diff --git a/providers/aws/cloudfront/lambdaedge.go b/providers/aws/cloudfront/lambdaedge.go index 694f45403..e26060ec7 100644 --- a/providers/aws/cloudfront/lambdaedge.go +++ b/providers/aws/cloudfront/lambdaedge.go @@ -26,12 +26,11 @@ func LambdaEdgeFunctions(ctx context.Context, client ProviderClient) ([]Resource resources := make([]Resource, 0) var config cloudfront.ListFunctionsInput cloudfrontClient := cloudfront.NewFromConfig(*client.AWSClient) - tempRegion := client.AWSClient.Region client.AWSClient.Region = "us-east-1" cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient) - client.AWSClient.Region = tempRegion pricingClient := pricing.NewFromConfig(*client.AWSClient) + client.AWSClient.Region = tempRegion pricingOutput, err := pricingClient.GetProducts(ctx, &pricing.GetProductsInput{ ServiceCode: aws.String("AmazonCloudFront"), From c4a576a24132f2e4408f530a7b11b4685b64ae25 Mon Sep 17 00:00:00 2001 From: bishal7679 Date: Tue, 31 Oct 2023 23:50:15 +0530 Subject: [PATCH 6/6] deleted println created for testing purpose for region Signed-off-by: bishal7679 --- providers/aws/cloudfront/distributions.go | 1 - 1 file changed, 1 deletion(-) diff --git a/providers/aws/cloudfront/distributions.go b/providers/aws/cloudfront/distributions.go index 34d910703..d132e85cd 100644 --- a/providers/aws/cloudfront/distributions.go +++ b/providers/aws/cloudfront/distributions.go @@ -60,7 +60,6 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro for { for region, edgelocation := range getRegions { if client.AWSClient.Region == region { - log.Println("matched region--------------------------", region) if priceMapForDataTransfer[edgelocation] != nil && priceMapForRequest[edgelocation] != nil { EdgeLocation = edgelocation }