-
Notifications
You must be signed in to change notification settings - Fork 431
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: Fix Cost Calculation for AWS CloudFront Distributions & Functions #1131
Changes from all commits
265b209
ad9362d
fe0a48e
4ddcb38
8fcfcc7
c4a576a
4832bb5
2dc22b6
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,6 +2,7 @@ package cloudfront | |
|
||
import ( | ||
"context" | ||
"encoding/json" | ||
"fmt" | ||
"time" | ||
|
||
|
@@ -11,22 +12,62 @@ import ( | |
"github.com/aws/aws-sdk-go-v2/service/cloudfront" | ||
"github.com/aws/aws-sdk-go-v2/service/cloudwatch" | ||
"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" | ||
"github.com/aws/aws-sdk-go-v2/service/pricing" | ||
|
||
// pricingTypes "github.com/aws/aws-sdk-go-v2/service/pricing/types" | ||
. "github.com/tailwarden/komiser/models" | ||
. "github.com/tailwarden/komiser/providers" | ||
awsUtils "github.com/tailwarden/komiser/providers/aws/utils" | ||
"github.com/tailwarden/komiser/utils" | ||
) | ||
|
||
const ( | ||
freeTierRequests = 10000000 | ||
freeTierUpload = 1099511627776 | ||
per10kRequest = 10000 | ||
) | ||
|
||
var EdgeLocation string | ||
|
||
func Distributions(ctx context.Context, client ProviderClient) ([]Resource, error) { | ||
resources := make([]Resource, 0) | ||
var config cloudfront.ListDistributionsInput | ||
cloudfrontClient := cloudfront.NewFromConfig(*client.AWSClient) | ||
|
||
tempRegion := client.AWSClient.Region | ||
client.AWSClient.Region = "us-east-1" | ||
cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient) | ||
pricingClient := pricing.NewFromConfig(*client.AWSClient) | ||
client.AWSClient.Region = tempRegion | ||
|
||
pricingOutput, err := pricingClient.GetProducts(ctx, &pricing.GetProductsInput{ | ||
ServiceCode: aws.String("AmazonCloudFront"), | ||
}) | ||
if err != nil { | ||
log.Errorf("ERROR: Couldn't fetch pricing info for AWS CloudFront: %v", err) | ||
} | ||
|
||
priceMapForDataTransfer, err := GetPriceMapCF(pricingOutput, "fromLocation") | ||
if err != nil { | ||
log.Errorf("ERROR: Failed to calculate cost per month: %v", err) | ||
} | ||
|
||
priceMapForRequest, err := GetPriceMapCF(pricingOutput, "location") | ||
if err != nil { | ||
log.Errorf("ERROR: Failed to calculate cost per month: %v", err) | ||
} | ||
|
||
getRegions := getRegionMapping() | ||
for { | ||
for region, edgelocation := range getRegions { | ||
if client.AWSClient.Region == region { | ||
if priceMapForDataTransfer[edgelocation] != nil && priceMapForRequest[edgelocation] != nil { | ||
EdgeLocation = edgelocation | ||
} | ||
|
||
} | ||
|
||
} | ||
|
||
output, err := cloudfrontClient.ListDistributions(ctx, &config) | ||
if err != nil { | ||
return resources, err | ||
|
@@ -39,7 +80,7 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro | |
MetricName: aws.String("BytesDownloaded"), | ||
Namespace: aws.String("AWS/CloudFront"), | ||
Dimensions: []types.Dimension{ | ||
types.Dimension{ | ||
{ | ||
Name: aws.String("DistributionId"), | ||
Value: distribution.Id, | ||
}, | ||
|
@@ -59,39 +100,13 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro | |
bytesDownloaded = *metricsBytesDownloadedOutput.Datapoints[0].Sum | ||
} | ||
|
||
metricsBytesUploadedOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ | ||
StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), | ||
EndTime: aws.Time(time.Now()), | ||
MetricName: aws.String("BytesUploaded"), | ||
Namespace: aws.String("AWS/CloudFront"), | ||
Dimensions: []types.Dimension{ | ||
types.Dimension{ | ||
Name: aws.String("DistributionId"), | ||
Value: distribution.Id, | ||
}, | ||
}, | ||
Period: aws.Int32(86400), | ||
Statistics: []types.Statistic{ | ||
types.StatisticSum, | ||
}, | ||
}) | ||
|
||
if err != nil { | ||
log.Warnf("Couldn't fetch invocations metric for %s", *distribution.Id) | ||
} | ||
|
||
bytesUploaded := 0.0 | ||
if metricsBytesUploadedOutput != nil && len(metricsBytesUploadedOutput.Datapoints) > 0 { | ||
bytesUploaded = *metricsBytesUploadedOutput.Datapoints[0].Sum | ||
} | ||
|
||
metricsRequestsOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ | ||
StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), | ||
EndTime: aws.Time(time.Now()), | ||
MetricName: aws.String("Requests"), | ||
Namespace: aws.String("AWS/CloudFront"), | ||
Dimensions: []types.Dimension{ | ||
types.Dimension{ | ||
{ | ||
Name: aws.String("DistributionId"), | ||
Value: distribution.Id, | ||
}, | ||
|
@@ -110,17 +125,15 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro | |
if metricsRequestsOutput != nil && len(metricsRequestsOutput.Datapoints) > 0 { | ||
requests = *metricsRequestsOutput.Datapoints[0].Sum | ||
} | ||
if requests > freeTierRequests { | ||
requests -= freeTierRequests | ||
} | ||
|
||
// calculate region data transfer out to internet | ||
dataTransferToInternet := (bytesUploaded / 1000000000) * 0.085 | ||
|
||
// calculate region data transfer out to origin | ||
dataTransferToOrigin := (bytesDownloaded / 1000000000) * 0.02 | ||
dataTransferToOriginCost := awsUtils.GetCost(priceMapForDataTransfer[EdgeLocation], (float64(bytesDownloaded)/1099511627776)*1024) | ||
|
||
// calculate requests cost | ||
requestsCost := requests * 0.000001 | ||
requestsCost := awsUtils.GetCost(priceMapForRequest[EdgeLocation], requests/per10kRequest) | ||
|
||
monthlyCost := dataTransferToInternet + dataTransferToOrigin + requestsCost | ||
monthlyCost := dataTransferToOriginCost + requestsCost | ||
|
||
outputTags, err := cloudfrontClient.ListTagsForResource(ctx, &cloudfront.ListTagsForResourceInput{ | ||
Resource: distribution.ARN, | ||
|
@@ -164,4 +177,66 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro | |
"resources": len(resources), | ||
}).Info("Fetched resources") | ||
return resources, nil | ||
|
||
} | ||
|
||
func getRegionMapping() map[string]string { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think it will be better to have this region mapping in an AWS util file so it can be reused by any AWS service needing to map regions. Also, we should cross-verify if this does not exist (or in a different format) in the codebase as of now There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yeap you are right but this mapping is cloudfront specific |
||
return map[string]string{ | ||
"us-east-1": "United States", | ||
"us-east-2": "United States", | ||
"us-west-1": "United States", | ||
"us-west-2": "United States", | ||
"ca-central-1": "Canada", | ||
"eu-north-1": "Europe", | ||
"eu-west-1": "Europe", | ||
"eu-west-2": "Europe", | ||
"eu-west-3": "Europe", | ||
"eu-central-1": "Europe", | ||
"ap-northeast-1": "Japan", | ||
"ap-northeast-2": "Asia Pacific", | ||
"ap-northeast-3": "Australia", | ||
"ap-southeast-1": "Asia Pacific", | ||
"ap-southeast-2": "Australia", | ||
"ap-south-1": "India", | ||
"sa-east-1": "South America", | ||
} | ||
} | ||
|
||
// GetPriceMapCF is modified functions from awsUtils.GetPriceMap to get CF distribution unit price based on location | ||
func GetPriceMapCF(pricingOutput *pricing.GetProductsOutput, field string) (map[string][]awsUtils.PriceDimensions, error) { | ||
priceMap := make(map[string][]awsUtils.PriceDimensions) | ||
|
||
if pricingOutput != nil && len(pricingOutput.PriceList) > 0 { | ||
for _, item := range pricingOutput.PriceList { | ||
price := awsUtils.ProductEntry{} | ||
err := json.Unmarshal([]byte(item), &price) | ||
if err != nil { | ||
return nil, fmt.Errorf("failed to unmarshal JSON: %w", err) | ||
} | ||
|
||
var key string | ||
switch field { | ||
case "fromLocation": | ||
if price.Product.Attributes.TransferType == "CloudFront to Origin" { | ||
|
||
key = price.Product.Attributes.FromLocation | ||
} | ||
case "location": | ||
if price.Product.Attributes.RequestType == "CloudFront-Request-HTTP-Proxy" { | ||
key = price.Product.Attributes.RequestLocation | ||
} | ||
} | ||
|
||
unitPrices := []awsUtils.PriceDimensions{} | ||
for _, pd := range price.Terms.OnDemand { | ||
for _, p := range pd.PriceDimensions { | ||
unitPrices = append(unitPrices, p) | ||
} | ||
} | ||
|
||
priceMap[key] = unitPrices | ||
} | ||
} | ||
|
||
return priceMap, nil | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,161 @@ | ||
package cloudfront | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"time" | ||
|
||
log "github.com/sirupsen/logrus" | ||
|
||
"github.com/aws/aws-sdk-go-v2/aws" | ||
"github.com/aws/aws-sdk-go-v2/service/cloudfront" | ||
"github.com/aws/aws-sdk-go-v2/service/cloudwatch" | ||
"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" | ||
"github.com/aws/aws-sdk-go-v2/service/pricing" | ||
pricingTypes "github.com/aws/aws-sdk-go-v2/service/pricing/types" | ||
. "github.com/tailwarden/komiser/models" | ||
. "github.com/tailwarden/komiser/providers" | ||
awsUtils "github.com/tailwarden/komiser/providers/aws/utils" | ||
"github.com/tailwarden/komiser/utils" | ||
) | ||
|
||
const ( | ||
perOneMillonRequest = 1000000 | ||
) | ||
func LambdaEdgeFunctions(ctx context.Context, client ProviderClient) ([]Resource, error) { | ||
resources := make([]Resource, 0) | ||
var config cloudfront.ListFunctionsInput | ||
cloudfrontClient := cloudfront.NewFromConfig(*client.AWSClient) | ||
tempRegion := client.AWSClient.Region | ||
client.AWSClient.Region = "us-east-1" | ||
cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient) | ||
pricingClient := pricing.NewFromConfig(*client.AWSClient) | ||
client.AWSClient.Region = tempRegion | ||
|
||
pricingOutput, err := pricingClient.GetProducts(ctx, &pricing.GetProductsInput{ | ||
ServiceCode: aws.String("AmazonCloudFront"), | ||
Filters: []pricingTypes.Filter{ | ||
{ | ||
Field: aws.String("regionCode"), | ||
Value: aws.String(client.AWSClient.Region), | ||
Type: pricingTypes.FilterTypeTermMatch, | ||
}, | ||
}, | ||
}) | ||
if err != nil { | ||
log.Errorf("ERROR: Couldn't fetch pricing info for AWS CloudFront: %v", err) | ||
} | ||
|
||
priceMap, err := awsUtils.GetPriceMap(pricingOutput, "group") | ||
if err != nil { | ||
log.Errorf("ERROR: Failed to calculate cost per month: %v", err) | ||
} | ||
|
||
for { | ||
output, err := cloudfrontClient.ListFunctions(ctx, &config) | ||
if err != nil { | ||
return resources, err | ||
} | ||
|
||
for _, function := range output.FunctionList.Items { | ||
metricsLambdaEdgeDurationOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ | ||
StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), | ||
EndTime: aws.Time(time.Now()), | ||
MetricName: aws.String("Duration"), | ||
Namespace: aws.String("AWS/CloudFront"), | ||
Dimensions: []types.Dimension{ | ||
{ | ||
Name: aws.String("FunctionName"), | ||
Value: function.Name, | ||
}, | ||
}, | ||
Period: aws.Int32(86400), | ||
Statistics: []types.Statistic{ | ||
types.StatisticAverage, | ||
}, | ||
}) | ||
|
||
if err != nil { | ||
log.Warnf("Couldn't fetch Lambda@Edge Duration metric for %s", *function.Name) | ||
} | ||
|
||
lambdaEdgeDuration := 0.0 | ||
if metricsLambdaEdgeDurationOutput != nil && len(metricsLambdaEdgeDurationOutput.Datapoints) > 0 { | ||
lambdaEdgeDuration = *metricsLambdaEdgeDurationOutput.Datapoints[0].Average | ||
} | ||
|
||
metricsLambdaEdgeRequestsOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ | ||
StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), | ||
EndTime: aws.Time(time.Now()), | ||
MetricName: aws.String("Requests"), | ||
Namespace: aws.String("AWS/CloudFront"), | ||
Dimensions: []types.Dimension{ | ||
{ | ||
Name: aws.String("FunctionName"), | ||
Value: function.Name, | ||
}, | ||
}, | ||
Period: aws.Int32(86400), | ||
Statistics: []types.Statistic{ | ||
types.StatisticSum, | ||
}, | ||
}) | ||
|
||
if err != nil { | ||
log.Warnf("Couldn't fetch Lambda@Edge Requests metric for %s", *function.Name) | ||
} | ||
|
||
lambdaEdgeRequests := 0.0 | ||
if metricsLambdaEdgeRequestsOutput != nil && len(metricsLambdaEdgeRequestsOutput.Datapoints) > 0 { | ||
lambdaEdgeRequests = *metricsLambdaEdgeRequestsOutput.Datapoints[0].Sum | ||
} | ||
|
||
lambdaEdgeDurationCost := awsUtils.GetCost(priceMap["AWS-Lambda-Edge-Duration"], lambdaEdgeDuration) | ||
|
||
lambdaEdgeRequestsCost := awsUtils.GetCost(priceMap["AWS-Lambda-Edge-Requests"], lambdaEdgeRequests/perOneMillonRequest) | ||
|
||
monthlyCost := lambdaEdgeDurationCost + lambdaEdgeRequestsCost | ||
|
||
outputTags, err := cloudfrontClient.ListTagsForResource(ctx, &cloudfront.ListTagsForResourceInput{ | ||
Resource: function.FunctionMetadata.FunctionARN, | ||
}) | ||
|
||
tags := make([]Tag, 0) | ||
|
||
if err == nil { | ||
for _, tag := range outputTags.Tags.Items { | ||
tags = append(tags, Tag{ | ||
Key: *tag.Key, | ||
Value: *tag.Value, | ||
}) | ||
} | ||
} | ||
|
||
resources = append(resources, Resource{ | ||
Provider: "AWS", | ||
Account: client.Name, | ||
Service: "CloudFront", | ||
ResourceId: *function.FunctionMetadata.FunctionARN, | ||
Region: client.AWSClient.Region, | ||
Name: *function.Name, | ||
Cost: monthlyCost, | ||
Tags: tags, | ||
FetchedAt: time.Now(), | ||
Link: fmt.Sprintf("https://%s.console.aws.amazon.com/cloudfront/v3/home?region=%s#/distributions/%s", client.AWSClient.Region, client.AWSClient.Region, *function.Name), | ||
}) | ||
} | ||
|
||
if aws.ToString(output.FunctionList.NextMarker) == "" { | ||
break | ||
} | ||
config.Marker = output.FunctionList.NextMarker | ||
} | ||
log.WithFields(log.Fields{ | ||
"provider": "AWS", | ||
"account": client.Name, | ||
"region": client.AWSClient.Region, | ||
"service": "CloudFront", | ||
"resources": len(resources), | ||
}).Info("Fetched resources") | ||
return resources, nil | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It would be better to use the
freeTierUpload
variable so anyone can tell by the name what this calculation should do You might also create a variable for1024
as well.