diff --git a/providers/aws/aws.go b/providers/aws/aws.go index aa0ba4338..327054732 100644 --- a/providers/aws/aws.go +++ b/providers/aws/aws.go @@ -58,7 +58,7 @@ func listOfSupportedServices() []providers.FetchDataFunction { ec2.Instances, eks.KubernetesClusters, cloudfront.Distributions, - cloudfront.Functions, + cloudfront.LambdaEdgeFunctions, dynamodb.Tables, ecs.Clusters, ecs.TaskDefinitions, diff --git a/providers/aws/cloudfront/distributions.go b/providers/aws/cloudfront/distributions.go index 92c56ce38..d132e85cd 100644 --- a/providers/aws/cloudfront/distributions.go +++ b/providers/aws/cloudfront/distributions.go @@ -2,6 +2,7 @@ package cloudfront import ( "context" + "encoding/json" "fmt" "time" @@ -11,22 +12,62 @@ import ( "github.com/aws/aws-sdk-go-v2/service/cloudfront" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" + "github.com/aws/aws-sdk-go-v2/service/pricing" + + // pricingTypes "github.com/aws/aws-sdk-go-v2/service/pricing/types" . "github.com/tailwarden/komiser/models" . "github.com/tailwarden/komiser/providers" + awsUtils "github.com/tailwarden/komiser/providers/aws/utils" "github.com/tailwarden/komiser/utils" ) +const ( + freeTierRequests = 10000000 + freeTierUpload = 1099511627776 + per10kRequest = 10000 +) + +var EdgeLocation string + func Distributions(ctx context.Context, client ProviderClient) ([]Resource, error) { resources := make([]Resource, 0) var config cloudfront.ListDistributionsInput cloudfrontClient := cloudfront.NewFromConfig(*client.AWSClient) - tempRegion := client.AWSClient.Region client.AWSClient.Region = "us-east-1" cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient) + pricingClient := pricing.NewFromConfig(*client.AWSClient) client.AWSClient.Region = tempRegion + pricingOutput, err := pricingClient.GetProducts(ctx, &pricing.GetProductsInput{ + ServiceCode: aws.String("AmazonCloudFront"), + }) + if err != nil { + log.Errorf("ERROR: Couldn't fetch pricing info for AWS CloudFront: %v", err) + } + + priceMapForDataTransfer, err := GetPriceMapCF(pricingOutput, "fromLocation") + if err != nil { + log.Errorf("ERROR: Failed to calculate cost per month: %v", err) + } + + priceMapForRequest, err := GetPriceMapCF(pricingOutput, "location") + if err != nil { + log.Errorf("ERROR: Failed to calculate cost per month: %v", err) + } + + getRegions := getRegionMapping() for { + for region, edgelocation := range getRegions { + if client.AWSClient.Region == region { + if priceMapForDataTransfer[edgelocation] != nil && priceMapForRequest[edgelocation] != nil { + EdgeLocation = edgelocation + } + + } + + } + output, err := cloudfrontClient.ListDistributions(ctx, &config) if err != nil { return resources, err @@ -39,7 +80,7 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro MetricName: aws.String("BytesDownloaded"), Namespace: aws.String("AWS/CloudFront"), Dimensions: []types.Dimension{ - types.Dimension{ + { Name: aws.String("DistributionId"), Value: distribution.Id, }, @@ -59,39 +100,13 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro bytesDownloaded = *metricsBytesDownloadedOutput.Datapoints[0].Sum } - metricsBytesUploadedOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ - StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), - EndTime: aws.Time(time.Now()), - MetricName: aws.String("BytesUploaded"), - Namespace: aws.String("AWS/CloudFront"), - Dimensions: []types.Dimension{ - types.Dimension{ - Name: aws.String("DistributionId"), - Value: distribution.Id, - }, - }, - Period: aws.Int32(86400), - Statistics: []types.Statistic{ - types.StatisticSum, - }, - }) - - if err != nil { - log.Warnf("Couldn't fetch invocations metric for %s", *distribution.Id) - } - - bytesUploaded := 0.0 - if metricsBytesUploadedOutput != nil && len(metricsBytesUploadedOutput.Datapoints) > 0 { - bytesUploaded = *metricsBytesUploadedOutput.Datapoints[0].Sum - } - metricsRequestsOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), EndTime: aws.Time(time.Now()), MetricName: aws.String("Requests"), Namespace: aws.String("AWS/CloudFront"), Dimensions: []types.Dimension{ - types.Dimension{ + { Name: aws.String("DistributionId"), Value: distribution.Id, }, @@ -110,17 +125,15 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro if metricsRequestsOutput != nil && len(metricsRequestsOutput.Datapoints) > 0 { requests = *metricsRequestsOutput.Datapoints[0].Sum } + if requests > freeTierRequests { + requests -= freeTierRequests + } - // calculate region data transfer out to internet - dataTransferToInternet := (bytesUploaded / 1000000000) * 0.085 - - // calculate region data transfer out to origin - dataTransferToOrigin := (bytesDownloaded / 1000000000) * 0.02 + dataTransferToOriginCost := awsUtils.GetCost(priceMapForDataTransfer[EdgeLocation], (float64(bytesDownloaded)/1099511627776)*1024) - // calculate requests cost - requestsCost := requests * 0.000001 + requestsCost := awsUtils.GetCost(priceMapForRequest[EdgeLocation], requests/per10kRequest) - monthlyCost := dataTransferToInternet + dataTransferToOrigin + requestsCost + monthlyCost := dataTransferToOriginCost + requestsCost outputTags, err := cloudfrontClient.ListTagsForResource(ctx, &cloudfront.ListTagsForResourceInput{ Resource: distribution.ARN, @@ -164,4 +177,66 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro "resources": len(resources), }).Info("Fetched resources") return resources, nil + +} + +func getRegionMapping() map[string]string { + return map[string]string{ + "us-east-1": "United States", + "us-east-2": "United States", + "us-west-1": "United States", + "us-west-2": "United States", + "ca-central-1": "Canada", + "eu-north-1": "Europe", + "eu-west-1": "Europe", + "eu-west-2": "Europe", + "eu-west-3": "Europe", + "eu-central-1": "Europe", + "ap-northeast-1": "Japan", + "ap-northeast-2": "Asia Pacific", + "ap-northeast-3": "Australia", + "ap-southeast-1": "Asia Pacific", + "ap-southeast-2": "Australia", + "ap-south-1": "India", + "sa-east-1": "South America", + } +} + +// GetPriceMapCF is modified functions from awsUtils.GetPriceMap to get CF distribution unit price based on location +func GetPriceMapCF(pricingOutput *pricing.GetProductsOutput, field string) (map[string][]awsUtils.PriceDimensions, error) { + priceMap := make(map[string][]awsUtils.PriceDimensions) + + if pricingOutput != nil && len(pricingOutput.PriceList) > 0 { + for _, item := range pricingOutput.PriceList { + price := awsUtils.ProductEntry{} + err := json.Unmarshal([]byte(item), &price) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal JSON: %w", err) + } + + var key string + switch field { + case "fromLocation": + if price.Product.Attributes.TransferType == "CloudFront to Origin" { + + key = price.Product.Attributes.FromLocation + } + case "location": + if price.Product.Attributes.RequestType == "CloudFront-Request-HTTP-Proxy" { + key = price.Product.Attributes.RequestLocation + } + } + + unitPrices := []awsUtils.PriceDimensions{} + for _, pd := range price.Terms.OnDemand { + for _, p := range pd.PriceDimensions { + unitPrices = append(unitPrices, p) + } + } + + priceMap[key] = unitPrices + } + } + + return priceMap, nil } diff --git a/providers/aws/cloudfront/lambdaedge.go b/providers/aws/cloudfront/lambdaedge.go new file mode 100644 index 000000000..e26060ec7 --- /dev/null +++ b/providers/aws/cloudfront/lambdaedge.go @@ -0,0 +1,161 @@ +package cloudfront + +import ( + "context" + "fmt" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/cloudfront" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" + "github.com/aws/aws-sdk-go-v2/service/pricing" + pricingTypes "github.com/aws/aws-sdk-go-v2/service/pricing/types" + . "github.com/tailwarden/komiser/models" + . "github.com/tailwarden/komiser/providers" + awsUtils "github.com/tailwarden/komiser/providers/aws/utils" + "github.com/tailwarden/komiser/utils" +) + +const ( + perOneMillonRequest = 1000000 +) +func LambdaEdgeFunctions(ctx context.Context, client ProviderClient) ([]Resource, error) { + resources := make([]Resource, 0) + var config cloudfront.ListFunctionsInput + cloudfrontClient := cloudfront.NewFromConfig(*client.AWSClient) + tempRegion := client.AWSClient.Region + client.AWSClient.Region = "us-east-1" + cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient) + pricingClient := pricing.NewFromConfig(*client.AWSClient) + client.AWSClient.Region = tempRegion + + pricingOutput, err := pricingClient.GetProducts(ctx, &pricing.GetProductsInput{ + ServiceCode: aws.String("AmazonCloudFront"), + Filters: []pricingTypes.Filter{ + { + Field: aws.String("regionCode"), + Value: aws.String(client.AWSClient.Region), + Type: pricingTypes.FilterTypeTermMatch, + }, + }, + }) + if err != nil { + log.Errorf("ERROR: Couldn't fetch pricing info for AWS CloudFront: %v", err) + } + + priceMap, err := awsUtils.GetPriceMap(pricingOutput, "group") + if err != nil { + log.Errorf("ERROR: Failed to calculate cost per month: %v", err) + } + + for { + output, err := cloudfrontClient.ListFunctions(ctx, &config) + if err != nil { + return resources, err + } + + for _, function := range output.FunctionList.Items { + metricsLambdaEdgeDurationOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ + StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), + EndTime: aws.Time(time.Now()), + MetricName: aws.String("Duration"), + Namespace: aws.String("AWS/CloudFront"), + Dimensions: []types.Dimension{ + { + Name: aws.String("FunctionName"), + Value: function.Name, + }, + }, + Period: aws.Int32(86400), + Statistics: []types.Statistic{ + types.StatisticAverage, + }, + }) + + if err != nil { + log.Warnf("Couldn't fetch Lambda@Edge Duration metric for %s", *function.Name) + } + + lambdaEdgeDuration := 0.0 + if metricsLambdaEdgeDurationOutput != nil && len(metricsLambdaEdgeDurationOutput.Datapoints) > 0 { + lambdaEdgeDuration = *metricsLambdaEdgeDurationOutput.Datapoints[0].Average + } + + metricsLambdaEdgeRequestsOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ + StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), + EndTime: aws.Time(time.Now()), + MetricName: aws.String("Requests"), + Namespace: aws.String("AWS/CloudFront"), + Dimensions: []types.Dimension{ + { + Name: aws.String("FunctionName"), + Value: function.Name, + }, + }, + Period: aws.Int32(86400), + Statistics: []types.Statistic{ + types.StatisticSum, + }, + }) + + if err != nil { + log.Warnf("Couldn't fetch Lambda@Edge Requests metric for %s", *function.Name) + } + + lambdaEdgeRequests := 0.0 + if metricsLambdaEdgeRequestsOutput != nil && len(metricsLambdaEdgeRequestsOutput.Datapoints) > 0 { + lambdaEdgeRequests = *metricsLambdaEdgeRequestsOutput.Datapoints[0].Sum + } + + lambdaEdgeDurationCost := awsUtils.GetCost(priceMap["AWS-Lambda-Edge-Duration"], lambdaEdgeDuration) + + lambdaEdgeRequestsCost := awsUtils.GetCost(priceMap["AWS-Lambda-Edge-Requests"], lambdaEdgeRequests/perOneMillonRequest) + + monthlyCost := lambdaEdgeDurationCost + lambdaEdgeRequestsCost + + outputTags, err := cloudfrontClient.ListTagsForResource(ctx, &cloudfront.ListTagsForResourceInput{ + Resource: function.FunctionMetadata.FunctionARN, + }) + + tags := make([]Tag, 0) + + if err == nil { + for _, tag := range outputTags.Tags.Items { + tags = append(tags, Tag{ + Key: *tag.Key, + Value: *tag.Value, + }) + } + } + + resources = append(resources, Resource{ + Provider: "AWS", + Account: client.Name, + Service: "CloudFront", + ResourceId: *function.FunctionMetadata.FunctionARN, + Region: client.AWSClient.Region, + Name: *function.Name, + Cost: monthlyCost, + Tags: tags, + FetchedAt: time.Now(), + Link: fmt.Sprintf("https://%s.console.aws.amazon.com/cloudfront/v3/home?region=%s#/distributions/%s", client.AWSClient.Region, client.AWSClient.Region, *function.Name), + }) + } + + if aws.ToString(output.FunctionList.NextMarker) == "" { + break + } + config.Marker = output.FunctionList.NextMarker + } + log.WithFields(log.Fields{ + "provider": "AWS", + "account": client.Name, + "region": client.AWSClient.Region, + "service": "CloudFront", + "resources": len(resources), + }).Info("Fetched resources") + return resources, nil +} \ No newline at end of file diff --git a/providers/aws/utils/utils.go b/providers/aws/utils/utils.go index 5ed566701..60f727069 100644 --- a/providers/aws/utils/utils.go +++ b/providers/aws/utils/utils.go @@ -15,8 +15,12 @@ type ProductEntry struct { Operation string `json:"operation"` GroupDescription string `json:"groupDescription"` RequestDescription string `json:"requestDescription"` + RequestType string `json:"requestType"` InstanceType string `json:"instanceType"` InstanceTypeFamily string `json:"instanceTypeFamily"` + TransferType string `json:"transferType"` + FromLocation string `json:"fromLocation"` + RequestLocation string `json:"location"` } `json:"attributes"` } `json:"product"` Terms struct { @@ -93,8 +97,8 @@ func GetPriceMap(pricingOutput *pricing.GetProductsOutput, field string) (map[st } func Int64PtrToFloat64(i *int64) float64 { - if i == nil { - return 0.0 // or any default value you prefer - } - return float64(*i) + if i == nil { + return 0.0 // or any default value you prefer + } + return float64(*i) }