diff --git a/AzureBillingExporter.Tests/DateEnumHelperTests.cs b/AzureBillingExporter.Tests/DateEnumHelperTests.cs index af20274..5999699 100644 --- a/AzureBillingExporter.Tests/DateEnumHelperTests.cs +++ b/AzureBillingExporter.Tests/DateEnumHelperTests.cs @@ -1,4 +1,5 @@ using System; +using AzureBillingExporter.PrometheusMetrics; using NUnit.Framework; namespace AzureBillingExporter.Tests diff --git a/README.md b/README.md index e9e65a6..9302f44 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Build](https://github.com/dodopizza/azure_billing_exporter/workflows/Build/badge.svg?branch=master)](https://github.com/dodopizza/azure_billing_exporter/actions?query=workflow%3ABuild) [![Docker Pulls](https://img.shields.io/docker/pulls/dodopizza/azure_billing_exporter)](https://hub.docker.com/r/dodopizza/azure_billing_exporter) -Expose Azure Billing data to prometheus format. Show daily, weekly, monthly cost by subscription. Also allow add custom billing query. +Expose Azure Billing data to prometheus format. Show daily, weekly, monthly cost by subscription. Also allow add custom billing query. ## Quick start. Docker images @@ -21,60 +21,89 @@ docker run\ ## How to run locally 1. Create ServicePrincipal -This SP should have access as Billing reader role [see Manage billing access](https://docs.microsoft.com/en-us/azure/cost-management-billing/manage/manage-billing-access) -2. Set configuration + This SP should have access as `Billing reader` role [see Manage billing access](https://docs.microsoft.com/en-us/azure/cost-management-billing/manage/manage-billing-access) -2.1. Environment Variables +1. Set configuration -```bash - EXPORT ApiSettings__SubscriptionId="YOUR_SUBSCRIPTION_ID" - EXPORT ApiSettings__TenantId="YOUR_TENANT_ID" - EXPORT ApiSettings__ClientId="YOUR_CLIENT_ID" - EXPORT ApiSettings__ClientSecret="CLIENT_SECRET_SP" -``` + 1. Environment Variables -2.2. `appsettings.json` -Using for local developing + ```bash + EXPORT ApiSettings__SubscriptionId="YOUR_SUBSCRIPTION_ID" + EXPORT ApiSettings__TenantId="YOUR_TENANT_ID" + EXPORT ApiSettings__ClientId="YOUR_CLIENT_ID" + EXPORT ApiSettings__ClientSecret="CLIENT_SECRET_SP" + ``` -```json - "ApiSettings": { - "SubscriptionId": "YOUR_SUBSCRIPTION_ID", - "TenantId": "YOUR_TENANT_ID", - "ClientId": "YOUR_CLIENT_ID", - "ClientSecret": "CLIENT_SECRET_SP" - }, -``` + 1. Configuration file `appsettings.json` -2.3 Tracing logs + Using for local developing -For trace all billing query and response set log level to trace info `appsettings.Development.json` + ```json + "ApiSettings": { + "SubscriptionId": "YOUR_SUBSCRIPTION_ID", + "TenantId": "YOUR_TENANT_ID", + "ClientId": "YOUR_CLIENT_ID", + "ClientSecret": "CLIENT_SECRET_SP" + }, + ``` -```json -{ - "Logging": { - "LogLevel": { - "Default": "Trace", -``` +1. Tracing logs -3. Install dotnet SDK -Download and install .NET Core 3.1 SDK or above - + For trace all billing query and response set log level to trace info `appsettings.Development.json` + ```json + "Serilog": { + "MinimumLevel": { + "Default": "Trace" + } + ``` -4. Run dotnet +1. Install dotnet SDK -```bash -dotnet run --project AzureBillingExporter/AzureBillingExporter.csproj -``` + Download and install .NET Core 3.1 SDK or above + -5. Open metrics +1. Run dotnet -```bash -curl http://localhost:5000/metrics -``` + ```bash + dotnet run --project AzureBillingExporter/AzureBillingExporter.csproj + ``` + +1. Open metrics + + ```bash + curl http://localhost:5000/metrics + ``` + +## Architecture + +According Microsoft [documentation](https://docs.microsoft.com/en-us/azure/cost-management-billing/costs/manage-automation#error-code-429---call-count-has-exceeded-rate-limits) application may create only 30 API calls per minute. +After that threshold application will get `Too Many Requests` response from API. + +> Error code 429 - Call count has exceeded rate limits +> +> To enable a consistent experience for all Cost Management subscribers, Cost Management APIs are rate limited. When you reach the limit, you receive the HTTP status code 429: Too many requests. The current throughput limits for our APIs are as follows: +> +> 30 calls per minute - It's done per scope, per user, or application. +> +> 200 calls per minute - It's done per tenant, per user, or application. -# Metrics +To avoid such errors, this exporter has background job to get data from API. +Received cost data placed in memory cache. Prometheus scriber calls on `/metrics` get data from cache and get quick response. + +In case of `Too Many Requests` errors, background job waits 1 minute before next calls. + +## Configuration + +| *Setting* | *Type* | *Description* | +|---|---|---| +| `LogsAtJsonFormat` | bool | Write logs in plain text or JSON format | +| `CollectPeriodInMinutes` | int | Period in minutes to make API call to the Azure, to get metrics | +| `CachePeriodInMinutes` | int | Period in minutes to cache API call results | +| `CustomCollectorsFilePath` | string | Path to YAML file with custom collectors (see [Custom Metrics](#Custom-Metrics)) | + +## Metrics | *Metrics Name* | *Description* | |---|---| @@ -83,9 +112,9 @@ curl http://localhost:5000/metrics | `azure_billing_daily_before_yesterday` | Day before yesterday all costs | | `azure_billing_monthly` | Costs by current month | -# Custom Metrics +## Custom Metrics -## Set custom metrics configs into `custom_collectors.yml` +### Set custom metrics configs into `custom_collectors.yml` ```yaml # A Prometheus metric with (optional) additional labels, value and labels populated from one query. @@ -103,14 +132,16 @@ metrics: replace_date_labels_to_enum: true # replace `05/01/2020 00:00:00` to `last_month`, `UsageDate="20200624"` to `yesterday`. Default false query_file: './custom_queries/azure_billing_by_resource_group.json' ``` -## You can set custom path to collectors.yaml file + +### You can set custom path to collectors.yaml file Into `appsettings.Development.json` (or env `CustomCollectorsFilePath`) set: + ```json "CustomCollectorsFilePath" : "./local/custom_collectors.yml", ``` -## Query to billing api +### Query to billing api ```json { @@ -144,9 +175,9 @@ Into `appsettings.Development.json` (or env `CustomCollectorsFilePath`) set: } ``` -## Datetime constants into query files +### Datetime constants into query files -You can use special constant into query file. For this use `{{ }}` template notation [Liquid Template Language](https://shopify.github.io/liquid/) . +You can use special constant into query file. For this use `{{ }}` template notation [Liquid Template Language](https://shopify.github.io/liquid/) . DateTime Constants (using server datetime). If today is '2020-06-23T08:12:45': | *Constant* | *Description* | *Example* | @@ -160,6 +191,7 @@ DateTime Constants (using server datetime). If today is '2020-06-23T08:12:45': | `YearAgo` | This month first day year ago. | '2019-06-01T00:00:00.0000000' | All this constants you can use into billing query json files: + ```json "timePeriod": { "from": "{{ PrevMonthStart }}", @@ -167,9 +199,7 @@ All this constants you can use into billing query json files: } ``` - - -# Try Azure Billing Query on sandbox +## Try Azure Billing Query on sandbox Go to docs: @@ -208,19 +238,11 @@ Body: } ``` - -# Notice +## Notice Request duration measuring for exporter: -``` -▶ curl -o /dev/null -s -w 'Total: %{time_total}s\n' http://localhost:5000/metrics -Total: 19.220319s - -~ -▶ curl -o /dev/null -s -w 'Total: %{time_total}s\n' http://localhost:5000/metrics -Total: 17.939426s -~ +```console ▶ curl -o /dev/null -s -w 'Total: %{time_total}s\n' http://localhost:5000/metrics -Total: 18.603152s -``` \ No newline at end of file +Total: 0.009669s +``` diff --git a/src/AzureApi/AccessTokenFactory.cs b/src/AzureApi/AccessTokenFactory.cs index 9a0071e..20eb957 100644 --- a/src/AzureApi/AccessTokenFactory.cs +++ b/src/AzureApi/AccessTokenFactory.cs @@ -3,6 +3,7 @@ using System.Threading; using System.Threading.Tasks; using System.Web; +using AzureBillingExporter.Cost; using Microsoft.Extensions.Logging; using Newtonsoft.Json; diff --git a/src/AzureApi/AzureCostManagementClient.cs b/src/AzureApi/AzureCostManagementClient.cs index b668621..4488df7 100644 --- a/src/AzureApi/AzureCostManagementClient.cs +++ b/src/AzureApi/AzureCostManagementClient.cs @@ -6,6 +6,7 @@ using System.Runtime.CompilerServices; using System.Text; using System.Threading; +using AzureBillingExporter.Cost; using Microsoft.Extensions.Logging; using Newtonsoft.Json; @@ -15,24 +16,27 @@ public class AzureCostManagementClient { private readonly ApiSettings _apiSettings; private readonly IAccessTokenProvider _accessTokenProvider; - private readonly ILogger _logger; + private readonly ILogger _logger; + private readonly IHttpClientFactory _httpClientFactory; public AzureCostManagementClient(ApiSettings apiSettings, - IAccessTokenProvider accessTokenProvider, - ILogger logger) + IAccessTokenProvider accessTokenProvider, + ILogger logger, + IHttpClientFactory httpClientFactory) { _apiSettings = apiSettings; _accessTokenProvider = accessTokenProvider; _logger = logger; + _httpClientFactory = httpClientFactory; } - - public async IAsyncEnumerable ExecuteBillingQuery(string billingQuery, [EnumeratorCancellation] CancellationToken cancel, BillingQueryClient billingQueryClient) + + public async IAsyncEnumerable ExecuteBillingQuery(string billingQuery, [EnumeratorCancellation] CancellationToken cancel) { + var client = _httpClientFactory.CreateClient(nameof(AzureCostManagementClient)); + var azureManagementUrl = $"https://management.azure.com/subscriptions/{_apiSettings.SubscriptionId}/providers/Microsoft.CostManagement/query?api-version=2019-10-01"; - using var httpClient = new HttpClient(); - _logger.LogTrace($"Billing query {billingQuery}"); var request = new HttpRequestMessage { @@ -44,18 +48,23 @@ public async IAsyncEnumerable ExecuteBillingQuery(string billing "application/json") }; request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - + var accessToken = _accessTokenProvider.GetAccessToken(); request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", accessToken); - var response = await httpClient.SendAsync(request, cancel); + var response = await client.SendAsync(request, cancel); if (response.StatusCode != HttpStatusCode.OK) { + if (response.StatusCode == HttpStatusCode.TooManyRequests) + { + throw new TooManyRequestsException($"{response.StatusCode} {await response.Content.ReadAsStringAsync()}"); + } + throw new Exception($"{response.StatusCode} {await response.Content.ReadAsStringAsync()}"); } - + var responseContent = await response.Content.ReadAsStringAsync(); _logger.LogTrace($"Billing query response result {responseContent}"); dynamic json = JsonConvert.DeserializeObject(responseContent); @@ -66,4 +75,4 @@ public async IAsyncEnumerable ExecuteBillingQuery(string billing } } } -} \ No newline at end of file +} diff --git a/src/AzureApi/BackgroundAccessTokenProviderHostedService.cs b/src/AzureApi/BackgroundAccessTokenProviderHostedService.cs index 7904df1..913396c 100644 --- a/src/AzureApi/BackgroundAccessTokenProviderHostedService.cs +++ b/src/AzureApi/BackgroundAccessTokenProviderHostedService.cs @@ -54,13 +54,13 @@ public async Task StopAsync(CancellationToken cancellationToken) { if (_executingTask == null) { - _logger.LogInformation("Devices API background access tokens refreshing hosted service - Already stopped."); + _logger.LogInformation("Azure Billing API background access tokens refreshing hosted service - Already stopped."); return; } try { - _logger.LogInformation("Devices API background access tokens refreshing hosted service - Stopping..."); + _logger.LogInformation("Azure Billing API background access tokens refreshing hosted service - Stopping..."); // Signal cancellation to the executing method _stoppingCts.Cancel(); } @@ -72,7 +72,7 @@ await Task.WhenAny( Task.Delay(Timeout.Infinite, cancellationToken)); } - _logger.LogInformation("Devices API background access tokens refreshing hosted service - Stopped."); + _logger.LogInformation("Azure Billing API background access tokens refreshing hosted service - Stopped."); } private async Task StartRefreshingAccessTokensInBackgroundAsync(CancellationToken cancellationToken) @@ -120,4 +120,4 @@ private async Task StartRefreshingAccessTokensInBackgroundAsync(CancellationToke } } } -} \ No newline at end of file +} diff --git a/src/AzureApi/TooManyRequestsException.cs b/src/AzureApi/TooManyRequestsException.cs new file mode 100644 index 0000000..598cc06 --- /dev/null +++ b/src/AzureApi/TooManyRequestsException.cs @@ -0,0 +1,11 @@ +using System; + +namespace AzureBillingExporter.AzureApi +{ + public class TooManyRequestsException : Exception + { + public TooManyRequestsException(string message) : base(message) + { + } + } +} diff --git a/src/AzureBillingExporter.csproj b/src/AzureBillingExporter.csproj index de0122f..d0d30d8 100644 --- a/src/AzureBillingExporter.csproj +++ b/src/AzureBillingExporter.csproj @@ -7,13 +7,19 @@ + - + + + + + + diff --git a/src/Configuration/CustomCollectorConfiguration.cs b/src/Configuration/CustomCollectorConfiguration.cs new file mode 100644 index 0000000..dba0097 --- /dev/null +++ b/src/Configuration/CustomCollectorConfiguration.cs @@ -0,0 +1,40 @@ +using System.IO; +using AzureBillingExporter.Configuration.Metrics; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace AzureBillingExporter.Configuration +{ + public class CustomCollectorConfiguration + { + private string CustomCollectorsConfigFile { get; } + public CustomCollectorConfiguration(string customCollectorsFilePath) + { + if (!string.IsNullOrEmpty(customCollectorsFilePath)) + { + CustomCollectorsConfigFile = customCollectorsFilePath; + } + else + { + CustomCollectorsConfigFile = "custom_collectors.yml"; + } + } + public CustomCollectorConfig GetCustomCollectorConfig() + { + if (!File.Exists(CustomCollectorsConfigFile)) + { + return null; + } + + var configText = File.ReadAllText(CustomCollectorsConfigFile); + var input = new StringReader(configText); + + var deserializer = new DeserializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .Build(); + + var config = deserializer.Deserialize(input); + return config; + } + } +} diff --git a/src/Configuration/EnvironmentConfiguration.cs b/src/Configuration/EnvironmentConfiguration.cs new file mode 100644 index 0000000..45783cd --- /dev/null +++ b/src/Configuration/EnvironmentConfiguration.cs @@ -0,0 +1,10 @@ +namespace AzureBillingExporter.Configuration +{ + public class EnvironmentConfiguration + { + public bool LogsAtJsonFormat { get; set; } + + public int CollectPeriodInMinutes { get; set; } = 5; + public int CachePeriodInMinutes { get; set; } = 10; + } +} diff --git a/src/Configuration/Metrics/CustomCollectorConfig.cs b/src/Configuration/Metrics/CustomCollectorConfig.cs new file mode 100644 index 0000000..89a9b78 --- /dev/null +++ b/src/Configuration/Metrics/CustomCollectorConfig.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; +using YamlDotNet.Serialization; + +namespace AzureBillingExporter.Configuration.Metrics +{ + public class CustomCollectorConfig + { + [YamlMember(Alias = "metrics", ApplyNamingConventions = false)] + public List Metrics { get; set; } + } +} diff --git a/src/Configuration/Metrics/MetricConfig.cs b/src/Configuration/Metrics/MetricConfig.cs new file mode 100644 index 0000000..c300b4e --- /dev/null +++ b/src/Configuration/Metrics/MetricConfig.cs @@ -0,0 +1,31 @@ +using System.Collections.Generic; +using YamlDotNet.Serialization; + +namespace AzureBillingExporter.Configuration.Metrics +{ + public class MetricConfig + { + [YamlMember(Alias = "metric_name", ApplyNamingConventions = false)] + public string MetricName { get; set; } + + public string Type { get; set; } + public string Help { get; set; } + + [YamlMember(Alias = "key_labels", ApplyNamingConventions = false)] + public List KeyLabels { get; set; } + + [YamlMember(Alias = "static_labels", ApplyNamingConventions = false)] + public Dictionary StaticLabel { get; set; } + + public string Value { get; set; } + + public int? Limit { get; set; } + + [YamlMember(Alias = "replace_date_labels_to_enum", ApplyNamingConventions = false)] + public bool ReplaceDateLabelsToEnum { get; set; } + + [YamlMember(Alias = "query_file", ApplyNamingConventions = false)] + public string QueryFilePath { get; set; } + + } +} diff --git a/src/Cost/BackgroundCostCollectorHostedService.cs b/src/Cost/BackgroundCostCollectorHostedService.cs new file mode 100644 index 0000000..fa4acd7 --- /dev/null +++ b/src/Cost/BackgroundCostCollectorHostedService.cs @@ -0,0 +1,213 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using AzureBillingExporter.AzureApi; +using AzureBillingExporter.Configuration; +using AzureBillingExporter.Configuration.Metrics; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace AzureBillingExporter.Cost +{ + public class BackgroundCostCollectorHostedService : IHostedService + { + const string DailyMetricKey = "daily"; + const string MonthlyMetricKey = "monthly"; + + const int MaxOldestTimeDriftInMinutes = 10; + const int ThrottleAzureApiTimeInMinutes = 1; + + private readonly Dictionary _metricsStats = new Dictionary(); + private readonly BillingQueryClient _billingQueryClient; + private readonly CostDataCache _costDataCache; + private readonly CustomCollectorConfiguration _customCollectorConfiguration; + private readonly EnvironmentConfiguration _environmentConfiguration; + private readonly ILogger _logger; + + private readonly CancellationTokenSource _stoppingCts = new CancellationTokenSource(); + private Task _executingTask = null!; + private CustomCollectorConfig _customCollectorConfig; + + public BackgroundCostCollectorHostedService( + BillingQueryClient billingQueryClient, + CostDataCache costDataCache, + CustomCollectorConfiguration customCollectorConfiguration, + EnvironmentConfiguration environmentConfiguration, + ILogger logger) + { + _billingQueryClient = billingQueryClient ?? throw new ArgumentNullException(nameof(billingQueryClient)); + _costDataCache = costDataCache ?? throw new ArgumentNullException(nameof(costDataCache)); + _customCollectorConfiguration = customCollectorConfiguration ?? throw new ArgumentNullException(nameof(customCollectorConfiguration)); + _environmentConfiguration = environmentConfiguration ?? throw new ArgumentNullException(nameof(environmentConfiguration)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public Task StartAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("Azure Billing data collector background hosted service - Starting..."); + + FillMetricsStats(); + + _logger.LogInformation($"Total metrics to process: {_metricsStats.Count}"); + + _executingTask = StartCollectingDataInBackgroundAsync(_stoppingCts.Token); + if (_executingTask.IsCompleted) + { + return _executingTask; + } + + _logger.LogInformation("Azure Billing data collector background hosted service - Has started."); + return Task.CompletedTask; + } + + private void FillMetricsStats() + { + var notActualTime = DateTime.UtcNow.AddMinutes(-1*_environmentConfiguration.CollectPeriodInMinutes); + _metricsStats.Add(DailyMetricKey, notActualTime); + _metricsStats.Add(MonthlyMetricKey, notActualTime); + + _customCollectorConfig = _customCollectorConfiguration.GetCustomCollectorConfig(); + foreach (var metric in _customCollectorConfig.Metrics) + { + _metricsStats.Add(metric.QueryFilePath, notActualTime); + } + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + if (_executingTask == null) + { + _logger.LogInformation("Azure Billing data collector background hosted service - Already stopped."); + return; + } + + try + { + _logger.LogInformation("Azure Billing data collector background hosted service - Stopping..."); + // Signal cancellation to the executing method + _stoppingCts.Cancel(); + } + finally + { + // Wait until the task completes or the stop token triggers + await Task.WhenAny( + _executingTask, + Task.Delay(Timeout.Infinite, cancellationToken)); + } + + _logger.LogInformation("Azure Billing data collector background hosted service - Stopped."); + } + + private async Task StartCollectingDataInBackgroundAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var timer = Stopwatch.StartNew(); + while (!cancellationToken.IsCancellationRequested) + { + cancellationToken.ThrowIfCancellationRequested(); + + var actualDateTimeUtc = DateTime.UtcNow.AddMinutes(-1*_environmentConfiguration.CollectPeriodInMinutes); + if (_metricsStats.All(x => x.Value > actualDateTimeUtc)) + { + var sleepTimeInMinutes = + Math.Ceiling(_environmentConfiguration.CollectPeriodInMinutes - timer.Elapsed.TotalMinutes); + _logger.Log(LogLevel.Debug, + $"Will sleep next {sleepTimeInMinutes} minute(s)"); + if (sleepTimeInMinutes > 0) + { + await Task.Delay(TimeSpan.FromMinutes(sleepTimeInMinutes), cancellationToken); + } + timer.Restart(); + } + + var oldestMetric = _metricsStats.OrderBy(x => x.Value).First(); + + WarnOnTooOldMetric(oldestMetric); + + try + { + await UpdateMetricData(oldestMetric.Key, cancellationToken); + } + catch (TooManyRequestsException ex) + { + _logger.LogError(ex, ex.Message); + _logger.LogWarning($"Too many requests to Azure Billing API. Let's sleep for {ThrottleAzureApiTimeInMinutes} minute(s)"); + await Task.Delay(TimeSpan.FromMinutes(ThrottleAzureApiTimeInMinutes), cancellationToken); + } + catch (Exception ex) + { + _logger.LogError(ex, ex.Message); + } + finally + { + _metricsStats[oldestMetric.Key] = DateTime.UtcNow; + } + } + } + + private void WarnOnTooOldMetric(KeyValuePair metric) + { + var maxOldestTimeInMinutes = _environmentConfiguration.CollectPeriodInMinutes + MaxOldestTimeDriftInMinutes; + if (metric.Value <= DateTime.UtcNow.AddMinutes(-1 * maxOldestTimeInMinutes)) + { + _logger.Log(LogLevel.Warning, + $"Metric {metric.Key} last update time was more than {maxOldestTimeInMinutes} minutes ago ({metric.Value})"); + } + } + + private async Task UpdateMetricData(string metricKey, CancellationToken cancellationToken) + { + switch (metricKey) + { + case DailyMetricKey: + { + _logger.Log(LogLevel.Debug, "Get daily metric data"); + var dailyData = new List(); + await foreach (var data in (await _billingQueryClient.GetDailyData(cancellationToken)) + .WithCancellation(cancellationToken)) + { + dailyData.Add(data); + } + + _costDataCache.SetDailyCost(dailyData); + + break; + } + case MonthlyMetricKey: + { + _logger.Log(LogLevel.Debug, "Get monthly metric data"); + var monthlyData = new List(); + await foreach (var data in (await _billingQueryClient.GetMonthlyData(cancellationToken)) + .WithCancellation(cancellationToken)) + { + monthlyData.Add(data); + } + + _costDataCache.SetMonthlyCost(monthlyData); + break; + } + default: + { + _logger.Log(LogLevel.Debug, $"Get custom metric {metricKey} data"); + + var metricConfig = + _customCollectorConfig.Metrics.Single(x => x.QueryFilePath == metricKey); + + var metricsData = new List(); + var data = await _billingQueryClient.GetCustomData(cancellationToken, + metricConfig.QueryFilePath); + await foreach (var customData in data.WithCancellation(cancellationToken)) + { + metricsData.Add(customData); + } + + _costDataCache.SetCostByKey(metricKey, metricsData); + break; + } + } + } + } +} diff --git a/src/BillingQueryClient.cs b/src/Cost/BillingQueryClient.cs similarity index 89% rename from src/BillingQueryClient.cs rename to src/Cost/BillingQueryClient.cs index ad3e7d2..c375b5d 100644 --- a/src/BillingQueryClient.cs +++ b/src/Cost/BillingQueryClient.cs @@ -6,29 +6,25 @@ using System.Threading.Tasks; using AzureBillingExporter.AzureApi; using DotLiquid; -using Microsoft.Extensions.Logging; -namespace AzureBillingExporter +namespace AzureBillingExporter.Cost { public class BillingQueryClient { private readonly AzureCostManagementClient _costManagementClient; - private readonly ILogger _logger; public BillingQueryClient( - AzureCostManagementClient costManagementClient, - ILogger logger) + AzureCostManagementClient costManagementClient) { _costManagementClient = costManagementClient; - _logger = logger; } public async Task> GetCustomData(CancellationToken cancel, string templateFileName) { var billingQuery = await GenerateBillingQuery(DateTime.MaxValue, DateTime.MaxValue, "None", templateFileName); - return _costManagementClient.ExecuteBillingQuery(billingQuery, cancel, this); + return _costManagementClient.ExecuteBillingQuery(billingQuery, cancel); } - + public async Task> GetDailyData(CancellationToken cancel) { var dateTimeNow = DateTime.Now; @@ -38,45 +34,45 @@ public async Task> GetDailyData(CancellationTok var granularity = "Daily"; var billingQuery = await GenerateBillingQuery(dateStart, dateEnd, granularity); - return _costManagementClient.ExecuteBillingQuery(billingQuery, cancel, this); + return _costManagementClient.ExecuteBillingQuery(billingQuery, cancel); } - + public async Task> GetMonthlyData(CancellationToken cancel) { var dateTimeNow = DateTime.Now; - + var dateStart = new DateTime(dateTimeNow.Year, dateTimeNow.Month, 1); dateStart = dateStart.AddMonths(-2); var dateEnd = new DateTime(dateTimeNow.Year, dateTimeNow.Month, dateTimeNow.Day, 23, 59, 59); var granularity = "Monthly"; var billingQuery = await GenerateBillingQuery(dateStart, dateEnd, granularity); - return _costManagementClient.ExecuteBillingQuery(billingQuery, cancel, this); + return _costManagementClient.ExecuteBillingQuery(billingQuery, cancel); } private async Task GenerateBillingQuery(DateTime dateStart, DateTime dateEnd, string granularity = "None", string templateFile = "./queries/get_daily_or_monthly_costs.json") { var dateTimeNow = DateTime.Now; - + var templateQuery =await File.ReadAllTextAsync(templateFile); var template = Template.Parse(templateQuery); - + var currentMonthStart = new DateTime(dateTimeNow.Year, dateTimeNow.Month, 1); - + var prevMonthStart = new DateTime(dateTimeNow.Year, dateTimeNow.Month, 1); prevMonthStart = prevMonthStart.AddMonths(-1); var beforePrevMonthStart = new DateTime(dateTimeNow.Year, dateTimeNow.Month, 1); beforePrevMonthStart = beforePrevMonthStart.AddMonths(-2); - + var todayEnd = new DateTime(dateTimeNow.Year, dateTimeNow.Month, dateTimeNow.Day, 23, 59, 59); - + var yesterdayStart = new DateTime(dateTimeNow.Year, dateTimeNow.Month, dateTimeNow.Day); yesterdayStart = yesterdayStart.AddDays(-1); - + var weekAgo = new DateTime(dateTimeNow.Year, dateTimeNow.Month, dateTimeNow.Day); weekAgo = weekAgo.AddDays(-7); - + //YearAgo var yearAgo = new DateTime(dateTimeNow.Year, dateTimeNow.Month, 1); yearAgo = yearAgo.AddYears(-1); @@ -84,7 +80,7 @@ private async Task GenerateBillingQuery(DateTime dateStart, DateTime dat return template.Render(Hash.FromAnonymousObject(new { - DayStart = dateStart.ToString("o", CultureInfo.InvariantCulture), + DayStart = dateStart.ToString("o", CultureInfo.InvariantCulture), DayEnd = dateEnd.ToString("o", CultureInfo.InvariantCulture), CurrentMonthStart = currentMonthStart.ToString("o", CultureInfo.InvariantCulture), PrevMonthStart = prevMonthStart.ToString("o", CultureInfo.InvariantCulture), @@ -97,4 +93,4 @@ private async Task GenerateBillingQuery(DateTime dateStart, DateTime dat })); } } -} \ No newline at end of file +} diff --git a/src/Cost/CostDataCache.cs b/src/Cost/CostDataCache.cs new file mode 100644 index 0000000..51726a1 --- /dev/null +++ b/src/Cost/CostDataCache.cs @@ -0,0 +1,83 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using AzureBillingExporter.Configuration; + +namespace AzureBillingExporter.Cost +{ + public class CostDataCache + { + private readonly ConcurrentDictionary _costDataCache = + new ConcurrentDictionary(); + + private readonly EnvironmentConfiguration _environmentConfiguration; + + public CostDataCache(EnvironmentConfiguration environmentConfiguration) + { + _environmentConfiguration = environmentConfiguration; + } + + public List GetDailyCost() + { + return GetCostByKey("Daily"); + } + + public void SetDailyCost(List data) + { + SetCostByKey("Daily", data); + } + + public List GetMonthlyCost() + { + return GetCostByKey("Monthly"); + } + + public void SetMonthlyCost(List data) + { + SetCostByKey("Monthly", data); + } + + public List GetCostByKey(string key) + { + var cacheItem = GetCacheItem(key); + if (cacheItem == null) + { + return new List(); + } + + return cacheItem.Data; + } + + public void SetCostByKey(string key, List data) + { + SetCacheItem(key, data); + } + + private CacheItem GetCacheItem(string key) + { + var expireDate = DateTime.UtcNow.Add(-1*TimeSpan.FromMinutes(_environmentConfiguration.CachePeriodInMinutes)); + var cacheItem = _costDataCache.ContainsKey(key) ? _costDataCache[key] : null; + if (cacheItem == null || cacheItem.Created <= expireDate) + { + return null; + } + + return _costDataCache[key]; + } + + private void SetCacheItem(string key, List data) + { + _costDataCache[key] = new CacheItem + { + Created = DateTime.UtcNow, + Data = data + }; + } + + class CacheItem + { + public DateTime Created { get; set; } + public List Data { get; set; } + } + } +} diff --git a/src/CostResultRows.cs b/src/Cost/CostResultRows.cs similarity index 94% rename from src/CostResultRows.cs rename to src/Cost/CostResultRows.cs index 97e31db..96848d2 100644 --- a/src/CostResultRows.cs +++ b/src/Cost/CostResultRows.cs @@ -3,7 +3,7 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; -namespace AzureBillingExporter +namespace AzureBillingExporter.Cost { public class CostResultRows { @@ -17,13 +17,13 @@ public double Cost { return 0; } - + return double.Parse(preTaxCosts); } } - public string Date + public string Date { get { return GetByColumnName("UsageDate"); } } @@ -44,23 +44,23 @@ public string GetByColumnName(string name) public double GetValueByColumnName(string name) { var strVal = GetByColumnName(name); - + if (string.IsNullOrEmpty(strVal)) { return 0; } - + return double.Parse(strVal); } public List ColumnNames { get; } = new List(); public List Values { get; } = new List(); - + public static CostResultRows Cast(dynamic columns, dynamic singleRow) { var parsedRow = new CostResultRows(); parsedRow.Values.AddRange(ClearParse(singleRow.ToString())); - + foreach (var column in columns) { parsedRow.ColumnNames.Add(column.name.ToString()); @@ -85,4 +85,4 @@ private static IEnumerable ClearParse(dynamic s) return res; } } -} \ No newline at end of file +} diff --git a/src/CustomCollectorConfiguration.cs b/src/CustomCollectorConfiguration.cs deleted file mode 100644 index 7f225ad..0000000 --- a/src/CustomCollectorConfiguration.cs +++ /dev/null @@ -1,127 +0,0 @@ -using System.Collections.Generic; -using System.IO; -using System.Linq; -using Prometheus; -using YamlDotNet.Serialization; -using YamlDotNet.Serialization.NamingConventions; - -namespace AzureBillingExporter -{ - public class MetricConfigCollections - { - [YamlMember(Alias = "metrics", ApplyNamingConventions = false)] - public List Metrics { get; set; } - } - - public class MetricConfig - { - [YamlMember(Alias = "metric_name", ApplyNamingConventions = false)] - public string MetricName { get; set; } - - public string Type { get; set; } - public string Help { get; set; } - - [YamlMember(Alias = "key_labels", ApplyNamingConventions = false)] - public List KeyLabels { get; set; } - - [YamlMember(Alias = "static_labels", ApplyNamingConventions = false)] - public Dictionary StaticLabel { get; set; } - - public string Value { get; set; } - - public int? Limit { get; set; } - - [YamlMember(Alias = "replace_date_labels_to_enum", ApplyNamingConventions = false)] - public bool ReplaceDateLabelsToEnum { get; set; } - - [YamlMember(Alias = "query_file", ApplyNamingConventions = false)] - public string QueryFilePath { get; set; } - - } - - public class CustomCollectorConfiguration - { - public readonly Dictionary CustomGaugeMetrics = new Dictionary(); // - - private string CustomCollectorsConfigFile { get; } - public CustomCollectorConfiguration(string customCollectorsFilePath) - { - if (!string.IsNullOrEmpty(customCollectorsFilePath)) - { - CustomCollectorsConfigFile = customCollectorsFilePath; - } - else - { - CustomCollectorsConfigFile = "custom_collectors.yml"; - } - } - public void ReadCustomCollectorConfig() - { - if (!File.Exists(CustomCollectorsConfigFile)) - { - return; - } - - var configText = File.ReadAllText(CustomCollectorsConfigFile); - var input = new StringReader(configText); - - var deserializer = new DeserializerBuilder() - .WithNamingConvention(CamelCaseNamingConvention.Instance) - .Build(); - - var metricCollections = deserializer.Deserialize(input); - - CreatePrometheusMetrics(metricCollections); - } - - private void CreatePrometheusMetrics(MetricConfigCollections metricCollection) - { - foreach (var metricConfig in metricCollection.Metrics) - { - // Labels - var labelNames = new List(); - foreach (var staticLabel in metricConfig.StaticLabel) - { - labelNames.Add(staticLabel.Key); - } - foreach (var keyLabel in metricConfig.KeyLabels) - { - labelNames.Add(keyLabel); - } - - if (metricConfig.Type?.ToLower() == "gauge") - { - var gauge = Metrics.CreateGauge(metricConfig.MetricName, metricConfig.Help, - new GaugeConfiguration() - { - LabelNames = labelNames.ToArray() - }); - - CustomGaugeMetrics.Add(metricConfig, gauge); - } - } - } - - public void SetValues(MetricConfig key, CostResultRows customData) - { - var gauge = CustomGaugeMetrics[key]; - - var labelValues = new List(); - labelValues.AddRange(key.StaticLabel.Select(x => x.Value)); - foreach (var keyLabel in key.KeyLabels) - { - var dataColumnByKeyLabel = customData.GetByColumnName(keyLabel); - - if (key.ReplaceDateLabelsToEnum) - { - dataColumnByKeyLabel = DateEnumHelper.ReplaceDateValueToEnums(dataColumnByKeyLabel); - } - labelValues.Add(dataColumnByKeyLabel); - } - - gauge - .WithLabels(labelValues.ToArray()) - .Set(customData.GetValueByColumnName(key.Value)); - } - } -} diff --git a/src/DateEnumHelper.cs b/src/DateEnumHelper.cs deleted file mode 100644 index 23a044b..0000000 --- a/src/DateEnumHelper.cs +++ /dev/null @@ -1,82 +0,0 @@ -using System; -using System.Globalization; - -public static class DateEnumHelper -{ - public static string ReplaceDateValueToEnums(string originalDate, DateTime? nowDate = null) - { - var todayDate = nowDate ?? DateTime.Now; - - //================================================= - // Month - //================================================= - var currentMonthStart = new DateTime(todayDate.Year, todayDate.Month, 1); - if (originalDate == currentMonthStart.ToString("yyyy-MM-01T00:00:00", CultureInfo.InvariantCulture)) - { - return "current_month"; - } - - var previousMonthStart = new DateTime(todayDate.Year, todayDate.Month, 1); - previousMonthStart = previousMonthStart.AddMonths(-1); - if (originalDate == previousMonthStart.ToString("yyyy-MM-01T00:00:00", CultureInfo.InvariantCulture)) - { - return "previous_month"; - } - - var beforePreviousMonthStart = new DateTime(todayDate.Year, todayDate.Month, 1); - beforePreviousMonthStart = beforePreviousMonthStart.AddMonths(-2); - if (originalDate == beforePreviousMonthStart.ToString("yyyy-MM-01T00:00:00", CultureInfo.InvariantCulture)) - { - return "before_previous_month"; - } - - for (int i = 0; i < 12; i++) - { - var firstDayTodayMonth = new DateTime(todayDate.Year, todayDate.Month, 1); - var monthMinus = firstDayTodayMonth.AddMonths(-i); - - if (originalDate == monthMinus.ToString("yyyy-MM-01T00:00:00", CultureInfo.InvariantCulture)) - { - return $"month_minus_{i+1:D2}"; - } - } - //================================================= - - //================================================= - // Day - //================================================= - // "20200624" -> "today" - if (originalDate == todayDate.ToString("yyyyMMdd", CultureInfo.InvariantCulture)) - { - return "today"; - } - - var yesterday = new DateTime(todayDate.Year, todayDate.Month, todayDate.Day); - yesterday = yesterday.AddDays(-1); - if (originalDate == yesterday.ToString("yyyyMMdd", CultureInfo.InvariantCulture)) - { - return "yesterday"; - } - - var beforeYesterday = new DateTime(todayDate.Year, todayDate.Month, todayDate.Day); - beforeYesterday = beforeYesterday.AddDays(-2); - if (originalDate == beforeYesterday.ToString("yyyyMMdd", CultureInfo.InvariantCulture)) - { - return "before_yesterday"; - } - - for (int i = 0; i < 367; i++) - { - var thisDayDate = new DateTime(todayDate.Year, todayDate.Month, todayDate.Day); - var dayMinus = thisDayDate.AddDays(-i); - - if (originalDate == dayMinus.ToString("yyyyMMdd", CultureInfo.InvariantCulture)) - { - return $"day_minus_{i+1:D3}"; - } - } - - return originalDate; - //================================================= - } -} diff --git a/src/MetricsGrapper.cs b/src/MetricsGrapper.cs deleted file mode 100644 index 2c8040b..0000000 --- a/src/MetricsGrapper.cs +++ /dev/null @@ -1,68 +0,0 @@ -using System.Threading; -using System.Threading.Tasks; -using Prometheus; - -namespace AzureBillingExporter -{ - public class AzureBillingMetricsGrapper - { - private static readonly Gauge DailyCosts = - Metrics.CreateGauge( - "azure_billing_daily", - "Daily cost by today, yesterday and day before yesterday", - new GaugeConfiguration - { - LabelNames = new [] {"DateEnum"} - }); - - private static readonly Gauge MonthlyCosts = - Metrics.CreateGauge( - "azure_billing_monthly", - "This month costs", - new GaugeConfiguration - { - LabelNames = new [] {"DateEnum"} - }); - - private BillingQueryClient _billingQueryClient; - private CustomCollectorConfiguration _customCollectorConfiguration; - - public AzureBillingMetricsGrapper(BillingQueryClient billingQueryClient, CustomCollectorConfiguration customCollectorConfiguration) - { - _billingQueryClient = billingQueryClient; - _customCollectorConfiguration = customCollectorConfiguration; - _customCollectorConfiguration.ReadCustomCollectorConfig(); - } - - public async Task DownloadFromApi(CancellationToken cancel) - { - // Daily, monthly costs - await foreach(var dayData in (await _billingQueryClient.GetDailyData(cancel)).WithCancellation(cancel)) - { - var dayEnum = DateEnumHelper.ReplaceDateValueToEnums(dayData.GetByColumnName("UsageDate")); - - DailyCosts - .WithLabels(dayEnum) - .Set(dayData.Cost); - } - - await foreach(var dayData in (await _billingQueryClient.GetMonthlyData(cancel)).WithCancellation(cancel)) - { - var monthEnum = DateEnumHelper.ReplaceDateValueToEnums(dayData.GetByColumnName("BillingMonth")); - - MonthlyCosts - .WithLabels(monthEnum) - .Set(dayData.Cost); - } - - foreach (var (key, _) in _customCollectorConfiguration.CustomGaugeMetrics) - { - await foreach (var customData in - (await _billingQueryClient.GetCustomData(cancel, key.QueryFilePath)).WithCancellation(cancel)) - { - _customCollectorConfiguration.SetValues(key, customData); - } - } - } - } -} diff --git a/src/Program.cs b/src/Program.cs index fe03be7..2a01525 100644 --- a/src/Program.cs +++ b/src/Program.cs @@ -1,23 +1,73 @@ +using System; +using System.Threading.Tasks; +using AzureBillingExporter.Configuration; using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; +using Serilog; +using Serilog.Formatting.Elasticsearch; namespace AzureBillingExporter { - public class Program + public static class Program { - public static void Main(string[] args) + public static async Task Main(string[] args) { - CreateHostBuilder(args).Build().Run(); + // The initial "bootstrap" logger is able to log errors during start-up. It's completely replaced by the + // logger configured in `UseSerilog()` below, once configuration and dependency-injection have both been + // set up successfully. + Log.Logger = new LoggerConfiguration() + .WriteTo.Console() + .CreateBootstrapLogger(); + + Log.Information("Starting up"); + + try + { + await CreateHostBuilder(args).Build().RunAsync(); + + Log.Information("Stopped cleanly"); + return 0; + } + catch (Exception ex) + { + Log.Fatal(ex, "An unhandled exception occured during bootstrapping"); + return 1; + } + finally + { + Log.CloseAndFlush(); + } } - public static IHostBuilder CreateHostBuilder(string[] args) => - Host.CreateDefaultBuilder(args) - .ConfigureLogging(logging => - { - logging.ClearProviders(); - logging.AddConsole(); - }) + private static IHostBuilder CreateHostBuilder(string[] args) + { + var builder = Host.CreateDefaultBuilder(args); + + builder.UseSerilog(ConfigureLogger) .ConfigureWebHostDefaults(webBuilder => { webBuilder.UseStartup(); }); + + return builder; + } + + private static void ConfigureLogger(HostBuilderContext context, IServiceProvider services, + LoggerConfiguration configuration) + { + var config = context.Configuration.Get(); + + configuration + .ReadFrom.Configuration(context.Configuration) + .ReadFrom.Services(services) + .Enrich.FromLogContext(); + + if (config.LogsAtJsonFormat) + { + configuration.WriteTo.Console(new ElasticsearchJsonFormatter(inlineFields: true)); + } + else + { + configuration.WriteTo.Console(); + } + } } -} \ No newline at end of file +} diff --git a/src/PrometheusMetrics/CustomMetricsService.cs b/src/PrometheusMetrics/CustomMetricsService.cs new file mode 100644 index 0000000..c232d6c --- /dev/null +++ b/src/PrometheusMetrics/CustomMetricsService.cs @@ -0,0 +1,71 @@ +using System.Collections.Generic; +using System.Linq; +using AzureBillingExporter.Configuration; +using AzureBillingExporter.Configuration.Metrics; +using AzureBillingExporter.Cost; +using Prometheus; + +namespace AzureBillingExporter.PrometheusMetrics +{ + public class CustomMetricsService + { + public readonly Dictionary CustomGaugeMetrics = new Dictionary(); + + private readonly CustomCollectorConfiguration _customCollectorConfiguration; + public CustomMetricsService(CustomCollectorConfiguration customCollectorConfiguration) + { + _customCollectorConfiguration = customCollectorConfiguration; + } + + public void CreatePrometheusMetrics() + { + var customCollector = _customCollectorConfiguration.GetCustomCollectorConfig(); + foreach (var metricConfig in customCollector.Metrics) + { + // Labels + var labelNames = new List(); + foreach (var staticLabel in metricConfig.StaticLabel) + { + labelNames.Add(staticLabel.Key); + } + foreach (var keyLabel in metricConfig.KeyLabels) + { + labelNames.Add(keyLabel); + } + + if (metricConfig.Type?.ToLower() == "gauge") + { + var gauge = Metrics.CreateGauge(metricConfig.MetricName, metricConfig.Help, + new GaugeConfiguration() + { + LabelNames = labelNames.ToArray() + }); + + CustomGaugeMetrics.Add(metricConfig, gauge); + } + } + } + + public void SetValues(MetricConfig key, CostResultRows customData) + { + var gauge = CustomGaugeMetrics[key]; + + var labelValues = new List(); + labelValues.AddRange(key.StaticLabel.Select(x => x.Value)); + foreach (var keyLabel in key.KeyLabels) + { + var dataColumnByKeyLabel = customData.GetByColumnName(keyLabel); + + if (key.ReplaceDateLabelsToEnum) + { + dataColumnByKeyLabel = DateEnumHelper.ReplaceDateValueToEnums(dataColumnByKeyLabel); + } + labelValues.Add(dataColumnByKeyLabel); + } + + gauge + .WithLabels(labelValues.ToArray()) + .Set(customData.GetValueByColumnName(key.Value)); + } + } +} diff --git a/src/PrometheusMetrics/DateEnumHelper.cs b/src/PrometheusMetrics/DateEnumHelper.cs new file mode 100644 index 0000000..440357c --- /dev/null +++ b/src/PrometheusMetrics/DateEnumHelper.cs @@ -0,0 +1,85 @@ +using System; +using System.Globalization; + +namespace AzureBillingExporter.PrometheusMetrics +{ + public static class DateEnumHelper + { + public static string ReplaceDateValueToEnums(string originalDate, DateTime? nowDate = null) + { + var todayDate = nowDate ?? DateTime.Now; + + //================================================= + // Month + //================================================= + var currentMonthStart = new DateTime(todayDate.Year, todayDate.Month, 1); + if (originalDate == currentMonthStart.ToString("yyyy-MM-01T00:00:00", CultureInfo.InvariantCulture)) + { + return "current_month"; + } + + var previousMonthStart = new DateTime(todayDate.Year, todayDate.Month, 1); + previousMonthStart = previousMonthStart.AddMonths(-1); + if (originalDate == previousMonthStart.ToString("yyyy-MM-01T00:00:00", CultureInfo.InvariantCulture)) + { + return "previous_month"; + } + + var beforePreviousMonthStart = new DateTime(todayDate.Year, todayDate.Month, 1); + beforePreviousMonthStart = beforePreviousMonthStart.AddMonths(-2); + if (originalDate == beforePreviousMonthStart.ToString("yyyy-MM-01T00:00:00", CultureInfo.InvariantCulture)) + { + return "before_previous_month"; + } + + for (int i = 0; i < 12; i++) + { + var firstDayTodayMonth = new DateTime(todayDate.Year, todayDate.Month, 1); + var monthMinus = firstDayTodayMonth.AddMonths(-i); + + if (originalDate == monthMinus.ToString("yyyy-MM-01T00:00:00", CultureInfo.InvariantCulture)) + { + return $"month_minus_{i+1:D2}"; + } + } + //================================================= + + //================================================= + // Day + //================================================= + // "20200624" -> "today" + if (originalDate == todayDate.ToString("yyyyMMdd", CultureInfo.InvariantCulture)) + { + return "today"; + } + + var yesterday = new DateTime(todayDate.Year, todayDate.Month, todayDate.Day); + yesterday = yesterday.AddDays(-1); + if (originalDate == yesterday.ToString("yyyyMMdd", CultureInfo.InvariantCulture)) + { + return "yesterday"; + } + + var beforeYesterday = new DateTime(todayDate.Year, todayDate.Month, todayDate.Day); + beforeYesterday = beforeYesterday.AddDays(-2); + if (originalDate == beforeYesterday.ToString("yyyyMMdd", CultureInfo.InvariantCulture)) + { + return "before_yesterday"; + } + + for (int i = 0; i < 367; i++) + { + var thisDayDate = new DateTime(todayDate.Year, todayDate.Month, todayDate.Day); + var dayMinus = thisDayDate.AddDays(-i); + + if (originalDate == dayMinus.ToString("yyyyMMdd", CultureInfo.InvariantCulture)) + { + return $"day_minus_{i+1:D3}"; + } + } + + return originalDate; + //================================================= + } + } +} diff --git a/src/PrometheusMetrics/MetricsUpdater.cs b/src/PrometheusMetrics/MetricsUpdater.cs new file mode 100644 index 0000000..5b17d9d --- /dev/null +++ b/src/PrometheusMetrics/MetricsUpdater.cs @@ -0,0 +1,74 @@ +using AzureBillingExporter.Cost; +using Prometheus; + +namespace AzureBillingExporter.PrometheusMetrics +{ + public class MetricsUpdater + { + private static readonly Gauge DailyCosts = + Metrics.CreateGauge( + "azure_billing_daily", + "Daily cost by today, yesterday and day before yesterday", + new GaugeConfiguration + { + LabelNames = new[] { "DateEnum" } + }); + + private static readonly Gauge MonthlyCosts = + Metrics.CreateGauge( + "azure_billing_monthly", + "This month costs", + new GaugeConfiguration + { + LabelNames = new[] { "DateEnum" } + }); + + private readonly CustomMetricsService _customMetricsService; + private readonly CostDataCache _costDataCache; + private bool _metricsCreated; + + public MetricsUpdater( + CustomMetricsService customMetricsService, + CostDataCache costDataCache) + { + _customMetricsService = customMetricsService; + _costDataCache = costDataCache; + } + + public void Update() + { + if (!_metricsCreated) + { + _customMetricsService.CreatePrometheusMetrics(); + _metricsCreated = true; + } + + foreach (var data in _costDataCache.GetDailyCost()) + { + var dayEnum = DateEnumHelper.ReplaceDateValueToEnums(data.GetByColumnName("UsageDate")); + + DailyCosts + .WithLabels(dayEnum) + .Set(data.Cost); + } + + foreach (var data in _costDataCache.GetMonthlyCost()) + { + var monthEnum = DateEnumHelper.ReplaceDateValueToEnums(data.GetByColumnName("BillingMonth")); + + MonthlyCosts + .WithLabels(monthEnum) + .Set(data.Cost); + } + + foreach (var metric in _customMetricsService.CustomGaugeMetrics) + { + var costs = _costDataCache.GetCostByKey(metric.Key.QueryFilePath); + foreach (var cost in costs) + { + _customMetricsService.SetValues(metric.Key, cost); + } + } + } + } +} diff --git a/src/Startup.cs b/src/Startup.cs index 5babbae..ee4704f 100644 --- a/src/Startup.cs +++ b/src/Startup.cs @@ -1,5 +1,9 @@ using System; using AzureBillingExporter.AzureApi; +using AzureBillingExporter.Configuration; +using AzureBillingExporter.Cost; +using AzureBillingExporter.PrometheusMetrics; +using Dodo.HttpClientResiliencePolicies; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.DependencyInjection; @@ -18,26 +22,32 @@ public Startup(IConfiguration configuration) Configuration = configuration; } - public IConfiguration Configuration { get; } + private IConfiguration Configuration { get; } // This method gets called by the runtime. Use this method to add services to the container. // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 public void ConfigureServices(IServiceCollection services) { + services.AddJsonClient( + new Uri("https://management.azure.com"), "AzureBillingExporter"); services.Configure(Configuration.GetSection("ApiSettings")); services.AddSingleton(resolver => resolver.GetRequiredService>().Value); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); - services.AddSingleton(serviceProvider=> + services.AddSingleton(serviceProvider => Configuration.Get()); + + services.AddSingleton(serviceProvider => { var customCollectorsFilePath = Configuration["CustomCollectorsFilePath"]; return new CustomCollectorConfiguration(customCollectorsFilePath); }); - services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddHostedService(resolver => { @@ -51,6 +61,21 @@ public void ConfigureServices(IServiceCollection services) logger, TimeSpan.FromSeconds(10)); }); + + services.AddHostedService(resolver => + { + var billingQueryClient = resolver.GetRequiredService(); + var costDataCache = resolver.GetRequiredService(); + var customCollectorConfiguration = resolver.GetRequiredService(); + var environmentConfiguration = resolver.GetRequiredService(); + var logger = resolver.GetRequiredService>(); + return new BackgroundCostCollectorHostedService( + billingQueryClient, + costDataCache, + customCollectorConfiguration, + environmentConfiguration, + logger); + }); } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. @@ -65,17 +90,11 @@ public void Configure( app.UseRouting(); - var billingGrapper = app.ApplicationServices.GetService(); - Metrics.DefaultRegistry.AddBeforeCollectCallback(async (cancel) => - { - await billingGrapper.DownloadFromApi(cancel); - }); + var metricsUpdater = app.ApplicationServices.GetService(); + Metrics.DefaultRegistry.AddBeforeCollectCallback(() => { metricsUpdater.Update(); }); // ASP.NET Core 3 or newer - app.UseEndpoints(endpoints => - { - endpoints.MapMetrics(); - }); + app.UseEndpoints(endpoints => { endpoints.MapMetrics(); }); } } } diff --git a/src/appsettings.json b/src/appsettings.json index 7a020b8..524fbfa 100644 --- a/src/appsettings.json +++ b/src/appsettings.json @@ -1,10 +1,14 @@ { - "Logging": { - "LogLevel": { + "Serilog": { + "MinimumLevel": { "Default": "Information", - "Microsoft": "Warning", - "Microsoft.Hosting.Lifetime": "Warning" + "Override": { + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Warning", + "System.Net.Http.HttpClient": "Warning" + } } }, + "LogsAtJsonFormat": true, "AllowedHosts": "*" }