From dcb5a640eed9bd183ff93d11df885ad65edc2f03 Mon Sep 17 00:00:00 2001 From: Renat Shajmardanov Date: Fri, 21 Jan 2022 10:19:12 +0300 Subject: [PATCH 1/8] #3057414 add json logs, run queries on parallel, add retry policy --- src/AzureApi/AzureCostManagementClient.cs | 25 +++++----- src/AzureBillingExporter.csproj | 9 +++- src/BillingQueryClient.cs | 35 +++++++------- src/MetricsGrapper.cs | 57 +++++++++++++++++++++-- src/Program.cs | 41 ++++++++++++++-- src/Startup.cs | 3 ++ src/appsettings.json | 11 +++-- 7 files changed, 137 insertions(+), 44 deletions(-) diff --git a/src/AzureApi/AzureCostManagementClient.cs b/src/AzureApi/AzureCostManagementClient.cs index b668621..3488f83 100644 --- a/src/AzureApi/AzureCostManagementClient.cs +++ b/src/AzureApi/AzureCostManagementClient.cs @@ -15,24 +15,27 @@ public class AzureCostManagementClient { private readonly ApiSettings _apiSettings; private readonly IAccessTokenProvider _accessTokenProvider; - private readonly ILogger _logger; + private readonly ILogger _logger; + private readonly IHttpClientFactory _httpClientFactory; public AzureCostManagementClient(ApiSettings apiSettings, - IAccessTokenProvider accessTokenProvider, - ILogger logger) + IAccessTokenProvider accessTokenProvider, + ILogger logger, + IHttpClientFactory httpClientFactory) { _apiSettings = apiSettings; _accessTokenProvider = accessTokenProvider; _logger = logger; + _httpClientFactory = httpClientFactory; } - - public async IAsyncEnumerable ExecuteBillingQuery(string billingQuery, [EnumeratorCancellation] CancellationToken cancel, BillingQueryClient billingQueryClient) + + public async IAsyncEnumerable ExecuteBillingQuery(string billingQuery, [EnumeratorCancellation] CancellationToken cancel) { + var client = _httpClientFactory.CreateClient(nameof(AzureCostManagementClient)); + var azureManagementUrl = $"https://management.azure.com/subscriptions/{_apiSettings.SubscriptionId}/providers/Microsoft.CostManagement/query?api-version=2019-10-01"; - using var httpClient = new HttpClient(); - _logger.LogTrace($"Billing query {billingQuery}"); var request = new HttpRequestMessage { @@ -44,18 +47,18 @@ public async IAsyncEnumerable ExecuteBillingQuery(string billing "application/json") }; request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - + var accessToken = _accessTokenProvider.GetAccessToken(); request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", accessToken); - var response = await httpClient.SendAsync(request, cancel); + var response = await client.SendAsync(request, cancel); if (response.StatusCode != HttpStatusCode.OK) { throw new Exception($"{response.StatusCode} {await response.Content.ReadAsStringAsync()}"); } - + var responseContent = await response.Content.ReadAsStringAsync(); _logger.LogTrace($"Billing query response result {responseContent}"); dynamic json = JsonConvert.DeserializeObject(responseContent); @@ -66,4 +69,4 @@ public async IAsyncEnumerable ExecuteBillingQuery(string billing } } } -} \ No newline at end of file +} diff --git a/src/AzureBillingExporter.csproj b/src/AzureBillingExporter.csproj index de0122f..dd090dd 100644 --- a/src/AzureBillingExporter.csproj +++ b/src/AzureBillingExporter.csproj @@ -7,13 +7,20 @@ + - + + + + + + + diff --git a/src/BillingQueryClient.cs b/src/BillingQueryClient.cs index ad3e7d2..96be1a8 100644 --- a/src/BillingQueryClient.cs +++ b/src/BillingQueryClient.cs @@ -13,22 +13,19 @@ namespace AzureBillingExporter public class BillingQueryClient { private readonly AzureCostManagementClient _costManagementClient; - private readonly ILogger _logger; public BillingQueryClient( - AzureCostManagementClient costManagementClient, - ILogger logger) + AzureCostManagementClient costManagementClient) { _costManagementClient = costManagementClient; - _logger = logger; } public async Task> GetCustomData(CancellationToken cancel, string templateFileName) { var billingQuery = await GenerateBillingQuery(DateTime.MaxValue, DateTime.MaxValue, "None", templateFileName); - return _costManagementClient.ExecuteBillingQuery(billingQuery, cancel, this); + return _costManagementClient.ExecuteBillingQuery(billingQuery, cancel); } - + public async Task> GetDailyData(CancellationToken cancel) { var dateTimeNow = DateTime.Now; @@ -38,45 +35,45 @@ public async Task> GetDailyData(CancellationTok var granularity = "Daily"; var billingQuery = await GenerateBillingQuery(dateStart, dateEnd, granularity); - return _costManagementClient.ExecuteBillingQuery(billingQuery, cancel, this); + return _costManagementClient.ExecuteBillingQuery(billingQuery, cancel); } - + public async Task> GetMonthlyData(CancellationToken cancel) { var dateTimeNow = DateTime.Now; - + var dateStart = new DateTime(dateTimeNow.Year, dateTimeNow.Month, 1); dateStart = dateStart.AddMonths(-2); var dateEnd = new DateTime(dateTimeNow.Year, dateTimeNow.Month, dateTimeNow.Day, 23, 59, 59); var granularity = "Monthly"; var billingQuery = await GenerateBillingQuery(dateStart, dateEnd, granularity); - return _costManagementClient.ExecuteBillingQuery(billingQuery, cancel, this); + return _costManagementClient.ExecuteBillingQuery(billingQuery, cancel); } private async Task GenerateBillingQuery(DateTime dateStart, DateTime dateEnd, string granularity = "None", string templateFile = "./queries/get_daily_or_monthly_costs.json") { var dateTimeNow = DateTime.Now; - + var templateQuery =await File.ReadAllTextAsync(templateFile); var template = Template.Parse(templateQuery); - + var currentMonthStart = new DateTime(dateTimeNow.Year, dateTimeNow.Month, 1); - + var prevMonthStart = new DateTime(dateTimeNow.Year, dateTimeNow.Month, 1); prevMonthStart = prevMonthStart.AddMonths(-1); var beforePrevMonthStart = new DateTime(dateTimeNow.Year, dateTimeNow.Month, 1); beforePrevMonthStart = beforePrevMonthStart.AddMonths(-2); - + var todayEnd = new DateTime(dateTimeNow.Year, dateTimeNow.Month, dateTimeNow.Day, 23, 59, 59); - + var yesterdayStart = new DateTime(dateTimeNow.Year, dateTimeNow.Month, dateTimeNow.Day); yesterdayStart = yesterdayStart.AddDays(-1); - + var weekAgo = new DateTime(dateTimeNow.Year, dateTimeNow.Month, dateTimeNow.Day); weekAgo = weekAgo.AddDays(-7); - + //YearAgo var yearAgo = new DateTime(dateTimeNow.Year, dateTimeNow.Month, 1); yearAgo = yearAgo.AddYears(-1); @@ -84,7 +81,7 @@ private async Task GenerateBillingQuery(DateTime dateStart, DateTime dat return template.Render(Hash.FromAnonymousObject(new { - DayStart = dateStart.ToString("o", CultureInfo.InvariantCulture), + DayStart = dateStart.ToString("o", CultureInfo.InvariantCulture), DayEnd = dateEnd.ToString("o", CultureInfo.InvariantCulture), CurrentMonthStart = currentMonthStart.ToString("o", CultureInfo.InvariantCulture), PrevMonthStart = prevMonthStart.ToString("o", CultureInfo.InvariantCulture), @@ -97,4 +94,4 @@ private async Task GenerateBillingQuery(DateTime dateStart, DateTime dat })); } } -} \ No newline at end of file +} diff --git a/src/MetricsGrapper.cs b/src/MetricsGrapper.cs index 2c8040b..328479d 100644 --- a/src/MetricsGrapper.cs +++ b/src/MetricsGrapper.cs @@ -1,11 +1,17 @@ +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using Prometheus; namespace AzureBillingExporter { public class AzureBillingMetricsGrapper { + private readonly ILogger _logger; + private static readonly Gauge DailyCosts = Metrics.CreateGauge( "azure_billing_daily", @@ -27,16 +33,24 @@ public class AzureBillingMetricsGrapper private BillingQueryClient _billingQueryClient; private CustomCollectorConfiguration _customCollectorConfiguration; - public AzureBillingMetricsGrapper(BillingQueryClient billingQueryClient, CustomCollectorConfiguration customCollectorConfiguration) + public AzureBillingMetricsGrapper(BillingQueryClient billingQueryClient, + CustomCollectorConfiguration customCollectorConfiguration, + ILogger logger) { _billingQueryClient = billingQueryClient; _customCollectorConfiguration = customCollectorConfiguration; + _logger = logger; _customCollectorConfiguration.ReadCustomCollectorConfig(); } public async Task DownloadFromApi(CancellationToken cancel) { + _logger.Log(LogLevel.Information, "Start getting data for metrics"); + var timer = new Stopwatch(); + timer.Start(); + // Daily, monthly costs + _logger.Log(LogLevel.Debug, "Get daily data"); await foreach(var dayData in (await _billingQueryClient.GetDailyData(cancel)).WithCancellation(cancel)) { var dayEnum = DateEnumHelper.ReplaceDateValueToEnums(dayData.GetByColumnName("UsageDate")); @@ -46,6 +60,7 @@ public async Task DownloadFromApi(CancellationToken cancel) .Set(dayData.Cost); } + _logger.Log(LogLevel.Debug, "Get monthly data"); await foreach(var dayData in (await _billingQueryClient.GetMonthlyData(cancel)).WithCancellation(cancel)) { var monthEnum = DateEnumHelper.ReplaceDateValueToEnums(dayData.GetByColumnName("BillingMonth")); @@ -55,14 +70,46 @@ public async Task DownloadFromApi(CancellationToken cancel) .Set(dayData.Cost); } - foreach (var (key, _) in _customCollectorConfiguration.CustomGaugeMetrics) + _logger.Log(LogLevel.Debug, "Get custom data in parallel"); + var customMetricsDataTasks = _customCollectorConfiguration.CustomGaugeMetrics.Keys + .Select(x => StartMetricDataTask(x, cancel)).ToArray(); + + Task.WaitAll(customMetricsDataTasks, cancel); + foreach (var task in customMetricsDataTasks) + { + if (task != null && task.Exception != null) + throw task.Exception; + } + + foreach (var task in customMetricsDataTasks) { - await foreach (var customData in - (await _billingQueryClient.GetCustomData(cancel, key.QueryFilePath)).WithCancellation(cancel)) + await foreach (var customData in task.Result.Data.WithCancellation(cancel)) { - _customCollectorConfiguration.SetValues(key, customData); + _customCollectorConfiguration.SetValues(task.Result.Config, customData); } } + + timer.Stop(); + _logger.Log(LogLevel.Debug, "Metrics get total time: " + timer.Elapsed); + + _logger.Log(LogLevel.Information, "Finish getting data for metrics"); } + + private async Task StartMetricDataTask(MetricConfig config, CancellationToken cancel) + { + _logger.Log(LogLevel.Debug, $"Get custom data for {config.MetricName} metric, query file path - {config.QueryFilePath}"); + var data = await _billingQueryClient.GetCustomData(cancel, config.QueryFilePath); + return new MetricsData + { + Config = config, + Data = data + }; + } + } + + class MetricsData + { + public MetricConfig Config { get; set; } + public IAsyncEnumerable Data { get; set; } } } diff --git a/src/Program.cs b/src/Program.cs index fe03be7..05b97a3 100644 --- a/src/Program.cs +++ b/src/Program.cs @@ -1,14 +1,43 @@ +using System; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Serilog; +using Serilog.Core; +using Serilog.Events; +using Serilog.Formatting.Elasticsearch; namespace AzureBillingExporter { public class Program { - public static void Main(string[] args) + public static int Main(string[] args) { - CreateHostBuilder(args).Build().Run(); + // The initial "bootstrap" logger is able to log errors during start-up. It's completely replaced by the + // logger configured in `UseSerilog()` below, once configuration and dependency-injection have both been + // set up successfully. + Log.Logger = new LoggerConfiguration() + .WriteTo.Console(new ElasticsearchJsonFormatter(inlineFields: true)) + .CreateBootstrapLogger(); + + Log.Information("Starting up"); + + try + { + CreateHostBuilder(args).Build().Run(); + + Log.Information("Stopped cleanly"); + return 0; + } + catch (Exception ex) + { + Log.Fatal(ex, "An unhandled exception occured during bootstrapping"); + return 1; + } + finally + { + Log.CloseAndFlush(); + } } public static IHostBuilder CreateHostBuilder(string[] args) => @@ -16,8 +45,12 @@ public static IHostBuilder CreateHostBuilder(string[] args) => .ConfigureLogging(logging => { logging.ClearProviders(); - logging.AddConsole(); }) + .UseSerilog((context, services, configuration) => configuration + .ReadFrom.Configuration(context.Configuration) + .ReadFrom.Services(services) + .Enrich.FromLogContext() + .WriteTo.Console(new ElasticsearchJsonFormatter(inlineFields: true))) .ConfigureWebHostDefaults(webBuilder => { webBuilder.UseStartup(); }); } -} \ No newline at end of file +} diff --git a/src/Startup.cs b/src/Startup.cs index 5babbae..e0cbdab 100644 --- a/src/Startup.cs +++ b/src/Startup.cs @@ -1,5 +1,6 @@ using System; using AzureBillingExporter.AzureApi; +using Dodo.HttpClientResiliencePolicies; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.DependencyInjection; @@ -24,6 +25,8 @@ public Startup(IConfiguration configuration) // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 public void ConfigureServices(IServiceCollection services) { + services.AddJsonClient( + new Uri("https://management.azure.com"), "AzureBillingExporter"); services.Configure(Configuration.GetSection("ApiSettings")); services.AddSingleton(resolver => resolver.GetRequiredService>().Value); services.AddSingleton(); diff --git a/src/appsettings.json b/src/appsettings.json index 7a020b8..59d1874 100644 --- a/src/appsettings.json +++ b/src/appsettings.json @@ -1,9 +1,12 @@ { - "Logging": { - "LogLevel": { + "Serilog": { + "MinimumLevel": { "Default": "Information", - "Microsoft": "Warning", - "Microsoft.Hosting.Lifetime": "Warning" + "Override": { + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Warning", + "System.Net.Http.HttpClient": "Warning" + } } }, "AllowedHosts": "*" From 0230549ee227b0e5c9f21316dc324a0edfb809d1 Mon Sep 17 00:00:00 2001 From: Renat Shajmardanov Date: Fri, 21 Jan 2022 12:25:31 +0300 Subject: [PATCH 2/8] #3057414 add LogsAtJsonFormat setting --- src/AzureBillingExporter.csproj | 1 - src/Configuration/EnvironmentConfiguration.cs | 7 +++ src/Program.cs | 48 +++++++++++++------ src/appsettings.json | 1 + 4 files changed, 42 insertions(+), 15 deletions(-) create mode 100644 src/Configuration/EnvironmentConfiguration.cs diff --git a/src/AzureBillingExporter.csproj b/src/AzureBillingExporter.csproj index dd090dd..d0d30d8 100644 --- a/src/AzureBillingExporter.csproj +++ b/src/AzureBillingExporter.csproj @@ -20,7 +20,6 @@ - diff --git a/src/Configuration/EnvironmentConfiguration.cs b/src/Configuration/EnvironmentConfiguration.cs new file mode 100644 index 0000000..aff7dd3 --- /dev/null +++ b/src/Configuration/EnvironmentConfiguration.cs @@ -0,0 +1,7 @@ +namespace AzureBillingExporter.Configuration +{ + public class EnvironmentConfiguration + { + public bool LogsAtJsonFormat { get; set; } + } +} diff --git a/src/Program.cs b/src/Program.cs index 05b97a3..09aa914 100644 --- a/src/Program.cs +++ b/src/Program.cs @@ -1,5 +1,8 @@ using System; +using System.Threading.Tasks; +using AzureBillingExporter.Configuration; using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Serilog; @@ -11,20 +14,20 @@ namespace AzureBillingExporter { public class Program { - public static int Main(string[] args) + public static async Task Main(string[] args) { // The initial "bootstrap" logger is able to log errors during start-up. It's completely replaced by the // logger configured in `UseSerilog()` below, once configuration and dependency-injection have both been // set up successfully. Log.Logger = new LoggerConfiguration() - .WriteTo.Console(new ElasticsearchJsonFormatter(inlineFields: true)) + .WriteTo.Console() .CreateBootstrapLogger(); Log.Information("Starting up"); try { - CreateHostBuilder(args).Build().Run(); + await CreateHostBuilder(args).Build().RunAsync(); Log.Information("Stopped cleanly"); return 0; @@ -40,17 +43,34 @@ public static int Main(string[] args) } } - public static IHostBuilder CreateHostBuilder(string[] args) => - Host.CreateDefaultBuilder(args) - .ConfigureLogging(logging => - { - logging.ClearProviders(); - }) - .UseSerilog((context, services, configuration) => configuration - .ReadFrom.Configuration(context.Configuration) - .ReadFrom.Services(services) - .Enrich.FromLogContext() - .WriteTo.Console(new ElasticsearchJsonFormatter(inlineFields: true))) + private static IHostBuilder CreateHostBuilder(string[] args) + { + var builder = Host.CreateDefaultBuilder(args); + + builder.UseSerilog(ConfigureLogger) .ConfigureWebHostDefaults(webBuilder => { webBuilder.UseStartup(); }); + + return builder; + } + + private static void ConfigureLogger(HostBuilderContext context, IServiceProvider services, + LoggerConfiguration configuration) + { + var config = context.Configuration.Get(); + + configuration + .ReadFrom.Configuration(context.Configuration) + .ReadFrom.Services(services) + .Enrich.FromLogContext(); + + if (config.LogsAtJsonFormat) + { + configuration.WriteTo.Console(new ElasticsearchJsonFormatter(inlineFields: true)); + } + else + { + configuration.WriteTo.Console(); + } + } } } diff --git a/src/appsettings.json b/src/appsettings.json index 59d1874..524fbfa 100644 --- a/src/appsettings.json +++ b/src/appsettings.json @@ -9,5 +9,6 @@ } } }, + "LogsAtJsonFormat": true, "AllowedHosts": "*" } From 69417233f5d4668e02cd3d0d3057ec906aa429b0 Mon Sep 17 00:00:00 2001 From: Renat Shajmardanov Date: Fri, 21 Jan 2022 14:05:27 +0300 Subject: [PATCH 3/8] #3057414 cal daily/monthly data at background process --- ...kgroundAccessTokenProviderHostedService.cs | 8 +- src/BackgroundCostCollectorHostedService.cs | 102 ++++++++++++++++++ src/CostDataCache.cs | 31 ++++++ src/MetricsGrapper.cs | 49 +++++---- src/Startup.cs | 12 +++ 5 files changed, 175 insertions(+), 27 deletions(-) create mode 100644 src/BackgroundCostCollectorHostedService.cs create mode 100644 src/CostDataCache.cs diff --git a/src/AzureApi/BackgroundAccessTokenProviderHostedService.cs b/src/AzureApi/BackgroundAccessTokenProviderHostedService.cs index 7904df1..913396c 100644 --- a/src/AzureApi/BackgroundAccessTokenProviderHostedService.cs +++ b/src/AzureApi/BackgroundAccessTokenProviderHostedService.cs @@ -54,13 +54,13 @@ public async Task StopAsync(CancellationToken cancellationToken) { if (_executingTask == null) { - _logger.LogInformation("Devices API background access tokens refreshing hosted service - Already stopped."); + _logger.LogInformation("Azure Billing API background access tokens refreshing hosted service - Already stopped."); return; } try { - _logger.LogInformation("Devices API background access tokens refreshing hosted service - Stopping..."); + _logger.LogInformation("Azure Billing API background access tokens refreshing hosted service - Stopping..."); // Signal cancellation to the executing method _stoppingCts.Cancel(); } @@ -72,7 +72,7 @@ await Task.WhenAny( Task.Delay(Timeout.Infinite, cancellationToken)); } - _logger.LogInformation("Devices API background access tokens refreshing hosted service - Stopped."); + _logger.LogInformation("Azure Billing API background access tokens refreshing hosted service - Stopped."); } private async Task StartRefreshingAccessTokensInBackgroundAsync(CancellationToken cancellationToken) @@ -120,4 +120,4 @@ private async Task StartRefreshingAccessTokensInBackgroundAsync(CancellationToke } } } -} \ No newline at end of file +} diff --git a/src/BackgroundCostCollectorHostedService.cs b/src/BackgroundCostCollectorHostedService.cs new file mode 100644 index 0000000..e736533 --- /dev/null +++ b/src/BackgroundCostCollectorHostedService.cs @@ -0,0 +1,102 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace AzureBillingExporter +{ + public class BackgroundCostCollectorHostedService : IHostedService + { + private readonly BillingQueryClient _billingQueryClient; + private readonly CostDataCache _costDataCache; + private readonly ILogger _logger; + + private readonly CancellationTokenSource _stoppingCts = new CancellationTokenSource(); + private Task _executingTask = null!; + + + public BackgroundCostCollectorHostedService( + BillingQueryClient billingQueryClient, + CostDataCache costDataCache, + ILogger logger) + { + _billingQueryClient = billingQueryClient ?? throw new ArgumentNullException(nameof(billingQueryClient)); + _costDataCache = costDataCache ?? throw new ArgumentNullException(nameof(costDataCache)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public Task StartAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("Azure Billing data collector background hosted service - Starting..."); + _executingTask = StartCollectingDataInBackgroundAsync(_stoppingCts.Token); + if (_executingTask.IsCompleted) + { + return _executingTask; + } + + _logger.LogInformation("Azure Billing data collector background hosted service - Has started."); + return Task.CompletedTask; + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + if (_executingTask == null) + { + _logger.LogInformation("Azure Billing data collector background hosted service - Already stopped."); + return; + } + + try + { + _logger.LogInformation("Azure Billing data collector background hosted service - Stopping..."); + // Signal cancellation to the executing method + _stoppingCts.Cancel(); + } + finally + { + // Wait until the task completes or the stop token triggers + await Task.WhenAny( + _executingTask, + Task.Delay(Timeout.Infinite, cancellationToken)); + } + + _logger.LogInformation("Azure Billing data collector background hosted service - Stopped."); + } + + private async Task StartCollectingDataInBackgroundAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + while (!cancellationToken.IsCancellationRequested) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + // Daily, monthly costs + _logger.Log(LogLevel.Debug, "Get daily data"); + var dailyData = new List(); + await foreach(var data in (await _billingQueryClient.GetDailyData(cancellationToken)).WithCancellation(cancellationToken)) + { + dailyData.Add(data); + } + _costDataCache.SetDailyCost(dailyData); + + _logger.Log(LogLevel.Debug, "Get monthly data"); + var monthlyData = new List(); + await foreach(var data in (await _billingQueryClient.GetMonthlyData(cancellationToken)).WithCancellation(cancellationToken)) + { + monthlyData.Add(data); + } + _costDataCache.SetMonthlyCost(monthlyData); + } + catch (Exception ex) + { + _logger.LogError(new EventId(0), ex, ex.Message); + } + + await Task.Delay(TimeSpan.FromMinutes(1), cancellationToken); + } + } + } +} diff --git a/src/CostDataCache.cs b/src/CostDataCache.cs new file mode 100644 index 0000000..b66edc2 --- /dev/null +++ b/src/CostDataCache.cs @@ -0,0 +1,31 @@ +using System.Collections.Concurrent; +using System.Collections.Generic; + +namespace AzureBillingExporter +{ + public class CostDataCache + { + private readonly ConcurrentDictionary> _costDataCache = + new ConcurrentDictionary>(); + + public List GetDailyCost() + { + return _costDataCache.ContainsKey("Daily") ? _costDataCache["Daily"] : new List(); + } + + public void SetDailyCost(List data) + { + _costDataCache["Daily"] = data; + } + + public List GetMonthlyCost() + { + return _costDataCache.ContainsKey("Monthly") ? _costDataCache["Monthly"] : new List(); + } + + public void SetMonthlyCost(List data) + { + _costDataCache["Monthly"] = data; + } + } +} diff --git a/src/MetricsGrapper.cs b/src/MetricsGrapper.cs index 328479d..2c92821 100644 --- a/src/MetricsGrapper.cs +++ b/src/MetricsGrapper.cs @@ -32,13 +32,16 @@ public class AzureBillingMetricsGrapper private BillingQueryClient _billingQueryClient; private CustomCollectorConfiguration _customCollectorConfiguration; + private CostDataCache _сostDataCache; public AzureBillingMetricsGrapper(BillingQueryClient billingQueryClient, CustomCollectorConfiguration customCollectorConfiguration, + CostDataCache сostDataCache, ILogger logger) { _billingQueryClient = billingQueryClient; _customCollectorConfiguration = customCollectorConfiguration; + _сostDataCache = сostDataCache; _logger = logger; _customCollectorConfiguration.ReadCustomCollectorConfig(); } @@ -51,43 +54,43 @@ public async Task DownloadFromApi(CancellationToken cancel) // Daily, monthly costs _logger.Log(LogLevel.Debug, "Get daily data"); - await foreach(var dayData in (await _billingQueryClient.GetDailyData(cancel)).WithCancellation(cancel)) + foreach(var data in _сostDataCache.GetDailyCost()) { - var dayEnum = DateEnumHelper.ReplaceDateValueToEnums(dayData.GetByColumnName("UsageDate")); + var dayEnum = DateEnumHelper.ReplaceDateValueToEnums(data.GetByColumnName("UsageDate")); DailyCosts .WithLabels(dayEnum) - .Set(dayData.Cost); + .Set(data.Cost); } _logger.Log(LogLevel.Debug, "Get monthly data"); - await foreach(var dayData in (await _billingQueryClient.GetMonthlyData(cancel)).WithCancellation(cancel)) + foreach(var data in _сostDataCache.GetMonthlyCost()) { - var monthEnum = DateEnumHelper.ReplaceDateValueToEnums(dayData.GetByColumnName("BillingMonth")); + var monthEnum = DateEnumHelper.ReplaceDateValueToEnums(data.GetByColumnName("BillingMonth")); MonthlyCosts .WithLabels(monthEnum) - .Set(dayData.Cost); + .Set(data.Cost); } _logger.Log(LogLevel.Debug, "Get custom data in parallel"); - var customMetricsDataTasks = _customCollectorConfiguration.CustomGaugeMetrics.Keys - .Select(x => StartMetricDataTask(x, cancel)).ToArray(); - - Task.WaitAll(customMetricsDataTasks, cancel); - foreach (var task in customMetricsDataTasks) - { - if (task != null && task.Exception != null) - throw task.Exception; - } - - foreach (var task in customMetricsDataTasks) - { - await foreach (var customData in task.Result.Data.WithCancellation(cancel)) - { - _customCollectorConfiguration.SetValues(task.Result.Config, customData); - } - } + // var customMetricsDataTasks = _customCollectorConfiguration.CustomGaugeMetrics.Keys + // .Select(x => StartMetricDataTask(x, cancel)).ToArray(); + // + // Task.WaitAll(customMetricsDataTasks, cancel); + // foreach (var task in customMetricsDataTasks) + // { + // if (task != null && task.Exception != null) + // throw task.Exception; + // } + // + // foreach (var task in customMetricsDataTasks) + // { + // await foreach (var customData in task.Result.Data.WithCancellation(cancel)) + // { + // _customCollectorConfiguration.SetValues(task.Result.Config, customData); + // } + // } timer.Stop(); _logger.Log(LogLevel.Debug, "Metrics get total time: " + timer.Elapsed); diff --git a/src/Startup.cs b/src/Startup.cs index e0cbdab..1ed69f9 100644 --- a/src/Startup.cs +++ b/src/Startup.cs @@ -32,6 +32,7 @@ public void ConfigureServices(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(serviceProvider=> { @@ -54,6 +55,17 @@ public void ConfigureServices(IServiceCollection services) logger, TimeSpan.FromSeconds(10)); }); + + services.AddHostedService(resolver => + { + var billingQueryClient = resolver.GetRequiredService(); + var costDataCache = resolver.GetRequiredService(); + var logger = resolver.GetRequiredService>(); + return new BackgroundCostCollectorHostedService( + billingQueryClient, + costDataCache, + logger); + }); } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. From 9628a085fda7f2db6516ac2c8dab214546d9a595 Mon Sep 17 00:00:00 2001 From: Renat Shajmardanov Date: Fri, 21 Jan 2022 16:21:01 +0300 Subject: [PATCH 4/8] #3057414 get metrics in background --- .../DateEnumHelperTests.cs | 1 + src/AzureApi/AccessTokenFactory.cs | 1 + src/AzureApi/AzureCostManagementClient.cs | 6 + src/AzureApi/TooManyRequestsException.cs | 11 ++ src/BackgroundCostCollectorHostedService.cs | 102 ---------- .../CustomCollectorConfiguration.cs | 40 ++++ .../Metrics/CustomCollectorConfig.cs | 11 ++ src/Configuration/Metrics/MetricConfig.cs | 31 +++ .../BackgroundCostCollectorHostedService.cs | 185 ++++++++++++++++++ src/{ => Cost}/BillingQueryClient.cs | 3 +- src/Cost/CostDataCache.cs | 77 ++++++++ src/{ => Cost}/CostResultRows.cs | 16 +- src/CostDataCache.cs | 31 --- src/CustomCollectorConfiguration.cs | 127 ------------ src/DateEnumHelper.cs | 82 -------- src/MetricsGrapper.cs | 118 ----------- src/Program.cs | 5 +- src/PrometheusMetrics/CustomMetricsService.cs | 71 +++++++ src/PrometheusMetrics/DateEnumHelper.cs | 85 ++++++++ src/PrometheusMetrics/MetricsUpdater.cs | 74 +++++++ src/Startup.cs | 34 ++-- 21 files changed, 620 insertions(+), 491 deletions(-) create mode 100644 src/AzureApi/TooManyRequestsException.cs delete mode 100644 src/BackgroundCostCollectorHostedService.cs create mode 100644 src/Configuration/CustomCollectorConfiguration.cs create mode 100644 src/Configuration/Metrics/CustomCollectorConfig.cs create mode 100644 src/Configuration/Metrics/MetricConfig.cs create mode 100644 src/Cost/BackgroundCostCollectorHostedService.cs rename src/{ => Cost}/BillingQueryClient.cs (98%) create mode 100644 src/Cost/CostDataCache.cs rename src/{ => Cost}/CostResultRows.cs (94%) delete mode 100644 src/CostDataCache.cs delete mode 100644 src/CustomCollectorConfiguration.cs delete mode 100644 src/DateEnumHelper.cs delete mode 100644 src/MetricsGrapper.cs create mode 100644 src/PrometheusMetrics/CustomMetricsService.cs create mode 100644 src/PrometheusMetrics/DateEnumHelper.cs create mode 100644 src/PrometheusMetrics/MetricsUpdater.cs diff --git a/AzureBillingExporter.Tests/DateEnumHelperTests.cs b/AzureBillingExporter.Tests/DateEnumHelperTests.cs index af20274..5999699 100644 --- a/AzureBillingExporter.Tests/DateEnumHelperTests.cs +++ b/AzureBillingExporter.Tests/DateEnumHelperTests.cs @@ -1,4 +1,5 @@ using System; +using AzureBillingExporter.PrometheusMetrics; using NUnit.Framework; namespace AzureBillingExporter.Tests diff --git a/src/AzureApi/AccessTokenFactory.cs b/src/AzureApi/AccessTokenFactory.cs index 9a0071e..20eb957 100644 --- a/src/AzureApi/AccessTokenFactory.cs +++ b/src/AzureApi/AccessTokenFactory.cs @@ -3,6 +3,7 @@ using System.Threading; using System.Threading.Tasks; using System.Web; +using AzureBillingExporter.Cost; using Microsoft.Extensions.Logging; using Newtonsoft.Json; diff --git a/src/AzureApi/AzureCostManagementClient.cs b/src/AzureApi/AzureCostManagementClient.cs index 3488f83..4488df7 100644 --- a/src/AzureApi/AzureCostManagementClient.cs +++ b/src/AzureApi/AzureCostManagementClient.cs @@ -6,6 +6,7 @@ using System.Runtime.CompilerServices; using System.Text; using System.Threading; +using AzureBillingExporter.Cost; using Microsoft.Extensions.Logging; using Newtonsoft.Json; @@ -56,6 +57,11 @@ public async IAsyncEnumerable ExecuteBillingQuery(string billing if (response.StatusCode != HttpStatusCode.OK) { + if (response.StatusCode == HttpStatusCode.TooManyRequests) + { + throw new TooManyRequestsException($"{response.StatusCode} {await response.Content.ReadAsStringAsync()}"); + } + throw new Exception($"{response.StatusCode} {await response.Content.ReadAsStringAsync()}"); } diff --git a/src/AzureApi/TooManyRequestsException.cs b/src/AzureApi/TooManyRequestsException.cs new file mode 100644 index 0000000..598cc06 --- /dev/null +++ b/src/AzureApi/TooManyRequestsException.cs @@ -0,0 +1,11 @@ +using System; + +namespace AzureBillingExporter.AzureApi +{ + public class TooManyRequestsException : Exception + { + public TooManyRequestsException(string message) : base(message) + { + } + } +} diff --git a/src/BackgroundCostCollectorHostedService.cs b/src/BackgroundCostCollectorHostedService.cs deleted file mode 100644 index e736533..0000000 --- a/src/BackgroundCostCollectorHostedService.cs +++ /dev/null @@ -1,102 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; - -namespace AzureBillingExporter -{ - public class BackgroundCostCollectorHostedService : IHostedService - { - private readonly BillingQueryClient _billingQueryClient; - private readonly CostDataCache _costDataCache; - private readonly ILogger _logger; - - private readonly CancellationTokenSource _stoppingCts = new CancellationTokenSource(); - private Task _executingTask = null!; - - - public BackgroundCostCollectorHostedService( - BillingQueryClient billingQueryClient, - CostDataCache costDataCache, - ILogger logger) - { - _billingQueryClient = billingQueryClient ?? throw new ArgumentNullException(nameof(billingQueryClient)); - _costDataCache = costDataCache ?? throw new ArgumentNullException(nameof(costDataCache)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public Task StartAsync(CancellationToken cancellationToken) - { - _logger.LogInformation("Azure Billing data collector background hosted service - Starting..."); - _executingTask = StartCollectingDataInBackgroundAsync(_stoppingCts.Token); - if (_executingTask.IsCompleted) - { - return _executingTask; - } - - _logger.LogInformation("Azure Billing data collector background hosted service - Has started."); - return Task.CompletedTask; - } - - public async Task StopAsync(CancellationToken cancellationToken) - { - if (_executingTask == null) - { - _logger.LogInformation("Azure Billing data collector background hosted service - Already stopped."); - return; - } - - try - { - _logger.LogInformation("Azure Billing data collector background hosted service - Stopping..."); - // Signal cancellation to the executing method - _stoppingCts.Cancel(); - } - finally - { - // Wait until the task completes or the stop token triggers - await Task.WhenAny( - _executingTask, - Task.Delay(Timeout.Infinite, cancellationToken)); - } - - _logger.LogInformation("Azure Billing data collector background hosted service - Stopped."); - } - - private async Task StartCollectingDataInBackgroundAsync(CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - while (!cancellationToken.IsCancellationRequested) - { - cancellationToken.ThrowIfCancellationRequested(); - try - { - // Daily, monthly costs - _logger.Log(LogLevel.Debug, "Get daily data"); - var dailyData = new List(); - await foreach(var data in (await _billingQueryClient.GetDailyData(cancellationToken)).WithCancellation(cancellationToken)) - { - dailyData.Add(data); - } - _costDataCache.SetDailyCost(dailyData); - - _logger.Log(LogLevel.Debug, "Get monthly data"); - var monthlyData = new List(); - await foreach(var data in (await _billingQueryClient.GetMonthlyData(cancellationToken)).WithCancellation(cancellationToken)) - { - monthlyData.Add(data); - } - _costDataCache.SetMonthlyCost(monthlyData); - } - catch (Exception ex) - { - _logger.LogError(new EventId(0), ex, ex.Message); - } - - await Task.Delay(TimeSpan.FromMinutes(1), cancellationToken); - } - } - } -} diff --git a/src/Configuration/CustomCollectorConfiguration.cs b/src/Configuration/CustomCollectorConfiguration.cs new file mode 100644 index 0000000..dba0097 --- /dev/null +++ b/src/Configuration/CustomCollectorConfiguration.cs @@ -0,0 +1,40 @@ +using System.IO; +using AzureBillingExporter.Configuration.Metrics; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace AzureBillingExporter.Configuration +{ + public class CustomCollectorConfiguration + { + private string CustomCollectorsConfigFile { get; } + public CustomCollectorConfiguration(string customCollectorsFilePath) + { + if (!string.IsNullOrEmpty(customCollectorsFilePath)) + { + CustomCollectorsConfigFile = customCollectorsFilePath; + } + else + { + CustomCollectorsConfigFile = "custom_collectors.yml"; + } + } + public CustomCollectorConfig GetCustomCollectorConfig() + { + if (!File.Exists(CustomCollectorsConfigFile)) + { + return null; + } + + var configText = File.ReadAllText(CustomCollectorsConfigFile); + var input = new StringReader(configText); + + var deserializer = new DeserializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .Build(); + + var config = deserializer.Deserialize(input); + return config; + } + } +} diff --git a/src/Configuration/Metrics/CustomCollectorConfig.cs b/src/Configuration/Metrics/CustomCollectorConfig.cs new file mode 100644 index 0000000..89a9b78 --- /dev/null +++ b/src/Configuration/Metrics/CustomCollectorConfig.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; +using YamlDotNet.Serialization; + +namespace AzureBillingExporter.Configuration.Metrics +{ + public class CustomCollectorConfig + { + [YamlMember(Alias = "metrics", ApplyNamingConventions = false)] + public List Metrics { get; set; } + } +} diff --git a/src/Configuration/Metrics/MetricConfig.cs b/src/Configuration/Metrics/MetricConfig.cs new file mode 100644 index 0000000..c300b4e --- /dev/null +++ b/src/Configuration/Metrics/MetricConfig.cs @@ -0,0 +1,31 @@ +using System.Collections.Generic; +using YamlDotNet.Serialization; + +namespace AzureBillingExporter.Configuration.Metrics +{ + public class MetricConfig + { + [YamlMember(Alias = "metric_name", ApplyNamingConventions = false)] + public string MetricName { get; set; } + + public string Type { get; set; } + public string Help { get; set; } + + [YamlMember(Alias = "key_labels", ApplyNamingConventions = false)] + public List KeyLabels { get; set; } + + [YamlMember(Alias = "static_labels", ApplyNamingConventions = false)] + public Dictionary StaticLabel { get; set; } + + public string Value { get; set; } + + public int? Limit { get; set; } + + [YamlMember(Alias = "replace_date_labels_to_enum", ApplyNamingConventions = false)] + public bool ReplaceDateLabelsToEnum { get; set; } + + [YamlMember(Alias = "query_file", ApplyNamingConventions = false)] + public string QueryFilePath { get; set; } + + } +} diff --git a/src/Cost/BackgroundCostCollectorHostedService.cs b/src/Cost/BackgroundCostCollectorHostedService.cs new file mode 100644 index 0000000..81c6327 --- /dev/null +++ b/src/Cost/BackgroundCostCollectorHostedService.cs @@ -0,0 +1,185 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using AzureBillingExporter.AzureApi; +using AzureBillingExporter.Configuration; +using AzureBillingExporter.Configuration.Metrics; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace AzureBillingExporter.Cost +{ + public class BackgroundCostCollectorHostedService : IHostedService + { + const string DailyMetricKey = "daily"; + const string MonthlyMetricKey = "monthly"; + const int MaxOldestTimeInMinutes = 10; + const int ThrottleAzureApiTimeInMinutes = 1; + const int ActualTimeInMinutes = 3; + const int SleepTimeInMinutes = 2; + + private readonly Dictionary _metricsStats = new Dictionary(); + private readonly BillingQueryClient _billingQueryClient; + private readonly CostDataCache _costDataCache; + private readonly CustomCollectorConfiguration _customCollectorConfiguration; + private readonly ILogger _logger; + + private readonly CancellationTokenSource _stoppingCts = new CancellationTokenSource(); + private Task _executingTask = null!; + private CustomCollectorConfig _customCollectorConfig; + + public BackgroundCostCollectorHostedService( + BillingQueryClient billingQueryClient, + CostDataCache costDataCache, + CustomCollectorConfiguration customCollectorConfiguration, + ILogger logger) + { + _billingQueryClient = billingQueryClient ?? throw new ArgumentNullException(nameof(billingQueryClient)); + _costDataCache = costDataCache ?? throw new ArgumentNullException(nameof(costDataCache)); + _customCollectorConfiguration = customCollectorConfiguration ?? throw new ArgumentNullException(nameof(customCollectorConfiguration)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public Task StartAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("Azure Billing data collector background hosted service - Starting..."); + + FillMetricsStats(); + + _executingTask = StartCollectingDataInBackgroundAsync(_stoppingCts.Token); + if (_executingTask.IsCompleted) + { + return _executingTask; + } + + _logger.LogInformation("Azure Billing data collector background hosted service - Has started."); + return Task.CompletedTask; + } + + private void FillMetricsStats() + { + var notActualTime = DateTime.UtcNow.AddMinutes(-1*ActualTimeInMinutes).AddSeconds(-1); + _metricsStats.Add(DailyMetricKey, notActualTime); + _metricsStats.Add(MonthlyMetricKey, notActualTime); + + _customCollectorConfig = _customCollectorConfiguration.GetCustomCollectorConfig(); + foreach (var metric in _customCollectorConfig.Metrics) + { + _metricsStats.Add(metric.QueryFilePath, notActualTime); + } + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + if (_executingTask == null) + { + _logger.LogInformation("Azure Billing data collector background hosted service - Already stopped."); + return; + } + + try + { + _logger.LogInformation("Azure Billing data collector background hosted service - Stopping..."); + // Signal cancellation to the executing method + _stoppingCts.Cancel(); + } + finally + { + // Wait until the task completes or the stop token triggers + await Task.WhenAny( + _executingTask, + Task.Delay(Timeout.Infinite, cancellationToken)); + } + + _logger.LogInformation("Azure Billing data collector background hosted service - Stopped."); + } + + private async Task StartCollectingDataInBackgroundAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + while (!cancellationToken.IsCancellationRequested) + { + cancellationToken.ThrowIfCancellationRequested(); + var oldestMetric = _metricsStats.OrderBy(x => x.Value).First(); + if (oldestMetric.Value <= DateTime.UtcNow.AddMinutes(-1 * MaxOldestTimeInMinutes)) + { + _logger.Log(LogLevel.Warning, + $"Metric {oldestMetric.Key} last update time was more than {MaxOldestTimeInMinutes} minutes ago ({oldestMetric.Value})"); + } + try + { + switch (oldestMetric.Key) + { + case DailyMetricKey: + { + _logger.Log(LogLevel.Debug, "Get daily metric data"); + var dailyData = new List(); + await foreach (var data in (await _billingQueryClient.GetDailyData(cancellationToken)) + .WithCancellation(cancellationToken)) + { + dailyData.Add(data); + } + + _costDataCache.SetDailyCost(dailyData); + + break; + } + case MonthlyMetricKey: + { + _logger.Log(LogLevel.Debug, "Get monthly metric data"); + var monthlyData = new List(); + await foreach (var data in (await _billingQueryClient.GetMonthlyData(cancellationToken)) + .WithCancellation(cancellationToken)) + { + monthlyData.Add(data); + } + + _costDataCache.SetMonthlyCost(monthlyData); + break; + } + default: + { + _logger.Log(LogLevel.Debug, $"Get custom metric {oldestMetric.Key} data"); + + var metricConfig = + _customCollectorConfig.Metrics.Single(x => x.QueryFilePath == oldestMetric.Key); + + var metricsData = new List(); + var data = await _billingQueryClient.GetCustomData(cancellationToken, + metricConfig.QueryFilePath); + await foreach (var customData in data.WithCancellation(cancellationToken)) + { + metricsData.Add(customData); + } + + _costDataCache.SetCostByKey(oldestMetric.Key, metricsData); + break; + } + } + } + catch (TooManyRequestsException ex) + { + _logger.LogError(ex, ex.Message); + _logger.LogInformation("Too many requests to Azure Billing API. Let's sleep for 1 minute"); + await Task.Delay(TimeSpan.FromMinutes(ThrottleAzureApiTimeInMinutes), cancellationToken); + } + catch (Exception ex) + { + _logger.LogError(ex, ex.Message); + } + finally + { + _metricsStats[oldestMetric.Key] = DateTime.UtcNow; + } + + var actualDateTimeUtc = DateTime.UtcNow.AddMinutes(-1*ActualTimeInMinutes); + if (_metricsStats.All(x => x.Value > actualDateTimeUtc)) + { + await Task.Delay(TimeSpan.FromMinutes(SleepTimeInMinutes), cancellationToken); + } + } + } + } +} diff --git a/src/BillingQueryClient.cs b/src/Cost/BillingQueryClient.cs similarity index 98% rename from src/BillingQueryClient.cs rename to src/Cost/BillingQueryClient.cs index 96be1a8..c375b5d 100644 --- a/src/BillingQueryClient.cs +++ b/src/Cost/BillingQueryClient.cs @@ -6,9 +6,8 @@ using System.Threading.Tasks; using AzureBillingExporter.AzureApi; using DotLiquid; -using Microsoft.Extensions.Logging; -namespace AzureBillingExporter +namespace AzureBillingExporter.Cost { public class BillingQueryClient { diff --git a/src/Cost/CostDataCache.cs b/src/Cost/CostDataCache.cs new file mode 100644 index 0000000..244583e --- /dev/null +++ b/src/Cost/CostDataCache.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; + +namespace AzureBillingExporter.Cost +{ + public class CostDataCache + { + const int CacheItemExpirationTimeInMinutes = 5; + + private readonly ConcurrentDictionary _costDataCache = + new ConcurrentDictionary(); + + public List GetDailyCost() + { + return GetCostByKey("Daily"); + } + + public void SetDailyCost(List data) + { + SetCostByKey("Daily", data); + } + + public List GetMonthlyCost() + { + return GetCostByKey("Monthly"); + } + + public void SetMonthlyCost(List data) + { + SetCostByKey("Monthly", data); + } + + public List GetCostByKey(string key) + { + var cacheItem = GetCacheItem(key); + if (cacheItem == null) + { + return new List(); + } + + return cacheItem.Data; + } + + public void SetCostByKey(string key, List data) + { + SetCacheItem(key, data); + } + + private CacheItem GetCacheItem(string key) + { + var expireDate = DateTime.UtcNow.Add(-1*TimeSpan.FromMinutes(CacheItemExpirationTimeInMinutes)); + var cacheItem = _costDataCache.ContainsKey(key) ? _costDataCache[key] : null; + if (cacheItem == null || cacheItem.Created <= expireDate) + { + return null; + } + + return _costDataCache[key]; + } + + private void SetCacheItem(string key, List data) + { + _costDataCache[key] = new CacheItem + { + Created = DateTime.UtcNow, + Data = data + }; + } + + class CacheItem + { + public DateTime Created { get; set; } + public List Data { get; set; } + } + } +} diff --git a/src/CostResultRows.cs b/src/Cost/CostResultRows.cs similarity index 94% rename from src/CostResultRows.cs rename to src/Cost/CostResultRows.cs index 97e31db..96848d2 100644 --- a/src/CostResultRows.cs +++ b/src/Cost/CostResultRows.cs @@ -3,7 +3,7 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; -namespace AzureBillingExporter +namespace AzureBillingExporter.Cost { public class CostResultRows { @@ -17,13 +17,13 @@ public double Cost { return 0; } - + return double.Parse(preTaxCosts); } } - public string Date + public string Date { get { return GetByColumnName("UsageDate"); } } @@ -44,23 +44,23 @@ public string GetByColumnName(string name) public double GetValueByColumnName(string name) { var strVal = GetByColumnName(name); - + if (string.IsNullOrEmpty(strVal)) { return 0; } - + return double.Parse(strVal); } public List ColumnNames { get; } = new List(); public List Values { get; } = new List(); - + public static CostResultRows Cast(dynamic columns, dynamic singleRow) { var parsedRow = new CostResultRows(); parsedRow.Values.AddRange(ClearParse(singleRow.ToString())); - + foreach (var column in columns) { parsedRow.ColumnNames.Add(column.name.ToString()); @@ -85,4 +85,4 @@ private static IEnumerable ClearParse(dynamic s) return res; } } -} \ No newline at end of file +} diff --git a/src/CostDataCache.cs b/src/CostDataCache.cs deleted file mode 100644 index b66edc2..0000000 --- a/src/CostDataCache.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System.Collections.Concurrent; -using System.Collections.Generic; - -namespace AzureBillingExporter -{ - public class CostDataCache - { - private readonly ConcurrentDictionary> _costDataCache = - new ConcurrentDictionary>(); - - public List GetDailyCost() - { - return _costDataCache.ContainsKey("Daily") ? _costDataCache["Daily"] : new List(); - } - - public void SetDailyCost(List data) - { - _costDataCache["Daily"] = data; - } - - public List GetMonthlyCost() - { - return _costDataCache.ContainsKey("Monthly") ? _costDataCache["Monthly"] : new List(); - } - - public void SetMonthlyCost(List data) - { - _costDataCache["Monthly"] = data; - } - } -} diff --git a/src/CustomCollectorConfiguration.cs b/src/CustomCollectorConfiguration.cs deleted file mode 100644 index 7f225ad..0000000 --- a/src/CustomCollectorConfiguration.cs +++ /dev/null @@ -1,127 +0,0 @@ -using System.Collections.Generic; -using System.IO; -using System.Linq; -using Prometheus; -using YamlDotNet.Serialization; -using YamlDotNet.Serialization.NamingConventions; - -namespace AzureBillingExporter -{ - public class MetricConfigCollections - { - [YamlMember(Alias = "metrics", ApplyNamingConventions = false)] - public List Metrics { get; set; } - } - - public class MetricConfig - { - [YamlMember(Alias = "metric_name", ApplyNamingConventions = false)] - public string MetricName { get; set; } - - public string Type { get; set; } - public string Help { get; set; } - - [YamlMember(Alias = "key_labels", ApplyNamingConventions = false)] - public List KeyLabels { get; set; } - - [YamlMember(Alias = "static_labels", ApplyNamingConventions = false)] - public Dictionary StaticLabel { get; set; } - - public string Value { get; set; } - - public int? Limit { get; set; } - - [YamlMember(Alias = "replace_date_labels_to_enum", ApplyNamingConventions = false)] - public bool ReplaceDateLabelsToEnum { get; set; } - - [YamlMember(Alias = "query_file", ApplyNamingConventions = false)] - public string QueryFilePath { get; set; } - - } - - public class CustomCollectorConfiguration - { - public readonly Dictionary CustomGaugeMetrics = new Dictionary(); // - - private string CustomCollectorsConfigFile { get; } - public CustomCollectorConfiguration(string customCollectorsFilePath) - { - if (!string.IsNullOrEmpty(customCollectorsFilePath)) - { - CustomCollectorsConfigFile = customCollectorsFilePath; - } - else - { - CustomCollectorsConfigFile = "custom_collectors.yml"; - } - } - public void ReadCustomCollectorConfig() - { - if (!File.Exists(CustomCollectorsConfigFile)) - { - return; - } - - var configText = File.ReadAllText(CustomCollectorsConfigFile); - var input = new StringReader(configText); - - var deserializer = new DeserializerBuilder() - .WithNamingConvention(CamelCaseNamingConvention.Instance) - .Build(); - - var metricCollections = deserializer.Deserialize(input); - - CreatePrometheusMetrics(metricCollections); - } - - private void CreatePrometheusMetrics(MetricConfigCollections metricCollection) - { - foreach (var metricConfig in metricCollection.Metrics) - { - // Labels - var labelNames = new List(); - foreach (var staticLabel in metricConfig.StaticLabel) - { - labelNames.Add(staticLabel.Key); - } - foreach (var keyLabel in metricConfig.KeyLabels) - { - labelNames.Add(keyLabel); - } - - if (metricConfig.Type?.ToLower() == "gauge") - { - var gauge = Metrics.CreateGauge(metricConfig.MetricName, metricConfig.Help, - new GaugeConfiguration() - { - LabelNames = labelNames.ToArray() - }); - - CustomGaugeMetrics.Add(metricConfig, gauge); - } - } - } - - public void SetValues(MetricConfig key, CostResultRows customData) - { - var gauge = CustomGaugeMetrics[key]; - - var labelValues = new List(); - labelValues.AddRange(key.StaticLabel.Select(x => x.Value)); - foreach (var keyLabel in key.KeyLabels) - { - var dataColumnByKeyLabel = customData.GetByColumnName(keyLabel); - - if (key.ReplaceDateLabelsToEnum) - { - dataColumnByKeyLabel = DateEnumHelper.ReplaceDateValueToEnums(dataColumnByKeyLabel); - } - labelValues.Add(dataColumnByKeyLabel); - } - - gauge - .WithLabels(labelValues.ToArray()) - .Set(customData.GetValueByColumnName(key.Value)); - } - } -} diff --git a/src/DateEnumHelper.cs b/src/DateEnumHelper.cs deleted file mode 100644 index 23a044b..0000000 --- a/src/DateEnumHelper.cs +++ /dev/null @@ -1,82 +0,0 @@ -using System; -using System.Globalization; - -public static class DateEnumHelper -{ - public static string ReplaceDateValueToEnums(string originalDate, DateTime? nowDate = null) - { - var todayDate = nowDate ?? DateTime.Now; - - //================================================= - // Month - //================================================= - var currentMonthStart = new DateTime(todayDate.Year, todayDate.Month, 1); - if (originalDate == currentMonthStart.ToString("yyyy-MM-01T00:00:00", CultureInfo.InvariantCulture)) - { - return "current_month"; - } - - var previousMonthStart = new DateTime(todayDate.Year, todayDate.Month, 1); - previousMonthStart = previousMonthStart.AddMonths(-1); - if (originalDate == previousMonthStart.ToString("yyyy-MM-01T00:00:00", CultureInfo.InvariantCulture)) - { - return "previous_month"; - } - - var beforePreviousMonthStart = new DateTime(todayDate.Year, todayDate.Month, 1); - beforePreviousMonthStart = beforePreviousMonthStart.AddMonths(-2); - if (originalDate == beforePreviousMonthStart.ToString("yyyy-MM-01T00:00:00", CultureInfo.InvariantCulture)) - { - return "before_previous_month"; - } - - for (int i = 0; i < 12; i++) - { - var firstDayTodayMonth = new DateTime(todayDate.Year, todayDate.Month, 1); - var monthMinus = firstDayTodayMonth.AddMonths(-i); - - if (originalDate == monthMinus.ToString("yyyy-MM-01T00:00:00", CultureInfo.InvariantCulture)) - { - return $"month_minus_{i+1:D2}"; - } - } - //================================================= - - //================================================= - // Day - //================================================= - // "20200624" -> "today" - if (originalDate == todayDate.ToString("yyyyMMdd", CultureInfo.InvariantCulture)) - { - return "today"; - } - - var yesterday = new DateTime(todayDate.Year, todayDate.Month, todayDate.Day); - yesterday = yesterday.AddDays(-1); - if (originalDate == yesterday.ToString("yyyyMMdd", CultureInfo.InvariantCulture)) - { - return "yesterday"; - } - - var beforeYesterday = new DateTime(todayDate.Year, todayDate.Month, todayDate.Day); - beforeYesterday = beforeYesterday.AddDays(-2); - if (originalDate == beforeYesterday.ToString("yyyyMMdd", CultureInfo.InvariantCulture)) - { - return "before_yesterday"; - } - - for (int i = 0; i < 367; i++) - { - var thisDayDate = new DateTime(todayDate.Year, todayDate.Month, todayDate.Day); - var dayMinus = thisDayDate.AddDays(-i); - - if (originalDate == dayMinus.ToString("yyyyMMdd", CultureInfo.InvariantCulture)) - { - return $"day_minus_{i+1:D3}"; - } - } - - return originalDate; - //================================================= - } -} diff --git a/src/MetricsGrapper.cs b/src/MetricsGrapper.cs deleted file mode 100644 index 2c92821..0000000 --- a/src/MetricsGrapper.cs +++ /dev/null @@ -1,118 +0,0 @@ -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Prometheus; - -namespace AzureBillingExporter -{ - public class AzureBillingMetricsGrapper - { - private readonly ILogger _logger; - - private static readonly Gauge DailyCosts = - Metrics.CreateGauge( - "azure_billing_daily", - "Daily cost by today, yesterday and day before yesterday", - new GaugeConfiguration - { - LabelNames = new [] {"DateEnum"} - }); - - private static readonly Gauge MonthlyCosts = - Metrics.CreateGauge( - "azure_billing_monthly", - "This month costs", - new GaugeConfiguration - { - LabelNames = new [] {"DateEnum"} - }); - - private BillingQueryClient _billingQueryClient; - private CustomCollectorConfiguration _customCollectorConfiguration; - private CostDataCache _сostDataCache; - - public AzureBillingMetricsGrapper(BillingQueryClient billingQueryClient, - CustomCollectorConfiguration customCollectorConfiguration, - CostDataCache сostDataCache, - ILogger logger) - { - _billingQueryClient = billingQueryClient; - _customCollectorConfiguration = customCollectorConfiguration; - _сostDataCache = сostDataCache; - _logger = logger; - _customCollectorConfiguration.ReadCustomCollectorConfig(); - } - - public async Task DownloadFromApi(CancellationToken cancel) - { - _logger.Log(LogLevel.Information, "Start getting data for metrics"); - var timer = new Stopwatch(); - timer.Start(); - - // Daily, monthly costs - _logger.Log(LogLevel.Debug, "Get daily data"); - foreach(var data in _сostDataCache.GetDailyCost()) - { - var dayEnum = DateEnumHelper.ReplaceDateValueToEnums(data.GetByColumnName("UsageDate")); - - DailyCosts - .WithLabels(dayEnum) - .Set(data.Cost); - } - - _logger.Log(LogLevel.Debug, "Get monthly data"); - foreach(var data in _сostDataCache.GetMonthlyCost()) - { - var monthEnum = DateEnumHelper.ReplaceDateValueToEnums(data.GetByColumnName("BillingMonth")); - - MonthlyCosts - .WithLabels(monthEnum) - .Set(data.Cost); - } - - _logger.Log(LogLevel.Debug, "Get custom data in parallel"); - // var customMetricsDataTasks = _customCollectorConfiguration.CustomGaugeMetrics.Keys - // .Select(x => StartMetricDataTask(x, cancel)).ToArray(); - // - // Task.WaitAll(customMetricsDataTasks, cancel); - // foreach (var task in customMetricsDataTasks) - // { - // if (task != null && task.Exception != null) - // throw task.Exception; - // } - // - // foreach (var task in customMetricsDataTasks) - // { - // await foreach (var customData in task.Result.Data.WithCancellation(cancel)) - // { - // _customCollectorConfiguration.SetValues(task.Result.Config, customData); - // } - // } - - timer.Stop(); - _logger.Log(LogLevel.Debug, "Metrics get total time: " + timer.Elapsed); - - _logger.Log(LogLevel.Information, "Finish getting data for metrics"); - } - - private async Task StartMetricDataTask(MetricConfig config, CancellationToken cancel) - { - _logger.Log(LogLevel.Debug, $"Get custom data for {config.MetricName} metric, query file path - {config.QueryFilePath}"); - var data = await _billingQueryClient.GetCustomData(cancel, config.QueryFilePath); - return new MetricsData - { - Config = config, - Data = data - }; - } - } - - class MetricsData - { - public MetricConfig Config { get; set; } - public IAsyncEnumerable Data { get; set; } - } -} diff --git a/src/Program.cs b/src/Program.cs index 09aa914..2a01525 100644 --- a/src/Program.cs +++ b/src/Program.cs @@ -4,15 +4,12 @@ using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; using Serilog; -using Serilog.Core; -using Serilog.Events; using Serilog.Formatting.Elasticsearch; namespace AzureBillingExporter { - public class Program + public static class Program { public static async Task Main(string[] args) { diff --git a/src/PrometheusMetrics/CustomMetricsService.cs b/src/PrometheusMetrics/CustomMetricsService.cs new file mode 100644 index 0000000..efdf1a4 --- /dev/null +++ b/src/PrometheusMetrics/CustomMetricsService.cs @@ -0,0 +1,71 @@ +using System.Collections.Generic; +using System.Linq; +using AzureBillingExporter.Configuration; +using AzureBillingExporter.Configuration.Metrics; +using AzureBillingExporter.Cost; +using Prometheus; + +namespace AzureBillingExporter.PrometheusMetrics +{ + public class CustomMetricsService + { + public readonly Dictionary CustomGaugeMetrics = new Dictionary(); // + + private readonly CustomCollectorConfiguration _customCollectorConfiguration; + public CustomMetricsService(CustomCollectorConfiguration customCollectorConfiguration) + { + _customCollectorConfiguration = customCollectorConfiguration; + } + + public void CreatePrometheusMetrics() + { + var customCollector = _customCollectorConfiguration.GetCustomCollectorConfig(); + foreach (var metricConfig in customCollector.Metrics) + { + // Labels + var labelNames = new List(); + foreach (var staticLabel in metricConfig.StaticLabel) + { + labelNames.Add(staticLabel.Key); + } + foreach (var keyLabel in metricConfig.KeyLabels) + { + labelNames.Add(keyLabel); + } + + if (metricConfig.Type?.ToLower() == "gauge") + { + var gauge = Metrics.CreateGauge(metricConfig.MetricName, metricConfig.Help, + new GaugeConfiguration() + { + LabelNames = labelNames.ToArray() + }); + + CustomGaugeMetrics.Add(metricConfig, gauge); + } + } + } + + public void SetValues(MetricConfig key, CostResultRows customData) + { + var gauge = CustomGaugeMetrics[key]; + + var labelValues = new List(); + labelValues.AddRange(key.StaticLabel.Select(x => x.Value)); + foreach (var keyLabel in key.KeyLabels) + { + var dataColumnByKeyLabel = customData.GetByColumnName(keyLabel); + + if (key.ReplaceDateLabelsToEnum) + { + dataColumnByKeyLabel = DateEnumHelper.ReplaceDateValueToEnums(dataColumnByKeyLabel); + } + labelValues.Add(dataColumnByKeyLabel); + } + + gauge + .WithLabels(labelValues.ToArray()) + .Set(customData.GetValueByColumnName(key.Value)); + } + } +} diff --git a/src/PrometheusMetrics/DateEnumHelper.cs b/src/PrometheusMetrics/DateEnumHelper.cs new file mode 100644 index 0000000..440357c --- /dev/null +++ b/src/PrometheusMetrics/DateEnumHelper.cs @@ -0,0 +1,85 @@ +using System; +using System.Globalization; + +namespace AzureBillingExporter.PrometheusMetrics +{ + public static class DateEnumHelper + { + public static string ReplaceDateValueToEnums(string originalDate, DateTime? nowDate = null) + { + var todayDate = nowDate ?? DateTime.Now; + + //================================================= + // Month + //================================================= + var currentMonthStart = new DateTime(todayDate.Year, todayDate.Month, 1); + if (originalDate == currentMonthStart.ToString("yyyy-MM-01T00:00:00", CultureInfo.InvariantCulture)) + { + return "current_month"; + } + + var previousMonthStart = new DateTime(todayDate.Year, todayDate.Month, 1); + previousMonthStart = previousMonthStart.AddMonths(-1); + if (originalDate == previousMonthStart.ToString("yyyy-MM-01T00:00:00", CultureInfo.InvariantCulture)) + { + return "previous_month"; + } + + var beforePreviousMonthStart = new DateTime(todayDate.Year, todayDate.Month, 1); + beforePreviousMonthStart = beforePreviousMonthStart.AddMonths(-2); + if (originalDate == beforePreviousMonthStart.ToString("yyyy-MM-01T00:00:00", CultureInfo.InvariantCulture)) + { + return "before_previous_month"; + } + + for (int i = 0; i < 12; i++) + { + var firstDayTodayMonth = new DateTime(todayDate.Year, todayDate.Month, 1); + var monthMinus = firstDayTodayMonth.AddMonths(-i); + + if (originalDate == monthMinus.ToString("yyyy-MM-01T00:00:00", CultureInfo.InvariantCulture)) + { + return $"month_minus_{i+1:D2}"; + } + } + //================================================= + + //================================================= + // Day + //================================================= + // "20200624" -> "today" + if (originalDate == todayDate.ToString("yyyyMMdd", CultureInfo.InvariantCulture)) + { + return "today"; + } + + var yesterday = new DateTime(todayDate.Year, todayDate.Month, todayDate.Day); + yesterday = yesterday.AddDays(-1); + if (originalDate == yesterday.ToString("yyyyMMdd", CultureInfo.InvariantCulture)) + { + return "yesterday"; + } + + var beforeYesterday = new DateTime(todayDate.Year, todayDate.Month, todayDate.Day); + beforeYesterday = beforeYesterday.AddDays(-2); + if (originalDate == beforeYesterday.ToString("yyyyMMdd", CultureInfo.InvariantCulture)) + { + return "before_yesterday"; + } + + for (int i = 0; i < 367; i++) + { + var thisDayDate = new DateTime(todayDate.Year, todayDate.Month, todayDate.Day); + var dayMinus = thisDayDate.AddDays(-i); + + if (originalDate == dayMinus.ToString("yyyyMMdd", CultureInfo.InvariantCulture)) + { + return $"day_minus_{i+1:D3}"; + } + } + + return originalDate; + //================================================= + } + } +} diff --git a/src/PrometheusMetrics/MetricsUpdater.cs b/src/PrometheusMetrics/MetricsUpdater.cs new file mode 100644 index 0000000..5b17d9d --- /dev/null +++ b/src/PrometheusMetrics/MetricsUpdater.cs @@ -0,0 +1,74 @@ +using AzureBillingExporter.Cost; +using Prometheus; + +namespace AzureBillingExporter.PrometheusMetrics +{ + public class MetricsUpdater + { + private static readonly Gauge DailyCosts = + Metrics.CreateGauge( + "azure_billing_daily", + "Daily cost by today, yesterday and day before yesterday", + new GaugeConfiguration + { + LabelNames = new[] { "DateEnum" } + }); + + private static readonly Gauge MonthlyCosts = + Metrics.CreateGauge( + "azure_billing_monthly", + "This month costs", + new GaugeConfiguration + { + LabelNames = new[] { "DateEnum" } + }); + + private readonly CustomMetricsService _customMetricsService; + private readonly CostDataCache _costDataCache; + private bool _metricsCreated; + + public MetricsUpdater( + CustomMetricsService customMetricsService, + CostDataCache costDataCache) + { + _customMetricsService = customMetricsService; + _costDataCache = costDataCache; + } + + public void Update() + { + if (!_metricsCreated) + { + _customMetricsService.CreatePrometheusMetrics(); + _metricsCreated = true; + } + + foreach (var data in _costDataCache.GetDailyCost()) + { + var dayEnum = DateEnumHelper.ReplaceDateValueToEnums(data.GetByColumnName("UsageDate")); + + DailyCosts + .WithLabels(dayEnum) + .Set(data.Cost); + } + + foreach (var data in _costDataCache.GetMonthlyCost()) + { + var monthEnum = DateEnumHelper.ReplaceDateValueToEnums(data.GetByColumnName("BillingMonth")); + + MonthlyCosts + .WithLabels(monthEnum) + .Set(data.Cost); + } + + foreach (var metric in _customMetricsService.CustomGaugeMetrics) + { + var costs = _costDataCache.GetCostByKey(metric.Key.QueryFilePath); + foreach (var cost in costs) + { + _customMetricsService.SetValues(metric.Key, cost); + } + } + } + } +} diff --git a/src/Startup.cs b/src/Startup.cs index 1ed69f9..7cff65b 100644 --- a/src/Startup.cs +++ b/src/Startup.cs @@ -1,5 +1,8 @@ using System; using AzureBillingExporter.AzureApi; +using AzureBillingExporter.Configuration; +using AzureBillingExporter.Cost; +using AzureBillingExporter.PrometheusMetrics; using Dodo.HttpClientResiliencePolicies; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; @@ -19,7 +22,7 @@ public Startup(IConfiguration configuration) Configuration = configuration; } - public IConfiguration Configuration { get; } + private IConfiguration Configuration { get; } // This method gets called by the runtime. Use this method to add services to the container. // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 @@ -29,19 +32,20 @@ public void ConfigureServices(IServiceCollection services) new Uri("https://management.azure.com"), "AzureBillingExporter"); services.Configure(Configuration.GetSection("ApiSettings")); services.AddSingleton(resolver => resolver.GetRequiredService>().Value); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); - services.AddSingleton(serviceProvider=> + services.AddSingleton(serviceProvider => { var customCollectorsFilePath = Configuration["CustomCollectorsFilePath"]; return new CustomCollectorConfiguration(customCollectorsFilePath); }); - services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddHostedService(resolver => { @@ -60,10 +64,12 @@ public void ConfigureServices(IServiceCollection services) { var billingQueryClient = resolver.GetRequiredService(); var costDataCache = resolver.GetRequiredService(); + var customCollectorConfiguration = resolver.GetRequiredService(); var logger = resolver.GetRequiredService>(); return new BackgroundCostCollectorHostedService( billingQueryClient, costDataCache, + customCollectorConfiguration, logger); }); } @@ -80,17 +86,11 @@ public void Configure( app.UseRouting(); - var billingGrapper = app.ApplicationServices.GetService(); - Metrics.DefaultRegistry.AddBeforeCollectCallback(async (cancel) => - { - await billingGrapper.DownloadFromApi(cancel); - }); + var azureBillingMetricsGrabber = app.ApplicationServices.GetService(); + Metrics.DefaultRegistry.AddBeforeCollectCallback(() => { azureBillingMetricsGrabber.Update(); }); // ASP.NET Core 3 or newer - app.UseEndpoints(endpoints => - { - endpoints.MapMetrics(); - }); + app.UseEndpoints(endpoints => { endpoints.MapMetrics(); }); } } } From aad63708657e48cec80063d05543a839b2695927 Mon Sep 17 00:00:00 2001 From: Renat Shajmardanov Date: Fri, 21 Jan 2022 16:30:10 +0300 Subject: [PATCH 5/8] #3057414 tune logs --- src/Cost/BackgroundCostCollectorHostedService.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Cost/BackgroundCostCollectorHostedService.cs b/src/Cost/BackgroundCostCollectorHostedService.cs index 81c6327..79233a0 100644 --- a/src/Cost/BackgroundCostCollectorHostedService.cs +++ b/src/Cost/BackgroundCostCollectorHostedService.cs @@ -48,6 +48,8 @@ public Task StartAsync(CancellationToken cancellationToken) FillMetricsStats(); + _logger.LogInformation($"Total metrics to process: {_metricsStats.Count}"); + _executingTask = StartCollectingDataInBackgroundAsync(_stoppingCts.Token); if (_executingTask.IsCompleted) { @@ -162,7 +164,7 @@ private async Task StartCollectingDataInBackgroundAsync(CancellationToken cancel catch (TooManyRequestsException ex) { _logger.LogError(ex, ex.Message); - _logger.LogInformation("Too many requests to Azure Billing API. Let's sleep for 1 minute"); + _logger.LogWarning($"Too many requests to Azure Billing API. Let's sleep for {ThrottleAzureApiTimeInMinutes} minute(s)"); await Task.Delay(TimeSpan.FromMinutes(ThrottleAzureApiTimeInMinutes), cancellationToken); } catch (Exception ex) From 2dae28bcf9a175147a6efc10d4075024e30c1958 Mon Sep 17 00:00:00 2001 From: Renat Shajmardanov Date: Fri, 21 Jan 2022 18:15:42 +0300 Subject: [PATCH 6/8] #3057414 move settings to the config --- src/Configuration/EnvironmentConfiguration.cs | 3 + .../BackgroundCostCollectorHostedService.cs | 144 +++++++++++------- src/Cost/CostDataCache.cs | 12 +- src/PrometheusMetrics/CustomMetricsService.cs | 2 +- src/Startup.cs | 4 + 5 files changed, 102 insertions(+), 63 deletions(-) diff --git a/src/Configuration/EnvironmentConfiguration.cs b/src/Configuration/EnvironmentConfiguration.cs index aff7dd3..45783cd 100644 --- a/src/Configuration/EnvironmentConfiguration.cs +++ b/src/Configuration/EnvironmentConfiguration.cs @@ -3,5 +3,8 @@ namespace AzureBillingExporter.Configuration public class EnvironmentConfiguration { public bool LogsAtJsonFormat { get; set; } + + public int CollectPeriodInMinutes { get; set; } = 5; + public int CachePeriodInMinutes { get; set; } = 10; } } diff --git a/src/Cost/BackgroundCostCollectorHostedService.cs b/src/Cost/BackgroundCostCollectorHostedService.cs index 79233a0..1c88654 100644 --- a/src/Cost/BackgroundCostCollectorHostedService.cs +++ b/src/Cost/BackgroundCostCollectorHostedService.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -15,15 +16,15 @@ public class BackgroundCostCollectorHostedService : IHostedService { const string DailyMetricKey = "daily"; const string MonthlyMetricKey = "monthly"; - const int MaxOldestTimeInMinutes = 10; + + const int MaxOldestTimeDriftInMinutes = 10; const int ThrottleAzureApiTimeInMinutes = 1; - const int ActualTimeInMinutes = 3; - const int SleepTimeInMinutes = 2; private readonly Dictionary _metricsStats = new Dictionary(); private readonly BillingQueryClient _billingQueryClient; private readonly CostDataCache _costDataCache; private readonly CustomCollectorConfiguration _customCollectorConfiguration; + private readonly EnvironmentConfiguration _environmentConfiguration; private readonly ILogger _logger; private readonly CancellationTokenSource _stoppingCts = new CancellationTokenSource(); @@ -34,11 +35,13 @@ public BackgroundCostCollectorHostedService( BillingQueryClient billingQueryClient, CostDataCache costDataCache, CustomCollectorConfiguration customCollectorConfiguration, + EnvironmentConfiguration environmentConfiguration, ILogger logger) { _billingQueryClient = billingQueryClient ?? throw new ArgumentNullException(nameof(billingQueryClient)); _costDataCache = costDataCache ?? throw new ArgumentNullException(nameof(costDataCache)); _customCollectorConfiguration = customCollectorConfiguration ?? throw new ArgumentNullException(nameof(customCollectorConfiguration)); + _environmentConfiguration = environmentConfiguration ?? throw new ArgumentNullException(nameof(environmentConfiguration)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -62,7 +65,7 @@ public Task StartAsync(CancellationToken cancellationToken) private void FillMetricsStats() { - var notActualTime = DateTime.UtcNow.AddMinutes(-1*ActualTimeInMinutes).AddSeconds(-1); + var notActualTime = DateTime.UtcNow.AddMinutes(-1*_environmentConfiguration.CollectPeriodInMinutes); _metricsStats.Add(DailyMetricKey, notActualTime); _metricsStats.Add(MonthlyMetricKey, notActualTime); @@ -101,65 +104,32 @@ await Task.WhenAny( private async Task StartCollectingDataInBackgroundAsync(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); + var timer = Stopwatch.StartNew(); while (!cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); - var oldestMetric = _metricsStats.OrderBy(x => x.Value).First(); - if (oldestMetric.Value <= DateTime.UtcNow.AddMinutes(-1 * MaxOldestTimeInMinutes)) + + var actualDateTimeUtc = DateTime.UtcNow.AddMinutes(-1*_environmentConfiguration.CollectPeriodInMinutes); + if (_metricsStats.All(x => x.Value > actualDateTimeUtc)) { - _logger.Log(LogLevel.Warning, - $"Metric {oldestMetric.Key} last update time was more than {MaxOldestTimeInMinutes} minutes ago ({oldestMetric.Value})"); + var sleepTimeInMinutes = + Math.Floor(_environmentConfiguration.CollectPeriodInMinutes - timer.Elapsed.TotalMinutes); + _logger.Log(LogLevel.Debug, + $"Will sleep next {sleepTimeInMinutes} minute(s)"); + if (sleepTimeInMinutes > 0) + { + await Task.Delay(TimeSpan.FromMinutes(sleepTimeInMinutes), cancellationToken); + } + timer.Restart(); } + + var oldestMetric = _metricsStats.OrderBy(x => x.Value).First(); + + WarnOnTooOldMetric(oldestMetric); + try { - switch (oldestMetric.Key) - { - case DailyMetricKey: - { - _logger.Log(LogLevel.Debug, "Get daily metric data"); - var dailyData = new List(); - await foreach (var data in (await _billingQueryClient.GetDailyData(cancellationToken)) - .WithCancellation(cancellationToken)) - { - dailyData.Add(data); - } - - _costDataCache.SetDailyCost(dailyData); - - break; - } - case MonthlyMetricKey: - { - _logger.Log(LogLevel.Debug, "Get monthly metric data"); - var monthlyData = new List(); - await foreach (var data in (await _billingQueryClient.GetMonthlyData(cancellationToken)) - .WithCancellation(cancellationToken)) - { - monthlyData.Add(data); - } - - _costDataCache.SetMonthlyCost(monthlyData); - break; - } - default: - { - _logger.Log(LogLevel.Debug, $"Get custom metric {oldestMetric.Key} data"); - - var metricConfig = - _customCollectorConfig.Metrics.Single(x => x.QueryFilePath == oldestMetric.Key); - - var metricsData = new List(); - var data = await _billingQueryClient.GetCustomData(cancellationToken, - metricConfig.QueryFilePath); - await foreach (var customData in data.WithCancellation(cancellationToken)) - { - metricsData.Add(customData); - } - - _costDataCache.SetCostByKey(oldestMetric.Key, metricsData); - break; - } - } + await UpdateMetricData(oldestMetric.Key, cancellationToken); } catch (TooManyRequestsException ex) { @@ -175,11 +145,67 @@ private async Task StartCollectingDataInBackgroundAsync(CancellationToken cancel { _metricsStats[oldestMetric.Key] = DateTime.UtcNow; } + } + } - var actualDateTimeUtc = DateTime.UtcNow.AddMinutes(-1*ActualTimeInMinutes); - if (_metricsStats.All(x => x.Value > actualDateTimeUtc)) + private void WarnOnTooOldMetric(KeyValuePair metric) + { + var maxOldestTimeInMinutes = _environmentConfiguration.CollectPeriodInMinutes + MaxOldestTimeDriftInMinutes; + if (metric.Value <= DateTime.UtcNow.AddMinutes(-1 * maxOldestTimeInMinutes)) + { + _logger.Log(LogLevel.Warning, + $"Metric {metric.Key} last update time was more than {maxOldestTimeInMinutes} minutes ago ({metric.Value})"); + } + } + + private async Task UpdateMetricData(string metricKey, CancellationToken cancellationToken) + { + switch (metricKey) + { + case DailyMetricKey: + { + _logger.Log(LogLevel.Debug, "Get daily metric data"); + var dailyData = new List(); + await foreach (var data in (await _billingQueryClient.GetDailyData(cancellationToken)) + .WithCancellation(cancellationToken)) + { + dailyData.Add(data); + } + + _costDataCache.SetDailyCost(dailyData); + + break; + } + case MonthlyMetricKey: { - await Task.Delay(TimeSpan.FromMinutes(SleepTimeInMinutes), cancellationToken); + _logger.Log(LogLevel.Debug, "Get monthly metric data"); + var monthlyData = new List(); + await foreach (var data in (await _billingQueryClient.GetMonthlyData(cancellationToken)) + .WithCancellation(cancellationToken)) + { + monthlyData.Add(data); + } + + _costDataCache.SetMonthlyCost(monthlyData); + break; + } + default: + { + _logger.Log(LogLevel.Debug, $"Get custom metric {metricKey} data"); + + var metricConfig = + _customCollectorConfig.Metrics.Single(x => x.QueryFilePath == metricKey); + + var metricsData = new List(); + var data = await _billingQueryClient.GetCustomData(cancellationToken, + metricConfig.QueryFilePath); + await foreach (var customData in data.WithCancellation(cancellationToken)) + { + metricsData.Add(customData); + } + + _costDataCache.SetCostByKey(metricKey, metricsData); + break; } } } diff --git a/src/Cost/CostDataCache.cs b/src/Cost/CostDataCache.cs index 244583e..51726a1 100644 --- a/src/Cost/CostDataCache.cs +++ b/src/Cost/CostDataCache.cs @@ -1,16 +1,22 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using AzureBillingExporter.Configuration; namespace AzureBillingExporter.Cost { public class CostDataCache { - const int CacheItemExpirationTimeInMinutes = 5; - private readonly ConcurrentDictionary _costDataCache = new ConcurrentDictionary(); + private readonly EnvironmentConfiguration _environmentConfiguration; + + public CostDataCache(EnvironmentConfiguration environmentConfiguration) + { + _environmentConfiguration = environmentConfiguration; + } + public List GetDailyCost() { return GetCostByKey("Daily"); @@ -49,7 +55,7 @@ public void SetCostByKey(string key, List data) private CacheItem GetCacheItem(string key) { - var expireDate = DateTime.UtcNow.Add(-1*TimeSpan.FromMinutes(CacheItemExpirationTimeInMinutes)); + var expireDate = DateTime.UtcNow.Add(-1*TimeSpan.FromMinutes(_environmentConfiguration.CachePeriodInMinutes)); var cacheItem = _costDataCache.ContainsKey(key) ? _costDataCache[key] : null; if (cacheItem == null || cacheItem.Created <= expireDate) { diff --git a/src/PrometheusMetrics/CustomMetricsService.cs b/src/PrometheusMetrics/CustomMetricsService.cs index efdf1a4..c232d6c 100644 --- a/src/PrometheusMetrics/CustomMetricsService.cs +++ b/src/PrometheusMetrics/CustomMetricsService.cs @@ -9,7 +9,7 @@ namespace AzureBillingExporter.PrometheusMetrics { public class CustomMetricsService { - public readonly Dictionary CustomGaugeMetrics = new Dictionary(); // + public readonly Dictionary CustomGaugeMetrics = new Dictionary(); private readonly CustomCollectorConfiguration _customCollectorConfiguration; public CustomMetricsService(CustomCollectorConfiguration customCollectorConfiguration) diff --git a/src/Startup.cs b/src/Startup.cs index 7cff65b..62bbe38 100644 --- a/src/Startup.cs +++ b/src/Startup.cs @@ -35,6 +35,8 @@ public void ConfigureServices(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(serviceProvider => Configuration.Get()); + services.AddSingleton(serviceProvider => { var customCollectorsFilePath = Configuration["CustomCollectorsFilePath"]; @@ -65,11 +67,13 @@ public void ConfigureServices(IServiceCollection services) var billingQueryClient = resolver.GetRequiredService(); var costDataCache = resolver.GetRequiredService(); var customCollectorConfiguration = resolver.GetRequiredService(); + var environmentConfiguration = resolver.GetRequiredService(); var logger = resolver.GetRequiredService>(); return new BackgroundCostCollectorHostedService( billingQueryClient, costDataCache, customCollectorConfiguration, + environmentConfiguration, logger); }); } From f994a4d80a573d82d9d7f683fc857656e914ec64 Mon Sep 17 00:00:00 2001 From: Renat Shajmardanov Date: Fri, 21 Jan 2022 18:50:15 +0300 Subject: [PATCH 7/8] #3057414 fix ceiling, update docs --- README.md | 144 ++++++++++-------- .../BackgroundCostCollectorHostedService.cs | 2 +- 2 files changed, 84 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index e9e65a6..9302f44 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Build](https://github.com/dodopizza/azure_billing_exporter/workflows/Build/badge.svg?branch=master)](https://github.com/dodopizza/azure_billing_exporter/actions?query=workflow%3ABuild) [![Docker Pulls](https://img.shields.io/docker/pulls/dodopizza/azure_billing_exporter)](https://hub.docker.com/r/dodopizza/azure_billing_exporter) -Expose Azure Billing data to prometheus format. Show daily, weekly, monthly cost by subscription. Also allow add custom billing query. +Expose Azure Billing data to prometheus format. Show daily, weekly, monthly cost by subscription. Also allow add custom billing query. ## Quick start. Docker images @@ -21,60 +21,89 @@ docker run\ ## How to run locally 1. Create ServicePrincipal -This SP should have access as Billing reader role [see Manage billing access](https://docs.microsoft.com/en-us/azure/cost-management-billing/manage/manage-billing-access) -2. Set configuration + This SP should have access as `Billing reader` role [see Manage billing access](https://docs.microsoft.com/en-us/azure/cost-management-billing/manage/manage-billing-access) -2.1. Environment Variables +1. Set configuration -```bash - EXPORT ApiSettings__SubscriptionId="YOUR_SUBSCRIPTION_ID" - EXPORT ApiSettings__TenantId="YOUR_TENANT_ID" - EXPORT ApiSettings__ClientId="YOUR_CLIENT_ID" - EXPORT ApiSettings__ClientSecret="CLIENT_SECRET_SP" -``` + 1. Environment Variables -2.2. `appsettings.json` -Using for local developing + ```bash + EXPORT ApiSettings__SubscriptionId="YOUR_SUBSCRIPTION_ID" + EXPORT ApiSettings__TenantId="YOUR_TENANT_ID" + EXPORT ApiSettings__ClientId="YOUR_CLIENT_ID" + EXPORT ApiSettings__ClientSecret="CLIENT_SECRET_SP" + ``` -```json - "ApiSettings": { - "SubscriptionId": "YOUR_SUBSCRIPTION_ID", - "TenantId": "YOUR_TENANT_ID", - "ClientId": "YOUR_CLIENT_ID", - "ClientSecret": "CLIENT_SECRET_SP" - }, -``` + 1. Configuration file `appsettings.json` -2.3 Tracing logs + Using for local developing -For trace all billing query and response set log level to trace info `appsettings.Development.json` + ```json + "ApiSettings": { + "SubscriptionId": "YOUR_SUBSCRIPTION_ID", + "TenantId": "YOUR_TENANT_ID", + "ClientId": "YOUR_CLIENT_ID", + "ClientSecret": "CLIENT_SECRET_SP" + }, + ``` -```json -{ - "Logging": { - "LogLevel": { - "Default": "Trace", -``` +1. Tracing logs -3. Install dotnet SDK -Download and install .NET Core 3.1 SDK or above - + For trace all billing query and response set log level to trace info `appsettings.Development.json` + ```json + "Serilog": { + "MinimumLevel": { + "Default": "Trace" + } + ``` -4. Run dotnet +1. Install dotnet SDK -```bash -dotnet run --project AzureBillingExporter/AzureBillingExporter.csproj -``` + Download and install .NET Core 3.1 SDK or above + -5. Open metrics +1. Run dotnet -```bash -curl http://localhost:5000/metrics -``` + ```bash + dotnet run --project AzureBillingExporter/AzureBillingExporter.csproj + ``` + +1. Open metrics + + ```bash + curl http://localhost:5000/metrics + ``` + +## Architecture + +According Microsoft [documentation](https://docs.microsoft.com/en-us/azure/cost-management-billing/costs/manage-automation#error-code-429---call-count-has-exceeded-rate-limits) application may create only 30 API calls per minute. +After that threshold application will get `Too Many Requests` response from API. + +> Error code 429 - Call count has exceeded rate limits +> +> To enable a consistent experience for all Cost Management subscribers, Cost Management APIs are rate limited. When you reach the limit, you receive the HTTP status code 429: Too many requests. The current throughput limits for our APIs are as follows: +> +> 30 calls per minute - It's done per scope, per user, or application. +> +> 200 calls per minute - It's done per tenant, per user, or application. -# Metrics +To avoid such errors, this exporter has background job to get data from API. +Received cost data placed in memory cache. Prometheus scriber calls on `/metrics` get data from cache and get quick response. + +In case of `Too Many Requests` errors, background job waits 1 minute before next calls. + +## Configuration + +| *Setting* | *Type* | *Description* | +|---|---|---| +| `LogsAtJsonFormat` | bool | Write logs in plain text or JSON format | +| `CollectPeriodInMinutes` | int | Period in minutes to make API call to the Azure, to get metrics | +| `CachePeriodInMinutes` | int | Period in minutes to cache API call results | +| `CustomCollectorsFilePath` | string | Path to YAML file with custom collectors (see [Custom Metrics](#Custom-Metrics)) | + +## Metrics | *Metrics Name* | *Description* | |---|---| @@ -83,9 +112,9 @@ curl http://localhost:5000/metrics | `azure_billing_daily_before_yesterday` | Day before yesterday all costs | | `azure_billing_monthly` | Costs by current month | -# Custom Metrics +## Custom Metrics -## Set custom metrics configs into `custom_collectors.yml` +### Set custom metrics configs into `custom_collectors.yml` ```yaml # A Prometheus metric with (optional) additional labels, value and labels populated from one query. @@ -103,14 +132,16 @@ metrics: replace_date_labels_to_enum: true # replace `05/01/2020 00:00:00` to `last_month`, `UsageDate="20200624"` to `yesterday`. Default false query_file: './custom_queries/azure_billing_by_resource_group.json' ``` -## You can set custom path to collectors.yaml file + +### You can set custom path to collectors.yaml file Into `appsettings.Development.json` (or env `CustomCollectorsFilePath`) set: + ```json "CustomCollectorsFilePath" : "./local/custom_collectors.yml", ``` -## Query to billing api +### Query to billing api ```json { @@ -144,9 +175,9 @@ Into `appsettings.Development.json` (or env `CustomCollectorsFilePath`) set: } ``` -## Datetime constants into query files +### Datetime constants into query files -You can use special constant into query file. For this use `{{ }}` template notation [Liquid Template Language](https://shopify.github.io/liquid/) . +You can use special constant into query file. For this use `{{ }}` template notation [Liquid Template Language](https://shopify.github.io/liquid/) . DateTime Constants (using server datetime). If today is '2020-06-23T08:12:45': | *Constant* | *Description* | *Example* | @@ -160,6 +191,7 @@ DateTime Constants (using server datetime). If today is '2020-06-23T08:12:45': | `YearAgo` | This month first day year ago. | '2019-06-01T00:00:00.0000000' | All this constants you can use into billing query json files: + ```json "timePeriod": { "from": "{{ PrevMonthStart }}", @@ -167,9 +199,7 @@ All this constants you can use into billing query json files: } ``` - - -# Try Azure Billing Query on sandbox +## Try Azure Billing Query on sandbox Go to docs: @@ -208,19 +238,11 @@ Body: } ``` - -# Notice +## Notice Request duration measuring for exporter: -``` -▶ curl -o /dev/null -s -w 'Total: %{time_total}s\n' http://localhost:5000/metrics -Total: 19.220319s - -~ -▶ curl -o /dev/null -s -w 'Total: %{time_total}s\n' http://localhost:5000/metrics -Total: 17.939426s -~ +```console ▶ curl -o /dev/null -s -w 'Total: %{time_total}s\n' http://localhost:5000/metrics -Total: 18.603152s -``` \ No newline at end of file +Total: 0.009669s +``` diff --git a/src/Cost/BackgroundCostCollectorHostedService.cs b/src/Cost/BackgroundCostCollectorHostedService.cs index 1c88654..fa4acd7 100644 --- a/src/Cost/BackgroundCostCollectorHostedService.cs +++ b/src/Cost/BackgroundCostCollectorHostedService.cs @@ -113,7 +113,7 @@ private async Task StartCollectingDataInBackgroundAsync(CancellationToken cancel if (_metricsStats.All(x => x.Value > actualDateTimeUtc)) { var sleepTimeInMinutes = - Math.Floor(_environmentConfiguration.CollectPeriodInMinutes - timer.Elapsed.TotalMinutes); + Math.Ceiling(_environmentConfiguration.CollectPeriodInMinutes - timer.Elapsed.TotalMinutes); _logger.Log(LogLevel.Debug, $"Will sleep next {sleepTimeInMinutes} minute(s)"); if (sleepTimeInMinutes > 0) From 177c87fb6adb0d7475319f40e7fd8327ade4aef5 Mon Sep 17 00:00:00 2001 From: Renat Shajmardanov Date: Fri, 21 Jan 2022 18:56:59 +0300 Subject: [PATCH 8/8] #3057414 fix ceiling, update docs --- src/Startup.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Startup.cs b/src/Startup.cs index 62bbe38..ee4704f 100644 --- a/src/Startup.cs +++ b/src/Startup.cs @@ -90,8 +90,8 @@ public void Configure( app.UseRouting(); - var azureBillingMetricsGrabber = app.ApplicationServices.GetService(); - Metrics.DefaultRegistry.AddBeforeCollectCallback(() => { azureBillingMetricsGrabber.Update(); }); + var metricsUpdater = app.ApplicationServices.GetService(); + Metrics.DefaultRegistry.AddBeforeCollectCallback(() => { metricsUpdater.Update(); }); // ASP.NET Core 3 or newer app.UseEndpoints(endpoints => { endpoints.MapMetrics(); });