Merge pull request #11 from dodopizza/sre-fix-too-many-requests-3057414

Fix too many requests error. Add JSON logs.

AzureBillingExporter.Tests/DateEnumHelperTests.cs

@@ -1,4 +1,5 @@
 using System;
+using AzureBillingExporter.PrometheusMetrics;
 using NUnit.Framework;
 
 namespace AzureBillingExporter.Tests

README.md

@@ -3,7 +3,7 @@
 [![Build](https://github.com/dodopizza/azure_billing_exporter/workflows/Build/badge.svg?branch=master)](https://github.com/dodopizza/azure_billing_exporter/actions?query=workflow%3ABuild)
 [![Docker Pulls](https://img.shields.io/docker/pulls/dodopizza/azure_billing_exporter)](https://hub.docker.com/r/dodopizza/azure_billing_exporter)
 
-Expose Azure Billing data to prometheus format. Show daily, weekly, monthly cost by subscription. Also allow add custom billing query. 
+Expose Azure Billing data to prometheus format. Show daily, weekly, monthly cost by subscription. Also allow add custom billing query.
 
 ## Quick start. Docker images
 
@@ -21,60 +21,89 @@ docker run\
 ## How to run locally
 
 1. Create ServicePrincipal
-This SP should have access as Billing reader role [see Manage billing access](https://docs.microsoft.com/en-us/azure/cost-management-billing/manage/manage-billing-access)
 
-2. Set configuration
+    This SP should have access as `Billing reader` role [see Manage billing access](https://docs.microsoft.com/en-us/azure/cost-management-billing/manage/manage-billing-access)
 
-2.1. Environment Variables
+1. Set configuration
 
-```bash
-    EXPORT ApiSettings__SubscriptionId="YOUR_SUBSCRIPTION_ID"
-    EXPORT ApiSettings__TenantId="YOUR_TENANT_ID"
-    EXPORT ApiSettings__ClientId="YOUR_CLIENT_ID"
-    EXPORT ApiSettings__ClientSecret="CLIENT_SECRET_SP"
-```
+    1. Environment Variables
 
-2.2. `appsettings.json`
-Using for local developing
+        ```bash
+            EXPORT ApiSettings__SubscriptionId="YOUR_SUBSCRIPTION_ID"
+            EXPORT ApiSettings__TenantId="YOUR_TENANT_ID"
+            EXPORT ApiSettings__ClientId="YOUR_CLIENT_ID"
+            EXPORT ApiSettings__ClientSecret="CLIENT_SECRET_SP"
+        ```
 
-```json
-  "ApiSettings": {
-    "SubscriptionId": "YOUR_SUBSCRIPTION_ID", 
-    "TenantId": "YOUR_TENANT_ID", 
-    "ClientId": "YOUR_CLIENT_ID", 
-    "ClientSecret": "CLIENT_SECRET_SP"
-  },
-```
+    1. Configuration file `appsettings.json`
 
-2.3 Tracing logs
+        Using for local developing
 
-For trace all billing query and response set log level to trace info `appsettings.Development.json`
+        ```json
+        "ApiSettings": {
+            "SubscriptionId": "YOUR_SUBSCRIPTION_ID",
+            "TenantId": "YOUR_TENANT_ID",
+            "ClientId": "YOUR_CLIENT_ID",
+            "ClientSecret": "CLIENT_SECRET_SP"
+        },
+        ```
 
-```json
-{
-  "Logging": {
-    "LogLevel": {
-      "Default": "Trace",   
-```
+1. Tracing logs
 
-3. Install dotnet SDK
-Download and install .NET Core 3.1 SDK or above 
-<https://dotnet.microsoft.com/download/dotnet-core/3.1>
+    For trace all billing query and response set log level to trace info `appsettings.Development.json`
 
+    ```json
+    "Serilog": {
+    "MinimumLevel": {
+        "Default": "Trace"
+    }
+    ```
 
-4. Run dotnet
+1. Install dotnet SDK
 
-```bash
-dotnet run --project AzureBillingExporter/AzureBillingExporter.csproj
-```
+    Download and install .NET Core 3.1 SDK or above
+    <https://dotnet.microsoft.com/download/dotnet-core/3.1>
 
-5. Open metrics
+1. Run dotnet
 
-```bash
-curl http://localhost:5000/metrics
-```
+    ```bash
+    dotnet run --project AzureBillingExporter/AzureBillingExporter.csproj
+    ```
+
+1. Open metrics
+
+    ```bash
+    curl http://localhost:5000/metrics
+    ```
+
+## Architecture
+
+According Microsoft [documentation](https://docs.microsoft.com/en-us/azure/cost-management-billing/costs/manage-automation#error-code-429---call-count-has-exceeded-rate-limits) application may create only 30 API calls per minute.
+After that threshold application will get `Too Many Requests` response from API.
+
+> Error code 429 - Call count has exceeded rate limits
+>
+> To enable a consistent experience for all Cost Management subscribers, Cost Management APIs are rate limited. When you reach the limit, you receive the HTTP status code 429: Too many requests. The current throughput limits for our APIs are as follows:
+>
+> 30 calls per minute - It's done per scope, per user, or application.
+>
+> 200 calls per minute - It's done per tenant, per user, or application.
 
-# Metrics
+To avoid such errors, this exporter has background job to get data from API.
+Received cost data placed in memory cache. Prometheus scriber calls on `/metrics` get data from cache and get quick response.
+
+In case of `Too Many Requests` errors, background job waits 1 minute before next calls.
+
+## Configuration
+
+| *Setting*  | *Type* | *Description* |
+|---|---|---|
+| `LogsAtJsonFormat` | bool | Write logs in plain text or JSON format |
+| `CollectPeriodInMinutes` | int | Period in minutes to make API call to the Azure, to get metrics |
+| `CachePeriodInMinutes` | int | Period in minutes to cache API call results |
+| `CustomCollectorsFilePath` | string | Path to YAML file with custom collectors (see [Custom Metrics](#Custom-Metrics)) |
+
+## Metrics
 
 | *Metrics Name*  | *Description* |
 |---|---|
@@ -83,9 +112,9 @@ curl http://localhost:5000/metrics
 | `azure_billing_daily_before_yesterday`  | Day before yesterday all costs |
 | `azure_billing_monthly`  | Costs by current month |
 
-# Custom Metrics
+## Custom Metrics
 
-## Set custom metrics configs into `custom_collectors.yml`
+### Set custom metrics configs into `custom_collectors.yml`
 
 ```yaml
 # A Prometheus metric with (optional) additional labels, value and labels populated from one query.
@@ -103,14 +132,16 @@ metrics:
     replace_date_labels_to_enum: true  # replace `05/01/2020 00:00:00` to `last_month`, `UsageDate="20200624"` to `yesterday`. Default false
     query_file: './custom_queries/azure_billing_by_resource_group.json'
 ```
-## You can set custom path to collectors.yaml file
+
+### You can set custom path to collectors.yaml file
 
 Into `appsettings.Development.json` (or env `CustomCollectorsFilePath`) set:
+
 ```json
   "CustomCollectorsFilePath" : "./local/custom_collectors.yml",
 ```
 
-## Query to billing api
+### Query to billing api
 
 ```json
 {
@@ -144,9 +175,9 @@ Into `appsettings.Development.json` (or env `CustomCollectorsFilePath`) set:
 }
 ```
 
-## Datetime constants into query files
+### Datetime constants into query files
 
-You can use special constant into query file. For this use `{{ }}` template notation [Liquid Template Language](https://shopify.github.io/liquid/) . 
+You can use special constant into query file. For this use `{{ }}` template notation [Liquid Template Language](https://shopify.github.io/liquid/) .
 DateTime Constants (using server datetime). If today is '2020-06-23T08:12:45':
 
 | *Constant*  | *Description* |  *Example* |
@@ -160,16 +191,15 @@ DateTime Constants (using server datetime). If today is '2020-06-23T08:12:45':
 | `YearAgo` |  This month first day year ago. | '2019-06-01T00:00:00.0000000' |
 
 All this constants you can use into billing query json files:
+
 ```json
   "timePeriod": {
     "from": "{{ PrevMonthStart }}",
     "to": "{{ TodayEnd }}"
   }
 ```
 
-
-
-# Try Azure Billing Query on sandbox
+## Try Azure Billing Query on sandbox
 
 Go to docs:
 <https://docs.microsoft.com/en-us/rest/api/cost-management/query/usage>
@@ -208,19 +238,11 @@ Body:
 }
 ```
 
-
-# Notice
+## Notice
 
 Request duration measuring for exporter:
-```
-▶ curl -o /dev/null -s -w 'Total: %{time_total}s\n' http://localhost:5000/metrics
-Total: 19.220319s
-
-~
-▶ curl -o /dev/null -s -w 'Total: %{time_total}s\n' http://localhost:5000/metrics
-Total: 17.939426s
 
-~
+```console
 ▶ curl -o /dev/null -s -w 'Total: %{time_total}s\n' http://localhost:5000/metrics
-Total: 18.603152s
-```
+Total: 0.009669s
+```

src/AzureApi/AccessTokenFactory.cs

@@ -3,6 +3,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using System.Web;
+using AzureBillingExporter.Cost;
 using Microsoft.Extensions.Logging;
 using Newtonsoft.Json;
 

src/AzureApi/AzureCostManagementClient.cs

@@ -6,6 +6,7 @@
 using System.Runtime.CompilerServices;
 using System.Text;
 using System.Threading;
+using AzureBillingExporter.Cost;
 using Microsoft.Extensions.Logging;
 using Newtonsoft.Json;
 
@@ -15,24 +16,27 @@ public class AzureCostManagementClient
     {
         private readonly ApiSettings _apiSettings;
         private readonly IAccessTokenProvider _accessTokenProvider;
-        private readonly ILogger<BillingQueryClient> _logger;
+        private readonly ILogger<AzureCostManagementClient> _logger;
+        private readonly IHttpClientFactory _httpClientFactory;
 
         public AzureCostManagementClient(ApiSettings apiSettings,
-            IAccessTokenProvider accessTokenProvider, 
-            ILogger<BillingQueryClient> logger)
+            IAccessTokenProvider accessTokenProvider,
+            ILogger<AzureCostManagementClient> logger,
+            IHttpClientFactory httpClientFactory)
         {
             _apiSettings = apiSettings;
             _accessTokenProvider = accessTokenProvider;
             _logger = logger;
+            _httpClientFactory = httpClientFactory;
         }
-        
-        public async IAsyncEnumerable<CostResultRows> ExecuteBillingQuery(string billingQuery, [EnumeratorCancellation] CancellationToken cancel, BillingQueryClient billingQueryClient)
+
+        public async IAsyncEnumerable<CostResultRows> ExecuteBillingQuery(string billingQuery, [EnumeratorCancellation] CancellationToken cancel)
         {
+            var client = _httpClientFactory.CreateClient(nameof(AzureCostManagementClient));
+
             var azureManagementUrl =
                 $"https://management.azure.com/subscriptions/{_apiSettings.SubscriptionId}/providers/Microsoft.CostManagement/query?api-version=2019-10-01";
 
-            using var httpClient = new HttpClient();
-
             _logger.LogTrace($"Billing query {billingQuery}");
             var request = new HttpRequestMessage
             {
@@ -44,18 +48,23 @@ public async IAsyncEnumerable<CostResultRows> ExecuteBillingQuery(string billing
                     "application/json")
             };
             request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
-            
+
             var accessToken = _accessTokenProvider.GetAccessToken();
             request.Headers.Authorization =
                 new AuthenticationHeaderValue("Bearer", accessToken);
 
-            var response = await httpClient.SendAsync(request, cancel);
+            var response = await client.SendAsync(request, cancel);
 
             if (response.StatusCode != HttpStatusCode.OK)
             {
+                if (response.StatusCode == HttpStatusCode.TooManyRequests)
+                {
+                    throw new TooManyRequestsException($"{response.StatusCode} {await response.Content.ReadAsStringAsync()}");
+                }
+
                 throw new Exception($"{response.StatusCode} {await response.Content.ReadAsStringAsync()}");
             }
-            
+
             var responseContent = await response.Content.ReadAsStringAsync();
             _logger.LogTrace($"Billing query response result {responseContent}");
             dynamic json = JsonConvert.DeserializeObject(responseContent);
@@ -66,4 +75,4 @@ public async IAsyncEnumerable<CostResultRows> ExecuteBillingQuery(string billing
             }
         }
     }
-}
+}

src/AzureApi/BackgroundAccessTokenProviderHostedService.cs

@@ -54,13 +54,13 @@ public async Task StopAsync(CancellationToken cancellationToken)
         {
             if (_executingTask == null)
             {
-                _logger.LogInformation("Devices API background access tokens refreshing hosted service - Already stopped.");
+                _logger.LogInformation("Azure Billing API background access tokens refreshing hosted service - Already stopped.");
                 return;
             }
 
             try
             {
-                _logger.LogInformation("Devices API background access tokens refreshing hosted service - Stopping...");
+                _logger.LogInformation("Azure Billing API background access tokens refreshing hosted service - Stopping...");
                 // Signal cancellation to the executing method
                 _stoppingCts.Cancel();
             }
@@ -72,7 +72,7 @@ await Task.WhenAny(
                     Task.Delay(Timeout.Infinite, cancellationToken));
             }
 
-            _logger.LogInformation("Devices API background access tokens refreshing hosted service - Stopped.");
+            _logger.LogInformation("Azure Billing API background access tokens refreshing hosted service - Stopped.");
         }
 
         private async Task StartRefreshingAccessTokensInBackgroundAsync(CancellationToken cancellationToken)
@@ -120,4 +120,4 @@ private async Task StartRefreshingAccessTokensInBackgroundAsync(CancellationToke
             }
         }
     }
-}
+}

src/AzureApi/TooManyRequestsException.cs

@@ -0,0 +1,11 @@
+using System;
+
+namespace AzureBillingExporter.AzureApi
+{
+    public class TooManyRequestsException : Exception
+    {
+        public TooManyRequestsException(string message) : base(message)
+        {
+        }
+    }
+}

src/AzureBillingExporter.csproj

@@ -7,13 +7,19 @@
     </PropertyGroup>
 
     <ItemGroup>
+      <PackageReference Include="Dodo.HttpClient.ResiliencePolicies" Version="2.0.1" />
       <PackageReference Include="DotLiquid" Version="2.0.358" />
-      <PackageReference Include="Microsoft.Extensions.Configuration" Version="3.1.5" />
+      <PackageReference Include="Microsoft.Extensions.Configuration" Version="6.0.0" />
       <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
       <PackageReference Include="prometheus-net" Version="3.6.0" />
       <PackageReference Include="prometheus-net.AspNetCore" Version="3.6.0" />
       <PackageReference Include="prometheus-net.AspNetCore.Grpc" Version="3.6.0" />
       <PackageReference Include="prometheus-net.AspNetCore.HealthChecks" Version="3.6.0" />
+      <PackageReference Include="Serilog" Version="2.10.0" />
+      <PackageReference Include="Serilog.AspNetCore" Version="4.1.0" />
+      <PackageReference Include="Serilog.Extensions.Logging" Version="3.1.0" />
+      <PackageReference Include="Serilog.Formatting.Elasticsearch" Version="8.4.1" />
+      <PackageReference Include="Serilog.Sinks.Console" Version="4.0.1" />
       <PackageReference Include="YamlDotNet" Version="8.1.2" />
     </ItemGroup>
 </Project>