Feature/rework data download (#171)

* Upgraded to .NET 8 and updated Nuget packages

* reimplement download of data

* add endpoint

* Add hangfire results downloader

* Update ElectionCategory.cs

* implement turnouts crawling

* improvements

* Add healthchecks

* Fix issues

* fix small issues

---------

Co-authored-by: Bogdan Bujdea <contact@bogdanbujdea.dev>

src/ElectionResults.API/Controllers/BallotsController.cs

@@ -2,16 +2,14 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
-using ElectionResults.Core.Configuration;
+using ElectionResults.API.Options;
 using ElectionResults.Core.Elections;
 using ElectionResults.Core.Endpoints.Query;
 using ElectionResults.Core.Endpoints.Response;
-using ElectionResults.Core.Entities;
 using ElectionResults.Core.Infrastructure;
 using ElectionResults.Core.Repositories;
 using LazyCache;
 using Microsoft.AspNetCore.Mvc;
-using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
 namespace ElectionResults.API.Controllers
@@ -20,23 +18,20 @@ namespace ElectionResults.API.Controllers
     [Route("api")]
     public class BallotsController : ControllerBase
     {
-        private readonly ILogger<BallotsController> _logger;
         private readonly IResultsAggregator _resultsAggregator;
         private readonly IAppCache _appCache;
         private readonly ITerritoryRepository _territoryRepository;
-        private readonly LiveElectionSettings _settings;
+        private readonly MemoryCacheSettings _cacheSettings;
 
-        public BallotsController(ILogger<BallotsController> logger,
-            IResultsAggregator resultsAggregator,
+        public BallotsController(IResultsAggregator resultsAggregator,
             IAppCache appCache,
             ITerritoryRepository territoryRepository,
-            IOptions<LiveElectionSettings> settings)
+            IOptions<MemoryCacheSettings> cacheSettings)
         {
-            _logger = logger;
             _resultsAggregator = resultsAggregator;
             _appCache = appCache;
             _territoryRepository = territoryRepository;
-            _settings = settings.Value;
+            _cacheSettings = cacheSettings.Value;
         }
 
         [HttpGet("ballots")]
@@ -45,8 +40,12 @@ public async Task<ActionResult<List<ElectionMeta>>> GetBallots()
             var result = await _appCache.GetOrAddAsync(
                 "ballots", () => _resultsAggregator.GetAllBallots(),
                 DateTimeOffset.Now.AddMinutes(120));
+
             if (result.IsSuccess)
+            {
                 return result.Value;
+            }
+
             return StatusCode(500, result.Error);
         }
 
@@ -56,16 +55,26 @@ public async Task<ActionResult<List<PartyList>>> GetCandidatesForBallot([FromQue
             try
             {
                 query.BallotId = ballotId;
+
                 if (query.LocalityId == 0)
+                {
                     query.LocalityId = null;
+                }
+
                 if (query.CountyId == 0)
+                {
                     query.CountyId = null;
+                }
+
                 if (query.Round == 0)
+                {
                     query.Round = null;
+                }
 
                 var result = await _appCache.GetOrAddAsync(
                     query.GetCacheKey(), () => _resultsAggregator.GetBallotCandidates(query),
                     DateTimeOffset.Now.AddMinutes(query.GetCacheDurationInMinutes()));
+
                 return result.Value;
             }
             catch (Exception e)
@@ -82,18 +91,29 @@ public async Task<ActionResult<ElectionResponse>> GetBallot([FromQuery] Election
             try
             {
                 if (query.LocalityId == 0)
+                {
                     query.LocalityId = null;
+                }
+
                 if (query.CountyId == 0)
+                {
                     query.CountyId = null;
+                }
+
                 if (query.Round == 0)
+                {
                     query.Round = null;
+                }
 
                 var expiration = GetExpirationDate(query);
+
                 var result = await _appCache.GetOrAddAsync(
                     query.GetCacheKey(), () => _resultsAggregator.GetBallotResults(query),
                     expiration);
+
                 var newsFeed = await _resultsAggregator.GetNewsFeed(query, result.Value.Meta.ElectionId);
                 result.Value.ElectionNews = newsFeed;
+
                 return result.Value;
             }
             catch (Exception e)
@@ -176,11 +196,11 @@ public async Task<ActionResult<List<LocationData>>> GetCountries([FromQuery] int
 
         private DateTimeOffset GetExpirationDate(ElectionResultsQuery electionResultsQuery)
         {
-            if (electionResultsQuery.BallotId < 110) // ballot older than parliament elections in 2020
+            if (electionResultsQuery.BallotId <= 110) // ballot older than parliament elections in 2020
             {
                 return DateTimeOffset.Now.AddDays(1);
             }
-            return DateTimeOffset.Now.AddMinutes(_settings.ResultsCacheInMinutes);
+            return DateTimeOffset.Now.AddMinutes(_cacheSettings.ResultsCacheInMinutes);
         }
     }
 }

src/ElectionResults.API/Converters/SnakeCaseNamingPolicy.cs

@@ -0,0 +1,21 @@
+using System.Text.Json;
+using System.Text.RegularExpressions;
+
+namespace ElectionResults.API.Converters
+{
+    public class SnakeCaseNamingPolicy : JsonNamingPolicy
+    {
+        public static SnakeCaseNamingPolicy Instance { get; } = new();
+        private static readonly Regex Pattern = new Regex(@"[A-Z]{2,}(?=[A-Z][a-z]+[0-9]*|\b)|[A-Z]?[a-z]+[0-9]*|[A-Z]|[0-9]+");
+
+        private static string ToSnakeCase(string str)
+        {
+            return string.Join("_", Pattern.Matches(str)).ToLower();
+        }
+
+        public override string ConvertName(string name)
+        {
+            return ToSnakeCase(name);
+        }
+    }
+}

src/ElectionResults.API/Dockerfile

@@ -11,7 +11,6 @@ ARG BUILD_CONFIGURATION=Release
 WORKDIR /src
 COPY ["ElectionResults.API/ElectionResults.API.csproj", "ElectionResults.API/"]
 COPY ["ElectionResults.Core/ElectionResults.Core.csproj", "ElectionResults.Core/"]
-COPY ["ElectionResults.Importer/ElectionResults.Importer.csproj", "ElectionResults.Importer/"]
 RUN dotnet restore "./ElectionResults.API/ElectionResults.API.csproj"
 COPY . .
 WORKDIR "/src/ElectionResults.API"

src/ElectionResults.API/ElectionResults.API.csproj

@@ -5,6 +5,7 @@
     <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
   </PropertyGroup>
   <ItemGroup>
+    <PackageReference Include="AspNetCore.HealthChecks.MySql" Version="8.0.1" />
     <PackageReference Include="AWSSDK.S3" Version="3.7.308.6" />
     <PackageReference Include="CSharpFunctionalExtensions" Version="2.42.0" />
     <PackageReference Include="Microsoft.Web.LibraryManager.Build" Version="2.1.175" />
@@ -33,6 +34,8 @@
     <None Include="wwwroot\*" />
   </ItemGroup>
   <ItemGroup>
+    <Folder Include="Import\" />
+    <Folder Include="Options\" />
     <Folder Include="wwwroot\lib\" />
     <Folder Include="wwwroot\upload\" />
   </ItemGroup>