From a1c30f6741bfa6234808b42c31e87be936a03ba3 Mon Sep 17 00:00:00 2001 From: Ion Dormenco Date: Thu, 11 Jul 2024 10:58:50 +0300 Subject: [PATCH] fix hangfire job, fix winners retrieval --- .../Elections/WinnersAggregator.cs | 63 +++++++++++++------ .../ElectionResults.Hangfire.csproj | 1 - .../DownloadAndProcessTurnoutResultsJob.cs | 59 ++++++++--------- .../Jobs/Installer.cs | 2 +- 4 files changed, 70 insertions(+), 55 deletions(-) diff --git a/src/ElectionResults.Core/Elections/WinnersAggregator.cs b/src/ElectionResults.Core/Elections/WinnersAggregator.cs index ec3a18d..5915756 100644 --- a/src/ElectionResults.Core/Elections/WinnersAggregator.cs +++ b/src/ElectionResults.Core/Elections/WinnersAggregator.cs @@ -31,7 +31,8 @@ public WinnersAggregator(ApplicationDbContext dbContext, _territoryRepository = territoryRepository; } - public async Task>> GetLocalityCityHallWinnersByCounty(int ballotId, int countyId, bool takeOnlyWinner = true) + public async Task>> GetLocalityCityHallWinnersByCounty(int ballotId, int countyId, + bool takeOnlyWinner = true) { var dbWinners = await GetWinners(ballotId, countyId, ElectionDivision.Locality); @@ -44,7 +45,8 @@ public async Task>> GetLocalityCityHallWinnersByCounty(int b var localities = await _dbContext .Localities - .Where(l => l.CountyId == countyId).ToListAsync(); + .Where(l => l.CountyId == countyId) + .ToListAsync(); var candidateResultsForCounty = await _dbContext.CandidateResults .Include(c => c.Ballot) @@ -66,19 +68,21 @@ public async Task>> GetLocalityCityHallWinnersByCounty(int b var localityWinner = results.FirstOrDefault(); - var turnoutForLocality = turnouts.Where(c => c.LocalityId == locality.LocalityId).FirstOrDefault(); + var turnoutForLocality = turnouts.FirstOrDefault(c => c.LocalityId == locality.LocalityId); if (localityWinner != null) { if (takeOnlyWinner) { - winningCandidates.Add(Winner.CreateLocalityWinner(ballotId, countyId, locality.LocalityId, localityWinner, turnoutForLocality)); + winningCandidates.Add(Winner.CreateLocalityWinner(ballotId, countyId, locality.LocalityId, + localityWinner, turnoutForLocality)); } else { foreach (var candidateResult in results) { - winningCandidates.Add(Winner.CreateLocalityWinner(ballotId, countyId, locality.LocalityId, candidateResult, turnoutForLocality)); + winningCandidates.Add(Winner.CreateLocalityWinner(ballotId, countyId, locality.LocalityId, + candidateResult, turnoutForLocality)); } } } @@ -96,8 +100,9 @@ private async Task> GetWinners(int ballotId, int? countyId, Electio .Where(w => w.BallotId == ballotId && w.Division == division && w.CountyId == countyId) - .FromCacheAsync(new MemoryCacheEntryOptions { SlidingExpiration = TimeSpan.FromMinutes(10) }, MemoryCache.CreateWinnersKey(ballotId, countyId, division)); - + .FromCacheAsync(new MemoryCacheEntryOptions { SlidingExpiration = TimeSpan.FromMinutes(10) }, + MemoryCache.CreateWinnersKey(ballotId, countyId, division)); + return winners.ToList(); } @@ -117,21 +122,22 @@ public async Task>> GetLocalityWinnersByCounty(in return Result.Failure>(winners.Error); } - private IIncludableQueryable CreateWinnersQuery() + private IQueryable CreateWinnersQuery() { return _dbContext.Winners - .Include(w => w.Candidate.Party) + .Include(w => w.Candidate) + .ThenInclude(x => x.Party) .Include(w => w.Turnout) .Include(w => w.Ballot) - .Include(w => w.Ballot.Election) - .Include(w => w.Party); + .Include(w => w.Ballot.Election); } private static ElectionMapWinner WinnerToElectionMapWinner(Winner winner, IEnumerable parties) { var divisionId = winner.Candidate.LocalityId ?? winner.CountyId ?? winner.LocalityId ?? winner.CountryId; - var electionMapWinner = CreateElectionMapWinner(divisionId, winner.Ballot, winner.Candidate, winner.Turnout); + var electionMapWinner = + CreateElectionMapWinner(divisionId, winner.Ballot, winner.Candidate, winner.Turnout); if (electionMapWinner.Winner.PartyColor.IsEmpty()) { @@ -150,7 +156,8 @@ public async Task>> GetCountryWinners(int ballotI if (dbWinners.Count > 0) return dbWinners.Select(winner => WinnerToElectionMapWinner(winner, parties.ToList())).ToList(); - QueryCacheManager.ExpireTag(MemoryCache.CreateWinnersKey(ballotId, null, ElectionDivision.Diaspora_Country)); + QueryCacheManager.ExpireTag(MemoryCache.CreateWinnersKey(ballotId, null, + ElectionDivision.Diaspora_Country)); var winners = new List(); var countries = await _territoryRepository.GetCountries(null); if (countries.IsFailure) @@ -186,11 +193,14 @@ public async Task>> GetCountryWinners(int ballotI var electionMapWinner = CreateElectionMapWinner(country.Id, ballot, countryWinner, turnoutForCountry); if (electionMapWinner.Winner.PartyColor.IsEmpty()) { - electionMapWinner.Winner.PartyColor = parties.ToList().GetMatchingParty(countryWinner.ShortName)?.Color ?? Consts.IndependentCandidateColor; + electionMapWinner.Winner.PartyColor = + parties.ToList().GetMatchingParty(countryWinner.ShortName)?.Color ?? + Consts.IndependentCandidateColor; } winners.Add(electionMapWinner); - winningCandidates.Add(Winner.CreateForDiasporaCountry(ballot.BallotId, country.Id, countryWinner, turnoutForCountry, electionMapWinner.Winner.Votes)); + winningCandidates.Add(Winner.CreateForDiasporaCountry(ballot.BallotId, country.Id, countryWinner, + turnoutForCountry, electionMapWinner.Winner.Votes)); } await SaveWinners(winningCandidates); @@ -231,8 +241,10 @@ public async Task>> GetCountyWinners(int ballotId { topResult = candidateResults.MaxBy(c => c.TotalSeats); } + list.Add(topResult); } + return list .Select(c => { @@ -241,21 +253,26 @@ public async Task>> GetCountyWinners(int ballotId { turnout.ValidVotes = c.TotalSeats; } + return CreateElectionMapWinner(c.CountyId, ballot, c, turnout); }).ToList(); } + var dbWinners = await GetWinners(ballotId, null, ElectionDivision.County); if (dbWinners.Count > 0) return dbWinners.Select(winner => WinnerToElectionMapWinner(winner, parties)).ToList(); - QueryCacheManager.ExpireTag(MemoryCache.CreateWinnersKey(ballotId, null, ElectionDivision.Diaspora_Country)); + QueryCacheManager.ExpireTag(MemoryCache.CreateWinnersKey(ballotId, null, ElectionDivision.County)); var winners = await AggregateCountyWinners(ballotId, parties); var ids = winners.Select(w => w.Id).ToList(); + winners = await _dbContext.Winners .AsNoTracking() .Include(w => w.Candidate) + .ThenInclude(x => x.Party) .Include(w => w.Ballot) .Include(w => w.Turnout) .Where(w => ids.Contains(w.Id)).ToListAsync(); + return Result.Success(winners.Select(winner => WinnerToElectionMapWinner(winner, parties)).ToList()); } @@ -290,11 +307,15 @@ private async Task> AggregateCountyWinners(int ballotId, List>> GetWinningCandidatesByCounty(in if (dbWinners.Count > 0) return dbWinners.Select(w => w.Candidate).ToList(); - QueryCacheManager.ExpireTag(MemoryCache.CreateWinnersKey(ballotId, null, ElectionDivision.Diaspora_Country)); + QueryCacheManager.ExpireTag(MemoryCache.CreateWinnersKey(ballotId, null, + ElectionDivision.Diaspora_Country)); var winners = await AggregateCountyWinners(ballotId, parties); return winners.Select(w => w.Candidate).ToList(); @@ -374,6 +396,7 @@ public List RetrieveWinners(List results, if (candidateResult.Party == null && candidateResult.PartyName.IsNotEmpty()) candidateResult.Party = new Party { Name = candidateResult.PartyName }; } + var groupedWinners = results .GroupBy(w => w.Party?.Name) .OrderByDescending(w => w.Count()) diff --git a/src/ElectionResults.Hangfire/ElectionResults.Hangfire.csproj b/src/ElectionResults.Hangfire/ElectionResults.Hangfire.csproj index 08d98f7..eee963e 100644 --- a/src/ElectionResults.Hangfire/ElectionResults.Hangfire.csproj +++ b/src/ElectionResults.Hangfire/ElectionResults.Hangfire.csproj @@ -28,7 +28,6 @@ - diff --git a/src/ElectionResults.Hangfire/Jobs/DownloadAndProcessTurnoutResultsJob.cs b/src/ElectionResults.Hangfire/Jobs/DownloadAndProcessTurnoutResultsJob.cs index 09c6514..44940e1 100644 --- a/src/ElectionResults.Hangfire/Jobs/DownloadAndProcessTurnoutResultsJob.cs +++ b/src/ElectionResults.Hangfire/Jobs/DownloadAndProcessTurnoutResultsJob.cs @@ -127,55 +127,46 @@ await context.Database.ExecuteSqlRawAsync( Dictionary> list = new Dictionary>(); if (ballot.BallotType == BallotType.LocalCouncil) { - list = countyResult.Value[ScopeCode.PRCNCT].Categories[CategoryCode.CL].GetTable().Values - .GroupBy(g => g.UatSiruta, (key, g) => new { Siruta = GetSiruta(county, key, g.First()), Votes = g.SelectMany(x => x.Votes).ToList() }) - .ToDictionary(x => x.Siruta, y => y.Votes.GroupBy(x => x.Candidate, (key, g) => new VoteModel() + list = countyResult.Value[ScopeCode.UAT].Categories[CategoryCode.CL].GetTable().Values + .ToDictionary(x => GetSiruta(county, x.UatSiruta, x), y => y.Votes.Select(v => new VoteModel() { - Candidate = key, - Party = g.FirstOrDefault().Party, - Votes = g.Sum(x => x.Votes ?? 0), - Mandates1 = g.Sum(x => x.Mandates1 ?? 0), - Mandates2 = g.Sum(x => x.Mandates2 ?? 0), + Candidate = v.Candidate, + Party = v.Party, + Votes = v.Votes ?? 0, + Mandates1 = v.Mandates1 ?? 0, + Mandates2 = v.Mandates2 ?? 0, }).ToList()); } else if (ballot.BallotType == BallotType.Mayor) { - list = countyResult.Value[ScopeCode.PRCNCT].Categories[CategoryCode.P].GetTable().Values - .GroupBy(g => GetUatSiruta(g, county), (key, g) => new { Siruta = GetSiruta(county, key, g.First()), Votes = g.SelectMany(x => x.Votes).ToList() }) - .ToDictionary(x => x.Siruta, y => y.Votes.GroupBy(x => x.Candidate, (key, g) => new VoteModel() + list = countyResult.Value[ScopeCode.UAT].Categories[CategoryCode.P].GetTable().Values + .ToDictionary(x => GetSiruta(county, x.UatSiruta, x), y => y.Votes.Select(v => new VoteModel() { - Candidate = key, - Party = g.FirstOrDefault().Party, - Votes = g.Sum(x => x.Votes ?? 0), - Mandates1 = g.Sum(x => x.Mandates1 ?? 0), - Mandates2 = g.Sum(x => x.Mandates2 ?? 0), + Candidate = v.Candidate, + Party = v.Party, + Votes = v.Votes ?? 0 }).ToList()); - } else if (ballot.BallotType == BallotType.CountyCouncil) { - list = countyResult.Value[ScopeCode.PRCNCT].Categories[CategoryCode.CJ].GetTable().Values - .GroupBy(g => g.CountyCode, (key, g) => new { CountyCode = key, Votes = g.SelectMany(x => x.Votes).ToList() }) - .ToDictionary(x => x.CountyCode, y => y.Votes.GroupBy(x => x.Candidate, (key, g) => new VoteModel() + list = countyResult.Value[ScopeCode.CNTY].Categories[CategoryCode.CJ].GetTable().Values + .ToDictionary(x => x.CountyCode, y => y.Votes.Select(v => new VoteModel() { - Candidate = key, - Party = g.FirstOrDefault().Party, - Votes = g.Sum(x => x.Votes ?? 0), - Mandates1 = g.Sum(x => x.Mandates1 ?? 0), - Mandates2 = g.Sum(x => x.Mandates2 ?? 0), + Candidate = v.Candidate, + Party = v.Party, + Votes = v.Votes ?? 0, + Mandates1 = v.Mandates1 ?? 0, + Mandates2 = v.Mandates2 ?? 0, }).ToList()); } else if (ballot.BallotType == BallotType.CountyCouncilPresident) { - list = countyResult.Value[ScopeCode.PRCNCT].Categories[CategoryCode.PCJ].GetTable().Values - .GroupBy(g => g.CountyCode, (key, g) => new { CountyCode = key, Votes = g.SelectMany(x => x.Votes).ToList() }) - .ToDictionary(x => x.CountyCode, y => y.Votes.GroupBy(x => x.Candidate, (key, g) => new VoteModel() + list = countyResult.Value[ScopeCode.CNTY].Categories[CategoryCode.PCJ].GetTable().Values + .ToDictionary(x => x.CountyCode, y => y.Votes.Select(v => new VoteModel() { - Candidate = key, - Party = g.FirstOrDefault().Party, - Votes = g.Sum(x => x.Votes ?? 0), - Mandates1 = g.Sum(x => x.Mandates1 ?? 0), - Mandates2 = g.Sum(x => x.Mandates2 ?? 0), + Candidate = v.Candidate, + Party = v.Party, + Votes = v.Votes ?? 0, }).ToList()); } @@ -219,7 +210,9 @@ await context.Database.ExecuteSqlRawAsync($@" DELETE FROM candidateresults WHERE ballotId = {ballot.BallotId}; "); var candidateResults = _candidates.Where(c => c.BallotId == ballot.BallotId).ToList(); + await BulkInsertCandidateResultsAsync(candidateResults); + await context.Database.ExecuteSqlRawAsync($@" INSERT INTO candidateresults (Votes, BallotId, Name, ShortName, PartyName, PartyId, YesVotes, NoVotes, SeatsGained, Division, CountyId, LocalityId, TotalSeats, Seats1, Seats2, OverElectoralThreshold, CountryId, BallotPosition) SELECT Votes, BallotId, Name, ShortName, PartyName, PartyId, YesVotes, NoVotes, SeatsGained, Division, CountyId, LocalityId, TotalSeats, Seats1, Seats2, OverElectoralThreshold, CountryId, BallotPosition diff --git a/src/ElectionResults.Hangfire/Jobs/Installer.cs b/src/ElectionResults.Hangfire/Jobs/Installer.cs index 8681eca..d9113ac 100644 --- a/src/ElectionResults.Hangfire/Jobs/Installer.cs +++ b/src/ElectionResults.Hangfire/Jobs/Installer.cs @@ -30,7 +30,7 @@ public static WebApplication WithJobs(this WebApplication app) backgroundJobClient.Enqueue(x => x.Run(CancellationToken.None)); - recurringJobManager.AddOrUpdate($"locale09062024-data-processor", x => x.Run("locale09062024", 50, false, StageCode.PART), "*/7 * * * *"); + recurringJobManager.AddOrUpdate($"locale09062024-data-processor", x => x.Run("locale09062024", 50, false, StageCode.FINAL), "* */12 * * *"); // recurringJobManager.AddOrUpdate($"europarlamentare09062024-data-processor", x => x.Run("europarlamentare09062024", 51, true, StageCode.PROV), "*/5 * * * *");