Skip to content

Commit

Permalink
Merge pull request #220 from code4romania/bugfi/hangfire
Browse files Browse the repository at this point in the history 
fix hangfire job, fix winners retrieval
  • Loading branch information
@idormenco
idormenco authored Jul 11, 2024
2 parents ec7cd2a + a1c30f6 commit e396bb3
Show file tree
Hide file tree
Showing 4 changed files with 70 additions and 55 deletions.
63 changes: 43 additions & 20 deletions src/ElectionResults.Core/Elections/WinnersAggregator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ public WinnersAggregator(ApplicationDbContext dbContext,
_territoryRepository = territoryRepository;
}

public async Task<Result<List<Winner>>> GetLocalityCityHallWinnersByCounty(int ballotId, int countyId, bool takeOnlyWinner = true)
public async Task<Result<List<Winner>>> GetLocalityCityHallWinnersByCounty(int ballotId, int countyId,
bool takeOnlyWinner = true)
{
var dbWinners = await GetWinners(ballotId, countyId, ElectionDivision.Locality);

Expand All @@ -44,7 +45,8 @@ public async Task<Result<List<Winner>>> GetLocalityCityHallWinnersByCounty(int b

var localities = await _dbContext
.Localities
.Where(l => l.CountyId == countyId).ToListAsync();
.Where(l => l.CountyId == countyId)
.ToListAsync();

var candidateResultsForCounty = await _dbContext.CandidateResults
.Include(c => c.Ballot)
Expand All @@ -66,19 +68,21 @@ public async Task<Result<List<Winner>>> GetLocalityCityHallWinnersByCounty(int b

var localityWinner = results.FirstOrDefault();

var turnoutForLocality = turnouts.Where(c => c.LocalityId == locality.LocalityId).FirstOrDefault();
var turnoutForLocality = turnouts.FirstOrDefault(c => c.LocalityId == locality.LocalityId);

if (localityWinner != null)
{
if (takeOnlyWinner)
{
winningCandidates.Add(Winner.CreateLocalityWinner(ballotId, countyId, locality.LocalityId, localityWinner, turnoutForLocality));
winningCandidates.Add(Winner.CreateLocalityWinner(ballotId, countyId, locality.LocalityId,
localityWinner, turnoutForLocality));
}
else
{
foreach (var candidateResult in results)
{
winningCandidates.Add(Winner.CreateLocalityWinner(ballotId, countyId, locality.LocalityId, candidateResult, turnoutForLocality));
winningCandidates.Add(Winner.CreateLocalityWinner(ballotId, countyId, locality.LocalityId,
candidateResult, turnoutForLocality));
}
}
}
Expand All @@ -96,8 +100,9 @@ private async Task<List<Winner>> GetWinners(int ballotId, int? countyId, Electio
.Where(w => w.BallotId == ballotId
&& w.Division == division
&& w.CountyId == countyId)
.FromCacheAsync(new MemoryCacheEntryOptions { SlidingExpiration = TimeSpan.FromMinutes(10) }, MemoryCache.CreateWinnersKey(ballotId, countyId, division));

.FromCacheAsync(new MemoryCacheEntryOptions { SlidingExpiration = TimeSpan.FromMinutes(10) },
MemoryCache.CreateWinnersKey(ballotId, countyId, division));

return winners.ToList();
}

Expand All @@ -117,21 +122,22 @@ public async Task<Result<List<ElectionMapWinner>>> GetLocalityWinnersByCounty(in
return Result.Failure<List<ElectionMapWinner>>(winners.Error);
}

private IIncludableQueryable<Winner, Party> CreateWinnersQuery()
private IQueryable<Winner> CreateWinnersQuery()
{
return _dbContext.Winners
.Include(w => w.Candidate.Party)
.Include(w => w.Candidate)
.ThenInclude(x => x.Party)
.Include(w => w.Turnout)
.Include(w => w.Ballot)
.Include(w => w.Ballot.Election)
.Include(w => w.Party);
.Include(w => w.Ballot.Election);
}

private static ElectionMapWinner WinnerToElectionMapWinner(Winner winner, IEnumerable<Party> parties)
{
var divisionId = winner.Candidate.LocalityId ?? winner.CountyId ?? winner.LocalityId ?? winner.CountryId;

var electionMapWinner = CreateElectionMapWinner(divisionId, winner.Ballot, winner.Candidate, winner.Turnout);
var electionMapWinner =
CreateElectionMapWinner(divisionId, winner.Ballot, winner.Candidate, winner.Turnout);

if (electionMapWinner.Winner.PartyColor.IsEmpty())
{
Expand All @@ -150,7 +156,8 @@ public async Task<Result<List<ElectionMapWinner>>> GetCountryWinners(int ballotI

if (dbWinners.Count > 0)
return dbWinners.Select(winner => WinnerToElectionMapWinner(winner, parties.ToList())).ToList();
QueryCacheManager.ExpireTag(MemoryCache.CreateWinnersKey(ballotId, null, ElectionDivision.Diaspora_Country));
QueryCacheManager.ExpireTag(MemoryCache.CreateWinnersKey(ballotId, null,
ElectionDivision.Diaspora_Country));
var winners = new List<ElectionMapWinner>();
var countries = await _territoryRepository.GetCountries(null);
if (countries.IsFailure)
Expand Down Expand Up @@ -186,11 +193,14 @@ public async Task<Result<List<ElectionMapWinner>>> GetCountryWinners(int ballotI
var electionMapWinner = CreateElectionMapWinner(country.Id, ballot, countryWinner, turnoutForCountry);
if (electionMapWinner.Winner.PartyColor.IsEmpty())
{
electionMapWinner.Winner.PartyColor = parties.ToList().GetMatchingParty(countryWinner.ShortName)?.Color ?? Consts.IndependentCandidateColor;
electionMapWinner.Winner.PartyColor =
parties.ToList().GetMatchingParty(countryWinner.ShortName)?.Color ??
Consts.IndependentCandidateColor;
}

winners.Add(electionMapWinner);
winningCandidates.Add(Winner.CreateForDiasporaCountry(ballot.BallotId, country.Id, countryWinner, turnoutForCountry, electionMapWinner.Winner.Votes));
winningCandidates.Add(Winner.CreateForDiasporaCountry(ballot.BallotId, country.Id, countryWinner,
turnoutForCountry, electionMapWinner.Winner.Votes));
}

await SaveWinners(winningCandidates);
Expand Down Expand Up @@ -231,8 +241,10 @@ public async Task<Result<List<ElectionMapWinner>>> GetCountyWinners(int ballotId
{
topResult = candidateResults.MaxBy(c => c.TotalSeats);
}

list.Add(topResult);
}

return list
.Select(c =>
{
Expand All @@ -241,21 +253,26 @@ public async Task<Result<List<ElectionMapWinner>>> GetCountyWinners(int ballotId
{
turnout.ValidVotes = c.TotalSeats;
}

return CreateElectionMapWinner(c.CountyId, ballot, c, turnout);
}).ToList();
}

var dbWinners = await GetWinners(ballotId, null, ElectionDivision.County);
if (dbWinners.Count > 0)
return dbWinners.Select(winner => WinnerToElectionMapWinner(winner, parties)).ToList();
QueryCacheManager.ExpireTag(MemoryCache.CreateWinnersKey(ballotId, null, ElectionDivision.Diaspora_Country));
QueryCacheManager.ExpireTag(MemoryCache.CreateWinnersKey(ballotId, null, ElectionDivision.County));
var winners = await AggregateCountyWinners(ballotId, parties);
var ids = winners.Select(w => w.Id).ToList();

winners = await _dbContext.Winners
.AsNoTracking()
.Include(w => w.Candidate)
.ThenInclude(x => x.Party)
.Include(w => w.Ballot)
.Include(w => w.Turnout)
.Where(w => ids.Contains(w.Id)).ToListAsync();

return Result.Success(winners.Select(winner => WinnerToElectionMapWinner(winner, parties)).ToList());
}

Expand Down Expand Up @@ -290,11 +307,15 @@ private async Task<List<Winner>> AggregateCountyWinners(int ballotId, List<Party

if (countyWinner == null || turnoutForCounty == null)
continue;
var electionMapWinner = CreateElectionMapWinner(county.CountyId, ballot, countyWinner, turnoutForCounty);
var electionMapWinner =
CreateElectionMapWinner(county.CountyId, ballot, countyWinner, turnoutForCounty);
if (electionMapWinner.Winner.PartyColor.IsEmpty())
electionMapWinner.Winner.PartyColor = parties.ToList().GetMatchingParty(countyWinner.ShortName)?.Color ?? Consts.IndependentCandidateColor;
electionMapWinner.Winner.PartyColor =
parties.ToList().GetMatchingParty(countyWinner.ShortName)?.Color ??
Consts.IndependentCandidateColor;

winningCandidates.Add(Winner.CreateForCounty(ballot, countyWinner, electionMapWinner, turnoutForCounty.Id,
winningCandidates.Add(Winner.CreateForCounty(ballot, countyWinner, electionMapWinner,
turnoutForCounty.Id,
county.CountyId));
}

Expand All @@ -309,7 +330,8 @@ public async Task<Result<List<CandidateResult>>> GetWinningCandidatesByCounty(in
if (dbWinners.Count > 0)
return dbWinners.Select(w => w.Candidate).ToList();

QueryCacheManager.ExpireTag(MemoryCache.CreateWinnersKey(ballotId, null, ElectionDivision.Diaspora_Country));
QueryCacheManager.ExpireTag(MemoryCache.CreateWinnersKey(ballotId, null,
ElectionDivision.Diaspora_Country));

var winners = await AggregateCountyWinners(ballotId, parties);
return winners.Select(w => w.Candidate).ToList();
Expand Down Expand Up @@ -374,6 +396,7 @@ public List<CandidateResult> RetrieveWinners(List<CandidateResult> results,
if (candidateResult.Party == null && candidateResult.PartyName.IsNotEmpty())
candidateResult.Party = new Party { Name = candidateResult.PartyName };
}

var groupedWinners = results
.GroupBy(w => w.Party?.Name)
.OrderByDescending(w => w.Count())
Expand Down
1 change: 0 additions & 1 deletion src/ElectionResults.Hangfire/ElectionResults.Hangfire.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
<PackageReference Include="Serilog.Extensions.Logging" Version="8.0.0" />
<PackageReference Include="Serilog.Settings.Configuration" Version="8.0.0" />
<PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
<PackageReference Include="Z.EntityFramework.Plus.EFCore" Version="8.102.2.5" />
</ItemGroup>

<ItemGroup>
Expand Down
59 changes: 26 additions & 33 deletions src/ElectionResults.Hangfire/Jobs/DownloadAndProcessTurnoutResultsJob.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,55 +127,46 @@ await context.Database.ExecuteSqlRawAsync(
Dictionary<string, List<VoteModel>> list = new Dictionary<string, List<VoteModel>>();
if (ballot.BallotType == BallotType.LocalCouncil)
{
list = countyResult.Value[ScopeCode.PRCNCT].Categories[CategoryCode.CL].GetTable().Values
.GroupBy(g => g.UatSiruta, (key, g) => new { Siruta = GetSiruta(county, key, g.First()), Votes = g.SelectMany(x => x.Votes).ToList() })
.ToDictionary(x => x.Siruta, y => y.Votes.GroupBy(x => x.Candidate, (key, g) => new VoteModel()
list = countyResult.Value[ScopeCode.UAT].Categories[CategoryCode.CL].GetTable().Values
.ToDictionary(x => GetSiruta(county, x.UatSiruta, x), y => y.Votes.Select(v => new VoteModel()
{
Candidate = key,
Party = g.FirstOrDefault().Party,
Votes = g.Sum(x => x.Votes ?? 0),
Mandates1 = g.Sum(x => x.Mandates1 ?? 0),
Mandates2 = g.Sum(x => x.Mandates2 ?? 0),
Candidate = v.Candidate,
Party = v.Party,
Votes = v.Votes ?? 0,
Mandates1 = v.Mandates1 ?? 0,
Mandates2 = v.Mandates2 ?? 0,
}).ToList());
}
else if (ballot.BallotType == BallotType.Mayor)
{
list = countyResult.Value[ScopeCode.PRCNCT].Categories[CategoryCode.P].GetTable().Values
.GroupBy(g => GetUatSiruta(g, county), (key, g) => new { Siruta = GetSiruta(county, key, g.First()), Votes = g.SelectMany(x => x.Votes).ToList() })
.ToDictionary(x => x.Siruta, y => y.Votes.GroupBy(x => x.Candidate, (key, g) => new VoteModel()
list = countyResult.Value[ScopeCode.UAT].Categories[CategoryCode.P].GetTable().Values
.ToDictionary(x => GetSiruta(county, x.UatSiruta, x), y => y.Votes.Select(v => new VoteModel()
{
Candidate = key,
Party = g.FirstOrDefault().Party,
Votes = g.Sum(x => x.Votes ?? 0),
Mandates1 = g.Sum(x => x.Mandates1 ?? 0),
Mandates2 = g.Sum(x => x.Mandates2 ?? 0),
Candidate = v.Candidate,
Party = v.Party,
Votes = v.Votes ?? 0
}).ToList());

}
else if (ballot.BallotType == BallotType.CountyCouncil)
{
list = countyResult.Value[ScopeCode.PRCNCT].Categories[CategoryCode.CJ].GetTable().Values
.GroupBy(g => g.CountyCode, (key, g) => new { CountyCode = key, Votes = g.SelectMany(x => x.Votes).ToList() })
.ToDictionary(x => x.CountyCode, y => y.Votes.GroupBy(x => x.Candidate, (key, g) => new VoteModel()
list = countyResult.Value[ScopeCode.CNTY].Categories[CategoryCode.CJ].GetTable().Values
.ToDictionary(x => x.CountyCode, y => y.Votes.Select(v => new VoteModel()
{
Candidate = key,
Party = g.FirstOrDefault().Party,
Votes = g.Sum(x => x.Votes ?? 0),
Mandates1 = g.Sum(x => x.Mandates1 ?? 0),
Mandates2 = g.Sum(x => x.Mandates2 ?? 0),
Candidate = v.Candidate,
Party = v.Party,
Votes = v.Votes ?? 0,
Mandates1 = v.Mandates1 ?? 0,
Mandates2 = v.Mandates2 ?? 0,
}).ToList());
}
else if (ballot.BallotType == BallotType.CountyCouncilPresident)
{
list = countyResult.Value[ScopeCode.PRCNCT].Categories[CategoryCode.PCJ].GetTable().Values
.GroupBy(g => g.CountyCode, (key, g) => new { CountyCode = key, Votes = g.SelectMany(x => x.Votes).ToList() })
.ToDictionary(x => x.CountyCode, y => y.Votes.GroupBy(x => x.Candidate, (key, g) => new VoteModel()
list = countyResult.Value[ScopeCode.CNTY].Categories[CategoryCode.PCJ].GetTable().Values
.ToDictionary(x => x.CountyCode, y => y.Votes.Select(v => new VoteModel()
{
Candidate = key,
Party = g.FirstOrDefault().Party,
Votes = g.Sum(x => x.Votes ?? 0),
Mandates1 = g.Sum(x => x.Mandates1 ?? 0),
Mandates2 = g.Sum(x => x.Mandates2 ?? 0),
Candidate = v.Candidate,
Party = v.Party,
Votes = v.Votes ?? 0,
}).ToList());
}

Expand Down Expand Up @@ -219,7 +210,9 @@ await context.Database.ExecuteSqlRawAsync($@"
DELETE FROM candidateresults WHERE ballotId = {ballot.BallotId};
");
var candidateResults = _candidates.Where(c => c.BallotId == ballot.BallotId).ToList();

await BulkInsertCandidateResultsAsync(candidateResults);

await context.Database.ExecuteSqlRawAsync($@"
INSERT INTO candidateresults (Votes, BallotId, Name, ShortName, PartyName, PartyId, YesVotes, NoVotes, SeatsGained, Division, CountyId, LocalityId, TotalSeats, Seats1, Seats2, OverElectoralThreshold, CountryId, BallotPosition)
SELECT Votes, BallotId, Name, ShortName, PartyName, PartyId, YesVotes, NoVotes, SeatsGained, Division, CountyId, LocalityId, TotalSeats, Seats1, Seats2, OverElectoralThreshold, CountryId, BallotPosition
Expand Down
2 changes: 1 addition & 1 deletion src/ElectionResults.Hangfire/Jobs/Installer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public static WebApplication WithJobs(this WebApplication app)

backgroundJobClient.Enqueue<SeedData>(x => x.Run(CancellationToken.None));

recurringJobManager.AddOrUpdate<DownloadAndProcessTurnoutResultsJob>($"locale09062024-data-processor", x => x.Run("locale09062024", 50, false, StageCode.PART), "*/7 * * * *");
recurringJobManager.AddOrUpdate<DownloadAndProcessTurnoutResultsJob>($"locale09062024-data-processor", x => x.Run("locale09062024", 50, false, StageCode.FINAL), "* */12 * * *");

// recurringJobManager.AddOrUpdate<DownloadAndProcessTurnoutResultsJob>($"europarlamentare09062024-data-processor", x => x.Run("europarlamentare09062024", 51, true, StageCode.PROV), "*/5 * * * *");

Expand Down

0 comments on commit e396bb3

Please sign in to comment.