diff --git a/ReplayBrowser/Controllers/DataController.cs b/ReplayBrowser/Controllers/DataController.cs index dc397ea..0dd48a7 100644 --- a/ReplayBrowser/Controllers/DataController.cs +++ b/ReplayBrowser/Controllers/DataController.cs @@ -1,6 +1,7 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using ReplayBrowser.Data; +using ReplayBrowser.Services; using ReplayBrowser.Services.ReplayParser; namespace ReplayBrowser.Controllers; @@ -10,10 +11,12 @@ namespace ReplayBrowser.Controllers; public class DataController : Controller { private readonly ReplayDbContext _context; + private readonly ProfilePregeneratorService _profilePregeneratorService; - public DataController(ReplayDbContext context) + public DataController(ReplayDbContext context, ProfilePregeneratorService profilePregeneratorService) { _context = context; + _profilePregeneratorService = profilePregeneratorService; } [HttpGet("username-completion")] @@ -41,7 +44,8 @@ public DownloadProgress GetDownloadProgress() { Progress = ReplayParserService.DownloadProgress.ToDictionary(x => x.Key, x => x.Value), Status = ReplayParserService.Status.ToFriendlyString(), - Details = ReplayParserService.Details + Details = ReplayParserService.Details, + PregenerationProgress = _profilePregeneratorService.PregenerationProgress }; } } @@ -52,4 +56,5 @@ public class DownloadProgress public required Dictionary Progress { get; set; } public required string Details { get; set; } + public required ProfilePregeneratorService.PreGenerationProgress PregenerationProgress { get; set; } } \ No newline at end of file diff --git a/ReplayBrowser/Pages/Downloads.razor b/ReplayBrowser/Pages/Downloads.razor index aec7467..9a42ffa 100644 --- a/ReplayBrowser/Pages/Downloads.razor +++ b/ReplayBrowser/Pages/Downloads.razor @@ -5,6 +5,11 @@

Downloads

Here you can see the progress of the internal downloads of the website.
The website looks for new replays every 10th minute. So 12:00, 12:10, 12:20, 12:30, etc.

+
+

Pregeneration: 0 / 0

+ +
+

Status: Waiting...

@@ -23,6 +28,8 @@ let Downloads = []; let Status = ""; let Details = ""; + let PregenerationCurrent = 0; + let PregenerationMax = 0; setInterval(() => { download(); @@ -30,9 +37,12 @@ function download() { $.get('/api/Data/download-progress', (data) => { + console.debug(data); Downloads = data.progress; Status = data.status; Details = data.details; + PregenerationCurrent = data.pregenerationProgress.current; + PregenerationMax = data.pregenerationProgress.max; updateDownloads(); }); } @@ -46,6 +56,12 @@ } else { document.getElementById('download-text-no').style.display = 'none'; } + + document.getElementById('pregenerationCurrent').innerText = PregenerationCurrent; + document.getElementById('pregenerationMax').innerText = PregenerationMax; + if (PregenerationMax !== 0) { // Prevent division by zero + document.getElementById('pregenerationProgress').value = PregenerationCurrent / PregenerationMax; + } document.getElementById('status-text').innerText = Status; document.getElementById('details').innerText = Details; diff --git a/ReplayBrowser/Pages/Shared/MainLayout.razor b/ReplayBrowser/Pages/Shared/MainLayout.razor index 40d422f..2a80b1e 100644 --- a/ReplayBrowser/Pages/Shared/MainLayout.razor +++ b/ReplayBrowser/Pages/Shared/MainLayout.razor @@ -2,6 +2,7 @@ @using ReplayBrowser.Data @using ReplayBrowser.Data.Models.Account @using ReplayBrowser.Helpers +@using ReplayBrowser.Services @using ReplayBrowser.Services.ReplayParser @using Serilog @inherits LayoutComponentBase @@ -10,6 +11,7 @@ @inject NavigationManager NavigationManager @inject NoticeHelper NoticeHelper @inject IHttpContextAccessor HttpContextAccessor +@inject ProfilePregeneratorService ProfilePregeneratorService @code { @@ -61,6 +63,14 @@ } + + @if(ProfilePregeneratorService.IsFirstRun) + { + + } @if (_user.Identity.IsAuthenticated) { diff --git a/ReplayBrowser/Services/LeaderboardService.cs b/ReplayBrowser/Services/LeaderboardService.cs index 70463d7..072df93 100644 --- a/ReplayBrowser/Services/LeaderboardService.cs +++ b/ReplayBrowser/Services/LeaderboardService.cs @@ -505,7 +505,7 @@ private async Task GenerateLeaderboard( } } stopwatch.Stop(); - Log.Information("Fetching player data took {Time}ms", stopwatch.ElapsedMilliseconds); + Log.Verbose("Fetching player data took {Time}ms", stopwatch.ElapsedMilliseconds); return returnValue; } diff --git a/ReplayBrowser/Services/ProfilePregeneratorService.cs b/ReplayBrowser/Services/ProfilePregeneratorService.cs index 1093aac..d6a42fe 100644 --- a/ReplayBrowser/Services/ProfilePregeneratorService.cs +++ b/ReplayBrowser/Services/ProfilePregeneratorService.cs @@ -2,7 +2,9 @@ using System.Security.Claims; using Microsoft.AspNetCore.Components.Authorization; using ReplayBrowser.Data; +using ReplayBrowser.Data.Models; using ReplayBrowser.Helpers; +using ReplayBrowser.Services.ReplayParser; using Serilog; namespace ReplayBrowser.Services; @@ -10,34 +12,88 @@ namespace ReplayBrowser.Services; /// /// Service that checks every watched profile for every user and pregenerates the profile if it is not already generated for better loading times. /// -public class ProfilePregeneratorService : IHostedService, IDisposable, IAsyncDisposable +public class ProfilePregeneratorService : IHostedService { - private Timer? _timer = null; private readonly IServiceScopeFactory _scopeFactory; + private ReplayParserService _replayParserService; + + public bool IsFirstRun { get; private set; }= true; + + /// + /// List of profiles that have been generated. + /// + private readonly List _generatedProfiles = new(); /// /// List of profiles which get generated even if they are not on a watched profile list. (example being leaderboards) /// public List AlwaysGenerateProfiles = new List(); - public ProfilePregeneratorService(IServiceScopeFactory scopeFactory) + /// + /// A var which tracks the progress of the pregeneration. + /// + public PreGenerationProgress PregenerationProgress { get; private set; } = new PreGenerationProgress(); + + private int _queuedPreGenerations = 0; + + public ProfilePregeneratorService(IServiceScopeFactory scopeFactory, ReplayParserService replayParserService) { _scopeFactory = scopeFactory; + _replayParserService = replayParserService; } public Task StartAsync(CancellationToken cancellationToken) { - _timer = new Timer(DoWork, null, TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(30)); + _replayParserService.OnReplaysFinishedParsing += OnReplaysParsed; + + // In one minute, start the pregeneration. + Task.Delay(TimeSpan.FromMinutes(1), cancellationToken).ContinueWith(t => QueuePregeneration(), cancellationToken); return Task.CompletedTask; } public Task StopAsync(CancellationToken cancellationToken) { - _timer?.Change(Timeout.Infinite, 0); + _replayParserService.OnReplaysFinishedParsing -= OnReplaysParsed; return Task.CompletedTask; } - - private async void DoWork(object? state) + + private void OnReplaysParsed(object? sender, List replays) + { + if (IsFirstRun) + { + // We are waiting for the first run to complete + Log.Warning("Triggered OnReplaysParsed before the first run has completed."); + return; + } + + foreach (var replay in replays) + { + if (replay.RoundEndPlayers == null) + continue; + + foreach (var player in replay.RoundEndPlayers) + { + if (!_generatedProfiles.Contains(player.PlayerGuid)) + // Profile is not getting autogenerated, ignore. + continue; + + // Profile is getting autogenerated, remove it from the list. + _generatedProfiles.Remove(player.PlayerGuid); + } + } + + QueuePregeneration(); + } + + private void QueuePregeneration() + { + _queuedPreGenerations++; + + if (_queuedPreGenerations == 1) // If we are the first one, start the pregeneration. + PregenerateProfiles(); + } + + private async void PregenerateProfiles() { var sw = new Stopwatch(); Log.Information("Starting profile pregeneration."); @@ -49,30 +105,48 @@ private async void DoWork(object? state) var profilesToGenerate = new List(); profilesToGenerate.AddRange(AlwaysGenerateProfiles); - dbContext.Accounts.Select(a => a.SavedProfiles).ToList().ForEach(profiles => + await foreach (var account in dbContext.Accounts) { - foreach (var profile in profiles.Where(profile => !profilesToGenerate.Contains(profile))) + foreach (var profile in account.SavedProfiles) { + if (profilesToGenerate.Contains(profile)) continue; profilesToGenerate.Add(profile); } - }); + } + + // Remove every profile already found in _generatedProfiles + profilesToGenerate = profilesToGenerate.Except(_generatedProfiles).ToList(); + PregenerationProgress.Max = profilesToGenerate.Count; foreach (var guid in profilesToGenerate) { await replayHelper.GetPlayerProfile(guid, new AuthenticationState(new ClaimsPrincipal()), true, true); Log.Information("Pregenerated profile for {Guid}.", guid); + _generatedProfiles.Add(guid); + PregenerationProgress.Current++; + } + + if (IsFirstRun) + { + IsFirstRun = false; + Log.Information("First run completed."); } + sw.Stop(); Log.Information("Profile pregeneration finished in {ElapsedMilliseconds}ms.", sw.ElapsedMilliseconds); + + _queuedPreGenerations--; + if (_queuedPreGenerations > 0) + { + PregenerateProfiles(); // this is stupid, but then again, this whole codebase is, and its not like someone else is going to read this. Right? + Log.Information("Pregeneration queued."); + } } - - public void Dispose() - { - _timer?.Dispose(); - } - - public async ValueTask DisposeAsync() + + public class PreGenerationProgress { - if (_timer != null) await _timer.DisposeAsync(); + public int Current { get; set; } = 0; + public int Max { get; set; } = 0; } } + diff --git a/ReplayBrowser/Services/ReplayParser/ReplayParserService.cs b/ReplayBrowser/Services/ReplayParser/ReplayParserService.cs index db47f16..bd24d14 100644 --- a/ReplayBrowser/Services/ReplayParser/ReplayParserService.cs +++ b/ReplayBrowser/Services/ReplayParser/ReplayParserService.cs @@ -27,6 +27,11 @@ public class ReplayParserService : IHostedService, IDisposable public CancellationTokenSource TokenSource = new(); + /// + /// Event that is fired when all replays have been parsed. + /// + public event EventHandler> OnReplaysFinishedParsing; + private readonly IConfiguration _configuration; private readonly IServiceScopeFactory _factory; @@ -107,6 +112,7 @@ private async Task ConsumeQueue(CancellationToken token) var total = Queue.Count; var completed = 0; + var parsedReplays = new List(); // Consume the queue. while (Queue.Count > 0) @@ -186,6 +192,7 @@ private async Task ConsumeQueue(CancellationToken token) await AddReplayToDb(parsedReplay); await AddParsedReplayToDb(replay); + parsedReplays.Add(parsedReplay); Log.Information("Parsed " + replay); } catch (Exception e) @@ -207,6 +214,8 @@ private async Task ConsumeQueue(CancellationToken token) Log.Warning("Parsing took too long for " + string.Join(", ", tasks.Select(x => x.Id))); } } + + OnReplaysFinishedParsing?.Invoke(this, parsedReplays); } ///