Make profile pregeneration more robust

ReplayBrowser/Controllers/DataController.cs

@@ -1,6 +1,7 @@
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.EntityFrameworkCore;
 using ReplayBrowser.Data;
+using ReplayBrowser.Services;
 using ReplayBrowser.Services.ReplayParser;
 
 namespace ReplayBrowser.Controllers;
@@ -10,10 +11,12 @@ namespace ReplayBrowser.Controllers;
 public class DataController : Controller
 {
     private readonly ReplayDbContext _context;
+    private readonly ProfilePregeneratorService _profilePregeneratorService;
 
-    public DataController(ReplayDbContext context)
+    public DataController(ReplayDbContext context, ProfilePregeneratorService profilePregeneratorService)
     {
         _context = context;
+        _profilePregeneratorService = profilePregeneratorService;
     }
 
     [HttpGet("username-completion")]
@@ -41,7 +44,8 @@ public DownloadProgress GetDownloadProgress()
         {
             Progress = ReplayParserService.DownloadProgress.ToDictionary(x => x.Key, x => x.Value),
             Status = ReplayParserService.Status.ToFriendlyString(),
-            Details = ReplayParserService.Details
+            Details = ReplayParserService.Details,
+            PregenerationProgress = _profilePregeneratorService.PregenerationProgress
         };
     }
 }
@@ -52,4 +56,5 @@ public class DownloadProgress
     public required Dictionary<string, double> Progress { get; set; }
 
     public required string Details { get; set; }
+    public required ProfilePregeneratorService.PreGenerationProgress PregenerationProgress { get; set; }
 }

ReplayBrowser/Pages/Downloads.razor

@@ -5,6 +5,11 @@
 
 <h3>Downloads</h3>
 <p>Here you can see the progress of the internal downloads of the website.<br/>The website looks for new replays every 10th minute. So 12:00, 12:10, 12:20, 12:30, etc.</p>
+<div id="pregenerationStatus">
+    <p>Pregeneration: <b id="pregenerationCurrent">0</b> / <b id="pregenerationMax">0</b></p>
+    <progress id="pregenerationProgress" value="0" max="1"></progress>
+</div>
+
 <p>Status: <b id="status-text">Waiting...</b> <span id="details"></span></p>
 
 <table class="table table-striped">
@@ -23,16 +28,21 @@
     let Downloads = [];
     let Status = "";
     let Details = "";
+    let PregenerationCurrent = 0;
+    let PregenerationMax = 0;
     
     setInterval(() => {
         download();
     }, 500);
     
     function download() {
         $.get('/api/Data/download-progress', (data) => {
+            console.debug(data);
             Downloads = data.progress;
             Status = data.status;
             Details = data.details;
+            PregenerationCurrent = data.pregenerationProgress.current;
+            PregenerationMax = data.pregenerationProgress.max;
             updateDownloads();
         });
     }
@@ -46,6 +56,12 @@
         } else {
             document.getElementById('download-text-no').style.display = 'none';
         }
+        
+        document.getElementById('pregenerationCurrent').innerText = PregenerationCurrent;
+        document.getElementById('pregenerationMax').innerText = PregenerationMax;
+        if (PregenerationMax !== 0) { // Prevent division by zero
+            document.getElementById('pregenerationProgress').value = PregenerationCurrent / PregenerationMax;
+        } 
 
         document.getElementById('status-text').innerText = Status;
         document.getElementById('details').innerText = Details;

ReplayBrowser/Pages/Shared/MainLayout.razor

@@ -2,6 +2,7 @@
 @using ReplayBrowser.Data
 @using ReplayBrowser.Data.Models.Account
 @using ReplayBrowser.Helpers
+@using ReplayBrowser.Services
 @using ReplayBrowser.Services.ReplayParser
 @using Serilog
 @inherits LayoutComponentBase
@@ -10,6 +11,7 @@
 @inject NavigationManager NavigationManager
 @inject NoticeHelper NoticeHelper
 @inject IHttpContextAccessor HttpContextAccessor
+@inject ProfilePregeneratorService ProfilePregeneratorService
 
 @code
 {
@@ -61,6 +63,14 @@
                 </button>
             </div>
         }
+
+        @if(ProfilePregeneratorService.IsFirstRun)
+        {
+            <div class="alert alert-warning" role="alert">
+                <h4 class="alert-heading">Profile pregeneration</h4>
+                <p>Profile pregeneration is currently running. This process will take a while, but it will make the website faster in the future. You can still use the website while this is running, but some features may be slower than expected. You can see the progress at the downloads page.</p>
+            </div>
+        }
 
         @if (_user.Identity.IsAuthenticated)
         {

ReplayBrowser/Services/LeaderboardService.cs

@@ -505,7 +505,7 @@ private async Task<Leaderboard> GenerateLeaderboard(
             }
         }
         stopwatch.Stop();
-        Log.Information("Fetching player data took {Time}ms", stopwatch.ElapsedMilliseconds);
+        Log.Verbose("Fetching player data took {Time}ms", stopwatch.ElapsedMilliseconds);
 
         return returnValue;
     }

ReplayBrowser/Services/ProfilePregeneratorService.cs

@@ -2,42 +2,98 @@
 using System.Security.Claims;
 using Microsoft.AspNetCore.Components.Authorization;
 using ReplayBrowser.Data;
+using ReplayBrowser.Data.Models;
 using ReplayBrowser.Helpers;
+using ReplayBrowser.Services.ReplayParser;
 using Serilog;
 
 namespace ReplayBrowser.Services;
 
 /// <summary>
 /// Service that checks every watched profile for every user and pregenerates the profile if it is not already generated for better loading times.
 /// </summary>
-public class ProfilePregeneratorService : IHostedService, IDisposable, IAsyncDisposable
+public class ProfilePregeneratorService : IHostedService
 {
-    private Timer? _timer = null;
     private readonly IServiceScopeFactory _scopeFactory;
+    private ReplayParserService _replayParserService;
+
+    public bool IsFirstRun { get; private set;  }= true;
+
+    /// <summary>
+    /// List of profiles that have been generated.
+    /// </summary>
+    private readonly List<Guid> _generatedProfiles = new(); 
 
     /// <summary>
     /// List of profiles which get generated even if they are not on a watched profile list. (example being leaderboards)
     /// </summary>
     public List<Guid> AlwaysGenerateProfiles = new List<Guid>();
 
-    public ProfilePregeneratorService(IServiceScopeFactory scopeFactory)
+    /// <summary>
+    /// A var which tracks the progress of the pregeneration.
+    /// </summary>
+    public PreGenerationProgress PregenerationProgress { get; private set; } = new PreGenerationProgress();
+
+    private int _queuedPreGenerations = 0;
+
+    public ProfilePregeneratorService(IServiceScopeFactory scopeFactory, ReplayParserService replayParserService)
     {
         _scopeFactory = scopeFactory;
+        _replayParserService = replayParserService;
     }
 
     public Task StartAsync(CancellationToken cancellationToken)
     {
-        _timer = new Timer(DoWork, null, TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(30));
+        _replayParserService.OnReplaysFinishedParsing += OnReplaysParsed;
+
+        // In one minute, start the pregeneration.
+        Task.Delay(TimeSpan.FromMinutes(1), cancellationToken).ContinueWith(t => QueuePregeneration(), cancellationToken);
         return Task.CompletedTask;
     }
 
     public Task StopAsync(CancellationToken cancellationToken)
     {
-        _timer?.Change(Timeout.Infinite, 0);
+        _replayParserService.OnReplaysFinishedParsing -= OnReplaysParsed;
         return Task.CompletedTask;
     }
-
-    private async void DoWork(object? state)
+
+    private void OnReplaysParsed(object? sender, List<Replay> replays)
+    {
+        if (IsFirstRun)
+        {
+            // We are waiting for the first run to complete
+            Log.Warning("Triggered OnReplaysParsed before the first run has completed.");
+            return;
+        }
+
+        foreach (var replay in replays)
+        {
+            if (replay.RoundEndPlayers == null)
+                continue;
+
+            foreach (var player in replay.RoundEndPlayers)
+            {
+                if (!_generatedProfiles.Contains(player.PlayerGuid))
+                    // Profile is not getting autogenerated, ignore.
+                    continue; 
+
+                // Profile is getting autogenerated, remove it from the list.
+                _generatedProfiles.Remove(player.PlayerGuid);
+            }
+        }
+
+        QueuePregeneration();
+    }
+
+    private void QueuePregeneration()
+    {
+        _queuedPreGenerations++;
+
+        if (_queuedPreGenerations == 1) // If we are the first one, start the pregeneration.
+            PregenerateProfiles();
+    }
+
+    private async void PregenerateProfiles()
     {
         var sw = new Stopwatch();
         Log.Information("Starting profile pregeneration.");
@@ -49,30 +105,48 @@ private async void DoWork(object? state)
         var profilesToGenerate = new List<Guid>();
         profilesToGenerate.AddRange(AlwaysGenerateProfiles);
 
-        dbContext.Accounts.Select(a => a.SavedProfiles).ToList().ForEach(profiles =>
+        await foreach (var account in dbContext.Accounts)
         {
-            foreach (var profile in profiles.Where(profile => !profilesToGenerate.Contains(profile)))
+            foreach (var profile in account.SavedProfiles)
             {
+                if (profilesToGenerate.Contains(profile)) continue;
                 profilesToGenerate.Add(profile);
             }
-        });
+        }
+
+        // Remove every profile already found in _generatedProfiles
+        profilesToGenerate = profilesToGenerate.Except(_generatedProfiles).ToList();
+        PregenerationProgress.Max = profilesToGenerate.Count;
 
         foreach (var guid in profilesToGenerate)
         {
             await replayHelper.GetPlayerProfile(guid, new AuthenticationState(new ClaimsPrincipal()), true, true);
             Log.Information("Pregenerated profile for {Guid}.", guid);
+            _generatedProfiles.Add(guid);
+            PregenerationProgress.Current++;
+        }
+
+        if (IsFirstRun)
+        {
+            IsFirstRun = false;
+            Log.Information("First run completed.");            
         }
+
         sw.Stop();
         Log.Information("Profile pregeneration finished in {ElapsedMilliseconds}ms.", sw.ElapsedMilliseconds);
+
+        _queuedPreGenerations--;
+        if (_queuedPreGenerations > 0)
+        {
+            PregenerateProfiles(); // this is stupid, but then again, this whole codebase is, and its not like someone else is going to read this. Right?
+            Log.Information("Pregeneration queued.");
+        }
     }
-
-    public void Dispose()
-    {
-        _timer?.Dispose();
-    }
-
-    public async ValueTask DisposeAsync()
+
+    public class PreGenerationProgress
     {
-        if (_timer != null) await _timer.DisposeAsync();
+        public int Current { get; set; } = 0;
+        public int Max { get; set; } = 0;
     }
 }
+

ReplayBrowser/Services/ReplayParser/ReplayParserService.cs

@@ -27,6 +27,11 @@ public class ReplayParserService : IHostedService, IDisposable
 
     public CancellationTokenSource TokenSource = new();
 
+    /// <summary>
+    /// Event that is fired when all replays have been parsed.
+    /// </summary>
+    public event EventHandler<List<Replay>> OnReplaysFinishedParsing;
+
     private readonly IConfiguration _configuration;
     private readonly IServiceScopeFactory _factory;
 
@@ -107,6 +112,7 @@ private async Task ConsumeQueue(CancellationToken token)
 
         var total = Queue.Count;
         var completed = 0;
+        var parsedReplays = new List<Replay>();
 
         // Consume the queue.
         while (Queue.Count > 0)
@@ -186,6 +192,7 @@ private async Task ConsumeQueue(CancellationToken token)
                             
                         await AddReplayToDb(parsedReplay);
                         await AddParsedReplayToDb(replay);
+                        parsedReplays.Add(parsedReplay);
                         Log.Information("Parsed " + replay);
                     }
                     catch (Exception e)
@@ -207,6 +214,8 @@ private async Task ConsumeQueue(CancellationToken token)
                 Log.Warning("Parsing took too long for " + string.Join(", ", tasks.Select(x => x.Id)));
             }
         }
+
+        OnReplaysFinishedParsing?.Invoke(this, parsedReplays);
     }
 
     /// <summary>