Add ability to load sources which are not a nginx file listing.

Server/Api/ReplayController.cs

@@ -34,7 +34,7 @@ public async Task<ActionResult> UploadReplay(IFormFile file)
         }
 
         var stream = file.OpenReadStream();
-        var replay = ReplayParser.ParseReplay(stream);
+        var replay = ReplayParser.ReplayParser.ParseReplay(stream);
         stream.Close();
 
         _context.Replays.Add(replay);

Server/Program.cs

@@ -6,6 +6,7 @@
 using Serilog.AspNetCore;
 using Server;
 using Server.Api;
+using Server.ReplayParser;
 
 Log.Logger = new LoggerConfiguration()
     .WriteTo.Console()
@@ -93,7 +94,7 @@
 
     // Run FetchReplays in a new thread.
     var tokens = new List<CancellationTokenSource>();
-    var URLs = builder.Configuration.GetSection("ReplayUrls").Get<string[]>();
+    var URLs = builder.Configuration.GetSection("ReplayUrls").Get<StorageUrl[]>();
     if (URLs == null)
     {
         throw new Exception("No replay URLs found in appsettings.json. Please set ReplayUrls to an array of URLs.");

Server/ReplayLoading/CaddyProvider.cs

@@ -0,0 +1,56 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Server.ReplayLoading;
+
+[ReplayProviderName("caddy")]
+public class CaddyProvider : ReplayProvider
+{
+    public override async Task RetrieveFilesRecursive(string directoryUrl, CancellationToken token)
+    {
+        var httpClient = ReplayParser.ReplayParser.CreateHttpClient();
+        httpClient.DefaultRequestHeaders.Add("Accept", "application/json");
+
+        var responseText = await httpClient.GetStringAsync(directoryUrl, token);
+        var response = JsonSerializer.Deserialize<CaddyResponse[]>(responseText);
+        if (response == null)
+        {
+            return;
+        }
+
+        foreach (var caddyResponse in response)
+        {
+            if (caddyResponse.Name.EndsWith(".zip", StringComparison.Ordinal))
+            {
+                if (caddyResponse.LastModified < ReplayParser.ReplayParser.CutOffDateTime)
+                {
+                    continue;
+                }
+
+                await ReplayParser.ReplayParser.AddReplayToQueue(directoryUrl + caddyResponse.Name);
+            }
+            else if (caddyResponse.IsDir)
+            {
+                await RetrieveFilesRecursive(directoryUrl + caddyResponse.Name, token);
+            }
+        }
+    }
+
+    internal class CaddyResponse
+    {
+        [JsonPropertyName("name")]
+        public string Name { get; set; }
+        [JsonPropertyName("size")]
+        public int Size { get; set; }
+        [JsonPropertyName("url")]
+        public string Url { get; set; }
+        [JsonPropertyName("mod_time")]
+        public DateTime LastModified { get; set; }
+        [JsonPropertyName("mode")]
+        public long Mode { get; set; }
+        [JsonPropertyName("is_dir")]
+        public bool IsDir { get; set; }
+        [JsonPropertyName("is_symlink")]
+        public bool IsSymlink { get; set; }
+    }
+}

Server/ReplayLoading/DummyProvider.cs

@@ -0,0 +1,14 @@
+namespace Server.ReplayLoading;
+
+/// <summary>
+/// Represents a replay provider that can retrieve replay files from a directory.
+/// This will never add any replays to the queue. It is used to temporarily disable some sources.
+/// </summary>
+[ReplayProviderName("dummy")]
+public class DummyProvider : ReplayProvider
+{
+    public override Task RetrieveFilesRecursive(string directoryUrl, CancellationToken token)
+    {
+        return Task.CompletedTask;
+    }
+}

Server/ReplayLoading/NginxProvider.cs

@@ -0,0 +1,54 @@
+using HtmlAgilityPack;
+using Serilog;
+
+namespace Server.ReplayLoading;
+
+[ReplayProviderName("nginx")]
+public class NginxProvider : ReplayProvider
+{
+    public override async Task RetrieveFilesRecursive(string directoryUrl, CancellationToken token)
+    {
+        Log.Information("Retrieving files from " + directoryUrl);
+        var client = ReplayParser.ReplayParser.CreateHttpClient();
+        var htmlContent = await client.GetStringAsync(directoryUrl, token);
+        var document = new HtmlDocument();
+        document.LoadHtml(htmlContent);
+
+        var links = document.DocumentNode.SelectNodes("//a[@href]");
+        if (links == null)
+        {
+            Log.Information("No links found on " + directoryUrl + ".");
+            return;
+        }
+
+        foreach (var link in links)
+        {
+            if (token.IsCancellationRequested)
+            {
+                return;
+            }
+
+            var href = link.Attributes["href"].Value;
+
+            if (href.StartsWith("..", StringComparison.Ordinal))
+            {
+                continue;
+            }
+
+            if (!Uri.TryCreate(href, UriKind.Absolute, out _))
+            {
+                href = new Uri(new Uri(directoryUrl), href).ToString();
+            }
+
+            if (href.EndsWith("/", StringComparison.Ordinal))
+            {
+                await RetrieveFilesRecursive(href, token);
+            }
+
+            if (href.EndsWith(".zip", StringComparison.Ordinal))
+            {
+                await ReplayParser.ReplayParser.AddReplayToQueue(href);
+            }
+        }
+    }
+}

Server/ReplayLoading/ReplayProvider.cs

@@ -0,0 +1,20 @@
+namespace Server.ReplayLoading;
+
+public abstract class ReplayProvider
+{
+    public abstract Task RetrieveFilesRecursive(string directoryUrl, CancellationToken token);
+}
+
+/// <summary>
+/// 
+/// </summary>
+[AttributeUsage(AttributeTargets.Class)]
+public class ReplayProviderNameAttribute : Attribute
+{
+    public string Name { get; }
+
+    public ReplayProviderNameAttribute(string name)
+    {
+        Name = name;
+    }
+}

Server/ReplayLoading/ReplayProviderFactory.cs

@@ -0,0 +1,22 @@
+using System.Reflection;
+
+namespace Server.ReplayLoading;
+
+public class ReplayProviderFactory
+{
+    public static ReplayProvider GetProvider(string providerName)
+    {
+        var type = Assembly.GetExecutingAssembly().GetTypes().FirstOrDefault(t => t.GetCustomAttribute<ReplayProviderNameAttribute>()?.Name == providerName);
+        if (type == null)
+        {
+            throw new ArgumentException("Invalid provider name.");
+        }
+
+        if (!typeof(ReplayProvider).IsAssignableFrom(type))
+        {
+            throw new ArgumentException("Invalid provider type.");
+        }
+
+        return (ReplayProvider) Activator.CreateInstance(type)!;
+    }
+}

Server/ReplayParser.cs → Server/ReplayParser/ReplayParser.cs

@@ -1,18 +1,14 @@
-using System.Diagnostics;
-using System.Globalization;
+using System.Globalization;
 using System.IO.Compression;
-using System.Text.RegularExpressions;
-using HtmlAgilityPack;
 using Microsoft.EntityFrameworkCore;
-using Microsoft.EntityFrameworkCore.Query;
-using Microsoft.Extensions.Caching.Memory;
 using Serilog;
 using Server.Api;
+using Server.ReplayLoading;
 using Shared;
 using Shared.Models;
 using YamlDotNet.Serialization;
 
-namespace Server;
+namespace Server.ReplayParser;
 
 public static class ReplayParser
 {
@@ -139,7 +135,7 @@ public static async Task ConsumeQueue(CancellationToken token)
     /// <summary>
     /// Handles fetching replays from the remote storage.
     /// </summary>
-    public static async Task FetchReplays(CancellationToken token, string[] storageUrls)
+    public static async Task FetchReplays(CancellationToken token, StorageUrl[] storageUrls)
     {
         while (!token.IsCancellationRequested)
         {
@@ -148,7 +144,8 @@ public static async Task FetchReplays(CancellationToken token, string[] storageU
                 Log.Information("Fetching replays from " + storageUrl);
                 try
                 {
-                    await RetrieveFilesRecursive(storageUrl, token);
+                    var provider = ReplayProviderFactory.GetProvider(storageUrl.Provider);
+                    await provider.RetrieveFilesRecursive(storageUrl.Url, token);
                 }
                 catch (Exception e)
                 {
@@ -164,80 +161,31 @@ public static async Task FetchReplays(CancellationToken token, string[] storageU
             await Task.Delay(delay, token);
         }
     }
-
-    private static async Task RetrieveFilesRecursive(string directoryUrl, CancellationToken token)
+    
+    public static async Task AddReplayToQueue(string replay)
     {
-        try
+        // Use regex to check and retrieve the date from the file name.
+        var fileName = Path.GetFileName(replay);
+        var match = RegexList.ReplayRegex.Match(fileName);
+        if (match.Success)
         {
-            Log.Information("Retrieving files from " + directoryUrl);
-            var client = CreateHttpClient();
-            var htmlContent = await client.GetStringAsync(directoryUrl, token);
-            var document = new HtmlDocument();
-            document.LoadHtml(htmlContent);
-
-            var links = document.DocumentNode.SelectNodes("//a[@href]");
-            if (links == null)
+            var date = DateTime.ParseExact(match.Groups[1].Value, "yyyy_MM_dd-HH_mm", CultureInfo.InvariantCulture);
+            if (date < CutOffDateTime)
             {
-                Log.Information("No links found on " + directoryUrl + ".");
                 return;
             }
-
-            foreach (var link in links)
-            {
-                if (token.IsCancellationRequested)
-                {
-                    return;
-                }
-
-                var href = link.Attributes["href"].Value;
-
-                if (href.StartsWith("..", StringComparison.Ordinal))
-                {
-                    continue;
-                }
-
-                if (!Uri.TryCreate(href, UriKind.Absolute, out _))
-                {
-                    href = new Uri(new Uri(directoryUrl), href).ToString();
-                }
-
-                if (href.EndsWith("/", StringComparison.Ordinal))
-                {
-                    await RetrieveFilesRecursive(href, token);
-                }
-
-                if (href.EndsWith(".zip", StringComparison.Ordinal))
-                {
-                    // Use regex to check and retrieve the date from the file name.
-                    var fileName = Path.GetFileName(href);
-                    var match = RegexList.ReplayRegex.Match(fileName);
-                    if (match.Success)
-                    {
-                        var date = DateTime.ParseExact(match.Groups[1].Value, "yyyy_MM_dd-HH_mm", CultureInfo.InvariantCulture);
-                        if (date < CutOffDateTime)
-                        {
-                            continue;
-                        }
-
-                        // If it's already in the database, skip it.
-                        if (await IsReplayParsed(href))
-                        {
-                            continue;
-                        }
-                        Log.Information("Adding " + href + " to the queue.");
-                        // Check if it's already in the queue.
-                        if (!Queue.Contains(href))
-                        {
-                            Queue.Add(href);
-                        }
-                    }
-                }
-            }
         }
-        catch (Exception e)
+
+        // If it's already in the database, skip it.
+        if (await IsReplayParsed(replay))
         {
-            Log.Error(e, "Error while retrieving files from " + directoryUrl);
-            // We don't care about the exception, we just want to return the files we have.
+            return;
+        }
+        Log.Information("Adding " + replay + " to the queue.");
+        // Check if it's already in the queue.
+        if (!Queue.Contains(replay))
+        {
+            Queue.Add(replay);
         }
     }
 
@@ -284,3 +232,14 @@ public static HttpClient CreateHttpClient()
         return client;
     }
 }
+
+public class StorageUrl
+{
+    public string Url { get; set; }
+    public string Provider { get; set; }
+
+    public override string ToString()
+    {
+        return Url;
+    }
+}

Server/appsettings.json

@@ -14,12 +14,33 @@
     }
   },
   "ReplayUrls": [
-    "https://moon.spacestation14.com/replays/leviathan/",
-    "https://moon.spacestation14.com/replays/lizard/",
-    "https://moon.spacestation14.com/replays/miros/",
-    "https://moon.spacestation14.com/replays/salamander/",
-    "https://moon.spacestation14.com/replays/vulture/",
-    "https://replays.delta-v.org/apoapsis/",
-    "https://replays.delta-v.org/periapsis/"
+    {
+      "url": "https://moon.spacestation14.com/replays/leviathan/",
+      "provider": "nginx"
+    },
+    {
+      "url": "https://moon.spacestation14.com/replays/lizard/",
+      "provider": "nginx"
+    },
+    {
+      "url": "https://moon.spacestation14.com/replays/miros/",
+      "provider": "nginx"
+    },
+    {
+      "url": "https://moon.spacestation14.com/replays/salamander/",
+      "provider": "nginx"
+    },
+    {
+      "url": "https://moon.spacestation14.com/replays/vulture/",
+      "provider": "nginx"
+    },
+    {
+      "url": "https://replays.delta-v.org/apoapsis/",
+      "provider": "nginx"
+    },
+    {
+      "url": "https://replays.delta-v.org/periapsis/",
+      "provider": "nginx"
+    }
   ]
 }