feat: Add middleware for handling + generating robots.txt path

add tests

chore: change max age to one day

chore: Linted code for plan-technology-for-your-school.sln solution

src/Dfe.PlanTech.Web/Middleware/RobotsTxtMiddleware.cs

@@ -0,0 +1,77 @@
+using System.Text;
+using Dfe.PlanTech.Web.Models;
+using Microsoft.Extensions.Options;
+
+namespace Dfe.PlanTech.Web.Middleware;
+
+/// <summary>
+/// Middleware to handle /robots.txt path and generate the file
+/// </summary>
+public class RobotsTxtMiddleware(RequestDelegate _, IOptions<RobotsConfiguration> options)
+{
+    private const string UserAgentKey = "User-agent";
+    private const string DisallowKey = "Disallow";
+    private const string ContentType = "text/plain";
+
+    public async Task InvokeAsync(HttpContext context)
+    {
+        await CreateRobotsTxtResponse(context);
+    }
+
+
+    /// <summary>
+    /// Generate and return Robots.txt
+    /// </summary>
+    private async Task CreateRobotsTxtResponse(HttpContext context)
+    {
+        context.Response.ContentType = ContentType;
+        context.Response.Headers.CacheControl = $"max-age={options.Value.CacheMaxAge}";
+
+        var result = GetResponseBody(options.Value);
+        await context.Response.Body.WriteAsync(result, context.RequestAborted);
+    }
+
+    private static ReadOnlyMemory<byte> GetResponseBody(RobotsConfiguration config)
+    => new(Encoding.UTF8.GetBytes(BuildRobotsTxt(config)));
+
+    private static string BuildRobotsTxt(RobotsConfiguration config)
+    {
+        var stringBuilder = new StringBuilder();
+
+        AppendUserAgent(config, stringBuilder);
+        foreach (var path in config.DisallowedPaths)
+        {
+            AppendDisallow(stringBuilder, path);
+        }
+
+        var result = stringBuilder.ToString();
+        return result;
+    }
+
+    private static void AppendKeyValue(StringBuilder stringBuilder, string key, string value)
+    {
+        stringBuilder.Append(key);
+        stringBuilder.Append(": ");
+        stringBuilder.Append(value);
+    }
+
+    private static void AppendUserAgent(RobotsConfiguration config, StringBuilder stringBuilder)
+    {
+        AppendKeyValue(stringBuilder, UserAgentKey, config.UserAgent);
+    }
+
+    private static void AppendDisallow(StringBuilder stringBuilder, string path)
+    {
+        stringBuilder.AppendLine();
+        AppendKeyValue(stringBuilder, DisallowKey, path);
+    }
+}
+
+public static class RobotsTxtMiddlewareExtensions
+{
+    public const string RobotsTxtPath = "/robots.txt";
+
+    public static bool IsRobotsPath(HttpContext context) => context.Request.Path == RobotsTxtPath;
+
+    public static IApplicationBuilder UseRobotsTxtMiddleware(this IApplicationBuilder builder) => builder.UseWhen(IsRobotsPath, app => app.UseMiddleware<RobotsTxtMiddleware>());
+}

src/Dfe.PlanTech.Web/Models/RobotsConfiguration.cs

@@ -0,0 +1,8 @@
+namespace Dfe.PlanTech.Web.Models;
+
+public record RobotsConfiguration
+{
+    public string UserAgent { get; init; } = "*";
+    public string[] DisallowedPaths { get; init; } = [];
+    public int CacheMaxAge { get; init; } = 86400; //One day
+}

src/Dfe.PlanTech.Web/Program.cs

@@ -6,6 +6,7 @@
 using Dfe.PlanTech.Web;
 using Dfe.PlanTech.Web.Configuration;
 using Dfe.PlanTech.Web.Middleware;
+using Dfe.PlanTech.Web.Models;
 using GovUk.Frontend.AspNetCore;
 
 var builder = WebApplication.CreateBuilder(args);
@@ -47,9 +48,13 @@
         .AddRedisServices(builder.Configuration);
 
 builder.Services.AddSingleton<ISystemTime, SystemTime>();
+builder.Services.Configure<RobotsConfiguration>(builder.Configuration.GetSection("Robots"));
+
 
 var app = builder.Build();
 
+app.UseRobotsTxtMiddleware();
+
 app.UseSecurityHeaders();
 app.UseMiddleware<HeadRequestMiddleware>();
 

src/Dfe.PlanTech.Web/appsettings.json

@@ -70,5 +70,9 @@
             "MaxRetryCount": 5,
             "MaxDelayInMilliseconds": 5000
         }
+    },
+    "Robots": {
+        "DisallowedPaths": ["/"],
+        "CacheMaxAge": 86400
     }
 }

tests/Dfe.PlanTech.Web.UnitTests/Middleware/RobotsTxtMiddlewareTests.cs

@@ -0,0 +1,61 @@
+using Dfe.PlanTech.Web.Middleware;
+using Dfe.PlanTech.Web.Models;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Options;
+using NSubstitute;
+using Xunit;
+
+namespace Dfe.PlanTech.Web.UnitTests.Middleware;
+
+public class RobotsTxtMiddlewareTests
+{
+    private IOptions<RobotsConfiguration> _options = Substitute.For<IOptions<RobotsConfiguration>>();
+    private readonly RobotsTxtMiddleware _middleware;
+    private readonly HttpContext _httpContext;
+    public RobotsTxtMiddlewareTests()
+    {
+        _httpContext = Substitute.For<HttpContext>();
+        var response = Substitute.For<HttpResponse>();
+        response.Body = new MemoryStream();
+
+        _httpContext.Response.Returns(response);
+
+        var next = (HttpContext hc) => Task.CompletedTask;
+        _middleware = new RobotsTxtMiddleware(new RequestDelegate(next), _options);
+    }
+
+
+    [Fact]
+    public async Task Should_Create_Valid_File()
+    {
+        var config = new RobotsConfiguration()
+        {
+            UserAgent = "example-user-agent",
+            DisallowedPaths = ["/", "2313", "/self-assessment"],
+        };
+
+        _options.Value.Returns(config);
+        await _middleware.InvokeAsync(_httpContext);
+
+        var response = await ReadResponseBodyAsync(_httpContext.Response);
+
+        Assert.Contains($"User-agent: {config.UserAgent}", response);
+
+        foreach (var path in config.DisallowedPaths)
+        {
+            Assert.Contains($"Disallow: {path}", response);
+        }
+
+        var cacheControlHeader = _httpContext.Response.Headers.CacheControl;
+
+        Assert.Equal($"max-age={config.CacheMaxAge}", cacheControlHeader);
+    }
+
+    protected static Task<string> ReadResponseBodyAsync(HttpResponse response)
+    {
+        response.Body.Position = 0;
+        var r = new StreamReader(response.Body);
+        return r.ReadToEndAsync();
+    }
+
+}