Add suggestions cleanup job

README.md

@@ -161,7 +161,7 @@ If you want to specify this yourself, add `IgnoredResourceExtensions` to the con
 
 ## Restricting access to the Admin UI
 
-By default, only users of `Administrators` role can access Admin UI. But you can configure you authorization policy when registrating the NotFound handler.
+By default, only users of `Administrators` role can access Admin UI. But you can configure your authorization policy when registering the NotFound handler.
 
 ```
  services.AddNotFoundHandler(o => { },
@@ -171,15 +171,15 @@ By default, only users of `Administrators` role can access Admin UI. But you can
             });
 ```
 
-You can setup any policy rules you want.
+You can set up any policy rules you want.
 
 ## Import
 
 For details see [Import redirects for 404 handler](https://getadigital.com/blog/import-redirects-for-404-handler/) article.
 
 # Custom 404 Page
 
-To setup 404 page, you can use any method ASP.NET Core provides.
+To set up 404 page, you can use any method ASP.NET Core provides.
 
 One of the simplest solutions is adding a controller and a view for it that would display an error page:
 
@@ -274,9 +274,37 @@ It will monitor primary, secondary and SEO URLs:
 
 Optimizely Content Cloud supports only primary URLs and Optimizely Commerce supports all three types of URLs.
 
-There are two scheduled jobs:
-- *[Geta NotFoundHandler] Index content URLs* - as mentioned before, this job indexes URLs of content. Usually, it is required to run this job only once. All new content is automatically indexed. But if for some reasons content publish events are not firing when creating new content (for example, during the import), then you should set this job to run frequently.
- - *[Geta NotFoundHandler] Register content move redirects* - this job creates redirects based on registered moved content. Normally, this job is not required at all, but there might be situations when content move is registered but redirect creation is not completed. This could happen during deployments. In this case, you can manually run this job or schedule it to run time to time to fix such issues.
+# Scheduled jobs
+
+Scheduled job - process that runs in background
+- Suggestions cleanup job - shipped with the package, contains process that cleans up suggestions table.
+This job is configured by default to remove records older than 14 days. You can adjust the retention period or timeout as needed.
+```
+services.AddNotFoundHandler(o =>
+{
+    o.SuggestionsCleanupOptions.DaysToKeep = 30;
+    o.SuggestionsCleanupOptions.Timeout = 30 * 60;
+});
+```
+
+Scheduler - mechanism that triggers scheduled jobs in a recurrent manner
+- InternalScheduler - default scheduler, included in the core package, a scheduler that uses [Coravel](https://docs.coravel.net/).
+  To enable the scheduler, you need to enable UseInternalScheduler flag. Additionally, you can adjust the scheduler run interval:
+```
+services.AddNotFoundHandler(o =>
+{
+    ...
+    o.UseInternalScheduler = true;
+    o.InternalSchedulerCronInterval = "0 0 * * *" // by default it's configured to run daily at midnight
+});
+```
+- OptimizelyScheduler - uses Optimizely to schedule job runs.
+An Optimizely scheduled job was added - <code>[Geta NotFoundHandler] Suggestions cleanup job</code>.
+
+Additionally, there are two optimizely scheduled jobs responsible for:
+- *[Geta NotFoundHandler] Index content URLs* - as mentioned before, this job indexes URLs of content. Usually, it is required to run this job only once. All new content is automatically indexed. But if for some reason content publish events are not firing when creating new content (for example, during the import), then you should set this job to run frequently.
+- *[Geta NotFoundHandler] Register content move redirects* - this job creates redirects based on registered moved content. Normally, this job is not required at all, but there might be situations when content move is registered but redirect creation is not completed. This could happen during deployments. In this case, you can manually run this job or schedule it to run time to time to fix such issues.
+
 
 # Troubleshooting
 
@@ -293,7 +321,7 @@ For example, if we have a redirect: `/a` to `/b`, then:
 - without wildcard setting it will redirect `/a/1` to `/b/1`
 
 # Sandbox App
-Sandbox application is testing poligon for pacakge new features and bug fixes.
+Sandbox application is testing polygon for package new features and bug fixes.
 
 CMS username: admin@example.com
 

src/Geta.NotFoundHandler.Optimizely/Core/Suggestions/Jobs/SuggestionsCleanupJob.cs

@@ -0,0 +1,30 @@
+// Copyright (c) Geta Digital. All rights reserved.
+// Licensed under Apache-2.0. See the LICENSE file in the project root for more information
+
+using EPiServer.PlugIn;
+using EPiServer.Scheduler;
+using Geta.NotFoundHandler.Core.Suggestions;
+
+namespace Geta.NotFoundHandler.Optimizely.Core.Suggestions.Jobs;
+
+[ScheduledPlugIn(DisplayName = "[Geta NotFoundHandler] Suggestions cleanup job",
+                 Description = "As suggestions table grow fast we should add a possibility to clean up old suggestions",
+                 GUID = "6AE19CEC-1052-4482-97DF-981076DDD6F2",
+                 SortIndex = 5555)]
+public class SuggestionsCleanupJob : ScheduledJobBase
+{
+    private readonly ISuggestionsCleanupService _suggestionsCleanupService;
+
+    public SuggestionsCleanupJob(ISuggestionsCleanupService suggestionsCleanupService)
+    {
+        IsStoppable = true;
+        _suggestionsCleanupService = suggestionsCleanupService;
+    }
+
+    public override string Execute()
+    {
+        _suggestionsCleanupService.Cleanup();
+
+        return string.Empty;
+    }
+}

src/Geta.NotFoundHandler.Optimizely/Infrastructure/Configuration/OptimizelyNotFoundHandlerOptions.cs

@@ -10,6 +10,7 @@ namespace Geta.NotFoundHandler.Optimizely.Infrastructure.Configuration
 {
     public class OptimizelyNotFoundHandlerOptions
     {
+        public const string Section = "Geta:NotFoundHandler:Optimizely";
         public const int CurrentDbVersion = 3;
 
         public bool AutomaticRedirectsEnabled { get; set; }

src/Geta.NotFoundHandler.Optimizely/Infrastructure/Configuration/ServiceCollectionExtensions.cs

@@ -74,7 +74,7 @@ public static IServiceCollection AddOptimizelyNotFoundHandler(
             services.AddOptions<OptimizelyNotFoundHandlerOptions>().Configure<IConfiguration>((options, configuration) =>
             {
                 setupAction(options);
-                configuration.GetSection("Geta:NotFoundHandler:Optimizely").Bind(options);
+                configuration.GetSection(OptimizelyNotFoundHandlerOptions.Section).Bind(options);
             });
 
             return services;

src/Geta.NotFoundHandler.Optimizely/Infrastructure/Initialization/ApplicationBuilderExtensions.cs

@@ -1,6 +1,9 @@
 // Copyright (c) Geta Digital. All rights reserved.
 // Licensed under Apache-2.0. See the LICENSE file in the project root for more information
 
+using Coravel.Scheduling.Schedule;
+using Coravel.Scheduling.Schedule.Interfaces;
+using Geta.NotFoundHandler.Core.ScheduledJobs.Suggestions;
 using Geta.NotFoundHandler.Optimizely.Core.AutomaticRedirects;
 using Geta.NotFoundHandler.Optimizely.Core.Events;
 using Microsoft.AspNetCore.Builder;
@@ -23,6 +26,10 @@ public static IApplicationBuilder UseOptimizelyNotFoundHandler(this IApplication
             var historyEvents = services.GetRequiredService<ContentUrlHistoryEvents>();
             historyEvents.Initialize();
 
+            // For optimizely we will use built-in scheduler for this job
+            var scheduler = services.GetService<IScheduler>();
+            (scheduler as Scheduler)?.TryUnschedule(nameof(SuggestionsCleanupJob));
+
             return app;
         }
     }

src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs

@@ -0,0 +1,37 @@
+// Copyright (c) Geta Digital. All rights reserved.
+// Licensed under Apache-2.0. See the LICENSE file in the project root for more information
+
+using Coravel;
+using Geta.NotFoundHandler.Core.ScheduledJobs.Suggestions;
+using Geta.NotFoundHandler.Infrastructure.Configuration;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace Geta.NotFoundHandler.Core.ScheduledJobs;
+
+public static class ApplicationBuilderExtensions
+{
+    public static IApplicationBuilder UseInternalScheduler(this IApplicationBuilder app)
+    {
+        var services = app.ApplicationServices;
+
+        var options = services.GetRequiredService<IOptions<NotFoundHandlerOptions>>().Value;
+        var logger = services.GetRequiredService<ILogger>();
+
+        services.UseScheduler(scheduler =>
+            {
+                scheduler
+                    .Schedule<SuggestionsCleanupJob>()
+                    .Cron(options.InternalSchedulerCronInterval)
+                    .PreventOverlapping(nameof(SuggestionsCleanupJob));
+            })
+            .OnError(x =>
+            {
+                logger.LogError(x, "Something went wrong, scheduled job failed with exception");
+            });
+
+        return app;
+    }
+}

src/Geta.NotFoundHandler/Core/ScheduledJobs/ServiceCollectionExtensions.cs

@@ -0,0 +1,28 @@
+// Copyright (c) Geta Digital. All rights reserved.
+// Licensed under Apache-2.0. See the LICENSE file in the project root for more information
+
+using Coravel;
+using Geta.NotFoundHandler.Core.ScheduledJobs.Suggestions;
+using Geta.NotFoundHandler.Infrastructure.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Options;
+
+namespace Geta.NotFoundHandler.Core.ScheduledJobs;
+
+public static class ServiceCollectionExtensions
+{
+    public static IServiceCollection EnableScheduler(this IServiceCollection services)
+    {
+        using var serviceProvider = services.BuildServiceProvider();
+        var options = serviceProvider.GetRequiredService<IOptions<NotFoundHandlerOptions>>().Value;
+
+        if (options.UseInternalScheduler)
+        {
+            services.AddScheduler();
+
+            services.AddTransient<SuggestionsCleanupJob>();
+        }
+
+        return services;
+    }
+}

src/Geta.NotFoundHandler/Core/ScheduledJobs/Suggestions/SuggestionsCleanupJob.cs

@@ -0,0 +1,23 @@
+// Copyright (c) Geta Digital. All rights reserved.
+// Licensed under Apache-2.0. See the LICENSE file in the project root for more information
+
+using System.Threading.Tasks;
+using Coravel.Invocable;
+using Geta.NotFoundHandler.Core.Suggestions;
+
+namespace Geta.NotFoundHandler.Core.ScheduledJobs.Suggestions;
+
+public class SuggestionsCleanupJob : IInvocable
+{
+    private readonly ISuggestionsCleanupService _suggestionsCleanupService;
+
+    public SuggestionsCleanupJob(ISuggestionsCleanupService suggestionsCleanupService)
+    {
+        _suggestionsCleanupService = suggestionsCleanupService;
+    }
+
+    public async Task Invoke()
+    {
+        _suggestionsCleanupService.Cleanup();
+    }
+}

src/Geta.NotFoundHandler/Core/Suggestions/ISuggestionsCleanupService.cs

@@ -0,0 +1,9 @@
+// Copyright (c) Geta Digital. All rights reserved.
+// Licensed under Apache-2.0. See the LICENSE file in the project root for more information
+
+namespace Geta.NotFoundHandler.Core.Suggestions;
+
+public interface ISuggestionsCleanupService
+{
+    void Cleanup();
+}

src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupOptions.cs

@@ -0,0 +1,10 @@
+// Copyright (c) Geta Digital. All rights reserved.
+// Licensed under Apache-2.0. See the LICENSE file in the project root for more information
+
+namespace Geta.NotFoundHandler.Core.Suggestions;
+
+public class SuggestionsCleanupOptions
+{
+    public int DaysToKeep { get; set; } = 14;
+    public int Timeout { get; set; } = 30 * 60;
+}

src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupService.cs

@@ -0,0 +1,60 @@
+// Copyright (c) Geta Digital. All rights reserved.
+// Licensed under Apache-2.0. See the LICENSE file in the project root for more information
+
+using System;
+using Geta.NotFoundHandler.Infrastructure.Configuration;
+using Microsoft.Data.SqlClient;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace Geta.NotFoundHandler.Core.Suggestions;
+
+public class SuggestionsCleanupService : ISuggestionsCleanupService
+{
+    private readonly IOptions<NotFoundHandlerOptions> _options;
+    private readonly ILogger<SuggestionsCleanupService> _logger;
+
+    public SuggestionsCleanupService(
+        IOptions<NotFoundHandlerOptions> options,
+        ILogger<SuggestionsCleanupService> logger
+    )
+    {
+        _options = options;
+        _logger = logger;
+    }
+
+    private string CleanupCommandText(int daysToKeep) => $@"
+        -- [NotFoundHandler.Suggestions]
+        IF OBJECT_ID('[NotFoundHandler.Suggestions]', 'U') IS NOT NULL
+        BEGIN
+            DELETE
+            FROM
+                [NotFoundHandler.Suggestions]
+            WHERE
+                [Requested] < DATEADD(day, -{daysToKeep}, GETDATE())
+
+            PRINT '* Deleted ' + CAST(@@ROWCOUNT AS nvarchar) + ' outdated records from table [NotFoundHandler.Suggestions].'
+        END
+        ";
+
+
+    public void Cleanup()
+    {
+        try
+        {
+            using var connection = new SqlConnection(_options.Value.ConnectionString);
+            connection.InfoMessage += (_, e) => _logger.LogInformation("{Message}", e.Message);
+
+            var command = new SqlCommand(CleanupCommandText(_options.Value.SuggestionsCleanupOptions.DaysToKeep), connection);
+            command.CommandTimeout = _options.Value.SuggestionsCleanupOptions.Timeout;
+            command.Connection.Open();
+            command.ExecuteNonQuery();
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "There was a problem while performing cleanup on connection");
+
+            throw;
+        }
+    }
+}

src/Geta.NotFoundHandler/Geta.NotFoundHandler.csproj

@@ -26,6 +26,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="Coravel" Version="5.0.4" />
     <PackageReference Include="CsvHelper" Version="33.0.0" />
     <PackageReference Include="Microsoft.Data.SqlClient" Version="2.1.1" />
     <PackageReference Include="X.PagedList" Version="8.4.3" />

src/Geta.NotFoundHandler/Infrastructure/Configuration/NotFoundHandlerOptions.cs

@@ -4,15 +4,21 @@
 using System;
 using System.Collections.Generic;
 using Geta.NotFoundHandler.Core;
+using Geta.NotFoundHandler.Core.Suggestions;
 using Geta.NotFoundHandler.Core.Redirects;
 
 namespace Geta.NotFoundHandler.Infrastructure.Configuration
 {
     public class NotFoundHandlerOptions
     {
+        public const string Section = "Geta:NotFoundHandler";
         public const int CurrentDbVersion = 2;
+
         public int BufferSize { get; set; } = 30;
         public int ThreshHold { get; set; } = 5;
+        public SuggestionsCleanupOptions SuggestionsCleanupOptions { get; set; } = new();
+        public bool UseInternalScheduler { get; set; }
+        public string InternalSchedulerCronInterval { get; set; } = "0 0 * * *";
         public FileNotFoundMode HandlerMode { get; set; } = FileNotFoundMode.On;
         public TimeSpan RegexTimeout { get; set; } = TimeSpan.FromMilliseconds(100);
 

src/Geta.NotFoundHandler/Infrastructure/Configuration/ServiceCollectionExtensions.cs

@@ -5,6 +5,7 @@
 using Geta.NotFoundHandler.Core;
 using Geta.NotFoundHandler.Core.Providers.RegexRedirects;
 using Geta.NotFoundHandler.Core.Redirects;
+using Geta.NotFoundHandler.Core.ScheduledJobs;
 using Geta.NotFoundHandler.Core.Suggestions;
 using Geta.NotFoundHandler.Data;
 using Geta.NotFoundHandler.Infrastructure.Initialization;
@@ -89,14 +90,18 @@ public static IServiceCollection AddNotFoundHandler(
             services.AddOptions<NotFoundHandlerOptions>().Configure<IConfiguration>((options, configuration) =>
             {
                 setupAction(options);
-                configuration.GetSection("Geta:NotFoundHandler").Bind(options);
+                configuration.GetSection(NotFoundHandlerOptions.Section).Bind(options);
             });
 
             services.AddAuthorization(options =>
             {
                 options.AddPolicy(Constants.PolicyName, configurePolicy);
             });
 
+            services.AddSingleton<ISuggestionsCleanupService, SuggestionsCleanupService>();
+
+            services.EnableScheduler();
+
             return services;
         }
     }