From 089d317954639050a694cf90429b10a811c69afc Mon Sep 17 00:00:00 2001 From: Jevgenij Puchko Date: Mon, 4 Nov 2024 10:54:47 +0200 Subject: [PATCH 1/5] Add suggestions cleanup job. --- README.md | 34 ++++++++++ .../Suggestions/Jobs/SuggestionsCleanupJob.cs | 28 +++++++++ .../OptimizelyNotFoundHandlerOptions.cs | 1 + .../ServiceCollectionExtensions.cs | 2 +- .../ApplicationBuilderExtensions.cs | 7 +++ .../ApplicationBuilderExtensions.cs | 30 +++++++++ .../ServiceCollectionExtensions.cs | 28 +++++++++ .../Suggestions/SuggestionsCleanupJob.cs | 23 +++++++ .../Suggestions/ISuggestionsCleanupService.cs | 9 +++ .../Suggestions/SuggestionsCleanupOptions.cs | 10 +++ .../Suggestions/SuggestionsCleanupService.cs | 62 +++++++++++++++++++ .../Geta.NotFoundHandler.csproj | 1 + .../Configuration/NotFoundHandlerOptions.cs | 5 ++ .../ServiceCollectionExtensions.cs | 7 ++- .../ApplicationBuilderExtensions.cs | 10 +++ 15 files changed, 255 insertions(+), 2 deletions(-) create mode 100644 src/Geta.NotFoundHandler.Optimizely/Core/Suggestions/Jobs/SuggestionsCleanupJob.cs create mode 100644 src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs create mode 100644 src/Geta.NotFoundHandler/Core/ScheduledJobs/ServiceCollectionExtensions.cs create mode 100644 src/Geta.NotFoundHandler/Core/ScheduledJobs/Suggestions/SuggestionsCleanupJob.cs create mode 100644 src/Geta.NotFoundHandler/Core/Suggestions/ISuggestionsCleanupService.cs create mode 100644 src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupOptions.cs create mode 100644 src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupService.cs diff --git a/README.md b/README.md index 391188ad..f2680880 100644 --- a/README.md +++ b/README.md @@ -278,6 +278,40 @@ There are two scheduled jobs: - *[Geta NotFoundHandler] Index content URLs* - as mentioned before, this job indexes URLs of content. Usually, it is required to run this job only once. All new content is automatically indexed. But if for some reasons content publish events are not firing when creating new content (for example, during the import), then you should set this job to run frequently. - *[Geta NotFoundHandler] Register content move redirects* - this job creates redirects based on registered moved content. Normally, this job is not required at all, but there might be situations when content move is registered but redirect creation is not completed. This could happen during deployments. In this case, you can manually run this job or schedule it to run time to time to fix such issues. +# Scheduled jobs + +Scheduled jobs powered by Coravel are included in the package, allowing jobs to run at set intervals. + +**Important** +Use this only if the package is not intended for an Optimizely site. Optimizely has built-in scheduled jobs mechanism. + +To enable scheduled jobs, you need to configure the following settings: +``` +services.AddNotFoundHandler(o => +{ + ... + o.ScheduledJobs = true; +}); +``` + +## Suggestions cleanup job +Practice shows that the suggestions table grows quickly in production, so a suggestions cleanup job was added to control its growth. + +This job is configured by default to run weekly, removing records older than 14 days. +You can adjust the retention period as needed. + +``` +services.AddNotFoundHandler(o => +{ + o.ScheduledJobs = true; + o.SuggestionsCleanupOptions.DaysToKeep = 30; +}); +``` +**Note** +For Optimizely was added job that is powered by built-in scheduled jobs mechanism. + +[Geta NotFoundHandler] Suggestions cleanup job + # Troubleshooting The module has extensive logging. Turn on debug logging for the `Geta.NotFoundHandler` namespace in your logging configuration. diff --git a/src/Geta.NotFoundHandler.Optimizely/Core/Suggestions/Jobs/SuggestionsCleanupJob.cs b/src/Geta.NotFoundHandler.Optimizely/Core/Suggestions/Jobs/SuggestionsCleanupJob.cs new file mode 100644 index 00000000..1fbda1e5 --- /dev/null +++ b/src/Geta.NotFoundHandler.Optimizely/Core/Suggestions/Jobs/SuggestionsCleanupJob.cs @@ -0,0 +1,28 @@ +// Copyright (c) Geta Digital. All rights reserved. +// Licensed under Apache-2.0. See the LICENSE file in the project root for more information + +using EPiServer.PlugIn; +using EPiServer.Scheduler; +using Geta.NotFoundHandler.Core.Suggestions; + +namespace Geta.NotFoundHandler.Optimizely.Core.Suggestions.Jobs; + +[ScheduledPlugIn(DisplayName = "[Geta NotFoundHandler] Suggestions cleanup job", + Description = "As suggestions table grow fast we should add a possibility to clean up old suggestions", + GUID = "6AE19CEC-1052-4482-97DF-981076DDD6F2", + SortIndex = 5555)] +public class SuggestionsCleanupJob : ScheduledJobBase +{ + private readonly ISuggestionsCleanupService _suggestionsCleanupService; + + public SuggestionsCleanupJob(ISuggestionsCleanupService suggestionsCleanupService) + { + IsStoppable = true; + _suggestionsCleanupService = suggestionsCleanupService; + } + + public override string Execute() + { + return _suggestionsCleanupService.Cleanup() ? "": "Unable to cleanup suggestions; please refer to the logs."; + } +} diff --git a/src/Geta.NotFoundHandler.Optimizely/Infrastructure/Configuration/OptimizelyNotFoundHandlerOptions.cs b/src/Geta.NotFoundHandler.Optimizely/Infrastructure/Configuration/OptimizelyNotFoundHandlerOptions.cs index 49d4bb1e..be865c1a 100644 --- a/src/Geta.NotFoundHandler.Optimizely/Infrastructure/Configuration/OptimizelyNotFoundHandlerOptions.cs +++ b/src/Geta.NotFoundHandler.Optimizely/Infrastructure/Configuration/OptimizelyNotFoundHandlerOptions.cs @@ -10,6 +10,7 @@ namespace Geta.NotFoundHandler.Optimizely.Infrastructure.Configuration { public class OptimizelyNotFoundHandlerOptions { + public const string Section = "Geta:NotFoundHandler:Optimizely"; public const int CurrentDbVersion = 3; public bool AutomaticRedirectsEnabled { get; set; } diff --git a/src/Geta.NotFoundHandler.Optimizely/Infrastructure/Configuration/ServiceCollectionExtensions.cs b/src/Geta.NotFoundHandler.Optimizely/Infrastructure/Configuration/ServiceCollectionExtensions.cs index ffe0954d..3c59ee60 100644 --- a/src/Geta.NotFoundHandler.Optimizely/Infrastructure/Configuration/ServiceCollectionExtensions.cs +++ b/src/Geta.NotFoundHandler.Optimizely/Infrastructure/Configuration/ServiceCollectionExtensions.cs @@ -74,7 +74,7 @@ public static IServiceCollection AddOptimizelyNotFoundHandler( services.AddOptions().Configure((options, configuration) => { setupAction(options); - configuration.GetSection("Geta:NotFoundHandler:Optimizely").Bind(options); + configuration.GetSection(OptimizelyNotFoundHandlerOptions.Section).Bind(options); }); return services; diff --git a/src/Geta.NotFoundHandler.Optimizely/Infrastructure/Initialization/ApplicationBuilderExtensions.cs b/src/Geta.NotFoundHandler.Optimizely/Infrastructure/Initialization/ApplicationBuilderExtensions.cs index a22ce6ca..4e8018ba 100644 --- a/src/Geta.NotFoundHandler.Optimizely/Infrastructure/Initialization/ApplicationBuilderExtensions.cs +++ b/src/Geta.NotFoundHandler.Optimizely/Infrastructure/Initialization/ApplicationBuilderExtensions.cs @@ -1,6 +1,9 @@ // Copyright (c) Geta Digital. All rights reserved. // Licensed under Apache-2.0. See the LICENSE file in the project root for more information +using Coravel.Scheduling.Schedule; +using Coravel.Scheduling.Schedule.Interfaces; +using Geta.NotFoundHandler.Core.ScheduledJobs.Suggestions; using Geta.NotFoundHandler.Optimizely.Core.AutomaticRedirects; using Geta.NotFoundHandler.Optimizely.Core.Events; using Microsoft.AspNetCore.Builder; @@ -23,6 +26,10 @@ public static IApplicationBuilder UseOptimizelyNotFoundHandler(this IApplication var historyEvents = services.GetRequiredService(); historyEvents.Initialize(); + // For optimizely we will use built-in scheduler for this job + var scheduler = services.GetService(); + (scheduler as Scheduler)?.TryUnschedule(nameof(SuggestionsCleanupJob)); + return app; } } diff --git a/src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs b/src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs new file mode 100644 index 00000000..c7e1d005 --- /dev/null +++ b/src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs @@ -0,0 +1,30 @@ +// Copyright (c) Geta Digital. All rights reserved. +// Licensed under Apache-2.0. See the LICENSE file in the project root for more information + +using Coravel; +using Geta.NotFoundHandler.Core.ScheduledJobs.Suggestions; +using Geta.NotFoundHandler.Infrastructure.Configuration; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +namespace Geta.NotFoundHandler.Core.ScheduledJobs; + +public static class ApplicationBuilderExtensions +{ + public static IApplicationBuilder UseScheduler(this IApplicationBuilder app) + { + var services = app.ApplicationServices; + var options = services.GetRequiredService>(); + + services.UseScheduler(scheduler => + { + scheduler + .Schedule() + .Weekly() + .PreventOverlapping(nameof(SuggestionsCleanupJob)); + }); + + return app; + } +} diff --git a/src/Geta.NotFoundHandler/Core/ScheduledJobs/ServiceCollectionExtensions.cs b/src/Geta.NotFoundHandler/Core/ScheduledJobs/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..94295010 --- /dev/null +++ b/src/Geta.NotFoundHandler/Core/ScheduledJobs/ServiceCollectionExtensions.cs @@ -0,0 +1,28 @@ +// Copyright (c) Geta Digital. All rights reserved. +// Licensed under Apache-2.0. See the LICENSE file in the project root for more information + +using Coravel; +using Geta.NotFoundHandler.Core.ScheduledJobs.Suggestions; +using Geta.NotFoundHandler.Infrastructure.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +namespace Geta.NotFoundHandler.Core.ScheduledJobs; + +public static class ServiceCollectionExtensions +{ + public static IServiceCollection EnableScheduler(this IServiceCollection services) + { + using var serviceProvider = services.BuildServiceProvider(); + var options = serviceProvider.GetRequiredService>().Value; + + if (options.UseScheduler) + { + services.AddScheduler(); + + services.AddTransient(); + } + + return services; + } +} diff --git a/src/Geta.NotFoundHandler/Core/ScheduledJobs/Suggestions/SuggestionsCleanupJob.cs b/src/Geta.NotFoundHandler/Core/ScheduledJobs/Suggestions/SuggestionsCleanupJob.cs new file mode 100644 index 00000000..aae5e394 --- /dev/null +++ b/src/Geta.NotFoundHandler/Core/ScheduledJobs/Suggestions/SuggestionsCleanupJob.cs @@ -0,0 +1,23 @@ +// Copyright (c) Geta Digital. All rights reserved. +// Licensed under Apache-2.0. See the LICENSE file in the project root for more information + +using System.Threading.Tasks; +using Coravel.Invocable; +using Geta.NotFoundHandler.Core.Suggestions; + +namespace Geta.NotFoundHandler.Core.ScheduledJobs.Suggestions; + +public class SuggestionsCleanupJob : IInvocable +{ + private readonly ISuggestionsCleanupService _suggestionsCleanupService; + + public SuggestionsCleanupJob(ISuggestionsCleanupService suggestionsCleanupService) + { + _suggestionsCleanupService = suggestionsCleanupService; + } + + public async Task Invoke() + { + _suggestionsCleanupService.Cleanup(); + } +} diff --git a/src/Geta.NotFoundHandler/Core/Suggestions/ISuggestionsCleanupService.cs b/src/Geta.NotFoundHandler/Core/Suggestions/ISuggestionsCleanupService.cs new file mode 100644 index 00000000..79cd13a3 --- /dev/null +++ b/src/Geta.NotFoundHandler/Core/Suggestions/ISuggestionsCleanupService.cs @@ -0,0 +1,9 @@ +// Copyright (c) Geta Digital. All rights reserved. +// Licensed under Apache-2.0. See the LICENSE file in the project root for more information + +namespace Geta.NotFoundHandler.Core.Suggestions; + +public interface ISuggestionsCleanupService +{ + bool Cleanup(); +} diff --git a/src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupOptions.cs b/src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupOptions.cs new file mode 100644 index 00000000..13dc5872 --- /dev/null +++ b/src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupOptions.cs @@ -0,0 +1,10 @@ +// Copyright (c) Geta Digital. All rights reserved. +// Licensed under Apache-2.0. See the LICENSE file in the project root for more information + +namespace Geta.NotFoundHandler.Core.Suggestions; + +public class SuggestionsCleanupOptions +{ + public int DaysToKeep { get; set; } = 14; + public int Timeout { get; set; } = 30 * 60; +} diff --git a/src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupService.cs b/src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupService.cs new file mode 100644 index 00000000..44763961 --- /dev/null +++ b/src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupService.cs @@ -0,0 +1,62 @@ +// Copyright (c) Geta Digital. All rights reserved. +// Licensed under Apache-2.0. See the LICENSE file in the project root for more information + +using System; +using Geta.NotFoundHandler.Infrastructure.Configuration; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Geta.NotFoundHandler.Core.Suggestions; + +public class SuggestionsCleanupService : ISuggestionsCleanupService +{ + private readonly IOptions _options; + private readonly ILogger _logger; + + public SuggestionsCleanupService( + IOptions options, + ILogger logger + ) + { + _options = options; + _logger = logger; + } + + private string CleanupCommandText(int daysToKeep) => $@" + -- [NotFoundHandler.Suggestions] + IF OBJECT_ID('[NotFoundHandler.Suggestions]', 'U') IS NOT NULL + BEGIN + DELETE + FROM + [NotFoundHandler.Suggestions] + WHERE + [Requested] < DATEADD(day, -{daysToKeep}, GETDATE()) + + PRINT '* Deleted ' + CAST(@@ROWCOUNT AS nvarchar) + ' outdated records from table [NotFoundHandler.Suggestions].' + END + "; + + + public bool Cleanup() + { + try + { + using var connection = new SqlConnection(_options.Value.ConnectionString); + connection.InfoMessage += (_, e) => _logger.LogInformation("{Message}", e.Message); + + var command = new SqlCommand(CleanupCommandText(_options.Value.SuggestionsCleanupOptions.DaysToKeep), connection); + command.CommandTimeout = _options.Value.SuggestionsCleanupOptions.Timeout; + command.Connection.Open(); + command.ExecuteNonQuery(); + + return true; + } + catch (Exception ex) + { + _logger.LogError(ex, "There was a problem while performing cleanup on connection"); + + return false; + } + } +} diff --git a/src/Geta.NotFoundHandler/Geta.NotFoundHandler.csproj b/src/Geta.NotFoundHandler/Geta.NotFoundHandler.csproj index 514cb523..ace405e6 100644 --- a/src/Geta.NotFoundHandler/Geta.NotFoundHandler.csproj +++ b/src/Geta.NotFoundHandler/Geta.NotFoundHandler.csproj @@ -26,6 +26,7 @@ + diff --git a/src/Geta.NotFoundHandler/Infrastructure/Configuration/NotFoundHandlerOptions.cs b/src/Geta.NotFoundHandler/Infrastructure/Configuration/NotFoundHandlerOptions.cs index 82a49736..d8f8397b 100644 --- a/src/Geta.NotFoundHandler/Infrastructure/Configuration/NotFoundHandlerOptions.cs +++ b/src/Geta.NotFoundHandler/Infrastructure/Configuration/NotFoundHandlerOptions.cs @@ -4,14 +4,19 @@ using System; using System.Collections.Generic; using Geta.NotFoundHandler.Core; +using Geta.NotFoundHandler.Core.Suggestions; namespace Geta.NotFoundHandler.Infrastructure.Configuration { public class NotFoundHandlerOptions { + public const string Section = "Geta:NotFoundHandler"; public const int CurrentDbVersion = 2; + public int BufferSize { get; set; } = 30; public int ThreshHold { get; set; } = 5; + public SuggestionsCleanupOptions SuggestionsCleanupOptions { get; set; } = new(); + public bool UseScheduler { get; set; } public FileNotFoundMode HandlerMode { get; set; } = FileNotFoundMode.On; public TimeSpan RegexTimeout { get; set; } = TimeSpan.FromMilliseconds(100); diff --git a/src/Geta.NotFoundHandler/Infrastructure/Configuration/ServiceCollectionExtensions.cs b/src/Geta.NotFoundHandler/Infrastructure/Configuration/ServiceCollectionExtensions.cs index ab55b59b..02a79a4e 100644 --- a/src/Geta.NotFoundHandler/Infrastructure/Configuration/ServiceCollectionExtensions.cs +++ b/src/Geta.NotFoundHandler/Infrastructure/Configuration/ServiceCollectionExtensions.cs @@ -5,6 +5,7 @@ using Geta.NotFoundHandler.Core; using Geta.NotFoundHandler.Core.Providers.RegexRedirects; using Geta.NotFoundHandler.Core.Redirects; +using Geta.NotFoundHandler.Core.ScheduledJobs; using Geta.NotFoundHandler.Core.Suggestions; using Geta.NotFoundHandler.Data; using Geta.NotFoundHandler.Infrastructure.Initialization; @@ -89,7 +90,7 @@ public static IServiceCollection AddNotFoundHandler( services.AddOptions().Configure((options, configuration) => { setupAction(options); - configuration.GetSection("Geta:NotFoundHandler").Bind(options); + configuration.GetSection(NotFoundHandlerOptions.Section).Bind(options); }); services.AddAuthorization(options => @@ -97,6 +98,10 @@ public static IServiceCollection AddNotFoundHandler( options.AddPolicy(Constants.PolicyName, configurePolicy); }); + services.AddSingleton(); + + services.EnableScheduler(); + return services; } } diff --git a/src/Geta.NotFoundHandler/Infrastructure/Initialization/ApplicationBuilderExtensions.cs b/src/Geta.NotFoundHandler/Infrastructure/Initialization/ApplicationBuilderExtensions.cs index 5cecb436..b52afac9 100644 --- a/src/Geta.NotFoundHandler/Infrastructure/Initialization/ApplicationBuilderExtensions.cs +++ b/src/Geta.NotFoundHandler/Infrastructure/Initialization/ApplicationBuilderExtensions.cs @@ -2,8 +2,11 @@ // Licensed under Apache-2.0. See the LICENSE file in the project root for more information using Geta.NotFoundHandler.Core.Redirects; +using Geta.NotFoundHandler.Core.ScheduledJobs; +using Geta.NotFoundHandler.Infrastructure.Configuration; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; namespace Geta.NotFoundHandler.Infrastructure.Initialization { @@ -21,6 +24,13 @@ public static IApplicationBuilder UseNotFoundHandler(this IApplicationBuilder ap app.UseMiddleware(); + var options = services.GetRequiredService>().Value; + + if (options.UseScheduler) + { + app.UseScheduler(); + } + return app; } } From 01ae2517604fb24beea4ab3084715eb862d87172 Mon Sep 17 00:00:00 2001 From: Jevgenij Puchko Date: Mon, 4 Nov 2024 10:58:15 +0200 Subject: [PATCH 2/5] Code review fixes --- .../Core/ScheduledJobs/ApplicationBuilderExtensions.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs b/src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs index c7e1d005..69a0166c 100644 --- a/src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs +++ b/src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs @@ -3,10 +3,7 @@ using Coravel; using Geta.NotFoundHandler.Core.ScheduledJobs.Suggestions; -using Geta.NotFoundHandler.Infrastructure.Configuration; using Microsoft.AspNetCore.Builder; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; namespace Geta.NotFoundHandler.Core.ScheduledJobs; @@ -15,7 +12,6 @@ public static class ApplicationBuilderExtensions public static IApplicationBuilder UseScheduler(this IApplicationBuilder app) { var services = app.ApplicationServices; - var options = services.GetRequiredService>(); services.UseScheduler(scheduler => { From b346274e1c0bf12e47311530715ccfc8b4c6e3ac Mon Sep 17 00:00:00 2001 From: Jevgenij Puchko Date: Mon, 4 Nov 2024 15:46:53 +0200 Subject: [PATCH 3/5] Add CronInterval option --- README.md | 13 +++++++------ .../ScheduledJobs/ApplicationBuilderExtensions.cs | 7 ++++++- .../Core/Suggestions/SuggestionsCleanupOptions.cs | 1 + 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index f2680880..c659d285 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ If you want to specify this yourself, add `IgnoredResourceExtensions` to the con ## Restricting access to the Admin UI -By default, only users of `Administrators` role can access Admin UI. But you can configure you authorization policy when registrating the NotFound handler. +By default, only users of `Administrators` role can access Admin UI. But you can configure your authorization policy when registering the NotFound handler. ``` services.AddNotFoundHandler(o => { }, @@ -171,7 +171,7 @@ By default, only users of `Administrators` role can access Admin UI. But you can }); ``` -You can setup any policy rules you want. +You can set up any policy rules you want. ## Import @@ -179,7 +179,7 @@ For details see [Import redirects for 404 handler](https://getadigital.com/blog/ # Custom 404 Page -To setup 404 page, you can use any method ASP.NET Core provides. +To set up 404 page, you can use any method ASP.NET Core provides. One of the simplest solutions is adding a controller and a view for it that would display an error page: @@ -275,7 +275,7 @@ It will monitor primary, secondary and SEO URLs: Optimizely Content Cloud supports only primary URLs and Optimizely Commerce supports all three types of URLs. There are two scheduled jobs: -- *[Geta NotFoundHandler] Index content URLs* - as mentioned before, this job indexes URLs of content. Usually, it is required to run this job only once. All new content is automatically indexed. But if for some reasons content publish events are not firing when creating new content (for example, during the import), then you should set this job to run frequently. +- *[Geta NotFoundHandler] Index content URLs* - as mentioned before, this job indexes URLs of content. Usually, it is required to run this job only once. All new content is automatically indexed. But if for some reason content publish events are not firing when creating new content (for example, during the import), then you should set this job to run frequently. - *[Geta NotFoundHandler] Register content move redirects* - this job creates redirects based on registered moved content. Normally, this job is not required at all, but there might be situations when content move is registered but redirect creation is not completed. This could happen during deployments. In this case, you can manually run this job or schedule it to run time to time to fix such issues. # Scheduled jobs @@ -297,13 +297,14 @@ services.AddNotFoundHandler(o => ## Suggestions cleanup job Practice shows that the suggestions table grows quickly in production, so a suggestions cleanup job was added to control its growth. -This job is configured by default to run weekly, removing records older than 14 days. +This job is configured by default to run daily at midnight, removing records older than 14 days. You can adjust the retention period as needed. ``` services.AddNotFoundHandler(o => { o.ScheduledJobs = true; + o.SuggestionsCleanupOptions.CronInterval = "0 0 * * 0" // weekly on Sunday midnight o.SuggestionsCleanupOptions.DaysToKeep = 30; }); ``` @@ -327,7 +328,7 @@ For example, if we have a redirect: `/a` to `/b`, then: - without wildcard setting it will redirect `/a/1` to `/b/1` # Sandbox App -Sandbox application is testing poligon for pacakge new features and bug fixes. +Sandbox application is testing polygon for package new features and bug fixes. CMS username: admin@example.com diff --git a/src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs b/src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs index 69a0166c..c86acf87 100644 --- a/src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs +++ b/src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs @@ -3,7 +3,10 @@ using Coravel; using Geta.NotFoundHandler.Core.ScheduledJobs.Suggestions; +using Geta.NotFoundHandler.Infrastructure.Configuration; using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; namespace Geta.NotFoundHandler.Core.ScheduledJobs; @@ -13,11 +16,13 @@ public static IApplicationBuilder UseScheduler(this IApplicationBuilder app) { var services = app.ApplicationServices; + var options = services.GetRequiredService>().Value; + services.UseScheduler(scheduler => { scheduler .Schedule() - .Weekly() + .Cron(options.SuggestionsCleanupOptions.CronInterval) .PreventOverlapping(nameof(SuggestionsCleanupJob)); }); diff --git a/src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupOptions.cs b/src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupOptions.cs index 13dc5872..5c5c1c2d 100644 --- a/src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupOptions.cs +++ b/src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupOptions.cs @@ -7,4 +7,5 @@ public class SuggestionsCleanupOptions { public int DaysToKeep { get; set; } = 14; public int Timeout { get; set; } = 30 * 60; + public string CronInterval { get; set; } = "0 0 * * *"; } From f15bb7d79ac4bc0b26226c27784a7f0733566041 Mon Sep 17 00:00:00 2001 From: Jevgenij Puchko Date: Wed, 6 Nov 2024 16:00:48 +0200 Subject: [PATCH 4/5] Code review fixes --- README.md | 34 +++++++------------ .../Suggestions/Jobs/SuggestionsCleanupJob.cs | 4 ++- .../ApplicationBuilderExtensions.cs | 20 +++++++---- .../ServiceCollectionExtensions.cs | 2 +- .../Suggestions/ISuggestionsCleanupService.cs | 2 +- .../Suggestions/SuggestionsCleanupOptions.cs | 1 - .../Suggestions/SuggestionsCleanupService.cs | 6 ++-- .../Configuration/NotFoundHandlerOptions.cs | 3 +- .../ApplicationBuilderExtensions.cs | 2 +- 9 files changed, 36 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index c659d285..0fd8724e 100644 --- a/README.md +++ b/README.md @@ -280,38 +280,30 @@ There are two scheduled jobs: # Scheduled jobs -Scheduled jobs powered by Coravel are included in the package, allowing jobs to run at set intervals. - -**Important** -Use this only if the package is not intended for an Optimizely site. Optimizely has built-in scheduled jobs mechanism. - -To enable scheduled jobs, you need to configure the following settings: +Scheduled job - process that runs in background +- Suggestions cleanup job - shipped with the package, contains process that cleans up suggestions table. +This job is configured by default to remove records older than 14 days. You can adjust the retention period or timeout as needed. ``` services.AddNotFoundHandler(o => { - ... - o.ScheduledJobs = true; + o.SuggestionsCleanupOptions.DaysToKeep = 30; + o.SuggestionsCleanupOptions.Timeout = 30 * 60; }); ``` -## Suggestions cleanup job -Practice shows that the suggestions table grows quickly in production, so a suggestions cleanup job was added to control its growth. - -This job is configured by default to run daily at midnight, removing records older than 14 days. -You can adjust the retention period as needed. - +Scheduler - mechanism that triggers scheduled jobs in a recurrent manner +- InternalScheduler - default scheduler, included in the core package, a scheduler that uses [Coravel](https://docs.coravel.net/). + To enable the scheduler, you need to enable UseInternalScheduler flag. Additionally, you can adjust the scheduler run interval: ``` services.AddNotFoundHandler(o => { - o.ScheduledJobs = true; - o.SuggestionsCleanupOptions.CronInterval = "0 0 * * 0" // weekly on Sunday midnight - o.SuggestionsCleanupOptions.DaysToKeep = 30; + ... + o.UseInternalScheduler = true; + o.InternalSchedulerCronInterval = "0 0 * * *" // by default it's configured to run daily at midnight }); ``` -**Note** -For Optimizely was added job that is powered by built-in scheduled jobs mechanism. - -[Geta NotFoundHandler] Suggestions cleanup job +- OptimizelyScheduler - uses Optimizely to schedule job runs. +An Optimizely scheduled job was added - [Geta NotFoundHandler] Suggestions cleanup job. # Troubleshooting diff --git a/src/Geta.NotFoundHandler.Optimizely/Core/Suggestions/Jobs/SuggestionsCleanupJob.cs b/src/Geta.NotFoundHandler.Optimizely/Core/Suggestions/Jobs/SuggestionsCleanupJob.cs index 1fbda1e5..e40de31c 100644 --- a/src/Geta.NotFoundHandler.Optimizely/Core/Suggestions/Jobs/SuggestionsCleanupJob.cs +++ b/src/Geta.NotFoundHandler.Optimizely/Core/Suggestions/Jobs/SuggestionsCleanupJob.cs @@ -23,6 +23,8 @@ public SuggestionsCleanupJob(ISuggestionsCleanupService suggestionsCleanupServic public override string Execute() { - return _suggestionsCleanupService.Cleanup() ? "": "Unable to cleanup suggestions; please refer to the logs."; + _suggestionsCleanupService.Cleanup(); + + return string.Empty; } } diff --git a/src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs b/src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs index c86acf87..2bfae1ec 100644 --- a/src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs +++ b/src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs @@ -6,6 +6,7 @@ using Geta.NotFoundHandler.Infrastructure.Configuration; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace Geta.NotFoundHandler.Core.ScheduledJobs; @@ -17,14 +18,19 @@ public static IApplicationBuilder UseScheduler(this IApplicationBuilder app) var services = app.ApplicationServices; var options = services.GetRequiredService>().Value; - + var logger = services.GetRequiredService(); + services.UseScheduler(scheduler => - { - scheduler - .Schedule() - .Cron(options.SuggestionsCleanupOptions.CronInterval) - .PreventOverlapping(nameof(SuggestionsCleanupJob)); - }); + { + scheduler + .Schedule() + .Cron(options.InternalSchedulerCronInterval) + .PreventOverlapping(nameof(SuggestionsCleanupJob)); + }) + .OnError(x => + { + logger.LogError(x, "Something went wrong, scheduled job fails with exception"); + }); return app; } diff --git a/src/Geta.NotFoundHandler/Core/ScheduledJobs/ServiceCollectionExtensions.cs b/src/Geta.NotFoundHandler/Core/ScheduledJobs/ServiceCollectionExtensions.cs index 94295010..a698cf7d 100644 --- a/src/Geta.NotFoundHandler/Core/ScheduledJobs/ServiceCollectionExtensions.cs +++ b/src/Geta.NotFoundHandler/Core/ScheduledJobs/ServiceCollectionExtensions.cs @@ -16,7 +16,7 @@ public static IServiceCollection EnableScheduler(this IServiceCollection service using var serviceProvider = services.BuildServiceProvider(); var options = serviceProvider.GetRequiredService>().Value; - if (options.UseScheduler) + if (options.UseInternalScheduler) { services.AddScheduler(); diff --git a/src/Geta.NotFoundHandler/Core/Suggestions/ISuggestionsCleanupService.cs b/src/Geta.NotFoundHandler/Core/Suggestions/ISuggestionsCleanupService.cs index 79cd13a3..8365c9eb 100644 --- a/src/Geta.NotFoundHandler/Core/Suggestions/ISuggestionsCleanupService.cs +++ b/src/Geta.NotFoundHandler/Core/Suggestions/ISuggestionsCleanupService.cs @@ -5,5 +5,5 @@ namespace Geta.NotFoundHandler.Core.Suggestions; public interface ISuggestionsCleanupService { - bool Cleanup(); + void Cleanup(); } diff --git a/src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupOptions.cs b/src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupOptions.cs index 5c5c1c2d..13dc5872 100644 --- a/src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupOptions.cs +++ b/src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupOptions.cs @@ -7,5 +7,4 @@ public class SuggestionsCleanupOptions { public int DaysToKeep { get; set; } = 14; public int Timeout { get; set; } = 30 * 60; - public string CronInterval { get; set; } = "0 0 * * *"; } diff --git a/src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupService.cs b/src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupService.cs index 44763961..ffb2844d 100644 --- a/src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupService.cs +++ b/src/Geta.NotFoundHandler/Core/Suggestions/SuggestionsCleanupService.cs @@ -38,7 +38,7 @@ IF OBJECT_ID('[NotFoundHandler.Suggestions]', 'U') IS NOT NULL "; - public bool Cleanup() + public void Cleanup() { try { @@ -49,14 +49,12 @@ public bool Cleanup() command.CommandTimeout = _options.Value.SuggestionsCleanupOptions.Timeout; command.Connection.Open(); command.ExecuteNonQuery(); - - return true; } catch (Exception ex) { _logger.LogError(ex, "There was a problem while performing cleanup on connection"); - return false; + throw; } } } diff --git a/src/Geta.NotFoundHandler/Infrastructure/Configuration/NotFoundHandlerOptions.cs b/src/Geta.NotFoundHandler/Infrastructure/Configuration/NotFoundHandlerOptions.cs index 4705ad7e..c8b1a80d 100644 --- a/src/Geta.NotFoundHandler/Infrastructure/Configuration/NotFoundHandlerOptions.cs +++ b/src/Geta.NotFoundHandler/Infrastructure/Configuration/NotFoundHandlerOptions.cs @@ -17,7 +17,8 @@ public class NotFoundHandlerOptions public int BufferSize { get; set; } = 30; public int ThreshHold { get; set; } = 5; public SuggestionsCleanupOptions SuggestionsCleanupOptions { get; set; } = new(); - public bool UseScheduler { get; set; } + public bool UseInternalScheduler { get; set; } + public string InternalSchedulerCronInterval { get; set; } = "0 0 * * *"; public FileNotFoundMode HandlerMode { get; set; } = FileNotFoundMode.On; public TimeSpan RegexTimeout { get; set; } = TimeSpan.FromMilliseconds(100); diff --git a/src/Geta.NotFoundHandler/Infrastructure/Initialization/ApplicationBuilderExtensions.cs b/src/Geta.NotFoundHandler/Infrastructure/Initialization/ApplicationBuilderExtensions.cs index b52afac9..07dc72bd 100644 --- a/src/Geta.NotFoundHandler/Infrastructure/Initialization/ApplicationBuilderExtensions.cs +++ b/src/Geta.NotFoundHandler/Infrastructure/Initialization/ApplicationBuilderExtensions.cs @@ -26,7 +26,7 @@ public static IApplicationBuilder UseNotFoundHandler(this IApplicationBuilder ap var options = services.GetRequiredService>().Value; - if (options.UseScheduler) + if (options.UseInternalScheduler) { app.UseScheduler(); } From 9eb600a6b077c9d1377977f85843a3d212fe2f90 Mon Sep 17 00:00:00 2001 From: Jevgenij Puchko Date: Tue, 26 Nov 2024 08:44:16 +0200 Subject: [PATCH 5/5] Code review fixes --- README.md | 9 +++++---- .../Core/ScheduledJobs/ApplicationBuilderExtensions.cs | 4 ++-- .../Initialization/ApplicationBuilderExtensions.cs | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 0fd8724e..eca37a0a 100644 --- a/README.md +++ b/README.md @@ -274,10 +274,6 @@ It will monitor primary, secondary and SEO URLs: Optimizely Content Cloud supports only primary URLs and Optimizely Commerce supports all three types of URLs. -There are two scheduled jobs: -- *[Geta NotFoundHandler] Index content URLs* - as mentioned before, this job indexes URLs of content. Usually, it is required to run this job only once. All new content is automatically indexed. But if for some reason content publish events are not firing when creating new content (for example, during the import), then you should set this job to run frequently. - - *[Geta NotFoundHandler] Register content move redirects* - this job creates redirects based on registered moved content. Normally, this job is not required at all, but there might be situations when content move is registered but redirect creation is not completed. This could happen during deployments. In this case, you can manually run this job or schedule it to run time to time to fix such issues. - # Scheduled jobs Scheduled job - process that runs in background @@ -305,6 +301,11 @@ services.AddNotFoundHandler(o => - OptimizelyScheduler - uses Optimizely to schedule job runs. An Optimizely scheduled job was added - [Geta NotFoundHandler] Suggestions cleanup job. +Additionally, there are two optimizely scheduled jobs responsible for: +- *[Geta NotFoundHandler] Index content URLs* - as mentioned before, this job indexes URLs of content. Usually, it is required to run this job only once. All new content is automatically indexed. But if for some reason content publish events are not firing when creating new content (for example, during the import), then you should set this job to run frequently. +- *[Geta NotFoundHandler] Register content move redirects* - this job creates redirects based on registered moved content. Normally, this job is not required at all, but there might be situations when content move is registered but redirect creation is not completed. This could happen during deployments. In this case, you can manually run this job or schedule it to run time to time to fix such issues. + + # Troubleshooting The module has extensive logging. Turn on debug logging for the `Geta.NotFoundHandler` namespace in your logging configuration. diff --git a/src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs b/src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs index 2bfae1ec..61cd25e8 100644 --- a/src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs +++ b/src/Geta.NotFoundHandler/Core/ScheduledJobs/ApplicationBuilderExtensions.cs @@ -13,7 +13,7 @@ namespace Geta.NotFoundHandler.Core.ScheduledJobs; public static class ApplicationBuilderExtensions { - public static IApplicationBuilder UseScheduler(this IApplicationBuilder app) + public static IApplicationBuilder UseInternalScheduler(this IApplicationBuilder app) { var services = app.ApplicationServices; @@ -29,7 +29,7 @@ public static IApplicationBuilder UseScheduler(this IApplicationBuilder app) }) .OnError(x => { - logger.LogError(x, "Something went wrong, scheduled job fails with exception"); + logger.LogError(x, "Something went wrong, scheduled job failed with exception"); }); return app; diff --git a/src/Geta.NotFoundHandler/Infrastructure/Initialization/ApplicationBuilderExtensions.cs b/src/Geta.NotFoundHandler/Infrastructure/Initialization/ApplicationBuilderExtensions.cs index 07dc72bd..73f21ee9 100644 --- a/src/Geta.NotFoundHandler/Infrastructure/Initialization/ApplicationBuilderExtensions.cs +++ b/src/Geta.NotFoundHandler/Infrastructure/Initialization/ApplicationBuilderExtensions.cs @@ -28,7 +28,7 @@ public static IApplicationBuilder UseNotFoundHandler(this IApplicationBuilder ap if (options.UseInternalScheduler) { - app.UseScheduler(); + app.UseInternalScheduler(); } return app;