From d77383a560e7ae0857d4abcc97df25b7b609f9d4 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 15 Apr 2024 10:59:07 +0800 Subject: [PATCH 01/11] Fix potential null sentry DSN causing startup failures --- osu.Server.QueueProcessor/QueueProcessor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Server.QueueProcessor/QueueProcessor.cs b/osu.Server.QueueProcessor/QueueProcessor.cs index 96ac5cf..575791b 100644 --- a/osu.Server.QueueProcessor/QueueProcessor.cs +++ b/osu.Server.QueueProcessor/QueueProcessor.cs @@ -191,7 +191,7 @@ public void Run(CancellationToken cancellation = default) private void setupSentry(SentryOptions options) { - options.Dsn = Environment.GetEnvironmentVariable("SENTRY_DSN"); + options.Dsn = Environment.GetEnvironmentVariable("SENTRY_DSN") ?? string.Empty; options.DefaultTags["queue"] = QueueName; } From 89addeab839218392e9baf45d935d6f707ba54a6 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 15 Apr 2024 11:01:41 +0800 Subject: [PATCH 02/11] Update project to net8 --- .../osu.Server.QueueProcessor.Tests.csproj | 2 +- osu.Server.QueueProcessor/osu.Server.QueueProcessor.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/osu.Server.QueueProcessor.Tests/osu.Server.QueueProcessor.Tests.csproj b/osu.Server.QueueProcessor.Tests/osu.Server.QueueProcessor.Tests.csproj index c0b9078..7f98bf3 100644 --- a/osu.Server.QueueProcessor.Tests/osu.Server.QueueProcessor.Tests.csproj +++ b/osu.Server.QueueProcessor.Tests/osu.Server.QueueProcessor.Tests.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 enable false diff --git a/osu.Server.QueueProcessor/osu.Server.QueueProcessor.csproj b/osu.Server.QueueProcessor/osu.Server.QueueProcessor.csproj index ce70aff..4976374 100644 --- a/osu.Server.QueueProcessor/osu.Server.QueueProcessor.csproj +++ b/osu.Server.QueueProcessor/osu.Server.QueueProcessor.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 OsuQueueProcessor ppy.osu.Server.OsuQueueProcessor Dean Herbert From 8992f713db081891d0968510036fcdde1b74bbd1 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 15 Apr 2024 11:03:13 +0800 Subject: [PATCH 03/11] Update packages --- .../osu.Server.QueueProcessor.Tests.csproj | 8 ++++---- .../osu.Server.QueueProcessor.csproj | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/osu.Server.QueueProcessor.Tests/osu.Server.QueueProcessor.Tests.csproj b/osu.Server.QueueProcessor.Tests/osu.Server.QueueProcessor.Tests.csproj index 7f98bf3..ed8af6b 100644 --- a/osu.Server.QueueProcessor.Tests/osu.Server.QueueProcessor.Tests.csproj +++ b/osu.Server.QueueProcessor.Tests/osu.Server.QueueProcessor.Tests.csproj @@ -7,13 +7,13 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/osu.Server.QueueProcessor/osu.Server.QueueProcessor.csproj b/osu.Server.QueueProcessor/osu.Server.QueueProcessor.csproj index 4976374..7a435a7 100644 --- a/osu.Server.QueueProcessor/osu.Server.QueueProcessor.csproj +++ b/osu.Server.QueueProcessor/osu.Server.QueueProcessor.csproj @@ -11,12 +11,12 @@ - - - - - - + + + + + + From 9fffd082ff30c3269d3c260087e7e336cc7a9641 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 15 Apr 2024 11:03:17 +0800 Subject: [PATCH 04/11] Add new dotsettings --- osu.Server.QueueProcessor.sln.DotSettings | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/osu.Server.QueueProcessor.sln.DotSettings b/osu.Server.QueueProcessor.sln.DotSettings index b9fc3de..b450f68 100644 --- a/osu.Server.QueueProcessor.sln.DotSettings +++ b/osu.Server.QueueProcessor.sln.DotSettings @@ -721,9 +721,19 @@ See the LICENCE file in the repository root for full licence text. <Policy Inspect="True" Prefix="" Suffix="" Style="aa_bb" /> <Policy Inspect="True" Prefix="" Suffix="" Style="AA_BB" /> <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> + <Policy><Descriptor Staticness="Static" AccessRightKinds="Private" Description="Static readonly fields (private)"><ElementKinds><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aa_bb" /></Policy> + <Policy><Descriptor Staticness="Any" AccessRightKinds="Private" Description="Constant fields (private)"><ElementKinds><Kind Name="CONSTANT_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aa_bb" /></Policy> + <Policy><Descriptor Staticness="Any" AccessRightKinds="Any" Description="Type parameters"><ElementKinds><Kind Name="TYPE_PARAMETER" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /></Policy> + <Policy><Descriptor Staticness="Instance" AccessRightKinds="Private" Description="Instance fields (private)"><ElementKinds><Kind Name="FIELD" /><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb"><ExtraRule Prefix="_" Suffix="" Style="aaBb" /></Policy></Policy> + <Policy><Descriptor Staticness="Any" AccessRightKinds="Protected, ProtectedInternal, Internal, Public, PrivateProtected" Description="Constant fields (not private)"><ElementKinds><Kind Name="CONSTANT_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="AA_BB" /></Policy> + <Policy><Descriptor Staticness="Any" AccessRightKinds="Any" Description="Local functions"><ElementKinds><Kind Name="LOCAL_FUNCTION" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> + <Policy><Descriptor Staticness="Any" AccessRightKinds="Any" Description="Enum members"><ElementKinds><Kind Name="ENUM_MEMBER" /></ElementKinds></Descriptor><Policy Inspect="False" Prefix="" Suffix="" Style="AaBb" /></Policy> <Policy><Descriptor Staticness="Static, Instance" AccessRightKinds="Private" Description="private methods"><ElementKinds><Kind Name="ASYNC_METHOD" /><Kind Name="METHOD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> <Policy><Descriptor Staticness="Static, Instance" AccessRightKinds="Protected, ProtectedInternal, Internal, Public" Description="internal/protected/public methods"><ElementKinds><Kind Name="ASYNC_METHOD" /><Kind Name="METHOD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /></Policy> <Policy><Descriptor Staticness="Static, Instance" AccessRightKinds="Private" Description="private properties"><ElementKinds><Kind Name="PROPERTY" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> + <Policy><Descriptor Staticness="Any" AccessRightKinds="Any" Description="Local constants"><ElementKinds><Kind Name="LOCAL_CONSTANT" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aa_bb" /></Policy> + <Policy><Descriptor Staticness="Static" AccessRightKinds="Protected, ProtectedInternal, Internal, Public, PrivateProtected" Description="Static readonly fields (not private)"><ElementKinds><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="AA_BB" /></Policy> + <Policy><Descriptor Staticness="Static" AccessRightKinds="Private" Description="Static fields (private)"><ElementKinds><Kind Name="FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> <Policy><Descriptor Staticness="Static, Instance" AccessRightKinds="Protected, ProtectedInternal, Internal, Public" Description="internal/protected/public properties"><ElementKinds><Kind Name="PROPERTY" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /></Policy> <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> @@ -767,6 +777,7 @@ See the LICENCE file in the repository root for full licence text. <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> + True True True True @@ -777,6 +788,7 @@ See the LICENCE file in the repository root for full licence text. True True True + True True True o!f – Object Initializer: Anchor&Origin From 49e563c54777c1ed52b81e9e39622cd48b993354 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 15 Apr 2024 11:14:39 +0800 Subject: [PATCH 05/11] Fix a few inspections --- osu.Server.QueueProcessor/QueueProcessor.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osu.Server.QueueProcessor/QueueProcessor.cs b/osu.Server.QueueProcessor/QueueProcessor.cs index 575791b..00db7d2 100644 --- a/osu.Server.QueueProcessor/QueueProcessor.cs +++ b/osu.Server.QueueProcessor/QueueProcessor.cs @@ -119,7 +119,7 @@ public void Run(CancellationToken cancellation = default) // null or empty check is required for redis 6.x. 7.x reports `null` instead. foreach (var redisItem in redisItems.Where(i => !i.IsNullOrEmpty)) - items.Add(JsonConvert.DeserializeObject(redisItem) ?? throw new InvalidOperationException("Dequeued item could not be deserialised.")); + items.Add(JsonConvert.DeserializeObject(redisItem!) ?? throw new InvalidOperationException("Dequeued item could not be deserialised.")); if (items.Count == 0) { @@ -254,7 +254,7 @@ public long GetQueueSize() => /// The type of message to be published. public void PublishMessage(string channelName, TMessage message) { - getRedisDatabase().Publish(channelName, JsonConvert.SerializeObject(message)); + getRedisDatabase().Publish(new RedisChannel(channelName, RedisChannel.PatternMode.Auto), JsonConvert.SerializeObject(message)); DogStatsd.Increment("messages_published", tags: new[] { $"channel:{channelName}", $"type:{typeof(TMessage).FullName}" }); } From 93ed4ccef908367608643f9a1044c999727a45ff Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 15 Apr 2024 11:15:55 +0800 Subject: [PATCH 06/11] Suppress some test inspections for now Fixing these is a bit annoying for a number of silly reasons, aka not worth the time atm. --- osu.Server.QueueProcessor.Tests/BatchProcessorTests.cs | 2 ++ osu.Server.QueueProcessor.Tests/InputOnlyQueueTests.cs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/osu.Server.QueueProcessor.Tests/BatchProcessorTests.cs b/osu.Server.QueueProcessor.Tests/BatchProcessorTests.cs index 60e07e4..a2bc17f 100644 --- a/osu.Server.QueueProcessor.Tests/BatchProcessorTests.cs +++ b/osu.Server.QueueProcessor.Tests/BatchProcessorTests.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Tasks; using Xunit; @@ -88,6 +89,7 @@ public void SendThenReceive_Multiple() /// If the processor is cancelled mid-operation, every item should either be processed or still in the queue. /// [Fact] + [SuppressMessage("Usage", "xUnit1031:Do not use blocking task operations in test method")] // For simplicity. public void EnsureCancellingDoesNotLoseItems() { var inFlightObjects = new List(); diff --git a/osu.Server.QueueProcessor.Tests/InputOnlyQueueTests.cs b/osu.Server.QueueProcessor.Tests/InputOnlyQueueTests.cs index 7427506..f47a1bd 100644 --- a/osu.Server.QueueProcessor.Tests/InputOnlyQueueTests.cs +++ b/osu.Server.QueueProcessor.Tests/InputOnlyQueueTests.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Tasks; using Xunit; @@ -106,6 +107,7 @@ public void SendThenReceive_MultipleUsingSingleCall() /// If the processor is cancelled mid-operation, every item should either be processed or still in the queue. /// [Fact] + [SuppressMessage("Usage", "xUnit1031:Do not use blocking task operations in test method")] // For simplicity. public void EnsureCancellingDoesNotLoseItems() { var inFlightObjects = new List(); From 154b5932e019dbe245156bc63b997c35bbfc543a Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 15 Apr 2024 18:09:55 +0800 Subject: [PATCH 07/11] Update .net version for workflows --- .github/workflows/ci.yml | 9 +++++---- .github/workflows/pack.yml | 11 +++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 58443bd..d4d0d70 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,11 +22,12 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v2 - - name: Setup .NET Core - uses: actions/setup-dotnet@v1 + - name: Checkout + uses: actions/checkout@v4 + - name: Install .NET 8.0.x + uses: actions/setup-dotnet@v4 with: - dotnet-version: "6.0.x" + dotnet-version: "8.0.x" - name: Install dependencies run: dotnet restore - name: Build diff --git a/.github/workflows/pack.yml b/.github/workflows/pack.yml index bf0595d..41323d4 100644 --- a/.github/workflows/pack.yml +++ b/.github/workflows/pack.yml @@ -9,13 +9,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - - name: Setup .NET Core - uses: actions/setup-dotnet@v1 + - name: Checkout + uses: actions/checkout@v4 + - name: Install .NET 8.0.x + uses: actions/setup-dotnet@v4 with: - dotnet-version: "6.0.x" - source-url: https://nuget.pkg.github.com/ppy/index.json + dotnet-version: "8.0.x" env: NUGET_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Get the version From ed4a53c41506b01b8780f2aa2db68029822103cf Mon Sep 17 00:00:00 2001 From: Dan Balasescu Date: Mon, 15 Apr 2024 19:36:55 +0900 Subject: [PATCH 08/11] Remove unnecessary container specification --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4d0d70..22b8212 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,9 +8,7 @@ on: jobs: unit-tests: - runs-on: ubuntu-latest - container: node:10.18-jessie services: redis: From 1ff5d73ab72738432029bf062668cadc0595c27e Mon Sep 17 00:00:00 2001 From: Dan Balasescu Date: Mon, 15 Apr 2024 19:42:02 +0900 Subject: [PATCH 09/11] Expose redis port --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22b8212..62c6570 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,8 @@ jobs: services: redis: image: redis + ports: + - 6379:6379 options: >- --health-cmd "redis-cli ping" --health-interval 10s From 0ce3c91092ebc957bc945f2e9d01ef2369fbef0e Mon Sep 17 00:00:00 2001 From: Dan Balasescu Date: Mon, 15 Apr 2024 19:51:22 +0900 Subject: [PATCH 10/11] Remove REDIS_HOST env variable --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62c6570..203f29a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,4 @@ jobs: - name: Build run: dotnet build --no-restore - name: Test - env: - REDIS_HOST: redis run: dotnet test --no-restore --verbosity normal From 45e3395f77d1399e41246ec24df6092c7c0b6cd0 Mon Sep 17 00:00:00 2001 From: Dan Balasescu Date: Mon, 15 Apr 2024 19:58:14 +0900 Subject: [PATCH 11/11] Fix inspections --- osu.Server.QueueProcessor/ConnectionMultiplexerExtensions.cs | 4 ++-- osu.Server.QueueProcessor/QueueProcessor.cs | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/osu.Server.QueueProcessor/ConnectionMultiplexerExtensions.cs b/osu.Server.QueueProcessor/ConnectionMultiplexerExtensions.cs index 6a94597..a859fda 100644 --- a/osu.Server.QueueProcessor/ConnectionMultiplexerExtensions.cs +++ b/osu.Server.QueueProcessor/ConnectionMultiplexerExtensions.cs @@ -34,7 +34,7 @@ public static void ClearCurrentSchema(this ConnectionMultiplexer connection) /// /// Get all active schemas (including past or future). /// - public static string[] GetActiveSchemas(this ConnectionMultiplexer connection) + public static string?[] GetActiveSchemas(this ConnectionMultiplexer connection) { return connection.GetDatabase().SetMembers(allActiveSchemasKey).ToStringArray(); } @@ -44,7 +44,7 @@ public static string[] GetActiveSchemas(this ConnectionMultiplexer connection) /// public static string GetCurrentSchema(this ConnectionMultiplexer connection) { - return connection.GetDatabase().StringGet(mainSchemaKey).ToString() ?? string.Empty; + return connection.GetDatabase().StringGet(mainSchemaKey).ToString(); } /// diff --git a/osu.Server.QueueProcessor/QueueProcessor.cs b/osu.Server.QueueProcessor/QueueProcessor.cs index e44d058..8fd2df0 100644 --- a/osu.Server.QueueProcessor/QueueProcessor.cs +++ b/osu.Server.QueueProcessor/QueueProcessor.cs @@ -107,6 +107,7 @@ public void Run(CancellationToken cancellation = default) var redisItems = database.ListRightPop(QueueName, config.BatchSize); + // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract (https://github.com/StackExchange/StackExchange.Redis/issues/2697) // queue doesn't exist. if (redisItems == null) {