Execute tests in sequence

By default, the tests are executed in parallel. While this doesn't seem
to be causing any type of issues, it's quite counter intuitive for the
type of tests implemented in this project. For example, some of the
tests invoke GC and expect side effects from GC.

This commit also adds diagnostic messages to help to debug hanging
tests. I implemented this after encountering a hanging test in one of
Github PRs.

Signed-off-by: Kristupas Antanavičius <kristupas.antanavicius@nordsec.com>

CONTRIBUTING.md

@@ -47,6 +47,15 @@ Running tests in Docker containers is easier, because manual `dotnet` installati
 ./docker_test_bindings.sh
 ```
 
+# Hanging tests
+
+To print test case names when a test starts, update `diagnosticMessages` to `true` in
+[xunit.runner.json](dotnet-tests/UniffiCS.BindingTests/xunit.runner.json). Xunit, the test framework
+used by this project, does not provide functionality to print test case names when a test starts,
+making it difficult to debug hanging tests without using a debugger.
+[CustomTestFramework.cs](dotnet-tests/UniffiCS.BindingTests/CustomTestFramework.cs) plugs into Xunit
+to print test case names when the tests start.
+
 # Directory structure
 
 | Directory                                | Description                                      |

dotnet-tests/UniffiCS.BindingTests/CustomTestFramework.cs

@@ -0,0 +1,121 @@
+// Print when tests start/end.
+// Xunit is quite inconvenient for hanging tests.. It doesn't come with an option to print a message before test starts.
+// Should we switch to another test framework??
+//
+// This code is ripped from:
+// https://andrewlock.net/tracking-down-a-hanging-xunit-test-in-ci-building-a-custom-test-framework/
+
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using System.Threading.Tasks;
+using System.Threading;
+using System;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+[assembly: Xunit.TestFramework("CustomTestFramework", "UniffiCS.BindingTests")]
+
+public class CustomTestFramework : XunitTestFramework
+{
+    public CustomTestFramework(IMessageSink messageSink)
+        : base(messageSink)
+    {
+        messageSink.OnMessage(new DiagnosticMessage("Using CustomTestFramework"));
+    }
+
+    protected override ITestFrameworkExecutor CreateExecutor(AssemblyName assemblyName)
+        => new CustomExecutor(assemblyName, SourceInformationProvider, DiagnosticMessageSink);
+
+    private class CustomExecutor : XunitTestFrameworkExecutor
+    {
+        public CustomExecutor(AssemblyName assemblyName, ISourceInformationProvider sourceInformationProvider, IMessageSink diagnosticMessageSink)
+            : base(assemblyName, sourceInformationProvider, diagnosticMessageSink)
+        {
+        }
+
+        protected override async void RunTestCases(IEnumerable<IXunitTestCase> testCases, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions)
+        {
+            using var assemblyRunner = new CustomAssemblyRunner(TestAssembly, testCases, DiagnosticMessageSink, executionMessageSink, executionOptions);
+            await assemblyRunner.RunAsync();
+        }
+    }
+
+    private class CustomAssemblyRunner : XunitTestAssemblyRunner
+    {
+        public CustomAssemblyRunner(ITestAssembly testAssembly, IEnumerable<IXunitTestCase> testCases, IMessageSink diagnosticMessageSink, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions)
+            : base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions)
+        {
+        }
+
+        protected override Task<RunSummary> RunTestCollectionAsync(IMessageBus messageBus, ITestCollection testCollection, IEnumerable<IXunitTestCase> testCases, CancellationTokenSource cancellationTokenSource)
+            => new CustomTestCollectionRunner(testCollection, testCases, DiagnosticMessageSink, messageBus, TestCaseOrderer, new ExceptionAggregator(Aggregator), cancellationTokenSource).RunAsync();
+    }
+
+    private class CustomTestCollectionRunner : XunitTestCollectionRunner
+    {
+        public CustomTestCollectionRunner(ITestCollection testCollection, IEnumerable<IXunitTestCase> testCases, IMessageSink diagnosticMessageSink, IMessageBus messageBus, ITestCaseOrderer testCaseOrderer, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource)
+            : base(testCollection, testCases, diagnosticMessageSink, messageBus, testCaseOrderer, aggregator, cancellationTokenSource)
+        {
+        }
+
+        protected override Task<RunSummary> RunTestClassAsync(ITestClass testClass, IReflectionTypeInfo @class, IEnumerable<IXunitTestCase> testCases)
+            => new CustomTestClassRunner(testClass, @class, testCases, DiagnosticMessageSink, MessageBus, TestCaseOrderer, new ExceptionAggregator(Aggregator), CancellationTokenSource, CollectionFixtureMappings)
+                .RunAsync();
+    }
+
+    private class CustomTestClassRunner : XunitTestClassRunner
+    {
+        public CustomTestClassRunner(ITestClass testClass, IReflectionTypeInfo @class, IEnumerable<IXunitTestCase> testCases, IMessageSink diagnosticMessageSink, IMessageBus messageBus, ITestCaseOrderer testCaseOrderer, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource, IDictionary<Type, object> collectionFixtureMappings)
+            : base(testClass, @class, testCases, diagnosticMessageSink, messageBus, testCaseOrderer, aggregator, cancellationTokenSource, collectionFixtureMappings)
+        {
+        }
+
+        protected override Task<RunSummary> RunTestMethodAsync(ITestMethod testMethod, IReflectionMethodInfo method, IEnumerable<IXunitTestCase> testCases, object[] constructorArguments)
+            => new CustomTestMethodRunner(testMethod, this.Class, method, testCases, this.DiagnosticMessageSink, this.MessageBus, new ExceptionAggregator(this.Aggregator), this.CancellationTokenSource, constructorArguments)
+                .RunAsync();
+    }
+
+    private class CustomTestMethodRunner : XunitTestMethodRunner
+    {
+        private readonly IMessageSink _diagnosticMessageSink;
+
+        public CustomTestMethodRunner(ITestMethod testMethod, IReflectionTypeInfo @class, IReflectionMethodInfo method, IEnumerable<IXunitTestCase> testCases, IMessageSink diagnosticMessageSink, IMessageBus messageBus, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource, object[] constructorArguments)
+            : base(testMethod, @class, method, testCases, diagnosticMessageSink, messageBus, aggregator, cancellationTokenSource, constructorArguments)
+        {
+            _diagnosticMessageSink = diagnosticMessageSink;
+        }
+
+        protected override async Task<RunSummary> RunTestCaseAsync(IXunitTestCase testCase)
+        {
+            var parameters = string.Empty;
+
+            if (testCase.TestMethodArguments != null)
+            {
+                parameters = string.Join(", ", testCase.TestMethodArguments.Select(a => a?.ToString() ?? "null"));
+            }
+
+            var test = $"{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}({parameters})";
+
+            _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"STARTED: {test}"));
+
+            try
+            {
+                var result = await base.RunTestCaseAsync(testCase);
+
+                var status = result.Failed > 0
+                    ? "FAILURE"
+                    : (result.Skipped > 0 ? "SKIPPED" : "SUCCESS");
+
+                _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"{status}: {test} ({result.Time}s)"));
+
+                return result;
+            }
+            catch (Exception ex)
+            {
+                _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"ERROR: {test} ({ex.Message})"));
+                throw;
+            }
+        }
+    }
+}

dotnet-tests/UniffiCS.BindingTests/UniffiCS.BindingTests.csproj

@@ -23,4 +23,10 @@
   <ItemGroup>
     <ProjectReference Include="..\UniffiCS\UniffiCS.csproj" />
   </ItemGroup>
+
+  <ItemGroup>
+    <None Update="xunit.runner.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
 </Project>

dotnet-tests/UniffiCS.BindingTests/xunit.runner.json

@@ -0,0 +1,5 @@
+{
+  "diagnosticMessages": false,
+  "parallelizeAssembly": false,
+  "parallelizeTestCollections": false
+}