From 816e7846d1d4c59273865814ae38ca6c1117aa1c Mon Sep 17 00:00:00 2001 From: Anthony Lloyd Date: Tue, 20 Aug 2024 10:25:06 +0100 Subject: [PATCH 01/15] update to .NET 8, update dependencies --- CsCheck/CsCheck.csproj | 10 +++++----- Tests/PCGTests.cs | 4 ++-- Tests/Tests.csproj | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/CsCheck/CsCheck.csproj b/CsCheck/CsCheck.csproj index c407e38..495e5ec 100644 --- a/CsCheck/CsCheck.csproj +++ b/CsCheck/CsCheck.csproj @@ -22,11 +22,11 @@ CsCheck also makes concurrency, performance and regression testing simple and fa http://github.com/AnthonyLloyd/CsCheck CsCheck.png quickcheck;random;model-based;metamorphic;concurrency;performance;causal-profiling;regression;testing - 3.2.2 + 4.0.0 -Added option to Dbg.Time stats for completed and running. +Update to .NET 8 - net6.0 + net8.0 preview true 9999 @@ -41,13 +41,13 @@ Added option to Dbg.Time stats for completed and running. true true snupkg - CS1591 + CS1591,MA0143 README.md All - + diff --git a/Tests/PCGTests.cs b/Tests/PCGTests.cs index 32cc830..cb9ca60 100644 --- a/Tests/PCGTests.cs +++ b/Tests/PCGTests.cs @@ -198,8 +198,8 @@ public void PCG_ToString_Roundtrip() [Fact] public void Double_Exp_Bug() { - var root2 = -6.3E-102; - var root3 = 6.6854976605820742; + const double root2 = -6.3E-102; + const double root3 = 6.6854976605820742; Gen.Double[root2, root3 * 2.0].Sample(_ => true); } } \ No newline at end of file diff --git a/Tests/Tests.csproj b/Tests/Tests.csproj index 06152de..9ed1447 100644 --- a/Tests/Tests.csproj +++ b/Tests/Tests.csproj @@ -14,11 +14,11 @@ xUnit1004 - - - - - + + + + + From 3042230911b5373e285f168eb26d463797afe14c Mon Sep 17 00:00:00 2001 From: Anthony Lloyd Date: Wed, 21 Aug 2024 09:00:42 +0100 Subject: [PATCH 02/15] refactor GenOperation --- CsCheck/Check.cs | 13 +++++++----- CsCheck/Gen.cs | 40 +++++++++++++++++++---------------- README.md | 12 +++++------ Tests/CheckTests.cs | 22 ++++--------------- Tests/IMToolsTests.cs | 8 +++---- Tests/SieveLruCacheTests.cs | 8 +++---- Tests/SlimCollectionsTests.cs | 25 +++++++++------------- 7 files changed, 56 insertions(+), 72 deletions(-) diff --git a/CsCheck/Check.cs b/CsCheck/Check.cs index 5c9df9d..6ca86d0 100644 --- a/CsCheck/Check.cs +++ b/CsCheck/Check.cs @@ -1277,9 +1277,9 @@ public static Task SampleAsync(this Gen<(T1, T2, Action? writeLine = null, string? seed = null, long iter = -1, int time = -1, int threads = -1, Func<(T1, T2, T3, T4, T5, T6, T7, T8), string>? print = null) => SampleAsync(gen, t => predicate(t.Item1, t.Item2, t.Item3, t.Item4, t.Item5, t.Item6, t.Item7, t.Item8), writeLine, seed, iter, time, threads, print); - sealed class ModelBasedData(Actual actualState, Model modelState, uint stream, ulong seed, (string, Action)[] operations) + sealed class ModelBasedData(Actual actualState, Model modelState, uint stream, ulong seed, (string, Action, Action)[] operations) { - public Actual ActualState = actualState; public Model ModelState = modelState; public uint Stream = stream; public ulong Seed = seed; public (string, Action)[] Operations = operations; public Exception? Exception; + public Actual ActualState = actualState; public Model ModelState = modelState; public uint Stream = stream; public ulong Seed = seed; public (string, Action, Action)[] Operations = operations; public Exception? Exception; } sealed class GenInitial(Gen<(Actual, Model)> initial) : Gen<(Actual Actual, Model Model, uint Stream, ulong Seed)> @@ -1316,12 +1316,12 @@ public static void SampleModelBased(this Gen<(Actual, Model)> ini printActual ??= Print; printModel ??= Print; - var opNameActions = new Gen<(string, Action)>[operations.Length]; + var opNameActions = new Gen<(string, Action, Action)>[operations.Length]; for (int i = 0; i < operations.Length; i++) { var op = operations[i]; var opName = "Op" + i; - opNameActions[i] = op.AddOpNumber ? op.Select(t => (opName + t.Item1, t.Item2)) : op; + opNameActions[i] = op.AddOpNumber ? op.Select(t => (opName + t.Item1, t.Item2, t.Item3)) : op; } new GenInitial(initial) @@ -1331,7 +1331,10 @@ public static void SampleModelBased(this Gen<(Actual, Model)> ini try { foreach (var operation in d.Operations) - operation.Item2(d.ActualState, d.ModelState); + { + operation.Item2(d.ActualState); + operation.Item3(d.ModelState); + } return equal(d.ActualState, d.ModelState); } catch (Exception e) diff --git a/CsCheck/Gen.cs b/CsCheck/Gen.cs index 703edb4..16c062f 100644 --- a/CsCheck/Gen.cs +++ b/CsCheck/Gen.cs @@ -87,10 +87,10 @@ public abstract class Gen : IGen public GenOperation Operation(Func name, Func async) => GenOperation.Create(this, name, (S s, T t) => async(s, t).GetAwaiter().GetResult()); public GenOperation Operation(Action action) => GenOperation.Create(this, action); public GenOperation Operation(Func async) => GenOperation.Create(this, (S s, T t) => async(s, t).GetAwaiter().GetResult()); - public GenOperation Operation(Func name, Action action) => GenOperation.Create(this, name, action); - public GenOperation Operation(Action action) => GenOperation.Create(this, action); - public GenMetamorphic Metamorphic(Func name, Action action1, Action action2) => GenOperation.Create(this, name, action1, action2); - public GenMetamorphic Metamorphic(Action action1, Action action2) => GenOperation.Create(this, Check.Print, action1, action2); + public GenOperation Operation(Func name, Action actual, Action model) => GenOperation.Create(this, name, actual, model); + public GenOperation Operation(Action actual, Action model) => GenOperation.Create(this, actual, model); + public GenMetamorphic Metamorphic(Func name, Action action1, Action action2) => GenMetamorphic.Create(this, name, action1, action2); + public GenMetamorphic Metamorphic(Action action1, Action action2) => GenMetamorphic.Create(this, Check.Print, action1, action2); /// Generator for an array of public GenArray Array => new(this); @@ -1456,8 +1456,8 @@ sealed class GenNull(Gen gen, uint nullLimit) : Gen where T : class public static GenOperation Operation(string name, Func async) => GenOperation.Create(name, (T t) => async(t).GetAwaiter().GetResult()); public static GenOperation Operation(Action action) => GenOperation.Create(action); public static GenOperation Operation(Func async) => GenOperation.Create((T t) => async(t).GetAwaiter().GetResult()); - public static GenOperation Operation(string name, Action action) => GenOperation.Create(name, action); - public static GenOperation Operation(Action action) => GenOperation.Create(action); + public static GenOperation Operation(string name, Action actual, Action model) => GenOperation.Create(name, actual, model); + public static GenOperation Operation(Action actual, Action model) => GenOperation.Create(actual, model); /// Generator for bool. public static readonly GenBool Bool = new(); @@ -2851,16 +2851,16 @@ internal GenOperation(Gen<(string, Action)> gen, bool addOpNumber) public override (string, Action) Generate(PCG pcg, Size? min, out Size size) => gen.Generate(pcg, min, out size); } -public sealed class GenOperation : Gen<(string, Action)> +public sealed class GenOperation : Gen<(string, Action, Action)> { + readonly Gen<(string, Action, Action)> gen; public bool AddOpNumber; - readonly Gen<(string, Action)> gen; - internal GenOperation(Gen<(string, Action)> gen, bool addOpNumber) + internal GenOperation(Gen<(string, Action, Action)> gen, bool addOpNumber) { this.gen = gen; AddOpNumber = addOpNumber; } - public override (string, Action) Generate(PCG pcg, Size? min, out Size size) => gen.Generate(pcg, min, out size); + public override (string, Action, Action) Generate(PCG pcg, Size? min, out Size size) => gen.Generate(pcg, min, out size); } public sealed class GenMetamorphic : Gen<(string, Action, Action)> @@ -2876,18 +2876,22 @@ public static GenOperation Create(Gen gen, Action action) => new(gen.Select)>(t => (" " + Check.Print(t), s => action(s, t))), true); public static GenOperation Create(Gen gen, Func name, Action action) => new(gen.Select)>(t => (name(t), s => action(s, t))), false); - public static GenOperation Create(Gen gen, Action action) => - new(gen.Select)>(t => (" " + Check.Print(t), (s1, s2) => action(s1, s2, t))), true); - public static GenOperation Create(Gen gen, Func name, Action action) => - new(gen.Select)>(t => (name(t), (s1, s2) => action(s1, s2, t))), false); + public static GenOperation Create(Gen gen, Action actual, Action model) => + new(gen.Select, Action)>(t => (" " + Check.Print(t), a => actual(a, t), m => model(m, t))), true); + public static GenOperation Create(Gen gen, Func name, Action actual, Action model) => + new(gen.Select, Action)>(t => (name(t), a => actual(a, t), m => model(m, t))), false); + public static GenOperation Create(Action actual, Action model) + => new(Gen.Const(("", actual, model)), true); + public static GenOperation Create(string name, Action actual, Action model) + => new(Gen.Const((name, actual, model)), false); public static GenOperation Create(Action action) => new(Gen.Const(("", action)), true); public static GenOperation Create(string name, Action action) => new(Gen.Const((name, action)), false); - public static GenOperation Create(Action action) - => new(Gen.Const(("", action)), true); - public static GenOperation Create(string name, Action action) - => new(Gen.Const((name, action)), false); +} + +public static class GenMetamorphic +{ public static GenMetamorphic Create(Gen gen, Func name, Action action1, Action action2) => new(gen.Select, Action)>(t => (name(t), s => action1(s, t), s => action2(s, t)))); } \ No newline at end of file diff --git a/README.md b/README.md index b87ec5d..ac5138e 100644 --- a/README.md +++ b/README.md @@ -229,15 +229,15 @@ SampleModelBased generates an initial actual and model and then applies a random ### SetSlim Add ```csharp [Fact] -public void SetSlim_ModelBased() +public void +SetSlim_ModelBased() { Gen.Int.Array.Select(a => (new SetSlim(a), new HashSet(a))) .SampleModelBased( - Gen.Int.Operation, HashSet>((ls, l, i) => - { - ls.Add(i); - l.Add(i); - }) + Gen.Int.Operation, HashSet>( + (ss, i) => ss.Add(i), + (hs, i) => hs.Add(i) + ) // ... other operations ); } diff --git a/Tests/CheckTests.cs b/Tests/CheckTests.cs index 5b77a6a..1bb4652 100644 --- a/Tests/CheckTests.cs +++ b/Tests/CheckTests.cs @@ -221,14 +221,10 @@ public void SampleModelBased_ConcurrentBag() { Gen.Int[0, 5].List.Select(l => (new ConcurrentBag(l), l)) .SampleModelBased( - Gen.Int.Operation, List>((bag, list, i) => - { - bag.Add(i); - list.Add(i); - }), - Gen.Operation, List>((bag, list) => Assert.Equal(bag.TryTake(out var i), list.Remove(i))) - , threads: 1 - ); + Gen.Int.Operation, List>((bag, i) => bag.Add(i), (list, i) => list.Add(i)), + Gen.Operation, List>(bag => bag.TryTake(out _), list => { if (list.Count > 0) list.RemoveAt(0); }), + equal: (bag, list) => bag.Count == list.Count + , threads: 1); } [Fact] @@ -241,16 +237,6 @@ public void SampleConcurrent_ConcurrentBag() ); } - //[Fact] - //public void SampleConcurrent_List() - //{ - // Gen.Int.List - // .SampleConcurrent( - // Gen.Int.Operation>(i => $"Add({i})", (list, i) => list.Add(i)) - // //Gen.Const<(string, Action>)>(("Remove()", list => list.RemoveAt(0))) - // ); - //} - [Fact] public void SampleConcurrent_ConcurrentDictionary() { diff --git a/Tests/IMToolsTests.cs b/Tests/IMToolsTests.cs index 30b7913..75828ff 100644 --- a/Tests/IMToolsTests.cs +++ b/Tests/IMToolsTests.cs @@ -107,11 +107,9 @@ public void AddOrUpdate_ModelBased() return (new ImHolder> { Im = d }, m); }) .SampleModelBased( - Gen.Int[0, upperBound].Select(Gen.Int).Operation>, Dictionary>((h, d, kv) => - { - h.Im = h.Im.AddOrUpdate(kv.Item1, kv.Item2); - d[kv.Item1] = kv.Item2; - }) + Gen.Int[0, upperBound].Select(Gen.Int).Operation>, Dictionary>( + (h, kv) => { h.Im = h.Im.AddOrUpdate(kv.Item1, kv.Item2); }, + (d, kv) => { d[kv.Item1] = kv.Item2; }) , equal: (h, d) => { var he = h.Im.Enumerate().Select(kv => (kv.Key, kv.Value)).ToList(); diff --git a/Tests/SieveLruCacheTests.cs b/Tests/SieveLruCacheTests.cs index 09eb121..498d7a2 100644 --- a/Tests/SieveLruCacheTests.cs +++ b/Tests/SieveLruCacheTests.cs @@ -89,11 +89,9 @@ public void SampleModelBased() { Check.SampleModelBased( Gen.Const(() => (new SieveLruCache(4), new SieveModel(4))), - Gen.Int[1, 5].Operation, SieveModel>((a, m, i) => - { - a.GetOrAdd(i, i => i); - m.GetOrAdd(i, i => i); - }), + Gen.Int[1, 5].Operation, SieveModel>( + (a, i) => a.GetOrAdd(i, i => i), + (m, i) => m.GetOrAdd(i, i => i)), equal: (a, m) => Check.Equal(a.Keys.ToHashSet(), m.Keys.ToHashSet()), printActual: a => Check.Print(a.Keys), printModel: m => Check.Print(m.Keys) diff --git a/Tests/SlimCollectionsTests.cs b/Tests/SlimCollectionsTests.cs index 386818b..c38157f 100644 --- a/Tests/SlimCollectionsTests.cs +++ b/Tests/SlimCollectionsTests.cs @@ -15,11 +15,9 @@ public void ListSlim_ModelBased() { Gen.Int.Array.Select(a => (new ListSlim(a), new List(a))) .SampleModelBased( - Gen.Int.Operation, List>((ls, l, i) => - { - ls.Add(i); - l.Add(i); - }) + Gen.Int.Operation, List>( + (ls, i) => ls.Add(i), + (l, i) => l.Add(i)) ); } @@ -40,11 +38,10 @@ public void SetSlim_ModelBased() { Gen.Int.Array.Select(a => (new SetSlim(a), new HashSet(a))) .SampleModelBased( - Gen.Int.Operation, HashSet>((ls, l, i) => - { - ls.Add(i); - l.Add(i); - }) + Gen.Int.Operation, HashSet>( + (ss, i) => ss.Add(i), + (hs, i) => hs.Add(i) + ) ); } @@ -100,11 +97,9 @@ public void MapSlim_ModelBased() Gen.Dictionary(Gen.Int, Gen.Byte) .Select(d => (new MapSlim(d), new Dictionary(d))) .SampleModelBased( - Gen.Select(Gen.Int[0, 100], Gen.Byte).Operation, Dictionary>((m, d, t) => - { - m[t.Item1] = t.Item2; - d[t.Item1] = t.Item2; - }) + Gen.Select(Gen.Int[0, 100], Gen.Byte).Operation, Dictionary>( + (m, t) => m[t.Item1] = t.Item2, + (d, t) => d[t.Item1] = t.Item2) ); } From c44a827f3d57ed3d26a924c1d57e0ab91c4fbc0c Mon Sep 17 00:00:00 2001 From: Anthony Lloyd Date: Wed, 21 Aug 2024 09:45:54 +0100 Subject: [PATCH 03/15] make maxParallelOperations a parameter, set min number of parallel to 2 --- CsCheck/Check.cs | 49 +++++++++++++++++++++++++----------------------- CsCheck/Utils.cs | 4 +--- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/CsCheck/Check.cs b/CsCheck/Check.cs index 6ca86d0..03ab43e 100644 --- a/CsCheck/Check.cs +++ b/CsCheck/Check.cs @@ -1575,8 +1575,6 @@ sealed class ConcurrentData(T state, uint stream, ulong seed, (string, Action public T State = state; public uint Stream = stream; public ulong Seed = seed; public (string, Action)[] Operations = operations; public int Threads = threads; public int[]? ThreadIds; public Exception? Exception; } - internal const int MAX_CONCURRENT_OPERATIONS = 10; - sealed class GenConcurrent(Gen initial) : Gen<(T Value, uint Stream, ulong Seed)> { public override (T, uint, ulong) Generate(PCG pcg, Size? min, out Size size) @@ -1595,6 +1593,7 @@ public override (T, uint, ulong) Generate(PCG pcg, Size? min, out Size size) /// The operation generators that can act on the state concurrently. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. + /// The maximum number of operations to run in parallel. /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). @@ -1602,7 +1601,7 @@ public override (T, uint, ulong) Generate(PCG pcg, Size? min, out Size size) /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. public static void SampleConcurrent(this Gen initial, GenOperation[] operations, Func? equal = null, string? seed = null, - long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) + int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) { equal ??= Equal; seed ??= Seed; @@ -1630,7 +1629,7 @@ public static void SampleConcurrent(this Gen initial, GenOperation[] op bool firstIteration = true; new GenConcurrent(initial) - .Select(Gen.OneOf(opNameActions).Array[1, MAX_CONCURRENT_OPERATIONS] + .Select(Gen.OneOf(opNameActions).Array[2, maxParallelOperations] .SelectMany(ops => Gen.Int[1, Math.Min(threads, ops.Length)].Select(i => (ops, i))), (a, b) => new ConcurrentData(a.Value, a.Stream, a.Seed, b.ops, b.i) ) @@ -1710,6 +1709,7 @@ public static void SampleConcurrent(this Gen initial, GenOperation[] op /// An operation generator that can act on the state concurrently. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. + /// The maximum number of operations to run in parallel. /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). @@ -1718,8 +1718,8 @@ public static void SampleConcurrent(this Gen initial, GenOperation[] op /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void SampleConcurrent(this Gen initial, GenOperation operation, Func? equal = null, string? seed = null, - long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) - => SampleConcurrent(initial, [operation], equal, seed, iter, time, threads, print, replay, writeLine); + int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) + => SampleConcurrent(initial, [operation], equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); /// Sample model-based operations on a random initial state concurrently. /// The result is compared against the result of the possible sequential permutations. @@ -1730,6 +1730,7 @@ public static void SampleConcurrent(this Gen initial, GenOperation oper /// An operation generator that can act on the state concurrently. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. + /// The maximum number of operations to run in parallel. /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). @@ -1737,9 +1738,9 @@ public static void SampleConcurrent(this Gen initial, GenOperation oper /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SampleConcurrent(this Gen initial, GenOperation operation1, GenOperation operation2, Func? equal = null, - string? seed = null, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) - => SampleConcurrent(initial, [operation1, operation2], equal, seed, iter, time, threads, print, replay, writeLine); + public static void SampleConcurrent(this Gen initial, GenOperation operation1, GenOperation operation2, Func? equal = null, string? seed = null, + int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) + => SampleConcurrent(initial, [operation1, operation2], equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); /// Sample model-based operations on a random initial state concurrently. /// The result is compared against the result of the possible sequential permutations. @@ -1751,6 +1752,7 @@ public static void SampleConcurrent(this Gen initial, GenOperation oper /// An operation generator that can act on the state concurrently. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. + /// The maximum number of operations to run in parallel. /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). @@ -1758,9 +1760,9 @@ public static void SampleConcurrent(this Gen initial, GenOperation oper /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SampleConcurrent(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, Func? equal = null, - string? seed = null, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) - => SampleConcurrent(initial, [operation1, operation2, operation3], equal, seed, iter, time, threads, print, replay, writeLine); + public static void SampleConcurrent(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, Func? equal = null, string? seed = null, + int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) + => SampleConcurrent(initial, [operation1, operation2, operation3], equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); /// Sample model-based operations on a random initial state concurrently. /// The result is compared against the result of the possible sequential permutations. @@ -1773,6 +1775,7 @@ public static void SampleConcurrent(this Gen initial, GenOperation oper /// An operation generator that can act on the state concurrently. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. + /// The maximum number of operations to run in parallel. /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). @@ -1780,10 +1783,9 @@ public static void SampleConcurrent(this Gen initial, GenOperation oper /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SampleConcurrent(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, - Func? equal = null, string? seed = null, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, - Action? writeLine = null) - => SampleConcurrent(initial, [operation1, operation2, operation3, operation4], equal, seed, iter, time, threads, print, replay, writeLine); + public static void SampleConcurrent(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, Func? equal = null, string? seed = null, + int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) + => SampleConcurrent(initial, [operation1, operation2, operation3, operation4], equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); /// Sample model-based operations on a random initial state concurrently. /// The result is compared against the result of the possible sequential permutations. @@ -1797,6 +1799,7 @@ public static void SampleConcurrent(this Gen initial, GenOperation oper /// An operation generator that can act on the state concurrently. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. + /// The maximum number of operations to run in parallel. /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). @@ -1804,11 +1807,11 @@ public static void SampleConcurrent(this Gen initial, GenOperation oper /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SampleConcurrent(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, - GenOperation operation5, Func? equal = null, string? seed = null, long iter = -1, int time = -1, int threads = -1, Func? print = null, + public static void SampleConcurrent(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, GenOperation operation5, Func? equal = null, string? seed = null, + int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) => SampleConcurrent(initial, [operation1, operation2, operation3, operation4, operation5], - equal, seed, iter, time, threads, print, replay, writeLine); + equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); /// Sample model-based operations on a random initial state concurrently. /// The result is compared against the result of the possible sequential permutations. @@ -1823,6 +1826,7 @@ public static void SampleConcurrent(this Gen initial, GenOperation oper /// An operation generator that can act on the state concurrently. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. + /// The maximum number of operations to run in parallel. /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). @@ -1830,11 +1834,10 @@ public static void SampleConcurrent(this Gen initial, GenOperation oper /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SampleConcurrent(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, - GenOperation operation5, GenOperation operation6, Func? equal = null, string? seed = null, long iter = -1, int time = -1, int threads = -1, - Func? print = null, int replay = -1, Action? writeLine = null) + public static void SampleConcurrent(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, GenOperation operation5, GenOperation operation6, Func? equal = null, string? seed = null, + int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) => SampleConcurrent(initial, [operation1, operation2, operation3, operation4, operation5, operation6], - equal, seed, iter, time, threads, print, replay, writeLine); + equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); /// Assert actual is in line with expected using a chi-squared test to sigma. /// The expected bin counts. diff --git a/CsCheck/Utils.cs b/CsCheck/Utils.cs index d0b5822..2295328 100644 --- a/CsCheck/Utils.cs +++ b/CsCheck/Utils.cs @@ -475,10 +475,8 @@ public static bool ModelEqual(T actual, M model) return actual.Equals(model); } - static readonly int[] DummyArray = new int[MAX_CONCURRENT_OPERATIONS]; internal static void Run(T concurrentState, (string, Action)[] operations, int threads, int[]? threadIds = null) { - threadIds ??= DummyArray; Exception? exception = null; var opId = -1; var runners = new Thread[threads]; @@ -489,7 +487,7 @@ internal static void Run(T concurrentState, (string, Action)[] operations, int i, tid = (int)threadId!; while ((i = Interlocked.Increment(ref opId)) < operations.Length) { - threadIds[i] = tid; + if (threadIds is not null) threadIds[i] = tid; try { operations[i].Item2(concurrentState); } catch (Exception e) { From 2696af008ee3b95e375970362ce8dd4c97ab396a Mon Sep 17 00:00:00 2001 From: Anthony Lloyd Date: Wed, 21 Aug 2024 10:30:31 +0100 Subject: [PATCH 04/15] rename SampleConcurrent to SampleParallel --- Comparison.md | 2 +- CsCheck/Check.cs | 108 +++++++++++++++++----------------- CsCheck/CsCheck.csproj | 6 +- CsCheck/Utils.cs | 8 +-- README.md | 18 +++--- Tests/CacheTests.cs | 4 +- Tests/CheckTests.cs | 12 ++-- Tests/SieveLruCacheTests.cs | 4 +- Tests/SlimCollectionsTests.cs | 12 ++-- Why.md | 2 +- 10 files changed, 88 insertions(+), 88 deletions(-) diff --git a/Comparison.md b/Comparison.md index f1c2dc1..5afdf3f 100644 --- a/Comparison.md +++ b/Comparison.md @@ -87,7 +87,7 @@ Size is also a better representation of comparison especially for collections or There are examples where increasing on one axis while decreasing on others can lead to smaller cases e.g. if Version fails for `2 * ma + mi + bu ≥ 255 * 2` CsCheck will be able to shrink to `255.0.0` but [Hedgehog](https://github.com/hedgehogqa) won't. -For concurrency testing random shrinkers also has an advantage. Concurrency tests may not fail deterministically. +For parallel testing random shrinkers also has an advantage. Parallel tests may not fail deterministically. This is a real problem for path explorer shrinkers. The only solution is to repeat each test multiple times (10 for QuickCheck) since they need to follow defined paths. For a random shrinker you can just continue testing different random cases until one fails and limit the size to that each time. diff --git a/CsCheck/Check.cs b/CsCheck/Check.cs index 03ab43e..fff4df9 100644 --- a/CsCheck/Check.cs +++ b/CsCheck/Check.cs @@ -25,7 +25,7 @@ public static partial class Check public static long Iter = ParseEnvironmentVariableToLong("CsCheck_Iter", 100); /// The number of seconds to run the sample. public static int Time = ParseEnvironmentVariableToInt("CsCheck_Time" , -1); - /// The number of times to retry the seed to reproduce a SampleConcurrent fail (default 100). + /// The number of times to retry the seed to reproduce a SampleParallel fail (default 100). public static int Replay = ParseEnvironmentVariableToInt("CsCheck_Replay", 100); /// The number of threads to run the sample on (default number logical CPUs). public static int Threads = ParseEnvironmentVariableToInt("CsCheck_Threads", Environment.ProcessorCount); @@ -1570,12 +1570,12 @@ public static void SampleMetamorphic(this Gen initial, GenMetamorphic o }); } - sealed class ConcurrentData(T state, uint stream, ulong seed, (string, Action)[] operations, int threads) + sealed class SampleParallelData(T state, uint stream, ulong seed, (string, Action)[] operations, int threads) { public T State = state; public uint Stream = stream; public ulong Seed = seed; public (string, Action)[] Operations = operations; public int Threads = threads; public int[]? ThreadIds; public Exception? Exception; } - sealed class GenConcurrent(Gen initial) : Gen<(T Value, uint Stream, ulong Seed)> + sealed class GenSampleParallel(Gen initial) : Gen<(T Value, uint Stream, ulong Seed)> { public override (T, uint, ulong) Generate(PCG pcg, Size? min, out Size size) { @@ -1585,12 +1585,12 @@ public override (T, uint, ulong) Generate(PCG pcg, Size? min, out Size size) } } - /// Sample model-based operations on a random initial state concurrently. + /// Sample operations on a random initial state in parallel. /// The result is compared against the result of the possible sequential permutations. - /// At least one of these permutations result must be equal for the concurrency to have been linearized successfully. + /// At least one of these permutations result must be equal for the parallel execution to have been linearized successfully. /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. /// The initial state generator. - /// The operation generators that can act on the state concurrently. + /// The operation generators that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. /// The maximum number of operations to run in parallel. @@ -1600,7 +1600,7 @@ public override (T, uint, ulong) Generate(PCG pcg, Size? min, out Size size) /// A function to convert the state to a string for error reporting (default Check.Print). /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. - public static void SampleConcurrent(this Gen initial, GenOperation[] operations, Func? equal = null, string? seed = null, + public static void SampleParallel(this Gen initial, GenOperation[] operations, Func? equal = null, string? seed = null, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) { equal ??= Equal; @@ -1628,10 +1628,10 @@ public static void SampleConcurrent(this Gen initial, GenOperation[] op bool firstIteration = true; - new GenConcurrent(initial) + new GenSampleParallel(initial) .Select(Gen.OneOf(opNameActions).Array[2, maxParallelOperations] .SelectMany(ops => Gen.Int[1, Math.Min(threads, ops.Length)].Select(i => (ops, i))), (a, b) => - new ConcurrentData(a.Value, a.Stream, a.Seed, b.ops, b.i) + new SampleParallelData(a.Value, a.Stream, a.Seed, b.ops, b.i) ) .Sample(cd => { @@ -1701,12 +1701,12 @@ public static void SampleConcurrent(this Gen initial, GenOperation[] op }); } - /// Sample model-based operations on a random initial state concurrently. + /// Sample operations on a random initial state in parallel. /// The result is compared against the result of the possible sequential permutations. - /// At least one of these permutations result must be equal for the concurrency to have been linearized successfully. + /// At least one of these permutations result must be equal for the parallel execution to have been linearized successfully. /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. /// The initial state generator. - /// An operation generator that can act on the state concurrently. + /// An operation generator that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. /// The maximum number of operations to run in parallel. @@ -1717,17 +1717,17 @@ public static void SampleConcurrent(this Gen initial, GenOperation[] op /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SampleConcurrent(this Gen initial, GenOperation operation, Func? equal = null, string? seed = null, + public static void SampleParallel(this Gen initial, GenOperation operation, Func? equal = null, string? seed = null, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) - => SampleConcurrent(initial, [operation], equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); + => SampleParallel(initial, [operation], equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); - /// Sample model-based operations on a random initial state concurrently. + /// Sample operations on a random initial state in parallel. /// The result is compared against the result of the possible sequential permutations. - /// At least one of these permutations result must be equal for the concurrency to have been linearized successfully. + /// At least one of these permutations result must be equal for the parallel execution to have been linearized successfully. /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. /// The initial state generator. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. /// The maximum number of operations to run in parallel. @@ -1738,18 +1738,18 @@ public static void SampleConcurrent(this Gen initial, GenOperation oper /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SampleConcurrent(this Gen initial, GenOperation operation1, GenOperation operation2, Func? equal = null, string? seed = null, + public static void SampleParallel(this Gen initial, GenOperation operation1, GenOperation operation2, Func? equal = null, string? seed = null, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) - => SampleConcurrent(initial, [operation1, operation2], equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); + => SampleParallel(initial, [operation1, operation2], equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); - /// Sample model-based operations on a random initial state concurrently. + /// Sample operations on a random initial state in parallel. /// The result is compared against the result of the possible sequential permutations. - /// At least one of these permutations result must be equal for the concurrency to have been linearized successfully. + /// At least one of these permutations result must be equal for the parallel execution to have been linearized successfully. /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. /// The initial state generator. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. /// The maximum number of operations to run in parallel. @@ -1760,19 +1760,19 @@ public static void SampleConcurrent(this Gen initial, GenOperation oper /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SampleConcurrent(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, Func? equal = null, string? seed = null, + public static void SampleParallel(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, Func? equal = null, string? seed = null, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) - => SampleConcurrent(initial, [operation1, operation2, operation3], equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); + => SampleParallel(initial, [operation1, operation2, operation3], equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); - /// Sample model-based operations on a random initial state concurrently. + /// Sample operations on a random initial state in parallel. /// The result is compared against the result of the possible sequential permutations. - /// At least one of these permutations result must be equal for the concurrency to have been linearized successfully. + /// At least one of these permutations result must be equal for the parallel execution to have been linearized successfully. /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. /// The initial state generator. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. /// The maximum number of operations to run in parallel. @@ -1783,20 +1783,20 @@ public static void SampleConcurrent(this Gen initial, GenOperation oper /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SampleConcurrent(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, Func? equal = null, string? seed = null, + public static void SampleParallel(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, Func? equal = null, string? seed = null, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) - => SampleConcurrent(initial, [operation1, operation2, operation3, operation4], equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); + => SampleParallel(initial, [operation1, operation2, operation3, operation4], equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); - /// Sample model-based operations on a random initial state concurrently. + /// Sample operations on a random initial state in parallel. /// The result is compared against the result of the possible sequential permutations. - /// At least one of these permutations result must be equal for the concurrency to have been linearized successfully. + /// At least one of these permutations result must be equal for the parallel execution to have been linearized successfully. /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. /// The initial state generator. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. /// The maximum number of operations to run in parallel. @@ -1807,23 +1807,23 @@ public static void SampleConcurrent(this Gen initial, GenOperation oper /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SampleConcurrent(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, GenOperation operation5, Func? equal = null, string? seed = null, + public static void SampleParallel(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, GenOperation operation5, Func? equal = null, string? seed = null, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) - => SampleConcurrent(initial, [operation1, operation2, operation3, operation4, operation5], + => SampleParallel(initial, [operation1, operation2, operation3, operation4, operation5], equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); - /// Sample model-based operations on a random initial state concurrently. + /// Sample operations on a random initial state in parallel. /// The result is compared against the result of the possible sequential permutations. - /// At least one of these permutations result must be equal for the concurrency to have been linearized successfully. + /// At least one of these permutations result must be equal for the parallel execution to have been linearized successfully. /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. /// The initial state generator. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. /// The maximum number of operations to run in parallel. @@ -1834,9 +1834,9 @@ public static void SampleConcurrent(this Gen initial, GenOperation oper /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SampleConcurrent(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, GenOperation operation5, GenOperation operation6, Func? equal = null, string? seed = null, + public static void SampleParallel(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, GenOperation operation5, GenOperation operation6, Func? equal = null, string? seed = null, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) - => SampleConcurrent(initial, [operation1, operation2, operation3, operation4, operation5, operation6], + => SampleParallel(initial, [operation1, operation2, operation3, operation4, operation5, operation6], equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); /// Assert actual is in line with expected using a chi-squared test to sigma. diff --git a/CsCheck/CsCheck.csproj b/CsCheck/CsCheck.csproj index 495e5ec..f92806a 100644 --- a/CsCheck/CsCheck.csproj +++ b/CsCheck/CsCheck.csproj @@ -11,9 +11,9 @@ This gives the following advantages: - Random testing and shrinking are parallelized. This and PCG make it very fast. - Shrunk cases have a seed value. Simpler examples can easily be reproduced. - Shrinking can be continued later to give simpler cases for high dimensional problems. -- Concurrency testing and random shrinking work well together. +- Parallel testing and random shrinking work well together. -CsCheck also makes concurrency, performance and regression testing simple and fast. +CsCheck also makes parallel, performance and regression testing simple and fast. Anthony Lloyd Anthony Lloyd @@ -21,7 +21,7 @@ CsCheck also makes concurrency, performance and regression testing simple and fa Apache-2.0 http://github.com/AnthonyLloyd/CsCheck CsCheck.png - quickcheck;random;model-based;metamorphic;concurrency;performance;causal-profiling;regression;testing + quickcheck;random;model-based;metamorphic;parallel;performance;causal-profiling;regression;testing 4.0.0 Update to .NET 8 diff --git a/CsCheck/Utils.cs b/CsCheck/Utils.cs index 2295328..77bee4e 100644 --- a/CsCheck/Utils.cs +++ b/CsCheck/Utils.cs @@ -475,7 +475,7 @@ public static bool ModelEqual(T actual, M model) return actual.Equals(model); } - internal static void Run(T concurrentState, (string, Action)[] operations, int threads, int[]? threadIds = null) + internal static void Run(T parallelState, (string, Action)[] operations, int threads, int[]? threadIds = null) { Exception? exception = null; var opId = -1; @@ -488,7 +488,7 @@ internal static void Run(T concurrentState, (string, Action)[] operations, while ((i = Interlocked.Increment(ref opId)) < operations.Length) { if (threadIds is not null) threadIds[i] = tid; - try { operations[i].Item2(concurrentState); } + try { operations[i].Item2(parallelState); } catch (Exception e) { if (exception is null) @@ -505,7 +505,7 @@ internal static void Run(T concurrentState, (string, Action)[] operations, if (exception is not null) throw exception; } - internal static void RunReplay(T concurrentState, (string, Action)[] operations, int threads, int[] threadIds) + internal static void RunReplay(T parallelState, (string, Action)[] operations, int threads, int[] threadIds) { Exception? exception = null; var runners = new Thread[threads]; @@ -518,7 +518,7 @@ internal static void RunReplay(T concurrentState, (string, Action)[] opera { if (threadIds[i] == tid) { - try { operations[i].Item2(concurrentState); } + try { operations[i].Item2(parallelState); } catch (Exception e) { if (exception is null) diff --git a/README.md b/README.md index ac5138e..662977e 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ This gives the following advantages over tree based shrinking libraries: - Random testing and shrinking are parallelized. This and PCG make it very fast. - Shrunk cases have a seed value. Simpler examples can easily be reproduced. - Shrinking can be continued later to give simpler cases for high dimensional problems. -- Concurrency testing and random shrinking work well together. Repeat is not needed. +- Parallel testing and random shrinking work well together. Repeat is not needed. See [why](https://github.com/AnthonyLloyd/CsCheck/blob/master/Why.md) you should use it, the [comparison](https://github.com/AnthonyLloyd/CsCheck/blob/master/Comparison.md) with other random testing libraries, or how CsCheck does in the [shrinking challenge](https://github.com/jlink/shrinking-challenge). In one [shrinking challenge test](https://github.com/jlink/shrinking-challenge/blob/main/challenges/binheap.md) CsCheck managed to shrink to a new smaller example than was thought possible and is not reached by any other testing library. @@ -21,7 +21,7 @@ CsCheck also has functionality to make multiple types of testing simple and fast - [Random testing](#Random-testing) - [Model-based testing](#Model-based-testing) - [Metamorphic testing](#Metamorphic-testing) -- [Concurrency testing](#Concurrency-testing) +- [Parallel testing](#Parallel-testing) - [Causal profiling](#Causal-profiling) - [Regression testing](#Regression-testing) - [Performance testing](#Performance-testing) @@ -267,21 +267,21 @@ public void MapSlim_Metamorphic() } ``` -## Concurrency testing +## Parallel testing -CsCheck has support for concurrency testing with full shrinking capability. -A concurrent sequence of operations are run on an initial state and the result is compared to all the possible linearized versions. -At least one of these must be equal to the concurrent version. +CsCheck has support for parallels testing with full shrinking capability. +A number of operations are run on an initial state in parallel and the result is compared to all the possible linearized versions. +At least one of these must be equal to the parallel result. Idea from John Hughes [talk](https://youtu.be/1LNEWF8s1hI?t=1603) and [paper](https://github.com/AnthonyLloyd/AnthonyLloyd.github.io/raw/master/public/cscheck/finding-race-conditions.pdf). This is actually easier to implement with CsCheck than QuickCheck because the random shrinking does not need to repeat each step as QuickCheck does (10 times by default) to make shrinking deterministic. ### SetSlim ```csharp [Fact] -public void SetSlim_Concurrency() +public void SetSlim_Parallel() { Gen.Byte.Array.Select(a => new SetSlim(a)) - .SampleConcurrent( + .SampleParallel( Gen.Byte.Operation>((l, i) => { lock (l) l.Add(i); }), Gen.Int.NonNegative.Operation>((l, i) => { if (i < l.Count) { var _ = l[i]; } }), Gen.Byte.Operation>((l, i) => { var _ = l.IndexOf(i); }), @@ -610,7 +610,7 @@ timeout - The timeout in seconds to use for Faster (default 60 seconds). print - A function to convert the state to a string for error reporting (default Check.Print). equal - A function to check if the two states are the same (default Check.Equal). sigma - For Faster sigma is the number of standard deviations from the null hypothesis (default 6). -replay - The number of times to retry the seed to reproduce a SampleConcurrent fail (default 100). +replay - The number of times to retry the seed to reproduce a SampleParallel fail (default 100). Global defaults can also be set via environment variables: diff --git a/Tests/CacheTests.cs b/Tests/CacheTests.cs index 3bfad01..7ffd8ac 100644 --- a/Tests/CacheTests.cs +++ b/Tests/CacheTests.cs @@ -14,9 +14,9 @@ class ConcurrentDictionaryCache : ConcurrentDictionary, ICache } [Fact] - public void GetOrAddAtomicAsync_SampleConcurrent() + public void GetOrAddAtomicAsync_SampleParallel() { - Check.SampleConcurrent( + Check.SampleParallel( Gen.Const(() => new ConcurrentDictionaryCache()), Gen.Int[1, 5].Operation>((d, i) => d.GetOrAddAtomicAsync(i, i => Task.FromResult(i)).AsTask()), equal: (a, b) => Check.Equal(a.Keys, b.Keys), diff --git a/Tests/CheckTests.cs b/Tests/CheckTests.cs index 1bb4652..8e036fc 100644 --- a/Tests/CheckTests.cs +++ b/Tests/CheckTests.cs @@ -228,20 +228,20 @@ public void SampleModelBased_ConcurrentBag() } [Fact] - public void SampleConcurrent_ConcurrentBag() + public void SampleParallel_ConcurrentBag() { Gen.Int.List[0, 5].Select(l => new ConcurrentBag(l)) - .SampleConcurrent( + .SampleParallel( Gen.Int.Operation>(i => $"Add({i})", (bag, i) => bag.Add(i)), Gen.Operation>("TryTake()", bag => bag.TryTake(out _)) ); } [Fact] - public void SampleConcurrent_ConcurrentDictionary() + public void SampleParallel_ConcurrentDictionary() { Gen.Dictionary(Gen.Int[0, 100], Gen.Byte)[0, 10].Select(l => new ConcurrentDictionary(l)) - .SampleConcurrent( + .SampleParallel( Gen.Int[0, 100].Select(Gen.Byte) .Operation>(t =>$"d[{t.Item1}] = {t.Item2}", (d, t) => d[t.Item1] = t.Item2), @@ -251,10 +251,10 @@ public void SampleConcurrent_ConcurrentDictionary() } [Fact] - public void SampleConcurrent_ConcurrentQueue() + public void SampleParallel_ConcurrentQueue() { Gen.Int.List[0, 5].Select(l => new ConcurrentQueue(l)) - .SampleConcurrent( + .SampleParallel( Gen.Int.Operation>(i => $"Enqueue({i})", (q, i) => q.Enqueue(i)), Gen.Operation>("TryDequeue()", q => q.TryDequeue(out _)) ); diff --git a/Tests/SieveLruCacheTests.cs b/Tests/SieveLruCacheTests.cs index 498d7a2..fb91323 100644 --- a/Tests/SieveLruCacheTests.cs +++ b/Tests/SieveLruCacheTests.cs @@ -99,9 +99,9 @@ public void SampleModelBased() } [Fact] - public void SampleConcurrent() + public void SampleParallel() { - Check.SampleConcurrent( + Check.SampleParallel( Gen.Const(() => new SieveLruCache(4)), Gen.Int[1, 5].Operation>((d, i) => d.GetOrAdd(i, i => i)), equal: (a, b) => Check.Equal(a.Keys, b.Keys), diff --git a/Tests/SlimCollectionsTests.cs b/Tests/SlimCollectionsTests.cs index c38157f..30dabd8 100644 --- a/Tests/SlimCollectionsTests.cs +++ b/Tests/SlimCollectionsTests.cs @@ -22,10 +22,10 @@ public void ListSlim_ModelBased() } [Fact] - public void ListSlim_Concurrency() + public void ListSlim_Parallel() { Gen.Byte.Array.Select(a => new ListSlim(a)) - .SampleConcurrent( + .SampleParallel( Gen.Byte.Operation>((l, i) => { lock (l) l.Add(i); }), Gen.Int.NonNegative.Operation>((l, i) => { if (i < l.Count) { var _ = l[i]; } }), Gen.Int.NonNegative.Select(Gen.Byte).Operation>((l, t) => { if (t.Item1 < l.Count) l[t.Item1] = t.Item2; }), @@ -46,10 +46,10 @@ public void SetSlim_ModelBased() } [Fact] - public void SetSlim_Concurrency() + public void SetSlim_Parallel() { Gen.Byte.Array.Select(a => new SetSlim(a)) - .SampleConcurrent( + .SampleParallel( Gen.Byte.Operation>((l, i) => { lock (l) l.Add(i); }), Gen.Int.NonNegative.Operation>((l, i) => { if (i < l.Count) { var _ = l[i]; } }), Gen.Byte.Operation>((l, i) => { var _ = l.IndexOf(i); }), @@ -116,10 +116,10 @@ public void MapSlim_Metamorphic() } [Fact] - public void MapSlim_Concurrency() + public void MapSlim_Parallel() { Gen.Dictionary(Gen.Int, Gen.Byte).Select(d => new MapSlim(d)) - .SampleConcurrent( + .SampleParallel( Gen.Int.Select(Gen.Byte).Operation>((m, t) => { lock (m) m[t.Item1] = t.Item2; }), Gen.Int.NonNegative.Operation>((m, i) => { if (i < m.Count) { var _ = m.Key(i); } }), Gen.Int.Operation>((m, i) => { var _ = m.IndexOf(i); }), diff --git a/Why.md b/Why.md index 782d88c..f70088c 100644 --- a/Why.md +++ b/Why.md @@ -40,7 +40,7 @@ Fluent style composition similar to LINQ is a much more robust and extensible op - Caches and collections - often a key part of server and client side code these can be tested against a suitable simplified test model with `Model Based` testing. - Calculations and algorithms - often possible to generalize examples for calculations and algorithms and check the result given the input. Algorithm often have properties they must guarantee. Rounding error issues automatically tested. - Code refactoring - keep a copy of the original code with the test, refactor for simplicity and performance, safe in the knowledge it still produces the same results. Pair with a `Faster` test to monitor the relative performance over a range of inputs. Or if a copy is not feasible create a `Regression` test to comprehensively make sure there is no change. -- Multithreading and concurrency - test on the same object instance across multiple threads and examples. Shrink even works for `Concurrency` testing. +- Multithreading - test on the same object instance across multiple threads and examples. Shrink even works for `Parallel` testing. ## Raise Your Game From 0c150b113221dd19d4489ba5745f2e431086532d Mon Sep 17 00:00:00 2001 From: Anthony Lloyd Date: Sat, 24 Aug 2024 08:23:25 +0100 Subject: [PATCH 05/15] add maxSequentialOperations parameter to SampleParallel --- CsCheck/Check.cs | 49 ++++++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/CsCheck/Check.cs b/CsCheck/Check.cs index fff4df9..2fc6bcb 100644 --- a/CsCheck/Check.cs +++ b/CsCheck/Check.cs @@ -1593,7 +1593,8 @@ public override (T, uint, ulong) Generate(PCG pcg, Size? min, out Size size) /// The operation generators that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. - /// The maximum number of operations to run in parallel. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). @@ -1601,7 +1602,7 @@ public override (T, uint, ulong) Generate(PCG pcg, Size? min, out Size size) /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. public static void SampleParallel(this Gen initial, GenOperation[] operations, Func? equal = null, string? seed = null, - int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) { equal ??= Equal; seed ??= Seed; @@ -1709,7 +1710,8 @@ public static void SampleParallel(this Gen initial, GenOperation[] oper /// An operation generator that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. - /// The maximum number of operations to run in parallel. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). @@ -1718,8 +1720,8 @@ public static void SampleParallel(this Gen initial, GenOperation[] oper /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void SampleParallel(this Gen initial, GenOperation operation, Func? equal = null, string? seed = null, - int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) - => SampleParallel(initial, [operation], equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, print, replay, writeLine); /// Sample operations on a random initial state in parallel. /// The result is compared against the result of the possible sequential permutations. @@ -1730,7 +1732,8 @@ public static void SampleParallel(this Gen initial, GenOperation operat /// An operation generator that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. - /// The maximum number of operations to run in parallel. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). @@ -1739,8 +1742,8 @@ public static void SampleParallel(this Gen initial, GenOperation operat /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void SampleParallel(this Gen initial, GenOperation operation1, GenOperation operation2, Func? equal = null, string? seed = null, - int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) - => SampleParallel(initial, [operation1, operation2], equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation1, operation2], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, print, replay, writeLine); /// Sample operations on a random initial state in parallel. /// The result is compared against the result of the possible sequential permutations. @@ -1752,7 +1755,8 @@ public static void SampleParallel(this Gen initial, GenOperation operat /// An operation generator that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. - /// The maximum number of operations to run in parallel. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). @@ -1761,8 +1765,8 @@ public static void SampleParallel(this Gen initial, GenOperation operat /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void SampleParallel(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, Func? equal = null, string? seed = null, - int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) - => SampleParallel(initial, [operation1, operation2, operation3], equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation1, operation2, operation3], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, print, replay, writeLine); /// Sample operations on a random initial state in parallel. /// The result is compared against the result of the possible sequential permutations. @@ -1775,7 +1779,8 @@ public static void SampleParallel(this Gen initial, GenOperation operat /// An operation generator that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. - /// The maximum number of operations to run in parallel. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). @@ -1784,8 +1789,8 @@ public static void SampleParallel(this Gen initial, GenOperation operat /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void SampleParallel(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, Func? equal = null, string? seed = null, - int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) - => SampleParallel(initial, [operation1, operation2, operation3, operation4], equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation1, operation2, operation3, operation4], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, print, replay, writeLine); /// Sample operations on a random initial state in parallel. /// The result is compared against the result of the possible sequential permutations. @@ -1799,7 +1804,8 @@ public static void SampleParallel(this Gen initial, GenOperation operat /// An operation generator that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. - /// The maximum number of operations to run in parallel. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). @@ -1808,10 +1814,9 @@ public static void SampleParallel(this Gen initial, GenOperation operat /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void SampleParallel(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, GenOperation operation5, Func? equal = null, string? seed = null, - int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) - => SampleParallel(initial, [operation1, operation2, operation3, operation4, operation5], - equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); + => SampleParallel(initial, [operation1, operation2, operation3, operation4, operation5], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, print, replay, writeLine); /// Sample operations on a random initial state in parallel. /// The result is compared against the result of the possible sequential permutations. @@ -1826,7 +1831,8 @@ public static void SampleParallel(this Gen initial, GenOperation operat /// An operation generator that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. - /// The maximum number of operations to run in parallel. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). @@ -1835,9 +1841,8 @@ public static void SampleParallel(this Gen initial, GenOperation operat /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void SampleParallel(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, GenOperation operation5, GenOperation operation6, Func? equal = null, string? seed = null, - int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) - => SampleParallel(initial, [operation1, operation2, operation3, operation4, operation5, operation6], - equal, seed, maxParallelOperations, iter, time, threads, print, replay, writeLine); + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation1, operation2, operation3, operation4, operation5, operation6], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, print, replay, writeLine); /// Assert actual is in line with expected using a chi-squared test to sigma. /// The expected bin counts. From 20497cbc7a535308ce01bfb2b7a4f989cf4c6d97 Mon Sep 17 00:00:00 2001 From: Anthony Lloyd Date: Sat, 24 Aug 2024 17:58:04 +0100 Subject: [PATCH 06/15] generate and run maxSequentialOperations --- CsCheck/Check.cs | 57 ++++++++++++++++++++++++++++-------------------- CsCheck/Utils.cs | 20 ++++++++++------- 2 files changed, 45 insertions(+), 32 deletions(-) diff --git a/CsCheck/Check.cs b/CsCheck/Check.cs index 2fc6bcb..a8b30d8 100644 --- a/CsCheck/Check.cs +++ b/CsCheck/Check.cs @@ -1570,9 +1570,16 @@ public static void SampleMetamorphic(this Gen initial, GenMetamorphic o }); } - sealed class SampleParallelData(T state, uint stream, ulong seed, (string, Action)[] operations, int threads) + sealed class SampleParallelData(T state, uint stream, ulong seed, (string, Action)[] sequencialOperations, (string, Action)[] parallelOperations, int threads) { - public T State = state; public uint Stream = stream; public ulong Seed = seed; public (string, Action)[] Operations = operations; public int Threads = threads; public int[]? ThreadIds; public Exception? Exception; + public T InitialState = state; + public uint Stream = stream; + public ulong Seed = seed; + public (string, Action)[] SequencialOperations = sequencialOperations; + public (string, Action)[] ParallelOperations = parallelOperations; + public int Threads = threads; + public int[]? ThreadIds; + public Exception? Exception; } sealed class GenSampleParallel(Gen initial) : Gen<(T Value, uint Stream, ulong Seed)> @@ -1629,12 +1636,14 @@ public static void SampleParallel(this Gen initial, GenOperation[] oper bool firstIteration = true; - new GenSampleParallel(initial) - .Select(Gen.OneOf(opNameActions).Array[2, maxParallelOperations] - .SelectMany(ops => Gen.Int[1, Math.Min(threads, ops.Length)].Select(i => (ops, i))), (a, b) => - new SampleParallelData(a.Value, a.Stream, a.Seed, b.ops, b.i) - ) - .Sample(cd => + var genOps = Gen.OneOf(opNameActions); + Gen.Int[2, maxParallelOperations] + .SelectMany(np => Gen.Int[2, Math.Min(threads, np)].Select(nt => (nt, np))) + .SelectMany((nt, np) => Gen.Int[0, maxSequentialOperations].Select(ns => (ns, nt, np))) + .SelectMany((ns, nt, np) => new GenSampleParallel(initial).Select(genOps.Array[ns], genOps.Array[np]) + .Select((initial, sequencial, parallel) => (initial, sequencial, nt, parallel))) + .Select((initial, sequencial, threads, parallel) => new SampleParallelData(initial.Value, initial.Stream, initial.Seed, sequencial, parallel, threads)) + .Sample(spd => { bool linearizable = false; do @@ -1642,22 +1651,22 @@ public static void SampleParallel(this Gen initial, GenOperation[] oper try { if (replayThreads is null) - Run(cd.State, cd.Operations, cd.Threads, cd.ThreadIds = new int[cd.Operations.Length]); + Run(spd.InitialState, spd.SequencialOperations, spd.ParallelOperations, spd.Threads, spd.ThreadIds = new int[spd.ParallelOperations.Length]); else - RunReplay(cd.State, cd.Operations, cd.Threads, cd.ThreadIds = replayThreads); + RunReplay(spd.InitialState, spd.SequencialOperations, spd.ParallelOperations, spd.Threads, spd.ThreadIds = replayThreads); } catch (Exception e) { - cd.Exception = e; + spd.Exception = e; break; } - Parallel.ForEach(Permutations(cd.ThreadIds, cd.Operations), (sequence, state) => + Parallel.ForEach(Permutations(spd.ThreadIds, spd.ParallelOperations), (sequence, state) => { - var linearState = initial.Generate(new PCG(cd.Stream, cd.Seed), null, out _); + var linearState = initial.Generate(new PCG(spd.Stream, spd.Seed), null, out _); try { - Run(linearState, sequence, 1); - if (equal(cd.State, linearState)) + Run(linearState, spd.SequencialOperations, sequence, 1); + if (equal(spd.InitialState, linearState)) { linearizable = true; state.Stop(); @@ -1669,23 +1678,23 @@ public static void SampleParallel(this Gen initial, GenOperation[] oper firstIteration = false; return linearizable; }, writeLine, seed, iter, time, threads: 1, - p => + spd => { print ??= Print; - if (p == null) return ""; + if (spd == null) return ""; var sb = new StringBuilder(); - sb.Append("\n Operations: ").Append(Print(p.Operations.Select(i => i.Item1).ToList())); - sb.Append("\n On Threads: ").Append(Print(p.ThreadIds)); - sb.Append("\nInitial state: ").Append(print(initial.Generate(new PCG(p.Stream, p.Seed), null, out _))); - sb.Append("\n Final state: ").Append(p.Exception is not null ? p.Exception.ToString() : print(p.State)); + sb.Append("\n Operations: ").Append(Print(spd.ParallelOperations.Select(i => i.Item1).ToList())); + sb.Append("\n On Threads: ").Append(Print(spd.ThreadIds)); + sb.Append("\nInitial state: ").Append(print(initial.Generate(new PCG(spd.Stream, spd.Seed), null, out _))); + sb.Append("\n Final state: ").Append(spd.Exception is not null ? spd.Exception.ToString() : print(spd.InitialState)); bool first = true; - foreach (var sequence in Permutations(p.ThreadIds!, p.Operations)) + foreach (var sequence in Permutations(spd.ThreadIds!, spd.ParallelOperations)) { - var linearState = initial.Generate(new PCG(p.Stream, p.Seed), null, out _); + var linearState = initial.Generate(new PCG(spd.Stream, spd.Seed), null, out _); string result; try { - Run(linearState, sequence, 1); + Run(linearState, spd.SequencialOperations, sequence, 1); result = print(linearState); } catch (Exception e) diff --git a/CsCheck/Utils.cs b/CsCheck/Utils.cs index 77bee4e..0b575d0 100644 --- a/CsCheck/Utils.cs +++ b/CsCheck/Utils.cs @@ -475,8 +475,10 @@ public static bool ModelEqual(T actual, M model) return actual.Equals(model); } - internal static void Run(T parallelState, (string, Action)[] operations, int threads, int[]? threadIds = null) + internal static void Run(T initialState, (string, Action)[] sequencialOperations, (string, Action)[] parallelOperations, int threads, int[]? threadIds = null) { + for (int i = 0; i < sequencialOperations.Length; i++) + sequencialOperations[i].Item2(initialState); Exception? exception = null; var opId = -1; var runners = new Thread[threads]; @@ -485,16 +487,16 @@ internal static void Run(T parallelState, (string, Action)[] operations, i runners[threads] = new Thread(threadId => { int i, tid = (int)threadId!; - while ((i = Interlocked.Increment(ref opId)) < operations.Length) + while ((i = Interlocked.Increment(ref opId)) < parallelOperations.Length) { if (threadIds is not null) threadIds[i] = tid; - try { operations[i].Item2(parallelState); } + try { parallelOperations[i].Item2(initialState); } catch (Exception e) { if (exception is null) { exception = e; - Interlocked.Exchange(ref opId, operations.Length); + Interlocked.Exchange(ref opId, parallelOperations.Length); } } } @@ -505,8 +507,10 @@ internal static void Run(T parallelState, (string, Action)[] operations, i if (exception is not null) throw exception; } - internal static void RunReplay(T parallelState, (string, Action)[] operations, int threads, int[] threadIds) + internal static void RunReplay(T initialState, (string, Action)[] sequencialOperations, (string, Action)[] parallelOperations, int threads, int[] threadIds) { + for (int i = 0; i < sequencialOperations.Length; i++) + sequencialOperations[i].Item2(initialState); Exception? exception = null; var runners = new Thread[threads]; while (--threads >= 0) @@ -514,17 +518,17 @@ internal static void RunReplay(T parallelState, (string, Action)[] operati runners[threads] = new Thread(threadId => { int opId = -1, i = -1, tid = (int)threadId!; - while ((i = Interlocked.Increment(ref opId)) < operations.Length) + while ((i = Interlocked.Increment(ref opId)) < parallelOperations.Length) { if (threadIds[i] == tid) { - try { operations[i].Item2(parallelState); } + try { parallelOperations[i].Item2(initialState); } catch (Exception e) { if (exception is null) { exception = e; - Interlocked.Exchange(ref opId, operations.Length); + Interlocked.Exchange(ref opId, parallelOperations.Length); } } } From 17c0931422ef31a9f5a4ff4fe523015cae021e14 Mon Sep 17 00:00:00 2001 From: Anthony Lloyd Date: Sun, 25 Aug 2024 14:47:13 +0100 Subject: [PATCH 07/15] add SampleParallel --- CsCheck/Check.cs | 152 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/CsCheck/Check.cs b/CsCheck/Check.cs index a8b30d8..7aa8b96 100644 --- a/CsCheck/Check.cs +++ b/CsCheck/Check.cs @@ -1582,6 +1582,19 @@ sealed class SampleParallelData(T state, uint stream, ulong seed, (string, Ac public Exception? Exception; } + sealed class SampleParallelData(Actual actual, Model model, uint stream, ulong seed, (string, Action, Action)[] sequencialOperations, (string, Action, Action)[] parallelOperations, int threads) + { + public Actual InitialActual = actual; + public Model InitialModel = model; + public uint Stream = stream; + public ulong Seed = seed; + public (string, Action, Action)[] SequencialOperations = sequencialOperations; + public (string, Action, Action)[] ParallelOperations = parallelOperations; + public int Threads = threads; + public int[]? ThreadIds; + public Exception? Exception; + } + sealed class GenSampleParallel(Gen initial) : Gen<(T Value, uint Stream, ulong Seed)> { public override (T, uint, ulong) Generate(PCG pcg, Size? min, out Size size) @@ -1592,6 +1605,17 @@ public override (T, uint, ulong) Generate(PCG pcg, Size? min, out Size size) } } + sealed class GenSampleParallel(Gen<(Actual, Model)> initial) : Gen<(Actual Actual, Model Model, uint Stream, ulong Seed)> + { + public override (Actual, Model, uint, ulong) Generate(PCG pcg, Size? min, out Size size) + { + var stream = pcg.Stream; + var seed = pcg.Seed; + var (actual, model) = initial.Generate(pcg, null, out size); + return (actual, model, stream, seed); + } + } + /// Sample operations on a random initial state in parallel. /// The result is compared against the result of the possible sequential permutations. /// At least one of these permutations result must be equal for the parallel execution to have been linearized successfully. @@ -1853,6 +1877,134 @@ public static void SampleParallel(this Gen initial, GenOperation operat int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) => SampleParallel(initial, [operation1, operation2, operation3, operation4, operation5, operation6], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, print, replay, writeLine); + /// Sample operations on a random initial actual state in parallel. + /// The actual result is compared against the model result of the possible sequential permutations. + /// At least one of these permutations model result must be equal for the parallel execution to have been linearized successfully. + /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. + /// There is no need for the model operations to be thread safe as they are only run sequencially. + /// The initial actual and model state generator. + /// The actual and model operation generators that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// A function to check if the actual and model are the same (default Check.ModelEqual). + /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). + /// The number of iterations to run in the sample (default 100). + /// The number of seconds to run the sample. + /// The number of threads to run the sample on (default number logical CPUs). + /// A function to convert the actual state to a string for error reporting (default Check.Print). + /// A function to convert the model state to a string for error reporting (default Check.Print). + /// The number of times to retry the seed to reproduce an initial fail (default 100). + /// WriteLine function to use for the summary total iterations output. + public static void SampleParallel(this Gen<(Actual, Model)> initial, GenOperation[] operations, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? printActual = null, Func? printModel = null, int replay = -1, Action? writeLine = null) + { + equal ??= ModelEqual; + seed ??= Seed; + if (iter == -1) iter = Iter; + if (time == -1) time = Time; + if (threads == -1) threads = Threads; + if (replay == -1) replay = Replay; + int[]? replayThreads = null; + printActual ??= Print; + printModel ??= Print; + if (seed?.Contains('[') == true) + { + int i = seed.IndexOf('['); + int j = seed.IndexOf(']', i + 1); + replayThreads = seed.Substring(i + 1, j - i - 1).Split(',').Select(int.Parse).ToArray(); + seed = seed[..i]; + } + + var opNameActions = new Gen<(string, Action, Action)>[operations.Length]; + for (int i = 0; i < operations.Length; i++) + { + var op = operations[i]; + var opName = "Op" + i; + opNameActions[i] = op.AddOpNumber ? op.Select((name, actual, model) => (opName + name, actual, model)) : op; + } + + bool firstIteration = true; + + var genOps = Gen.OneOf(opNameActions); + Gen.Int[2, maxParallelOperations] + .SelectMany(np => Gen.Int[2, Math.Min(threads, np)].Select(nt => (nt, np))) + .SelectMany((nt, np) => Gen.Int[0, maxSequentialOperations].Select(ns => (ns, nt, np))) + .SelectMany((ns, nt, np) => new GenSampleParallel(initial).Select(genOps.Array[ns], genOps.Array[np]) + .Select((initial, sequencial, parallel) => (initial, sequencial, nt, parallel))) + .Select((initial, sequencial, threads, parallel) => new SampleParallelData(initial.Actual, initial.Model, initial.Stream, initial.Seed, sequencial, parallel, threads)) + .Sample(spd => + { + bool linearizable = false; + do + { + var actualSequencialOperations = Array.ConvertAll(spd.SequencialOperations, i => (i.Item1, i.Item2)); + var actualParallelOperations = Array.ConvertAll(spd.ParallelOperations, i => (i.Item1, i.Item2)); + try + { + if (replayThreads is null) + Run(spd.InitialActual, actualSequencialOperations, actualParallelOperations, spd.Threads, spd.ThreadIds = new int[spd.ParallelOperations.Length]); + else + RunReplay(spd.InitialActual, actualSequencialOperations, actualParallelOperations, spd.Threads, spd.ThreadIds = replayThreads); + } + catch (Exception e) + { + spd.Exception = e; + break; + } + var modelSequencialOperations = Array.ConvertAll(spd.SequencialOperations, i => (i.Item1, i.Item3)); + var modelParallelOperations = Array.ConvertAll(spd.ParallelOperations, i => (i.Item1, i.Item3)); + Parallel.ForEach(Permutations(spd.ThreadIds, modelParallelOperations), (sequence, state) => + { + var (_, initialModel) = initial.Generate(new PCG(spd.Stream, spd.Seed), null, out _); + try + { + Run(initialModel, modelSequencialOperations, sequence, 1); + if (equal(spd.InitialActual, initialModel)) + { + linearizable = true; + state.Stop(); + } + } + catch { state.Stop(); } + }); + } while (linearizable && firstIteration && seed is not null && --replay > 0); + firstIteration = false; + return linearizable; + }, writeLine, seed, iter, time, threads: 1, + spd => + { + if (spd == null) return ""; + var sb = new StringBuilder(); + sb.Append("\n Operations: ").Append(Print(spd.ParallelOperations.Select(i => i.Item1).ToList())); + sb.Append("\n On Threads: ").Append(Print(spd.ThreadIds)); + sb.Append("\nInitial state: ").Append(printActual(initial.Generate(new PCG(spd.Stream, spd.Seed), null, out _).Item1)); + sb.Append("\n Final state: ").Append(spd.Exception is not null ? spd.Exception.ToString() : printActual(spd.InitialActual)); + var modelSequencialOperations = Array.ConvertAll(spd.SequencialOperations, i => (i.Item1, i.Item3)); + var modelParallelOperations = Array.ConvertAll(spd.ParallelOperations, i => (i.Item1, i.Item3)); + bool first = true; + foreach (var sequence in Permutations(spd.ThreadIds!, modelParallelOperations)) + { + var (_, initialModel) = initial.Generate(new PCG(spd.Stream, spd.Seed), null, out _); + string result; + try + { + Run(initialModel, modelSequencialOperations, sequence, 1); + result = printModel(initialModel); + } + catch (Exception e) + { + result = e.ToString(); + } + sb.Append(first ? "\n Linearized: " : "\n : "); + sb.Append(Print(sequence.Select(i => i.Item1).ToList())); + sb.Append(" -> "); + sb.Append(result); + first = false; + } + return sb.ToString(); + }); + } + /// Assert actual is in line with expected using a chi-squared test to sigma. /// The expected bin counts. /// The actual bin counts. From eab044576024ce932789e8081ee2ba58edf14dc6 Mon Sep 17 00:00:00 2001 From: Anthony Lloyd Date: Sun, 25 Aug 2024 15:11:25 +0100 Subject: [PATCH 08/15] add SampleParallel overloads --- CsCheck/Check.cs | 147 +++++++++++++++++++++++++++++++++++++++++++-- Tests/Tests.csproj | 2 +- 2 files changed, 144 insertions(+), 5 deletions(-) diff --git a/CsCheck/Check.cs b/CsCheck/Check.cs index 7aa8b96..a6cd583 100644 --- a/CsCheck/Check.cs +++ b/CsCheck/Check.cs @@ -1877,11 +1877,9 @@ public static void SampleParallel(this Gen initial, GenOperation operat int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) => SampleParallel(initial, [operation1, operation2, operation3, operation4, operation5, operation6], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, print, replay, writeLine); - /// Sample operations on a random initial actual state in parallel. - /// The actual result is compared against the model result of the possible sequential permutations. + /// Sample operations on the random initial actual state in parallel and compare to all the possible linearized operations run sequencially on the initial model state. /// At least one of these permutations model result must be equal for the parallel execution to have been linearized successfully. - /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. - /// There is no need for the model operations to be thread safe as they are only run sequencially. + /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. /// The initial actual and model state generator. /// The actual and model operation generators that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. /// A function to check if the actual and model are the same (default Check.ModelEqual). @@ -2005,6 +2003,147 @@ public static void SampleParallel(this Gen<(Actual, Model)> initi }); } + /// Sample operations on the random initial actual state in parallel and compare to all the possible linearized operations run sequencially on the initial model state. + /// At least one of these permutations model result must be equal for the parallel execution to have been linearized successfully. + /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. + /// The initial actual and model state generator. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// A function to check if the actual and model are the same (default Check.ModelEqual). + /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). + /// The number of iterations to run in the sample (default 100). + /// The number of seconds to run the sample. + /// The number of threads to run the sample on (default number logical CPUs). + /// A function to convert the actual state to a string for error reporting (default Check.Print). + /// A function to convert the model state to a string for error reporting (default Check.Print). + /// The number of times to retry the seed to reproduce an initial fail (default 100). + /// WriteLine function to use for the summary total iterations output. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SampleParallel(this Gen<(Actual, Model)> initial, GenOperation operation, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? printActual = null, Func? printModel = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, printActual, printModel, replay, writeLine); + + /// Sample operations on the random initial actual state in parallel and compare to all the possible linearized operations run sequencially on the initial model state. + /// At least one of these permutations model result must be equal for the parallel execution to have been linearized successfully. + /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. + /// The initial actual and model state generator. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// A function to check if the actual and model are the same (default Check.ModelEqual). + /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). + /// The number of iterations to run in the sample (default 100). + /// The number of seconds to run the sample. + /// The number of threads to run the sample on (default number logical CPUs). + /// A function to convert the actual state to a string for error reporting (default Check.Print). + /// A function to convert the model state to a string for error reporting (default Check.Print). + /// The number of times to retry the seed to reproduce an initial fail (default 100). + /// WriteLine function to use for the summary total iterations output. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SampleParallel(this Gen<(Actual, Model)> initial, GenOperation operation1, GenOperation operation2, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? printActual = null, Func? printModel = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation1, operation2], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, printActual, printModel, replay, writeLine); + + /// Sample operations on the random initial actual state in parallel and compare to all the possible linearized operations run sequencially on the initial model state. + /// At least one of these permutations model result must be equal for the parallel execution to have been linearized successfully. + /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. + /// The initial actual and model state generator. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// A function to check if the actual and model are the same (default Check.ModelEqual). + /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). + /// The number of iterations to run in the sample (default 100). + /// The number of seconds to run the sample. + /// The number of threads to run the sample on (default number logical CPUs). + /// A function to convert the actual state to a string for error reporting (default Check.Print). + /// A function to convert the model state to a string for error reporting (default Check.Print). + /// The number of times to retry the seed to reproduce an initial fail (default 100). + /// WriteLine function to use for the summary total iterations output. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SampleParallel(this Gen<(Actual, Model)> initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? printActual = null, Func? printModel = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation1, operation2, operation3], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, printActual, printModel, replay, writeLine); + + /// Sample operations on the random initial actual state in parallel and compare to all the possible linearized operations run sequencially on the initial model state. + /// At least one of these permutations model result must be equal for the parallel execution to have been linearized successfully. + /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. + /// The initial actual and model state generator. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// A function to check if the actual and model are the same (default Check.ModelEqual). + /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). + /// The number of iterations to run in the sample (default 100). + /// The number of seconds to run the sample. + /// The number of threads to run the sample on (default number logical CPUs). + /// A function to convert the actual state to a string for error reporting (default Check.Print). + /// A function to convert the model state to a string for error reporting (default Check.Print). + /// The number of times to retry the seed to reproduce an initial fail (default 100). + /// WriteLine function to use for the summary total iterations output. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SampleParallel(this Gen<(Actual, Model)> initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? printActual = null, Func? printModel = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation1, operation2, operation3, operation4], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, printActual, printModel, replay, writeLine); + + /// Sample operations on the random initial actual state in parallel and compare to all the possible linearized operations run sequencially on the initial model state. + /// At least one of these permutations model result must be equal for the parallel execution to have been linearized successfully. + /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. + /// The initial actual and model state generator. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// A function to check if the actual and model are the same (default Check.ModelEqual). + /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). + /// The number of iterations to run in the sample (default 100). + /// The number of seconds to run the sample. + /// The number of threads to run the sample on (default number logical CPUs). + /// A function to convert the actual state to a string for error reporting (default Check.Print). + /// A function to convert the model state to a string for error reporting (default Check.Print). + /// The number of times to retry the seed to reproduce an initial fail (default 100). + /// WriteLine function to use for the summary total iterations output. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SampleParallel(this Gen<(Actual, Model)> initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, GenOperation operation5, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? printActual = null, Func? printModel = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation1, operation2, operation3, operation4, operation5], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, printActual, printModel, replay, writeLine); + + /// Sample operations on the random initial actual state in parallel and compare to all the possible linearized operations run sequencially on the initial model state. + /// At least one of these permutations model result must be equal for the parallel execution to have been linearized successfully. + /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. + /// The initial actual and model state generator. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// A function to check if the actual and model are the same (default Check.ModelEqual). + /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). + /// The number of iterations to run in the sample (default 100). + /// The number of seconds to run the sample. + /// The number of threads to run the sample on (default number logical CPUs). + /// A function to convert the actual state to a string for error reporting (default Check.Print). + /// A function to convert the model state to a string for error reporting (default Check.Print). + /// The number of times to retry the seed to reproduce an initial fail (default 100). + /// WriteLine function to use for the summary total iterations output. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SampleParallel(this Gen<(Actual, Model)> initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, GenOperation operation5, GenOperation operation6, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? printActual = null, Func? printModel = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation1, operation2, operation3, operation4, operation5, operation6], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, printActual, printModel, replay, writeLine); + /// Assert actual is in line with expected using a chi-squared test to sigma. /// The expected bin counts. /// The actual bin counts. diff --git a/Tests/Tests.csproj b/Tests/Tests.csproj index 9ed1447..2dc7f73 100644 --- a/Tests/Tests.csproj +++ b/Tests/Tests.csproj @@ -14,7 +14,7 @@ xUnit1004 - + From e1219a012dce6f22e61ef07e0f647abfe6564e05 Mon Sep 17 00:00:00 2001 From: Anthony Lloyd Date: Sun, 25 Aug 2024 18:41:45 +0100 Subject: [PATCH 09/15] add SampleParallel queue test --- Tests/CheckTests.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Tests/CheckTests.cs b/Tests/CheckTests.cs index 8e036fc..483c3f1 100644 --- a/Tests/CheckTests.cs +++ b/Tests/CheckTests.cs @@ -260,6 +260,16 @@ public void SampleParallel_ConcurrentQueue() ); } + [Fact] + public void SampleParallelModel_ConcurrentQueue() + { + Gen.Const(() => (new ConcurrentQueue(), new Queue())) + .SampleParallel( + Gen.Int.Operation, Queue>(i => $"Enqueue({i})", (q, i) => q.Enqueue(i), (q, i) => q.Enqueue(i)), + Gen.Operation, Queue>("TryDequeue()", q => q.TryDequeue(out _), q => q.TryDequeue(out _)) + ); + } + [Fact] public void Equality() { From de12a55c5c3d21220b0ad767814063a8a90ff7cf Mon Sep 17 00:00:00 2001 From: Anthony Lloyd Date: Mon, 26 Aug 2024 08:16:59 +0100 Subject: [PATCH 10/15] add release notes --- CsCheck/CsCheck.csproj | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CsCheck/CsCheck.csproj b/CsCheck/CsCheck.csproj index f92806a..bab8c75 100644 --- a/CsCheck/CsCheck.csproj +++ b/CsCheck/CsCheck.csproj @@ -24,7 +24,14 @@ CsCheck also makes parallel, performance and regression testing simple and fast. quickcheck;random;model-based;metamorphic;parallel;performance;causal-profiling;regression;testing 4.0.0 -Update to .NET 8 +Added SampleParallel overloads using a sequencially run model for linearization. +Added running operations sequentially before parallel to SampleParallel. +Added maxSequentialOperations and maxParallelOperations parameters to SampleParallel. + +BREAKING CHANGES: +- Now targets net8.0, version 3.2.2 was the last to target net6.0. +- SampleConcurrent renamed to SampleParallel. +- GenOperation<Actual,Model> takes separate actions for actual and model. net8.0 preview From 2e187fbd1705f321b4126b75db73b48acbd50abe Mon Sep 17 00:00:00 2001 From: Anthony Lloyd Date: Mon, 26 Aug 2024 17:20:17 +0100 Subject: [PATCH 11/15] add sequencial to fail output --- CsCheck/Check.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CsCheck/Check.cs b/CsCheck/Check.cs index a6cd583..b7bc154 100644 --- a/CsCheck/Check.cs +++ b/CsCheck/Check.cs @@ -1973,10 +1973,11 @@ public static void SampleParallel(this Gen<(Actual, Model)> initi { if (spd == null) return ""; var sb = new StringBuilder(); - sb.Append("\n Operations: ").Append(Print(spd.ParallelOperations.Select(i => i.Item1).ToList())); - sb.Append("\n On Threads: ").Append(Print(spd.ThreadIds)); - sb.Append("\nInitial state: ").Append(printActual(initial.Generate(new PCG(spd.Stream, spd.Seed), null, out _).Item1)); - sb.Append("\n Final state: ").Append(spd.Exception is not null ? spd.Exception.ToString() : printActual(spd.InitialActual)); + sb.Append("\n Initial state: ").Append(printActual(initial.Generate(new PCG(spd.Stream, spd.Seed), null, out _).Item1)); + sb.Append("\nSequencial Operations: ").Append(Print(spd.SequencialOperations.Select(i => i.Item1).ToList())); + sb.Append("\n Parallel Operations: ").Append(Print(spd.ParallelOperations.Select(i => i.Item1).ToList())); + sb.Append("\n On Threads: ").Append(Print(spd.ThreadIds)); + sb.Append("\n Final state: ").Append(spd.Exception is not null ? spd.Exception.ToString() : printActual(spd.InitialActual)); var modelSequencialOperations = Array.ConvertAll(spd.SequencialOperations, i => (i.Item1, i.Item3)); var modelParallelOperations = Array.ConvertAll(spd.ParallelOperations, i => (i.Item1, i.Item3)); bool first = true; From 3346ffc1cfc2d7625521a5602960290822272d8f Mon Sep 17 00:00:00 2001 From: Anthony Lloyd Date: Mon, 26 Aug 2024 22:47:52 +0100 Subject: [PATCH 12/15] improve parallel Run and RunReplay --- CsCheck/Utils.cs | 95 ++++++++++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 40 deletions(-) diff --git a/CsCheck/Utils.cs b/CsCheck/Utils.cs index 0b575d0..2636991 100644 --- a/CsCheck/Utils.cs +++ b/CsCheck/Utils.cs @@ -21,6 +21,8 @@ namespace CsCheck; using System.Collections.Concurrent; using System.Diagnostics; using System.Runtime.InteropServices; +using System.Threading; +using System; public sealed class CsCheckException : Exception { @@ -475,69 +477,82 @@ public static bool ModelEqual(T actual, M model) return actual.Equals(model); } - internal static void Run(T initialState, (string, Action)[] sequencialOperations, (string, Action)[] parallelOperations, int threads, int[]? threadIds = null) + sealed class RunWorker(T state, (string, Action)[] parallelOperations, int[]? threadIds) : IThreadPoolWorkItem { - for (int i = 0; i < sequencialOperations.Length; i++) - sequencialOperations[i].Item2(initialState); - Exception? exception = null; - var opId = -1; - var runners = new Thread[threads]; - while (--threads >= 0) + int opId = -1, threadId = -1; + public volatile bool Hold = true; + public Exception? Exception; + public void Execute() { - runners[threads] = new Thread(threadId => + int i, tid = Interlocked.Increment(ref threadId); + while (Hold) { } + while ((i = Interlocked.Increment(ref opId)) < parallelOperations.Length) { - int i, tid = (int)threadId!; - while ((i = Interlocked.Increment(ref opId)) < parallelOperations.Length) + if (threadIds is not null) threadIds[i] = tid; + try { parallelOperations[i].Item2(state); } + catch (Exception e) { - if (threadIds is not null) threadIds[i] = tid; - try { parallelOperations[i].Item2(initialState); } - catch (Exception e) + if (Exception is null) { - if (exception is null) - { - exception = e; - Interlocked.Exchange(ref opId, parallelOperations.Length); - } + Exception = e; + opId = 1_000_000; } } - }); + } } - for (int i = 0; i < runners.Length; i++) runners[i].Start(i); - for (int i = 0; i < runners.Length; i++) runners[i].Join(); - if (exception is not null) throw exception; } - internal static void RunReplay(T initialState, (string, Action)[] sequencialOperations, (string, Action)[] parallelOperations, int threads, int[] threadIds) + internal static void Run(T state, (string, Action)[] sequencialOperations, (string, Action)[] parallelOperations, int threads, int[]? threadIds = null) { for (int i = 0; i < sequencialOperations.Length; i++) - sequencialOperations[i].Item2(initialState); - Exception? exception = null; + sequencialOperations[i].Item2(state); + var worker = new RunWorker(state, parallelOperations, threadIds); var runners = new Thread[threads]; - while (--threads >= 0) + for (int i = 0; i < runners.Length; i++) runners[i] = new Thread(worker.Execute); + for (int i = 0; i < runners.Length; i++) runners[i].Start(); + worker.Hold = false; + for (int i = 0; i < runners.Length; i++) runners[i].Join(); + if (worker.Exception is not null) throw worker.Exception; + } + + sealed class RunReplayWorker(T state, (string, Action)[] parallelOperations, int[] threadIds) : IThreadPoolWorkItem + { + int threadId = -1; + public volatile bool Hold = true; + public Exception? Exception; + public void Execute() { - runners[threads] = new Thread(threadId => + int i, opId = -1, tid = Interlocked.Increment(ref threadId); + while (Hold) { } + while ((i = Interlocked.Increment(ref opId)) < parallelOperations.Length) { - int opId = -1, i = -1, tid = (int)threadId!; - while ((i = Interlocked.Increment(ref opId)) < parallelOperations.Length) + if (threadIds[i] == tid) { - if (threadIds[i] == tid) + try { parallelOperations[i].Item2(state); } + catch (Exception e) { - try { parallelOperations[i].Item2(initialState); } - catch (Exception e) + if (Exception is null) { - if (exception is null) - { - exception = e; - Interlocked.Exchange(ref opId, parallelOperations.Length); - } + Exception = e; + opId = 1_000_000; } } } - }); + } } - for (int i = 0; i < runners.Length; i++) runners[i].Start(i); + } + + internal static void RunReplay(T state, (string, Action)[] sequencialOperations, (string, Action)[] parallelOperations, int threads, int[] threadIds) + { + for (int i = 0; i < sequencialOperations.Length; i++) + sequencialOperations[i].Item2(state); + var worker = new RunReplayWorker(state, parallelOperations, threadIds); + var runners = new Thread[threads]; + for (int i = 0; i < runners.Length; i++) runners[i] = new Thread(worker.Execute); + for (int i = 0; i < runners.Length; i++) runners[i].Start(); + worker.Hold = false; for (int i = 0; i < runners.Length; i++) runners[i].Join(); - if (exception is not null) throw exception; + if (worker.Exception is not null) throw worker.Exception; } internal static IEnumerable Permutations(int[] threadIds, T[] sequence) From f646dfbf508a0ba7fe7318879023e31c01d30b84 Mon Sep 17 00:00:00 2001 From: Anthony Lloyd Date: Tue, 27 Aug 2024 16:35:18 +0100 Subject: [PATCH 13/15] SampleParallel fail output format fix, fix other tests --- CsCheck/Check.cs | 13 +++++++------ CsCheck/Gen.cs | 3 ++- Tests/Allocator_Tests.cs | 2 +- Tests/CheckTests.cs | 10 ---------- Tests/SieveLruCacheTests.cs | 11 ----------- Tests/SlimCollectionsTests.cs | 8 ++++---- 6 files changed, 14 insertions(+), 33 deletions(-) diff --git a/CsCheck/Check.cs b/CsCheck/Check.cs index b7bc154..8a6d184 100644 --- a/CsCheck/Check.cs +++ b/CsCheck/Check.cs @@ -1707,10 +1707,11 @@ public static void SampleParallel(this Gen initial, GenOperation[] oper print ??= Print; if (spd == null) return ""; var sb = new StringBuilder(); - sb.Append("\n Operations: ").Append(Print(spd.ParallelOperations.Select(i => i.Item1).ToList())); - sb.Append("\n On Threads: ").Append(Print(spd.ThreadIds)); - sb.Append("\nInitial state: ").Append(print(initial.Generate(new PCG(spd.Stream, spd.Seed), null, out _))); - sb.Append("\n Final state: ").Append(spd.Exception is not null ? spd.Exception.ToString() : print(spd.InitialState)); + sb.Append("\n Initial state: ").Append(print(initial.Generate(new PCG(spd.Stream, spd.Seed), null, out _))); + sb.Append("\nSequencial Operations: ").Append(Print(spd.SequencialOperations.Select(i => i.Item1).ToList())); + sb.Append("\n Parallel Operations: ").Append(Print(spd.ParallelOperations.Select(i => i.Item1).ToList())); + sb.Append("\n On Threads: ").Append(Print(spd.ThreadIds)); + sb.Append("\n Final state: ").Append(spd.Exception is not null ? spd.Exception.ToString() : print(spd.InitialState)); bool first = true; foreach (var sequence in Permutations(spd.ThreadIds!, spd.ParallelOperations)) { @@ -1725,7 +1726,7 @@ public static void SampleParallel(this Gen initial, GenOperation[] oper { result = e.ToString(); } - sb.Append(first ? "\n Linearized: " : "\n : "); + sb.Append(first ? "\n Linearized: " : "\n : "); sb.Append(Print(sequence.Select(i => i.Item1).ToList())); sb.Append(" -> "); sb.Append(result); @@ -1994,7 +1995,7 @@ public static void SampleParallel(this Gen<(Actual, Model)> initi { result = e.ToString(); } - sb.Append(first ? "\n Linearized: " : "\n : "); + sb.Append(first ? "\n Linearized: " : "\n : "); sb.Append(Print(sequence.Select(i => i.Item1).ToList())); sb.Append(" -> "); sb.Append(result); diff --git a/CsCheck/Gen.cs b/CsCheck/Gen.cs index 16c062f..0dbf719 100644 --- a/CsCheck/Gen.cs +++ b/CsCheck/Gen.cs @@ -1939,7 +1939,8 @@ static Gen GenInt(float start, float finish) lower--; var rational = Gen.Int[lower + 1, denominator] .SelectMany(den => GenInt(start * den, finish * den) - .Select(num => (float)num / den)); + .Select(num => (float)num / den)) + .Where(r => r >= start && r <= finish); myGens[1] = (1, rational); } Gen? exponential = null; diff --git a/Tests/Allocator_Tests.cs b/Tests/Allocator_Tests.cs index c3a35ea..5d2d57c 100644 --- a/Tests/Allocator_Tests.cs +++ b/Tests/Allocator_Tests.cs @@ -51,7 +51,7 @@ public void Allocate_BalinskiYoung_Long_TotalsCorrectly() static bool BetweenFloorAndCeiling(long quantity, W[] weights, Func allocate) { - var sumWeights = weights.Sum(w => Convert.ToDouble(w)); + var sumWeights = weights.Select(w => Convert.ToDouble(w)).ToArray().FSum(compress: true); var allocations = allocate(quantity, weights); for (int i = 0; i < allocations.Length; i++) { diff --git a/Tests/CheckTests.cs b/Tests/CheckTests.cs index 483c3f1..a1635e2 100644 --- a/Tests/CheckTests.cs +++ b/Tests/CheckTests.cs @@ -227,16 +227,6 @@ public void SampleModelBased_ConcurrentBag() , threads: 1); } - [Fact] - public void SampleParallel_ConcurrentBag() - { - Gen.Int.List[0, 5].Select(l => new ConcurrentBag(l)) - .SampleParallel( - Gen.Int.Operation>(i => $"Add({i})", (bag, i) => bag.Add(i)), - Gen.Operation>("TryTake()", bag => bag.TryTake(out _)) - ); - } - [Fact] public void SampleParallel_ConcurrentDictionary() { diff --git a/Tests/SieveLruCacheTests.cs b/Tests/SieveLruCacheTests.cs index fb91323..eef9c4d 100644 --- a/Tests/SieveLruCacheTests.cs +++ b/Tests/SieveLruCacheTests.cs @@ -97,17 +97,6 @@ public void SampleModelBased() printModel: m => Check.Print(m.Keys) ); } - - [Fact] - public void SampleParallel() - { - Check.SampleParallel( - Gen.Const(() => new SieveLruCache(4)), - Gen.Int[1, 5].Operation>((d, i) => d.GetOrAdd(i, i => i)), - equal: (a, b) => Check.Equal(a.Keys, b.Keys), - print: a => Check.Print(a.Keys) - ); - } } internal static class SieveLruCacheExtensions diff --git a/Tests/SlimCollectionsTests.cs b/Tests/SlimCollectionsTests.cs index 30dabd8..2b9b641 100644 --- a/Tests/SlimCollectionsTests.cs +++ b/Tests/SlimCollectionsTests.cs @@ -26,10 +26,10 @@ public void ListSlim_Parallel() { Gen.Byte.Array.Select(a => new ListSlim(a)) .SampleParallel( - Gen.Byte.Operation>((l, i) => { lock (l) l.Add(i); }), - Gen.Int.NonNegative.Operation>((l, i) => { if (i < l.Count) { var _ = l[i]; } }), - Gen.Int.NonNegative.Select(Gen.Byte).Operation>((l, t) => { if (t.Item1 < l.Count) l[t.Item1] = t.Item2; }), - Gen.Operation>(l => l.ToArray()) + Gen.Byte.Operation>(i => $"Add {i}", (l, i) => { lock (l) l.Add(i); }), + Gen.Int.NonNegative.Operation>(i => $"Get {i}", (l, i) => { if (i < l.Count) { var _ = l[i]; } }), + Gen.Int.NonNegative.Select(Gen.Byte).Operation>(t => $"Set {t}", (l, t) => { if (t.Item1 < l.Count) l[t.Item1] = t.Item2; }), + Gen.Operation>("ToArray", l => l.ToArray()) ); } From c435b3bfece49fbedde43d3159675b7e6a0470dd Mon Sep 17 00:00:00 2001 From: Anthony Lloyd Date: Tue, 27 Aug 2024 21:29:38 +0100 Subject: [PATCH 14/15] add further tests --- Tests/CheckTests.cs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Tests/CheckTests.cs b/Tests/CheckTests.cs index a1635e2..369cbf9 100644 --- a/Tests/CheckTests.cs +++ b/Tests/CheckTests.cs @@ -260,6 +260,26 @@ public void SampleParallelModel_ConcurrentQueue() ); } + [Fact] + public void SampleParallelModel_ConcurrentStack() + { + Gen.Const(() => (new ConcurrentStack(), new Stack())) + .SampleParallel( + Gen.Int.Operation, Stack>(i => $"Push({i})", (q, i) => q.Push(i), (q, i) => q.Push(i)), + Gen.Operation, Stack>("TryPop()", q => q.TryPop(out _), q => q.TryPop(out _)) + ); + } + + [Fact] + public void SampleParallelModel_ConcurrentDictionary() + { + Gen.Const(() => (new ConcurrentDictionary(), new Dictionary())) + .SampleParallel( + Gen.Int[1, 5].Operation, Dictionary>(i => $"Set ({i})", (q, i) => q[i] = i, (q, i) => q[i] = i), + Gen.Int[1, 5].Operation, Dictionary>(i => $"TryRemove ({i})", (q, i) => q.TryRemove(i, out _), (q, i) => q.Remove(i)) + ); + } + [Fact] public void Equality() { From cf5d22c6349b74fe8d5c96ef71d36911dac68fcb Mon Sep 17 00:00:00 2001 From: Anthony Lloyd Date: Tue, 27 Aug 2024 21:47:30 +0100 Subject: [PATCH 15/15] skip Example05 --- Tests/AllocatorMany_Tests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/AllocatorMany_Tests.cs b/Tests/AllocatorMany_Tests.cs index 4d2934d..89aa6f3 100644 --- a/Tests/AllocatorMany_Tests.cs +++ b/Tests/AllocatorMany_Tests.cs @@ -154,7 +154,7 @@ public void Example04() ], actual.Solution); } - [Fact] + [Fact(Skip ="Takes too long")] public void Example05() { var actual = AllocatorMany.Allocate(