diff --git a/Comparison.md b/Comparison.md index f1c2dc1..5afdf3f 100644 --- a/Comparison.md +++ b/Comparison.md @@ -87,7 +87,7 @@ Size is also a better representation of comparison especially for collections or There are examples where increasing on one axis while decreasing on others can lead to smaller cases e.g. if Version fails for `2 * ma + mi + bu ≥ 255 * 2` CsCheck will be able to shrink to `255.0.0` but [Hedgehog](https://github.com/hedgehogqa) won't. -For concurrency testing random shrinkers also has an advantage. Concurrency tests may not fail deterministically. +For parallel testing random shrinkers also has an advantage. Parallel tests may not fail deterministically. This is a real problem for path explorer shrinkers. The only solution is to repeat each test multiple times (10 for QuickCheck) since they need to follow defined paths. For a random shrinker you can just continue testing different random cases until one fails and limit the size to that each time. diff --git a/CsCheck/Check.cs b/CsCheck/Check.cs index 5c9df9d..8a6d184 100644 --- a/CsCheck/Check.cs +++ b/CsCheck/Check.cs @@ -25,7 +25,7 @@ public static partial class Check public static long Iter = ParseEnvironmentVariableToLong("CsCheck_Iter", 100); /// The number of seconds to run the sample. public static int Time = ParseEnvironmentVariableToInt("CsCheck_Time" , -1); - /// The number of times to retry the seed to reproduce a SampleConcurrent fail (default 100). + /// The number of times to retry the seed to reproduce a SampleParallel fail (default 100). public static int Replay = ParseEnvironmentVariableToInt("CsCheck_Replay", 100); /// The number of threads to run the sample on (default number logical CPUs). public static int Threads = ParseEnvironmentVariableToInt("CsCheck_Threads", Environment.ProcessorCount); @@ -1277,9 +1277,9 @@ public static Task SampleAsync(this Gen<(T1, T2, Action? writeLine = null, string? seed = null, long iter = -1, int time = -1, int threads = -1, Func<(T1, T2, T3, T4, T5, T6, T7, T8), string>? print = null) => SampleAsync(gen, t => predicate(t.Item1, t.Item2, t.Item3, t.Item4, t.Item5, t.Item6, t.Item7, t.Item8), writeLine, seed, iter, time, threads, print); - sealed class ModelBasedData(Actual actualState, Model modelState, uint stream, ulong seed, (string, Action)[] operations) + sealed class ModelBasedData(Actual actualState, Model modelState, uint stream, ulong seed, (string, Action, Action)[] operations) { - public Actual ActualState = actualState; public Model ModelState = modelState; public uint Stream = stream; public ulong Seed = seed; public (string, Action)[] Operations = operations; public Exception? Exception; + public Actual ActualState = actualState; public Model ModelState = modelState; public uint Stream = stream; public ulong Seed = seed; public (string, Action, Action)[] Operations = operations; public Exception? Exception; } sealed class GenInitial(Gen<(Actual, Model)> initial) : Gen<(Actual Actual, Model Model, uint Stream, ulong Seed)> @@ -1316,12 +1316,12 @@ public static void SampleModelBased(this Gen<(Actual, Model)> ini printActual ??= Print; printModel ??= Print; - var opNameActions = new Gen<(string, Action)>[operations.Length]; + var opNameActions = new Gen<(string, Action, Action)>[operations.Length]; for (int i = 0; i < operations.Length; i++) { var op = operations[i]; var opName = "Op" + i; - opNameActions[i] = op.AddOpNumber ? op.Select(t => (opName + t.Item1, t.Item2)) : op; + opNameActions[i] = op.AddOpNumber ? op.Select(t => (opName + t.Item1, t.Item2, t.Item3)) : op; } new GenInitial(initial) @@ -1331,7 +1331,10 @@ public static void SampleModelBased(this Gen<(Actual, Model)> ini try { foreach (var operation in d.Operations) - operation.Item2(d.ActualState, d.ModelState); + { + operation.Item2(d.ActualState); + operation.Item3(d.ModelState); + } return equal(d.ActualState, d.ModelState); } catch (Exception e) @@ -1567,14 +1570,32 @@ public static void SampleMetamorphic(this Gen initial, GenMetamorphic o }); } - sealed class ConcurrentData(T state, uint stream, ulong seed, (string, Action)[] operations, int threads) + sealed class SampleParallelData(T state, uint stream, ulong seed, (string, Action)[] sequencialOperations, (string, Action)[] parallelOperations, int threads) { - public T State = state; public uint Stream = stream; public ulong Seed = seed; public (string, Action)[] Operations = operations; public int Threads = threads; public int[]? ThreadIds; public Exception? Exception; + public T InitialState = state; + public uint Stream = stream; + public ulong Seed = seed; + public (string, Action)[] SequencialOperations = sequencialOperations; + public (string, Action)[] ParallelOperations = parallelOperations; + public int Threads = threads; + public int[]? ThreadIds; + public Exception? Exception; } - internal const int MAX_CONCURRENT_OPERATIONS = 10; + sealed class SampleParallelData(Actual actual, Model model, uint stream, ulong seed, (string, Action, Action)[] sequencialOperations, (string, Action, Action)[] parallelOperations, int threads) + { + public Actual InitialActual = actual; + public Model InitialModel = model; + public uint Stream = stream; + public ulong Seed = seed; + public (string, Action, Action)[] SequencialOperations = sequencialOperations; + public (string, Action, Action)[] ParallelOperations = parallelOperations; + public int Threads = threads; + public int[]? ThreadIds; + public Exception? Exception; + } - sealed class GenConcurrent(Gen initial) : Gen<(T Value, uint Stream, ulong Seed)> + sealed class GenSampleParallel(Gen initial) : Gen<(T Value, uint Stream, ulong Seed)> { public override (T, uint, ulong) Generate(PCG pcg, Size? min, out Size size) { @@ -1584,22 +1605,35 @@ public override (T, uint, ulong) Generate(PCG pcg, Size? min, out Size size) } } - /// Sample model-based operations on a random initial state concurrently. + sealed class GenSampleParallel(Gen<(Actual, Model)> initial) : Gen<(Actual Actual, Model Model, uint Stream, ulong Seed)> + { + public override (Actual, Model, uint, ulong) Generate(PCG pcg, Size? min, out Size size) + { + var stream = pcg.Stream; + var seed = pcg.Seed; + var (actual, model) = initial.Generate(pcg, null, out size); + return (actual, model, stream, seed); + } + } + + /// Sample operations on a random initial state in parallel. /// The result is compared against the result of the possible sequential permutations. - /// At least one of these permutations result must be equal for the concurrency to have been linearized successfully. + /// At least one of these permutations result must be equal for the parallel execution to have been linearized successfully. /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. /// The initial state generator. - /// The operation generators that can act on the state concurrently. + /// The operation generators that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). /// A function to convert the state to a string for error reporting (default Check.Print). /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. - public static void SampleConcurrent(this Gen initial, GenOperation[] operations, Func? equal = null, string? seed = null, - long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) + public static void SampleParallel(this Gen initial, GenOperation[] operations, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) { equal ??= Equal; seed ??= Seed; @@ -1626,12 +1660,14 @@ public static void SampleConcurrent(this Gen initial, GenOperation[] op bool firstIteration = true; - new GenConcurrent(initial) - .Select(Gen.OneOf(opNameActions).Array[1, MAX_CONCURRENT_OPERATIONS] - .SelectMany(ops => Gen.Int[1, Math.Min(threads, ops.Length)].Select(i => (ops, i))), (a, b) => - new ConcurrentData(a.Value, a.Stream, a.Seed, b.ops, b.i) - ) - .Sample(cd => + var genOps = Gen.OneOf(opNameActions); + Gen.Int[2, maxParallelOperations] + .SelectMany(np => Gen.Int[2, Math.Min(threads, np)].Select(nt => (nt, np))) + .SelectMany((nt, np) => Gen.Int[0, maxSequentialOperations].Select(ns => (ns, nt, np))) + .SelectMany((ns, nt, np) => new GenSampleParallel(initial).Select(genOps.Array[ns], genOps.Array[np]) + .Select((initial, sequencial, parallel) => (initial, sequencial, nt, parallel))) + .Select((initial, sequencial, threads, parallel) => new SampleParallelData(initial.Value, initial.Stream, initial.Seed, sequencial, parallel, threads)) + .Sample(spd => { bool linearizable = false; do @@ -1639,22 +1675,22 @@ public static void SampleConcurrent(this Gen initial, GenOperation[] op try { if (replayThreads is null) - Run(cd.State, cd.Operations, cd.Threads, cd.ThreadIds = new int[cd.Operations.Length]); + Run(spd.InitialState, spd.SequencialOperations, spd.ParallelOperations, spd.Threads, spd.ThreadIds = new int[spd.ParallelOperations.Length]); else - RunReplay(cd.State, cd.Operations, cd.Threads, cd.ThreadIds = replayThreads); + RunReplay(spd.InitialState, spd.SequencialOperations, spd.ParallelOperations, spd.Threads, spd.ThreadIds = replayThreads); } catch (Exception e) { - cd.Exception = e; + spd.Exception = e; break; } - Parallel.ForEach(Permutations(cd.ThreadIds, cd.Operations), (sequence, state) => + Parallel.ForEach(Permutations(spd.ThreadIds, spd.ParallelOperations), (sequence, state) => { - var linearState = initial.Generate(new PCG(cd.Stream, cd.Seed), null, out _); + var linearState = initial.Generate(new PCG(spd.Stream, spd.Seed), null, out _); try { - Run(linearState, sequence, 1); - if (equal(cd.State, linearState)) + Run(linearState, spd.SequencialOperations, sequence, 1); + if (equal(spd.InitialState, linearState)) { linearizable = true; state.Stop(); @@ -1666,30 +1702,31 @@ public static void SampleConcurrent(this Gen initial, GenOperation[] op firstIteration = false; return linearizable; }, writeLine, seed, iter, time, threads: 1, - p => + spd => { print ??= Print; - if (p == null) return ""; + if (spd == null) return ""; var sb = new StringBuilder(); - sb.Append("\n Operations: ").Append(Print(p.Operations.Select(i => i.Item1).ToList())); - sb.Append("\n On Threads: ").Append(Print(p.ThreadIds)); - sb.Append("\nInitial state: ").Append(print(initial.Generate(new PCG(p.Stream, p.Seed), null, out _))); - sb.Append("\n Final state: ").Append(p.Exception is not null ? p.Exception.ToString() : print(p.State)); + sb.Append("\n Initial state: ").Append(print(initial.Generate(new PCG(spd.Stream, spd.Seed), null, out _))); + sb.Append("\nSequencial Operations: ").Append(Print(spd.SequencialOperations.Select(i => i.Item1).ToList())); + sb.Append("\n Parallel Operations: ").Append(Print(spd.ParallelOperations.Select(i => i.Item1).ToList())); + sb.Append("\n On Threads: ").Append(Print(spd.ThreadIds)); + sb.Append("\n Final state: ").Append(spd.Exception is not null ? spd.Exception.ToString() : print(spd.InitialState)); bool first = true; - foreach (var sequence in Permutations(p.ThreadIds!, p.Operations)) + foreach (var sequence in Permutations(spd.ThreadIds!, spd.ParallelOperations)) { - var linearState = initial.Generate(new PCG(p.Stream, p.Seed), null, out _); + var linearState = initial.Generate(new PCG(spd.Stream, spd.Seed), null, out _); string result; try { - Run(linearState, sequence, 1); + Run(linearState, spd.SequencialOperations, sequence, 1); result = print(linearState); } catch (Exception e) { result = e.ToString(); } - sb.Append(first ? "\n Linearized: " : "\n : "); + sb.Append(first ? "\n Linearized: " : "\n : "); sb.Append(Print(sequence.Select(i => i.Item1).ToList())); sb.Append(" -> "); sb.Append(result); @@ -1699,14 +1736,16 @@ public static void SampleConcurrent(this Gen initial, GenOperation[] op }); } - /// Sample model-based operations on a random initial state concurrently. + /// Sample operations on a random initial state in parallel. /// The result is compared against the result of the possible sequential permutations. - /// At least one of these permutations result must be equal for the concurrency to have been linearized successfully. + /// At least one of these permutations result must be equal for the parallel execution to have been linearized successfully. /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. /// The initial state generator. - /// An operation generator that can act on the state concurrently. + /// An operation generator that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). @@ -1714,19 +1753,21 @@ public static void SampleConcurrent(this Gen initial, GenOperation[] op /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SampleConcurrent(this Gen initial, GenOperation operation, Func? equal = null, string? seed = null, - long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) - => SampleConcurrent(initial, [operation], equal, seed, iter, time, threads, print, replay, writeLine); + public static void SampleParallel(this Gen initial, GenOperation operation, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, print, replay, writeLine); - /// Sample model-based operations on a random initial state concurrently. + /// Sample operations on a random initial state in parallel. /// The result is compared against the result of the possible sequential permutations. - /// At least one of these permutations result must be equal for the concurrency to have been linearized successfully. + /// At least one of these permutations result must be equal for the parallel execution to have been linearized successfully. /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. /// The initial state generator. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). @@ -1734,20 +1775,22 @@ public static void SampleConcurrent(this Gen initial, GenOperation oper /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SampleConcurrent(this Gen initial, GenOperation operation1, GenOperation operation2, Func? equal = null, - string? seed = null, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) - => SampleConcurrent(initial, [operation1, operation2], equal, seed, iter, time, threads, print, replay, writeLine); + public static void SampleParallel(this Gen initial, GenOperation operation1, GenOperation operation2, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation1, operation2], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, print, replay, writeLine); - /// Sample model-based operations on a random initial state concurrently. + /// Sample operations on a random initial state in parallel. /// The result is compared against the result of the possible sequential permutations. - /// At least one of these permutations result must be equal for the concurrency to have been linearized successfully. + /// At least one of these permutations result must be equal for the parallel execution to have been linearized successfully. /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. /// The initial state generator. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). @@ -1755,21 +1798,23 @@ public static void SampleConcurrent(this Gen initial, GenOperation oper /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SampleConcurrent(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, Func? equal = null, - string? seed = null, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) - => SampleConcurrent(initial, [operation1, operation2, operation3], equal, seed, iter, time, threads, print, replay, writeLine); + public static void SampleParallel(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation1, operation2, operation3], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, print, replay, writeLine); - /// Sample model-based operations on a random initial state concurrently. + /// Sample operations on a random initial state in parallel. /// The result is compared against the result of the possible sequential permutations. - /// At least one of these permutations result must be equal for the concurrency to have been linearized successfully. + /// At least one of these permutations result must be equal for the parallel execution to have been linearized successfully. /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. /// The initial state generator. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). @@ -1777,23 +1822,24 @@ public static void SampleConcurrent(this Gen initial, GenOperation oper /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SampleConcurrent(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, - Func? equal = null, string? seed = null, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, - Action? writeLine = null) - => SampleConcurrent(initial, [operation1, operation2, operation3, operation4], equal, seed, iter, time, threads, print, replay, writeLine); + public static void SampleParallel(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation1, operation2, operation3, operation4], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, print, replay, writeLine); - /// Sample model-based operations on a random initial state concurrently. + /// Sample operations on a random initial state in parallel. /// The result is compared against the result of the possible sequential permutations. - /// At least one of these permutations result must be equal for the concurrency to have been linearized successfully. + /// At least one of these permutations result must be equal for the parallel execution to have been linearized successfully. /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. /// The initial state generator. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). @@ -1801,25 +1847,26 @@ public static void SampleConcurrent(this Gen initial, GenOperation oper /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SampleConcurrent(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, - GenOperation operation5, Func? equal = null, string? seed = null, long iter = -1, int time = -1, int threads = -1, Func? print = null, + public static void SampleParallel(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, GenOperation operation5, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) - => SampleConcurrent(initial, [operation1, operation2, operation3, operation4, operation5], - equal, seed, iter, time, threads, print, replay, writeLine); + => SampleParallel(initial, [operation1, operation2, operation3, operation4, operation5], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, print, replay, writeLine); - /// Sample model-based operations on a random initial state concurrently. + /// Sample operations on a random initial state in parallel. /// The result is compared against the result of the possible sequential permutations. - /// At least one of these permutations result must be equal for the concurrency to have been linearized successfully. + /// At least one of these permutations result must be equal for the parallel execution to have been linearized successfully. /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. /// The initial state generator. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. - /// An operation generator that can act on the state concurrently. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. + /// An operation generator that can act on the state in parallel. /// A function to check if the two states are the same (default Check.Equal). /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). /// The number of iterations to run in the sample (default 100). /// The number of seconds to run the sample. /// The number of threads to run the sample on (default number logical CPUs). @@ -1827,11 +1874,277 @@ public static void SampleConcurrent(this Gen initial, GenOperation oper /// The number of times to retry the seed to reproduce an initial fail (default 100). /// WriteLine function to use for the summary total iterations output. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SampleConcurrent(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, - GenOperation operation5, GenOperation operation6, Func? equal = null, string? seed = null, long iter = -1, int time = -1, int threads = -1, - Func? print = null, int replay = -1, Action? writeLine = null) - => SampleConcurrent(initial, [operation1, operation2, operation3, operation4, operation5, operation6], - equal, seed, iter, time, threads, print, replay, writeLine); + public static void SampleParallel(this Gen initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, GenOperation operation5, GenOperation operation6, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? print = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation1, operation2, operation3, operation4, operation5, operation6], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, print, replay, writeLine); + + /// Sample operations on the random initial actual state in parallel and compare to all the possible linearized operations run sequencially on the initial model state. + /// At least one of these permutations model result must be equal for the parallel execution to have been linearized successfully. + /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. + /// The initial actual and model state generator. + /// The actual and model operation generators that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// A function to check if the actual and model are the same (default Check.ModelEqual). + /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). + /// The number of iterations to run in the sample (default 100). + /// The number of seconds to run the sample. + /// The number of threads to run the sample on (default number logical CPUs). + /// A function to convert the actual state to a string for error reporting (default Check.Print). + /// A function to convert the model state to a string for error reporting (default Check.Print). + /// The number of times to retry the seed to reproduce an initial fail (default 100). + /// WriteLine function to use for the summary total iterations output. + public static void SampleParallel(this Gen<(Actual, Model)> initial, GenOperation[] operations, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? printActual = null, Func? printModel = null, int replay = -1, Action? writeLine = null) + { + equal ??= ModelEqual; + seed ??= Seed; + if (iter == -1) iter = Iter; + if (time == -1) time = Time; + if (threads == -1) threads = Threads; + if (replay == -1) replay = Replay; + int[]? replayThreads = null; + printActual ??= Print; + printModel ??= Print; + if (seed?.Contains('[') == true) + { + int i = seed.IndexOf('['); + int j = seed.IndexOf(']', i + 1); + replayThreads = seed.Substring(i + 1, j - i - 1).Split(',').Select(int.Parse).ToArray(); + seed = seed[..i]; + } + + var opNameActions = new Gen<(string, Action, Action)>[operations.Length]; + for (int i = 0; i < operations.Length; i++) + { + var op = operations[i]; + var opName = "Op" + i; + opNameActions[i] = op.AddOpNumber ? op.Select((name, actual, model) => (opName + name, actual, model)) : op; + } + + bool firstIteration = true; + + var genOps = Gen.OneOf(opNameActions); + Gen.Int[2, maxParallelOperations] + .SelectMany(np => Gen.Int[2, Math.Min(threads, np)].Select(nt => (nt, np))) + .SelectMany((nt, np) => Gen.Int[0, maxSequentialOperations].Select(ns => (ns, nt, np))) + .SelectMany((ns, nt, np) => new GenSampleParallel(initial).Select(genOps.Array[ns], genOps.Array[np]) + .Select((initial, sequencial, parallel) => (initial, sequencial, nt, parallel))) + .Select((initial, sequencial, threads, parallel) => new SampleParallelData(initial.Actual, initial.Model, initial.Stream, initial.Seed, sequencial, parallel, threads)) + .Sample(spd => + { + bool linearizable = false; + do + { + var actualSequencialOperations = Array.ConvertAll(spd.SequencialOperations, i => (i.Item1, i.Item2)); + var actualParallelOperations = Array.ConvertAll(spd.ParallelOperations, i => (i.Item1, i.Item2)); + try + { + if (replayThreads is null) + Run(spd.InitialActual, actualSequencialOperations, actualParallelOperations, spd.Threads, spd.ThreadIds = new int[spd.ParallelOperations.Length]); + else + RunReplay(spd.InitialActual, actualSequencialOperations, actualParallelOperations, spd.Threads, spd.ThreadIds = replayThreads); + } + catch (Exception e) + { + spd.Exception = e; + break; + } + var modelSequencialOperations = Array.ConvertAll(spd.SequencialOperations, i => (i.Item1, i.Item3)); + var modelParallelOperations = Array.ConvertAll(spd.ParallelOperations, i => (i.Item1, i.Item3)); + Parallel.ForEach(Permutations(spd.ThreadIds, modelParallelOperations), (sequence, state) => + { + var (_, initialModel) = initial.Generate(new PCG(spd.Stream, spd.Seed), null, out _); + try + { + Run(initialModel, modelSequencialOperations, sequence, 1); + if (equal(spd.InitialActual, initialModel)) + { + linearizable = true; + state.Stop(); + } + } + catch { state.Stop(); } + }); + } while (linearizable && firstIteration && seed is not null && --replay > 0); + firstIteration = false; + return linearizable; + }, writeLine, seed, iter, time, threads: 1, + spd => + { + if (spd == null) return ""; + var sb = new StringBuilder(); + sb.Append("\n Initial state: ").Append(printActual(initial.Generate(new PCG(spd.Stream, spd.Seed), null, out _).Item1)); + sb.Append("\nSequencial Operations: ").Append(Print(spd.SequencialOperations.Select(i => i.Item1).ToList())); + sb.Append("\n Parallel Operations: ").Append(Print(spd.ParallelOperations.Select(i => i.Item1).ToList())); + sb.Append("\n On Threads: ").Append(Print(spd.ThreadIds)); + sb.Append("\n Final state: ").Append(spd.Exception is not null ? spd.Exception.ToString() : printActual(spd.InitialActual)); + var modelSequencialOperations = Array.ConvertAll(spd.SequencialOperations, i => (i.Item1, i.Item3)); + var modelParallelOperations = Array.ConvertAll(spd.ParallelOperations, i => (i.Item1, i.Item3)); + bool first = true; + foreach (var sequence in Permutations(spd.ThreadIds!, modelParallelOperations)) + { + var (_, initialModel) = initial.Generate(new PCG(spd.Stream, spd.Seed), null, out _); + string result; + try + { + Run(initialModel, modelSequencialOperations, sequence, 1); + result = printModel(initialModel); + } + catch (Exception e) + { + result = e.ToString(); + } + sb.Append(first ? "\n Linearized: " : "\n : "); + sb.Append(Print(sequence.Select(i => i.Item1).ToList())); + sb.Append(" -> "); + sb.Append(result); + first = false; + } + return sb.ToString(); + }); + } + + /// Sample operations on the random initial actual state in parallel and compare to all the possible linearized operations run sequencially on the initial model state. + /// At least one of these permutations model result must be equal for the parallel execution to have been linearized successfully. + /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. + /// The initial actual and model state generator. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// A function to check if the actual and model are the same (default Check.ModelEqual). + /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). + /// The number of iterations to run in the sample (default 100). + /// The number of seconds to run the sample. + /// The number of threads to run the sample on (default number logical CPUs). + /// A function to convert the actual state to a string for error reporting (default Check.Print). + /// A function to convert the model state to a string for error reporting (default Check.Print). + /// The number of times to retry the seed to reproduce an initial fail (default 100). + /// WriteLine function to use for the summary total iterations output. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SampleParallel(this Gen<(Actual, Model)> initial, GenOperation operation, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? printActual = null, Func? printModel = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, printActual, printModel, replay, writeLine); + + /// Sample operations on the random initial actual state in parallel and compare to all the possible linearized operations run sequencially on the initial model state. + /// At least one of these permutations model result must be equal for the parallel execution to have been linearized successfully. + /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. + /// The initial actual and model state generator. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// A function to check if the actual and model are the same (default Check.ModelEqual). + /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). + /// The number of iterations to run in the sample (default 100). + /// The number of seconds to run the sample. + /// The number of threads to run the sample on (default number logical CPUs). + /// A function to convert the actual state to a string for error reporting (default Check.Print). + /// A function to convert the model state to a string for error reporting (default Check.Print). + /// The number of times to retry the seed to reproduce an initial fail (default 100). + /// WriteLine function to use for the summary total iterations output. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SampleParallel(this Gen<(Actual, Model)> initial, GenOperation operation1, GenOperation operation2, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? printActual = null, Func? printModel = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation1, operation2], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, printActual, printModel, replay, writeLine); + + /// Sample operations on the random initial actual state in parallel and compare to all the possible linearized operations run sequencially on the initial model state. + /// At least one of these permutations model result must be equal for the parallel execution to have been linearized successfully. + /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. + /// The initial actual and model state generator. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// A function to check if the actual and model are the same (default Check.ModelEqual). + /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). + /// The number of iterations to run in the sample (default 100). + /// The number of seconds to run the sample. + /// The number of threads to run the sample on (default number logical CPUs). + /// A function to convert the actual state to a string for error reporting (default Check.Print). + /// A function to convert the model state to a string for error reporting (default Check.Print). + /// The number of times to retry the seed to reproduce an initial fail (default 100). + /// WriteLine function to use for the summary total iterations output. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SampleParallel(this Gen<(Actual, Model)> initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? printActual = null, Func? printModel = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation1, operation2, operation3], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, printActual, printModel, replay, writeLine); + + /// Sample operations on the random initial actual state in parallel and compare to all the possible linearized operations run sequencially on the initial model state. + /// At least one of these permutations model result must be equal for the parallel execution to have been linearized successfully. + /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. + /// The initial actual and model state generator. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// A function to check if the actual and model are the same (default Check.ModelEqual). + /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). + /// The number of iterations to run in the sample (default 100). + /// The number of seconds to run the sample. + /// The number of threads to run the sample on (default number logical CPUs). + /// A function to convert the actual state to a string for error reporting (default Check.Print). + /// A function to convert the model state to a string for error reporting (default Check.Print). + /// The number of times to retry the seed to reproduce an initial fail (default 100). + /// WriteLine function to use for the summary total iterations output. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SampleParallel(this Gen<(Actual, Model)> initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? printActual = null, Func? printModel = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation1, operation2, operation3, operation4], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, printActual, printModel, replay, writeLine); + + /// Sample operations on the random initial actual state in parallel and compare to all the possible linearized operations run sequencially on the initial model state. + /// At least one of these permutations model result must be equal for the parallel execution to have been linearized successfully. + /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. + /// The initial actual and model state generator. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// A function to check if the actual and model are the same (default Check.ModelEqual). + /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). + /// The number of iterations to run in the sample (default 100). + /// The number of seconds to run the sample. + /// The number of threads to run the sample on (default number logical CPUs). + /// A function to convert the actual state to a string for error reporting (default Check.Print). + /// A function to convert the model state to a string for error reporting (default Check.Print). + /// The number of times to retry the seed to reproduce an initial fail (default 100). + /// WriteLine function to use for the summary total iterations output. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SampleParallel(this Gen<(Actual, Model)> initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, GenOperation operation5, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? printActual = null, Func? printModel = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation1, operation2, operation3, operation4, operation5], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, printActual, printModel, replay, writeLine); + + /// Sample operations on the random initial actual state in parallel and compare to all the possible linearized operations run sequencially on the initial model state. + /// At least one of these permutations model result must be equal for the parallel execution to have been linearized successfully. + /// If not the failing initial state and sequence will be shrunk down to the shortest and simplest. + /// The initial actual and model state generator. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// An actual and model operation generator that can act on the state in parallel. There is no need for the model operations to be thread safe as they are only run sequencially. + /// A function to check if the actual and model are the same (default Check.ModelEqual). + /// The initial seed to use for the first iteration. + /// The maximum number of operations to run sequentially before the parallel operations (default of 10). + /// The maximum number of operations to run in parallel (default of 5). + /// The number of iterations to run in the sample (default 100). + /// The number of seconds to run the sample. + /// The number of threads to run the sample on (default number logical CPUs). + /// A function to convert the actual state to a string for error reporting (default Check.Print). + /// A function to convert the model state to a string for error reporting (default Check.Print). + /// The number of times to retry the seed to reproduce an initial fail (default 100). + /// WriteLine function to use for the summary total iterations output. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SampleParallel(this Gen<(Actual, Model)> initial, GenOperation operation1, GenOperation operation2, GenOperation operation3, GenOperation operation4, GenOperation operation5, GenOperation operation6, Func? equal = null, string? seed = null, + int maxSequentialOperations = 10, int maxParallelOperations = 5, long iter = -1, int time = -1, int threads = -1, Func? printActual = null, Func? printModel = null, int replay = -1, Action? writeLine = null) + => SampleParallel(initial, [operation1, operation2, operation3, operation4, operation5, operation6], equal, seed, maxSequentialOperations, maxParallelOperations, iter, time, threads, printActual, printModel, replay, writeLine); /// Assert actual is in line with expected using a chi-squared test to sigma. /// The expected bin counts. diff --git a/CsCheck/CsCheck.csproj b/CsCheck/CsCheck.csproj index c407e38..bab8c75 100644 --- a/CsCheck/CsCheck.csproj +++ b/CsCheck/CsCheck.csproj @@ -11,9 +11,9 @@ This gives the following advantages: - Random testing and shrinking are parallelized. This and PCG make it very fast. - Shrunk cases have a seed value. Simpler examples can easily be reproduced. - Shrinking can be continued later to give simpler cases for high dimensional problems. -- Concurrency testing and random shrinking work well together. +- Parallel testing and random shrinking work well together. -CsCheck also makes concurrency, performance and regression testing simple and fast. +CsCheck also makes parallel, performance and regression testing simple and fast. Anthony Lloyd Anthony Lloyd @@ -21,12 +21,19 @@ CsCheck also makes concurrency, performance and regression testing simple and fa Apache-2.0 http://github.com/AnthonyLloyd/CsCheck CsCheck.png - quickcheck;random;model-based;metamorphic;concurrency;performance;causal-profiling;regression;testing - 3.2.2 + quickcheck;random;model-based;metamorphic;parallel;performance;causal-profiling;regression;testing + 4.0.0 -Added option to Dbg.Time stats for completed and running. +Added SampleParallel overloads using a sequencially run model for linearization. +Added running operations sequentially before parallel to SampleParallel. +Added maxSequentialOperations and maxParallelOperations parameters to SampleParallel. + +BREAKING CHANGES: +- Now targets net8.0, version 3.2.2 was the last to target net6.0. +- SampleConcurrent renamed to SampleParallel. +- GenOperation<Actual,Model> takes separate actions for actual and model. - net6.0 + net8.0 preview true 9999 @@ -41,13 +48,13 @@ Added option to Dbg.Time stats for completed and running. true true snupkg - CS1591 + CS1591,MA0143 README.md All - + diff --git a/CsCheck/Gen.cs b/CsCheck/Gen.cs index 703edb4..0dbf719 100644 --- a/CsCheck/Gen.cs +++ b/CsCheck/Gen.cs @@ -87,10 +87,10 @@ public abstract class Gen : IGen public GenOperation Operation(Func name, Func async) => GenOperation.Create(this, name, (S s, T t) => async(s, t).GetAwaiter().GetResult()); public GenOperation Operation(Action action) => GenOperation.Create(this, action); public GenOperation Operation(Func async) => GenOperation.Create(this, (S s, T t) => async(s, t).GetAwaiter().GetResult()); - public GenOperation Operation(Func name, Action action) => GenOperation.Create(this, name, action); - public GenOperation Operation(Action action) => GenOperation.Create(this, action); - public GenMetamorphic Metamorphic(Func name, Action action1, Action action2) => GenOperation.Create(this, name, action1, action2); - public GenMetamorphic Metamorphic(Action action1, Action action2) => GenOperation.Create(this, Check.Print, action1, action2); + public GenOperation Operation(Func name, Action actual, Action model) => GenOperation.Create(this, name, actual, model); + public GenOperation Operation(Action actual, Action model) => GenOperation.Create(this, actual, model); + public GenMetamorphic Metamorphic(Func name, Action action1, Action action2) => GenMetamorphic.Create(this, name, action1, action2); + public GenMetamorphic Metamorphic(Action action1, Action action2) => GenMetamorphic.Create(this, Check.Print, action1, action2); /// Generator for an array of public GenArray Array => new(this); @@ -1456,8 +1456,8 @@ sealed class GenNull(Gen gen, uint nullLimit) : Gen where T : class public static GenOperation Operation(string name, Func async) => GenOperation.Create(name, (T t) => async(t).GetAwaiter().GetResult()); public static GenOperation Operation(Action action) => GenOperation.Create(action); public static GenOperation Operation(Func async) => GenOperation.Create((T t) => async(t).GetAwaiter().GetResult()); - public static GenOperation Operation(string name, Action action) => GenOperation.Create(name, action); - public static GenOperation Operation(Action action) => GenOperation.Create(action); + public static GenOperation Operation(string name, Action actual, Action model) => GenOperation.Create(name, actual, model); + public static GenOperation Operation(Action actual, Action model) => GenOperation.Create(actual, model); /// Generator for bool. public static readonly GenBool Bool = new(); @@ -1939,7 +1939,8 @@ static Gen GenInt(float start, float finish) lower--; var rational = Gen.Int[lower + 1, denominator] .SelectMany(den => GenInt(start * den, finish * den) - .Select(num => (float)num / den)); + .Select(num => (float)num / den)) + .Where(r => r >= start && r <= finish); myGens[1] = (1, rational); } Gen? exponential = null; @@ -2851,16 +2852,16 @@ internal GenOperation(Gen<(string, Action)> gen, bool addOpNumber) public override (string, Action) Generate(PCG pcg, Size? min, out Size size) => gen.Generate(pcg, min, out size); } -public sealed class GenOperation : Gen<(string, Action)> +public sealed class GenOperation : Gen<(string, Action, Action)> { + readonly Gen<(string, Action, Action)> gen; public bool AddOpNumber; - readonly Gen<(string, Action)> gen; - internal GenOperation(Gen<(string, Action)> gen, bool addOpNumber) + internal GenOperation(Gen<(string, Action, Action)> gen, bool addOpNumber) { this.gen = gen; AddOpNumber = addOpNumber; } - public override (string, Action) Generate(PCG pcg, Size? min, out Size size) => gen.Generate(pcg, min, out size); + public override (string, Action, Action) Generate(PCG pcg, Size? min, out Size size) => gen.Generate(pcg, min, out size); } public sealed class GenMetamorphic : Gen<(string, Action, Action)> @@ -2876,18 +2877,22 @@ public static GenOperation Create(Gen gen, Action action) => new(gen.Select)>(t => (" " + Check.Print(t), s => action(s, t))), true); public static GenOperation Create(Gen gen, Func name, Action action) => new(gen.Select)>(t => (name(t), s => action(s, t))), false); - public static GenOperation Create(Gen gen, Action action) => - new(gen.Select)>(t => (" " + Check.Print(t), (s1, s2) => action(s1, s2, t))), true); - public static GenOperation Create(Gen gen, Func name, Action action) => - new(gen.Select)>(t => (name(t), (s1, s2) => action(s1, s2, t))), false); + public static GenOperation Create(Gen gen, Action actual, Action model) => + new(gen.Select, Action)>(t => (" " + Check.Print(t), a => actual(a, t), m => model(m, t))), true); + public static GenOperation Create(Gen gen, Func name, Action actual, Action model) => + new(gen.Select, Action)>(t => (name(t), a => actual(a, t), m => model(m, t))), false); + public static GenOperation Create(Action actual, Action model) + => new(Gen.Const(("", actual, model)), true); + public static GenOperation Create(string name, Action actual, Action model) + => new(Gen.Const((name, actual, model)), false); public static GenOperation Create(Action action) => new(Gen.Const(("", action)), true); public static GenOperation Create(string name, Action action) => new(Gen.Const((name, action)), false); - public static GenOperation Create(Action action) - => new(Gen.Const(("", action)), true); - public static GenOperation Create(string name, Action action) - => new(Gen.Const((name, action)), false); +} + +public static class GenMetamorphic +{ public static GenMetamorphic Create(Gen gen, Func name, Action action1, Action action2) => new(gen.Select, Action)>(t => (name(t), s => action1(s, t), s => action2(s, t)))); } \ No newline at end of file diff --git a/CsCheck/Utils.cs b/CsCheck/Utils.cs index d0b5822..2636991 100644 --- a/CsCheck/Utils.cs +++ b/CsCheck/Utils.cs @@ -21,6 +21,8 @@ namespace CsCheck; using System.Collections.Concurrent; using System.Diagnostics; using System.Runtime.InteropServices; +using System.Threading; +using System; public sealed class CsCheckException : Exception { @@ -475,67 +477,82 @@ public static bool ModelEqual(T actual, M model) return actual.Equals(model); } - static readonly int[] DummyArray = new int[MAX_CONCURRENT_OPERATIONS]; - internal static void Run(T concurrentState, (string, Action)[] operations, int threads, int[]? threadIds = null) + sealed class RunWorker(T state, (string, Action)[] parallelOperations, int[]? threadIds) : IThreadPoolWorkItem { - threadIds ??= DummyArray; - Exception? exception = null; - var opId = -1; - var runners = new Thread[threads]; - while (--threads >= 0) + int opId = -1, threadId = -1; + public volatile bool Hold = true; + public Exception? Exception; + public void Execute() { - runners[threads] = new Thread(threadId => + int i, tid = Interlocked.Increment(ref threadId); + while (Hold) { } + while ((i = Interlocked.Increment(ref opId)) < parallelOperations.Length) { - int i, tid = (int)threadId!; - while ((i = Interlocked.Increment(ref opId)) < operations.Length) + if (threadIds is not null) threadIds[i] = tid; + try { parallelOperations[i].Item2(state); } + catch (Exception e) { - threadIds[i] = tid; - try { operations[i].Item2(concurrentState); } - catch (Exception e) + if (Exception is null) { - if (exception is null) - { - exception = e; - Interlocked.Exchange(ref opId, operations.Length); - } + Exception = e; + opId = 1_000_000; } } - }); + } } - for (int i = 0; i < runners.Length; i++) runners[i].Start(i); - for (int i = 0; i < runners.Length; i++) runners[i].Join(); - if (exception is not null) throw exception; } - internal static void RunReplay(T concurrentState, (string, Action)[] operations, int threads, int[] threadIds) + internal static void Run(T state, (string, Action)[] sequencialOperations, (string, Action)[] parallelOperations, int threads, int[]? threadIds = null) { - Exception? exception = null; + for (int i = 0; i < sequencialOperations.Length; i++) + sequencialOperations[i].Item2(state); + var worker = new RunWorker(state, parallelOperations, threadIds); var runners = new Thread[threads]; - while (--threads >= 0) + for (int i = 0; i < runners.Length; i++) runners[i] = new Thread(worker.Execute); + for (int i = 0; i < runners.Length; i++) runners[i].Start(); + worker.Hold = false; + for (int i = 0; i < runners.Length; i++) runners[i].Join(); + if (worker.Exception is not null) throw worker.Exception; + } + + sealed class RunReplayWorker(T state, (string, Action)[] parallelOperations, int[] threadIds) : IThreadPoolWorkItem + { + int threadId = -1; + public volatile bool Hold = true; + public Exception? Exception; + public void Execute() { - runners[threads] = new Thread(threadId => + int i, opId = -1, tid = Interlocked.Increment(ref threadId); + while (Hold) { } + while ((i = Interlocked.Increment(ref opId)) < parallelOperations.Length) { - int opId = -1, i = -1, tid = (int)threadId!; - while ((i = Interlocked.Increment(ref opId)) < operations.Length) + if (threadIds[i] == tid) { - if (threadIds[i] == tid) + try { parallelOperations[i].Item2(state); } + catch (Exception e) { - try { operations[i].Item2(concurrentState); } - catch (Exception e) + if (Exception is null) { - if (exception is null) - { - exception = e; - Interlocked.Exchange(ref opId, operations.Length); - } + Exception = e; + opId = 1_000_000; } } } - }); + } } - for (int i = 0; i < runners.Length; i++) runners[i].Start(i); + } + + internal static void RunReplay(T state, (string, Action)[] sequencialOperations, (string, Action)[] parallelOperations, int threads, int[] threadIds) + { + for (int i = 0; i < sequencialOperations.Length; i++) + sequencialOperations[i].Item2(state); + var worker = new RunReplayWorker(state, parallelOperations, threadIds); + var runners = new Thread[threads]; + for (int i = 0; i < runners.Length; i++) runners[i] = new Thread(worker.Execute); + for (int i = 0; i < runners.Length; i++) runners[i].Start(); + worker.Hold = false; for (int i = 0; i < runners.Length; i++) runners[i].Join(); - if (exception is not null) throw exception; + if (worker.Exception is not null) throw worker.Exception; } internal static IEnumerable Permutations(int[] threadIds, T[] sequence) diff --git a/README.md b/README.md index b87ec5d..662977e 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ This gives the following advantages over tree based shrinking libraries: - Random testing and shrinking are parallelized. This and PCG make it very fast. - Shrunk cases have a seed value. Simpler examples can easily be reproduced. - Shrinking can be continued later to give simpler cases for high dimensional problems. -- Concurrency testing and random shrinking work well together. Repeat is not needed. +- Parallel testing and random shrinking work well together. Repeat is not needed. See [why](https://github.com/AnthonyLloyd/CsCheck/blob/master/Why.md) you should use it, the [comparison](https://github.com/AnthonyLloyd/CsCheck/blob/master/Comparison.md) with other random testing libraries, or how CsCheck does in the [shrinking challenge](https://github.com/jlink/shrinking-challenge). In one [shrinking challenge test](https://github.com/jlink/shrinking-challenge/blob/main/challenges/binheap.md) CsCheck managed to shrink to a new smaller example than was thought possible and is not reached by any other testing library. @@ -21,7 +21,7 @@ CsCheck also has functionality to make multiple types of testing simple and fast - [Random testing](#Random-testing) - [Model-based testing](#Model-based-testing) - [Metamorphic testing](#Metamorphic-testing) -- [Concurrency testing](#Concurrency-testing) +- [Parallel testing](#Parallel-testing) - [Causal profiling](#Causal-profiling) - [Regression testing](#Regression-testing) - [Performance testing](#Performance-testing) @@ -229,15 +229,15 @@ SampleModelBased generates an initial actual and model and then applies a random ### SetSlim Add ```csharp [Fact] -public void SetSlim_ModelBased() +public void +SetSlim_ModelBased() { Gen.Int.Array.Select(a => (new SetSlim(a), new HashSet(a))) .SampleModelBased( - Gen.Int.Operation, HashSet>((ls, l, i) => - { - ls.Add(i); - l.Add(i); - }) + Gen.Int.Operation, HashSet>( + (ss, i) => ss.Add(i), + (hs, i) => hs.Add(i) + ) // ... other operations ); } @@ -267,21 +267,21 @@ public void MapSlim_Metamorphic() } ``` -## Concurrency testing +## Parallel testing -CsCheck has support for concurrency testing with full shrinking capability. -A concurrent sequence of operations are run on an initial state and the result is compared to all the possible linearized versions. -At least one of these must be equal to the concurrent version. +CsCheck has support for parallels testing with full shrinking capability. +A number of operations are run on an initial state in parallel and the result is compared to all the possible linearized versions. +At least one of these must be equal to the parallel result. Idea from John Hughes [talk](https://youtu.be/1LNEWF8s1hI?t=1603) and [paper](https://github.com/AnthonyLloyd/AnthonyLloyd.github.io/raw/master/public/cscheck/finding-race-conditions.pdf). This is actually easier to implement with CsCheck than QuickCheck because the random shrinking does not need to repeat each step as QuickCheck does (10 times by default) to make shrinking deterministic. ### SetSlim ```csharp [Fact] -public void SetSlim_Concurrency() +public void SetSlim_Parallel() { Gen.Byte.Array.Select(a => new SetSlim(a)) - .SampleConcurrent( + .SampleParallel( Gen.Byte.Operation>((l, i) => { lock (l) l.Add(i); }), Gen.Int.NonNegative.Operation>((l, i) => { if (i < l.Count) { var _ = l[i]; } }), Gen.Byte.Operation>((l, i) => { var _ = l.IndexOf(i); }), @@ -610,7 +610,7 @@ timeout - The timeout in seconds to use for Faster (default 60 seconds). print - A function to convert the state to a string for error reporting (default Check.Print). equal - A function to check if the two states are the same (default Check.Equal). sigma - For Faster sigma is the number of standard deviations from the null hypothesis (default 6). -replay - The number of times to retry the seed to reproduce a SampleConcurrent fail (default 100). +replay - The number of times to retry the seed to reproduce a SampleParallel fail (default 100). Global defaults can also be set via environment variables: diff --git a/Tests/AllocatorMany_Tests.cs b/Tests/AllocatorMany_Tests.cs index 4d2934d..89aa6f3 100644 --- a/Tests/AllocatorMany_Tests.cs +++ b/Tests/AllocatorMany_Tests.cs @@ -154,7 +154,7 @@ public void Example04() ], actual.Solution); } - [Fact] + [Fact(Skip ="Takes too long")] public void Example05() { var actual = AllocatorMany.Allocate( diff --git a/Tests/Allocator_Tests.cs b/Tests/Allocator_Tests.cs index c3a35ea..5d2d57c 100644 --- a/Tests/Allocator_Tests.cs +++ b/Tests/Allocator_Tests.cs @@ -51,7 +51,7 @@ public void Allocate_BalinskiYoung_Long_TotalsCorrectly() static bool BetweenFloorAndCeiling(long quantity, W[] weights, Func allocate) { - var sumWeights = weights.Sum(w => Convert.ToDouble(w)); + var sumWeights = weights.Select(w => Convert.ToDouble(w)).ToArray().FSum(compress: true); var allocations = allocate(quantity, weights); for (int i = 0; i < allocations.Length; i++) { diff --git a/Tests/CacheTests.cs b/Tests/CacheTests.cs index 3bfad01..7ffd8ac 100644 --- a/Tests/CacheTests.cs +++ b/Tests/CacheTests.cs @@ -14,9 +14,9 @@ class ConcurrentDictionaryCache : ConcurrentDictionary, ICache } [Fact] - public void GetOrAddAtomicAsync_SampleConcurrent() + public void GetOrAddAtomicAsync_SampleParallel() { - Check.SampleConcurrent( + Check.SampleParallel( Gen.Const(() => new ConcurrentDictionaryCache()), Gen.Int[1, 5].Operation>((d, i) => d.GetOrAddAtomicAsync(i, i => Task.FromResult(i)).AsTask()), equal: (a, b) => Check.Equal(a.Keys, b.Keys), diff --git a/Tests/CheckTests.cs b/Tests/CheckTests.cs index 5b77a6a..369cbf9 100644 --- a/Tests/CheckTests.cs +++ b/Tests/CheckTests.cs @@ -221,41 +221,17 @@ public void SampleModelBased_ConcurrentBag() { Gen.Int[0, 5].List.Select(l => (new ConcurrentBag(l), l)) .SampleModelBased( - Gen.Int.Operation, List>((bag, list, i) => - { - bag.Add(i); - list.Add(i); - }), - Gen.Operation, List>((bag, list) => Assert.Equal(bag.TryTake(out var i), list.Remove(i))) - , threads: 1 - ); - } - - [Fact] - public void SampleConcurrent_ConcurrentBag() - { - Gen.Int.List[0, 5].Select(l => new ConcurrentBag(l)) - .SampleConcurrent( - Gen.Int.Operation>(i => $"Add({i})", (bag, i) => bag.Add(i)), - Gen.Operation>("TryTake()", bag => bag.TryTake(out _)) - ); + Gen.Int.Operation, List>((bag, i) => bag.Add(i), (list, i) => list.Add(i)), + Gen.Operation, List>(bag => bag.TryTake(out _), list => { if (list.Count > 0) list.RemoveAt(0); }), + equal: (bag, list) => bag.Count == list.Count + , threads: 1); } - //[Fact] - //public void SampleConcurrent_List() - //{ - // Gen.Int.List - // .SampleConcurrent( - // Gen.Int.Operation>(i => $"Add({i})", (list, i) => list.Add(i)) - // //Gen.Const<(string, Action>)>(("Remove()", list => list.RemoveAt(0))) - // ); - //} - [Fact] - public void SampleConcurrent_ConcurrentDictionary() + public void SampleParallel_ConcurrentDictionary() { Gen.Dictionary(Gen.Int[0, 100], Gen.Byte)[0, 10].Select(l => new ConcurrentDictionary(l)) - .SampleConcurrent( + .SampleParallel( Gen.Int[0, 100].Select(Gen.Byte) .Operation>(t =>$"d[{t.Item1}] = {t.Item2}", (d, t) => d[t.Item1] = t.Item2), @@ -265,15 +241,45 @@ public void SampleConcurrent_ConcurrentDictionary() } [Fact] - public void SampleConcurrent_ConcurrentQueue() + public void SampleParallel_ConcurrentQueue() { Gen.Int.List[0, 5].Select(l => new ConcurrentQueue(l)) - .SampleConcurrent( + .SampleParallel( Gen.Int.Operation>(i => $"Enqueue({i})", (q, i) => q.Enqueue(i)), Gen.Operation>("TryDequeue()", q => q.TryDequeue(out _)) ); } + [Fact] + public void SampleParallelModel_ConcurrentQueue() + { + Gen.Const(() => (new ConcurrentQueue(), new Queue())) + .SampleParallel( + Gen.Int.Operation, Queue>(i => $"Enqueue({i})", (q, i) => q.Enqueue(i), (q, i) => q.Enqueue(i)), + Gen.Operation, Queue>("TryDequeue()", q => q.TryDequeue(out _), q => q.TryDequeue(out _)) + ); + } + + [Fact] + public void SampleParallelModel_ConcurrentStack() + { + Gen.Const(() => (new ConcurrentStack(), new Stack())) + .SampleParallel( + Gen.Int.Operation, Stack>(i => $"Push({i})", (q, i) => q.Push(i), (q, i) => q.Push(i)), + Gen.Operation, Stack>("TryPop()", q => q.TryPop(out _), q => q.TryPop(out _)) + ); + } + + [Fact] + public void SampleParallelModel_ConcurrentDictionary() + { + Gen.Const(() => (new ConcurrentDictionary(), new Dictionary())) + .SampleParallel( + Gen.Int[1, 5].Operation, Dictionary>(i => $"Set ({i})", (q, i) => q[i] = i, (q, i) => q[i] = i), + Gen.Int[1, 5].Operation, Dictionary>(i => $"TryRemove ({i})", (q, i) => q.TryRemove(i, out _), (q, i) => q.Remove(i)) + ); + } + [Fact] public void Equality() { diff --git a/Tests/IMToolsTests.cs b/Tests/IMToolsTests.cs index 30b7913..75828ff 100644 --- a/Tests/IMToolsTests.cs +++ b/Tests/IMToolsTests.cs @@ -107,11 +107,9 @@ public void AddOrUpdate_ModelBased() return (new ImHolder> { Im = d }, m); }) .SampleModelBased( - Gen.Int[0, upperBound].Select(Gen.Int).Operation>, Dictionary>((h, d, kv) => - { - h.Im = h.Im.AddOrUpdate(kv.Item1, kv.Item2); - d[kv.Item1] = kv.Item2; - }) + Gen.Int[0, upperBound].Select(Gen.Int).Operation>, Dictionary>( + (h, kv) => { h.Im = h.Im.AddOrUpdate(kv.Item1, kv.Item2); }, + (d, kv) => { d[kv.Item1] = kv.Item2; }) , equal: (h, d) => { var he = h.Im.Enumerate().Select(kv => (kv.Key, kv.Value)).ToList(); diff --git a/Tests/PCGTests.cs b/Tests/PCGTests.cs index 32cc830..cb9ca60 100644 --- a/Tests/PCGTests.cs +++ b/Tests/PCGTests.cs @@ -198,8 +198,8 @@ public void PCG_ToString_Roundtrip() [Fact] public void Double_Exp_Bug() { - var root2 = -6.3E-102; - var root3 = 6.6854976605820742; + const double root2 = -6.3E-102; + const double root3 = 6.6854976605820742; Gen.Double[root2, root3 * 2.0].Sample(_ => true); } } \ No newline at end of file diff --git a/Tests/SieveLruCacheTests.cs b/Tests/SieveLruCacheTests.cs index 09eb121..eef9c4d 100644 --- a/Tests/SieveLruCacheTests.cs +++ b/Tests/SieveLruCacheTests.cs @@ -89,27 +89,14 @@ public void SampleModelBased() { Check.SampleModelBased( Gen.Const(() => (new SieveLruCache(4), new SieveModel(4))), - Gen.Int[1, 5].Operation, SieveModel>((a, m, i) => - { - a.GetOrAdd(i, i => i); - m.GetOrAdd(i, i => i); - }), + Gen.Int[1, 5].Operation, SieveModel>( + (a, i) => a.GetOrAdd(i, i => i), + (m, i) => m.GetOrAdd(i, i => i)), equal: (a, m) => Check.Equal(a.Keys.ToHashSet(), m.Keys.ToHashSet()), printActual: a => Check.Print(a.Keys), printModel: m => Check.Print(m.Keys) ); } - - [Fact] - public void SampleConcurrent() - { - Check.SampleConcurrent( - Gen.Const(() => new SieveLruCache(4)), - Gen.Int[1, 5].Operation>((d, i) => d.GetOrAdd(i, i => i)), - equal: (a, b) => Check.Equal(a.Keys, b.Keys), - print: a => Check.Print(a.Keys) - ); - } } internal static class SieveLruCacheExtensions diff --git a/Tests/SlimCollectionsTests.cs b/Tests/SlimCollectionsTests.cs index 386818b..2b9b641 100644 --- a/Tests/SlimCollectionsTests.cs +++ b/Tests/SlimCollectionsTests.cs @@ -15,23 +15,21 @@ public void ListSlim_ModelBased() { Gen.Int.Array.Select(a => (new ListSlim(a), new List(a))) .SampleModelBased( - Gen.Int.Operation, List>((ls, l, i) => - { - ls.Add(i); - l.Add(i); - }) + Gen.Int.Operation, List>( + (ls, i) => ls.Add(i), + (l, i) => l.Add(i)) ); } [Fact] - public void ListSlim_Concurrency() + public void ListSlim_Parallel() { Gen.Byte.Array.Select(a => new ListSlim(a)) - .SampleConcurrent( - Gen.Byte.Operation>((l, i) => { lock (l) l.Add(i); }), - Gen.Int.NonNegative.Operation>((l, i) => { if (i < l.Count) { var _ = l[i]; } }), - Gen.Int.NonNegative.Select(Gen.Byte).Operation>((l, t) => { if (t.Item1 < l.Count) l[t.Item1] = t.Item2; }), - Gen.Operation>(l => l.ToArray()) + .SampleParallel( + Gen.Byte.Operation>(i => $"Add {i}", (l, i) => { lock (l) l.Add(i); }), + Gen.Int.NonNegative.Operation>(i => $"Get {i}", (l, i) => { if (i < l.Count) { var _ = l[i]; } }), + Gen.Int.NonNegative.Select(Gen.Byte).Operation>(t => $"Set {t}", (l, t) => { if (t.Item1 < l.Count) l[t.Item1] = t.Item2; }), + Gen.Operation>("ToArray", l => l.ToArray()) ); } @@ -40,19 +38,18 @@ public void SetSlim_ModelBased() { Gen.Int.Array.Select(a => (new SetSlim(a), new HashSet(a))) .SampleModelBased( - Gen.Int.Operation, HashSet>((ls, l, i) => - { - ls.Add(i); - l.Add(i); - }) + Gen.Int.Operation, HashSet>( + (ss, i) => ss.Add(i), + (hs, i) => hs.Add(i) + ) ); } [Fact] - public void SetSlim_Concurrency() + public void SetSlim_Parallel() { Gen.Byte.Array.Select(a => new SetSlim(a)) - .SampleConcurrent( + .SampleParallel( Gen.Byte.Operation>((l, i) => { lock (l) l.Add(i); }), Gen.Int.NonNegative.Operation>((l, i) => { if (i < l.Count) { var _ = l[i]; } }), Gen.Byte.Operation>((l, i) => { var _ = l.IndexOf(i); }), @@ -100,11 +97,9 @@ public void MapSlim_ModelBased() Gen.Dictionary(Gen.Int, Gen.Byte) .Select(d => (new MapSlim(d), new Dictionary(d))) .SampleModelBased( - Gen.Select(Gen.Int[0, 100], Gen.Byte).Operation, Dictionary>((m, d, t) => - { - m[t.Item1] = t.Item2; - d[t.Item1] = t.Item2; - }) + Gen.Select(Gen.Int[0, 100], Gen.Byte).Operation, Dictionary>( + (m, t) => m[t.Item1] = t.Item2, + (d, t) => d[t.Item1] = t.Item2) ); } @@ -121,10 +116,10 @@ public void MapSlim_Metamorphic() } [Fact] - public void MapSlim_Concurrency() + public void MapSlim_Parallel() { Gen.Dictionary(Gen.Int, Gen.Byte).Select(d => new MapSlim(d)) - .SampleConcurrent( + .SampleParallel( Gen.Int.Select(Gen.Byte).Operation>((m, t) => { lock (m) m[t.Item1] = t.Item2; }), Gen.Int.NonNegative.Operation>((m, i) => { if (i < m.Count) { var _ = m.Key(i); } }), Gen.Int.Operation>((m, i) => { var _ = m.IndexOf(i); }), diff --git a/Tests/Tests.csproj b/Tests/Tests.csproj index 06152de..2dc7f73 100644 --- a/Tests/Tests.csproj +++ b/Tests/Tests.csproj @@ -14,11 +14,11 @@ xUnit1004 - - - - - + + + + + diff --git a/Why.md b/Why.md index 782d88c..f70088c 100644 --- a/Why.md +++ b/Why.md @@ -40,7 +40,7 @@ Fluent style composition similar to LINQ is a much more robust and extensible op - Caches and collections - often a key part of server and client side code these can be tested against a suitable simplified test model with `Model Based` testing. - Calculations and algorithms - often possible to generalize examples for calculations and algorithms and check the result given the input. Algorithm often have properties they must guarantee. Rounding error issues automatically tested. - Code refactoring - keep a copy of the original code with the test, refactor for simplicity and performance, safe in the knowledge it still produces the same results. Pair with a `Faster` test to monitor the relative performance over a range of inputs. Or if a copy is not feasible create a `Regression` test to comprehensively make sure there is no change. -- Multithreading and concurrency - test on the same object instance across multiple threads and examples. Shrink even works for `Concurrency` testing. +- Multithreading - test on the same object instance across multiple threads and examples. Shrink even works for `Parallel` testing. ## Raise Your Game