diff --git a/Expecto.FsCheck/FsCheck.fs b/Expecto.FsCheck/FsCheck.fs index 3758cd20..cc08c729 100644 --- a/Expecto.FsCheck/FsCheck.fs +++ b/Expecto.FsCheck/FsCheck.fs @@ -71,7 +71,7 @@ module ExpectoFsCheck = (String.concat " " data.Labels) let focus = - sprintf "Focus on error:\n\t%s (%A, %A) \"%s\"" methodName (uint64 std) (uint64 gen) name + sprintf "Focus on error:\n\t%s (%A, %A, 0) \"%s\"" methodName (uint64 std) (uint64 gen) name sprintf "Failed after %s. %s%s\nResult:\n\t%A\n%s%s%s" (numTests data.NumberOfTests) parameters shrunk @@ -92,7 +92,7 @@ module ExpectoFsCheck = MaxFail = 1000 // We're converting uint64s to a smaller type, but it shouldn't be an issue because users are only using the // values given in the test output, which are only ints when running FsCheck 2 - Replay = Option.map Random.StdGen (config.replay |> Option.map (fun (seed, gamma) -> int seed, int gamma)) + Replay = Option.map Random.StdGen (config.replay |> Option.map (fun (seed, gamma, _) -> int seed, int gamma)) Name = name StartSize = config.startSize EndSize = config.endSize diff --git a/Expecto.FsCheck3/FsCheck3.fs b/Expecto.FsCheck3/FsCheck3.fs index 55c8e4c4..56e66d47 100644 --- a/Expecto.FsCheck3/FsCheck3.fs +++ b/Expecto.FsCheck3/FsCheck3.fs @@ -47,7 +47,7 @@ module ExpectoFsCheck = | TestResult.Failed (_,_,_, Outcome.Failed (:? IgnoreException as e),_,_,_) -> raise e - | TestResult.Failed (data, original, shrunk, outcome,originalSeed,_finalSeed,size) -> + | TestResult.Failed (data, original, shrunk, outcome,originalSeed,finalSeed,size) -> let parameters = original |> List.map (sprintf "%A") @@ -70,12 +70,14 @@ module ExpectoFsCheck = | _ -> sprintf "Labels of failing property (one or more is failing): %s\n" (String.concat " " data.Labels) + let original = + sprintf "Original seed: (%A, %A)" originalSeed.Seed originalSeed.Gamma let focus = - sprintf "Focus on error:\n\t%s (%A, %A) \"%s\"" methodName originalSeed.Seed originalSeed.Gamma name + sprintf "Focus on error:\n\t%s (%A, %A, %A) \"%s\"" methodName finalSeed.Seed finalSeed.Gamma size name - sprintf "Failed after %s. %s%s\nResult:\n\t%A\n%s%s%s" + sprintf "Failed after %s. %s%s\nResult:\n\t%A\n%s%s%s\n%s" (numTests data.NumberOfTests) parameters shrunk - outcome labels (stampsToString data.Stamps) focus + outcome labels (stampsToString data.Stamps) original focus |> FailedException |> raise @@ -91,7 +93,7 @@ module ExpectoFsCheck = let config = Config.Default .WithMaxTest(config.maxTest) - .WithReplay(Option.map (fun (seed,gamma) -> {Rnd = Rnd(seed, gamma); Size = None}) config.replay) + .WithReplay(Option.map (fun (seed,gamma,size) -> {Rnd = Rnd(seed, gamma); Size = Some(size)}) config.replay) .WithName(name) .WithStartSize(config.startSize) .WithEndSize(config.endSize) diff --git a/Expecto.Tests.FsCheck3/FsCheck3Tests.fs b/Expecto.Tests.FsCheck3/FsCheck3Tests.fs index f1bbeb2b..5cee077a 100644 --- a/Expecto.Tests.FsCheck3/FsCheck3Tests.fs +++ b/Expecto.Tests.FsCheck3/FsCheck3Tests.fs @@ -60,7 +60,7 @@ let focused = testList "FsCheck focused" [ testCase "ignore me" <| ignore - etestProperty (1UL,3UL) "Deliberately failing test" <| + etestProperty (1UL,3UL,50) "Deliberately failing test" <| fun a b c -> // wrong on purpose to test failures a * (b + c) = a * a + a * c @@ -87,14 +87,15 @@ let runFsCheckFocusedTests = match (getResult ["FsCheck focused";"Deliberately failing test"]).result with | TestResult.Failed actual -> let expected = " -Failed after 3 tests. Parameters: - -1 1 2 -Shrunk 2 times to: - -1 0 0 +Failed after 1 test. Parameters: + 47 43 -38 +Shrunk 9 times to: + 1 0 0 Result: Failed System.Exception: Expected true, got false. +Original seed: (1UL, 3UL) Focus on error: - etestProperty (1UL, 3UL) \"Deliberately failing test\"" + etestProperty (1UL, 3UL, 50) \"Deliberately failing test\"" Expect.equal actual expected "It should fail with the right message" | x -> failtestf "Expected Failed, actual was: %A" x @@ -104,7 +105,7 @@ let config = testList "FsCheck config" [ testCase "ignore me" ignore - etestPropertyWithConfig (1UL,3UL) FsCheckConfig.defaultConfig + etestPropertyWithConfig (1UL,3UL,50) FsCheckConfig.defaultConfig "Deliberately failing test" <| fun a b c -> // wrong on purpose to test failures @@ -132,14 +133,15 @@ let runFsCheckConfigTests = match (getResult ["FsCheck config";"Deliberately failing test"]).result with | TestResult.Failed actual -> let expected = " -Failed after 3 tests. Parameters: - -1 1 2 -Shrunk 2 times to: - -1 0 0 +Failed after 1 test. Parameters: + 47 43 -38 +Shrunk 9 times to: + 1 0 0 Result: Failed System.Exception: Expected true, got false. +Original seed: (1UL, 3UL) Focus on error: - etestPropertyWithConfig (1UL, 3UL) \"Deliberately failing test\"" + etestPropertyWithConfig (1UL, 3UL, 50) \"Deliberately failing test\"" Expect.equal actual expected "It should fail with the right message." | x -> diff --git a/Expecto.Tests/FsCheckTests.fs b/Expecto.Tests/FsCheckTests.fs index a6059aca..ee1fa4bd 100644 --- a/Expecto.Tests/FsCheckTests.fs +++ b/Expecto.Tests/FsCheckTests.fs @@ -60,7 +60,7 @@ let focused = testList "FsCheck focused" [ testCase "ignore me" <| ignore - etestProperty (1UL,2UL) "Deliberately failing test" <| + etestProperty (1UL,2UL,0) "Deliberately failing test" <| fun a b c -> // wrong on purpose to test failures a * (b + c) = a * a + a * c @@ -94,7 +94,7 @@ Shrunk 4 times to: Result: False Focus on error: - etestProperty (1UL, 2UL) \"Deliberately failing test\"" + etestProperty (1UL, 2UL, 0) \"Deliberately failing test\"" Expect.equal actual expected "It should fail with the right message" | x -> failtestf "Expected Failed, actual was: %A" x @@ -104,7 +104,7 @@ let config = testList "FsCheck config" [ testCase "ignore me" ignore - etestPropertyWithConfig (1UL,2UL) FsCheckConfig.defaultConfig + etestPropertyWithConfig (1UL,2UL,0) FsCheckConfig.defaultConfig "Deliberately failing test" <| fun a b c -> // wrong on purpose to test failures @@ -139,7 +139,7 @@ Shrunk 4 times to: Result: False Focus on error: - etestPropertyWithConfig (1UL, 2UL) \"Deliberately failing test\"" + etestPropertyWithConfig (1UL, 2UL, 0) \"Deliberately failing test\"" Expect.equal actual expected "It should fail with the right message." | x -> diff --git a/Expecto/Expecto.Impl.fs b/Expecto/Expecto.Impl.fs index 7987a07d..a8b384df 100644 --- a/Expecto/Expecto.Impl.fs +++ b/Expecto/Expecto.Impl.fs @@ -498,6 +498,8 @@ module Impl = listStates : FocusState list /// Allows the test printer to be parametised to your liking. printer : TestPrinters + /// Whether to show skipped tests in the output. + printSkippedTests : bool /// Verbosity level (default: Info). verbosity : LogLevel /// Process name to log under (default: "Expecto") @@ -536,6 +538,7 @@ module Impl = TestPrinters.defaultPrinter else TestPrinters.teamCityPrinter TestPrinters.defaultPrinter + printSkippedTests = true verbosity = Info logName = None locate = fun _ -> SourceLocation.empty @@ -648,13 +651,18 @@ module Impl = let beforeEach (test:FlatTest) = let name = config.joinWith.format test.name - config.printer.beforeEach name + if config.printSkippedTests || test.shouldSkipEvaluation.IsNone then + config.printer.beforeEach name + else + async.Zero() async { let! beforeAsync = beforeEach test |> Async.StartChild let! result = execTestAsync ct config test do! beforeAsync - do! TestPrinters.printResult config test result + + if config.printSkippedTests || test.shouldSkipEvaluation.IsNone then + do! TestPrinters.printResult config test result if progressStarted && Option.isNone test.shouldSkipEvaluation then Fraction (Interlocked.Increment testsCompleted, testLength) diff --git a/Expecto/Expecto.fs b/Expecto/Expecto.fs index e8d6be2d..b0ba26c0 100644 --- a/Expecto/Expecto.fs +++ b/Expecto/Expecto.fs @@ -449,6 +449,8 @@ module Tests = | Colours of int /// Adds a test printer. | Printer of TestPrinters + /// Whether to show skipped tests in the output. + | PrintSkippedTests of bool /// Sets the verbosity level. | Verbosity of LogLevel /// Append a summary handler. @@ -540,6 +542,7 @@ module Tests = | _ -> JoinWith.Dot } | Printer p -> fun o -> { o with printer = p } + | PrintSkippedTests b -> fun o -> { o with printSkippedTests = b } | Verbosity l -> fun o -> { o with verbosity = l } | Append_Summary_Handler (SummaryHandler h) -> fun o -> o.appendSummaryHandler h diff --git a/Expecto/Model.fs b/Expecto/Model.fs index aeb3c7dd..1c52c4ce 100644 --- a/Expecto/Model.fs +++ b/Expecto/Model.fs @@ -16,7 +16,7 @@ type FsCheckConfig = /// The size to use for the last test, when all the tests are passing. The size increases linearly between Start- and EndSize. endSize: int /// If set, the seed to use to start testing. Allows reproduction of previous runs. - replay: (uint64 * uint64) option + replay: (uint64 * uint64 * int) option /// The Arbitrary instances on this class will be merged in back to front order, i.e. instances for the same generated type at the front /// of the list will override those at the back. The instances on Arb.Default are always known, and are at the back (so they can always be /// overridden) diff --git a/README.md b/README.md index f1b33efd..043192b2 100644 --- a/README.md +++ b/README.md @@ -1372,5 +1372,5 @@ This might be due to how terminals/the locking thereof work: try running your te ## Migration notes ### 11.0.0 -- Any usages of the `replay` (a.k.a `stdGen` with `etestProperty*` functions) config with FsCheck tests will need to be updated to use `uint64` by appending `UL` to the literals, e.g. from `(1865288075, 296281834)` to `(1865288075UL, 296281834UL)`. +- Any usages of the `replay` (a.k.a `stdGen` with `etestProperty*` functions) config with FsCheck tests will need to be updated to use `uint64` by appending `UL` to the literals. They will also now require a third item indicating the size. E.g. from `(1865288075, 296281834)` to `(1865288075UL, 296281834UL, 3)`. - FsCheck 2 is no longer supported, so we're switching Expecto.FsCheck to use FsCheck 3 by default, even though FsCheck 3 is still in release candidate state. If you still want FsCheck2, we will continue to release FsCheck2 support for the time being using a version suffix, e.g. [11.0.0-fscheck2](https://www.nuget.org/packages/Expecto.FsCheck/11.0.0-alpha1-fscheck2) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 61192c29..ce3bedd1 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,10 @@ +### 11.0.0-alpha4 - 2024-10-15 +* Add option to not print skipped tests +* Breaking change: Add third item to `FsCheckConfig.replay` indicating the size + * Fixes issue where the replay seed without the size was playing all tests leading up to the failure, making debugging + more difficult + * Existing FsCheck 2 users can enter any number for the size, because it is ignored. + ### 11.0.0-alpha3 - 2024-10-13 * Add testParamAsync and testParamTask (#512), thanks @1eyewonder