Add option to skip printing skipped tests + fix FsCheck3 replay

Expecto.FsCheck/FsCheck.fs

@@ -71,7 +71,7 @@ module ExpectoFsCheck =
                           (String.concat " " data.Labels)
 
               let focus =
-                sprintf "Focus on error:\n\t%s (%A, %A) \"%s\"" methodName (uint64 std) (uint64 gen) name
+                sprintf "Focus on error:\n\t%s (%A, %A, 0) \"%s\"" methodName (uint64 std) (uint64 gen) name
 
               sprintf "Failed after %s. %s%s\nResult:\n\t%A\n%s%s%s"
                       (numTests data.NumberOfTests) parameters shrunk
@@ -92,7 +92,7 @@ module ExpectoFsCheck =
           MaxFail = 1000
           // We're converting uint64s to a smaller type, but it shouldn't be an issue because users are only using the
           // values given in the test output, which are only ints when running FsCheck 2
-          Replay = Option.map Random.StdGen (config.replay |> Option.map (fun (seed, gamma) -> int seed, int gamma))
+          Replay = Option.map Random.StdGen (config.replay |> Option.map (fun (seed, gamma, _) -> int seed, int gamma))
           Name = name
           StartSize = config.startSize
           EndSize = config.endSize

Expecto.FsCheck3/FsCheck3.fs

@@ -47,7 +47,7 @@ module ExpectoFsCheck =
             | TestResult.Failed (_,_,_, Outcome.Failed (:? IgnoreException as e),_,_,_) ->
               raise e
 
-            | TestResult.Failed (data, original, shrunk, outcome,originalSeed,_finalSeed,size) ->
+            | TestResult.Failed (data, original, shrunk, outcome,originalSeed,finalSeed,size) ->
               let parameters =
                 original
                 |> List.map (sprintf "%A")
@@ -70,12 +70,14 @@ module ExpectoFsCheck =
                 | _ -> sprintf "Labels of failing property (one or more is failing): %s\n"
                           (String.concat " " data.Labels)
 
+              let original =
+                sprintf "Original seed: (%A, %A)" originalSeed.Seed originalSeed.Gamma
               let focus =
-                sprintf "Focus on error:\n\t%s (%A, %A) \"%s\"" methodName originalSeed.Seed originalSeed.Gamma name
+                sprintf "Focus on error:\n\t%s (%A, %A, %A) \"%s\"" methodName finalSeed.Seed finalSeed.Gamma size name
 
-              sprintf "Failed after %s. %s%s\nResult:\n\t%A\n%s%s%s"
+              sprintf "Failed after %s. %s%s\nResult:\n\t%A\n%s%s%s\n%s"
                       (numTests data.NumberOfTests) parameters shrunk
-                      outcome labels (stampsToString data.Stamps) focus
+                      outcome labels (stampsToString data.Stamps) original focus
               |> FailedException
               |> raise
 
@@ -91,7 +93,7 @@ module ExpectoFsCheck =
       let config =
         Config.Default
             .WithMaxTest(config.maxTest)
-            .WithReplay(Option.map (fun (seed,gamma) -> {Rnd = Rnd(seed, gamma); Size = None}) config.replay)
+            .WithReplay(Option.map (fun (seed,gamma,size) -> {Rnd = Rnd(seed, gamma); Size = Some(size)}) config.replay)
             .WithName(name)
             .WithStartSize(config.startSize)
             .WithEndSize(config.endSize)

Expecto.Tests.FsCheck3/FsCheck3Tests.fs

@@ -60,7 +60,7 @@ let focused =
   testList "FsCheck focused" [
     testCase "ignore me" <| ignore
 
-    etestProperty (1UL,3UL) "Deliberately failing test" <|
+    etestProperty (1UL,3UL,50) "Deliberately failing test" <|
       fun a b c ->
         // wrong on purpose to test failures
         a * (b + c) = a * a + a * c
@@ -87,14 +87,15 @@ let runFsCheckFocusedTests =
     match (getResult ["FsCheck focused";"Deliberately failing test"]).result with
     | TestResult.Failed actual ->
       let expected = "
-Failed after 3 tests. Parameters:
-	-1 1 2
-Shrunk 2 times to:
-	-1 0 0
+Failed after 1 test. Parameters:
+	47 43 -38
+Shrunk 9 times to:
+	1 0 0
 Result:
 	Failed System.Exception: Expected true, got false.
+Original seed: (1UL, 3UL)
 Focus on error:
-	etestProperty (1UL, 3UL) \"Deliberately failing test\""
+	etestProperty (1UL, 3UL, 50) \"Deliberately failing test\""
       Expect.equal actual expected "It should fail with the right message"
     | x ->
       failtestf "Expected Failed, actual was: %A" x
@@ -104,7 +105,7 @@ let config =
   testList "FsCheck config" [
     testCase "ignore me" ignore
 
-    etestPropertyWithConfig (1UL,3UL) FsCheckConfig.defaultConfig
+    etestPropertyWithConfig (1UL,3UL,50) FsCheckConfig.defaultConfig
       "Deliberately failing test" <|
       fun a b c ->
         // wrong on purpose to test failures
@@ -132,14 +133,15 @@ let runFsCheckConfigTests =
     match (getResult ["FsCheck config";"Deliberately failing test"]).result with
     | TestResult.Failed actual ->
       let expected = "
-Failed after 3 tests. Parameters:
-	-1 1 2
-Shrunk 2 times to:
-	-1 0 0
+Failed after 1 test. Parameters:
+	47 43 -38
+Shrunk 9 times to:
+	1 0 0
 Result:
 	Failed System.Exception: Expected true, got false.
+Original seed: (1UL, 3UL)
 Focus on error:
-	etestPropertyWithConfig (1UL, 3UL) \"Deliberately failing test\""
+	etestPropertyWithConfig (1UL, 3UL, 50) \"Deliberately failing test\""
       Expect.equal actual expected "It should fail with the right message."
 
     | x ->

Expecto.Tests/FsCheckTests.fs

@@ -60,7 +60,7 @@ let focused =
   testList "FsCheck focused" [
     testCase "ignore me" <| ignore
 
-    etestProperty (1UL,2UL) "Deliberately failing test" <|
+    etestProperty (1UL,2UL,0) "Deliberately failing test" <|
       fun a b c ->
         // wrong on purpose to test failures
         a * (b + c) = a * a + a * c
@@ -94,7 +94,7 @@ Shrunk 4 times to:
 Result:
 	False
 Focus on error:
-	etestProperty (1UL, 2UL) \"Deliberately failing test\""
+	etestProperty (1UL, 2UL, 0) \"Deliberately failing test\""
       Expect.equal actual expected "It should fail with the right message"
     | x ->
       failtestf "Expected Failed, actual was: %A" x
@@ -104,7 +104,7 @@ let config =
   testList "FsCheck config" [
     testCase "ignore me" ignore
 
-    etestPropertyWithConfig (1UL,2UL) FsCheckConfig.defaultConfig
+    etestPropertyWithConfig (1UL,2UL,0) FsCheckConfig.defaultConfig
       "Deliberately failing test" <|
       fun a b c ->
         // wrong on purpose to test failures
@@ -139,7 +139,7 @@ Shrunk 4 times to:
 Result:
 	False
 Focus on error:
-	etestPropertyWithConfig (1UL, 2UL) \"Deliberately failing test\""
+	etestPropertyWithConfig (1UL, 2UL, 0) \"Deliberately failing test\""
       Expect.equal actual expected "It should fail with the right message."
 
     | x ->

Expecto/Expecto.Impl.fs

@@ -498,6 +498,8 @@ module Impl =
       listStates : FocusState list
       /// Allows the test printer to be parametised to your liking.
       printer : TestPrinters
+      /// Whether to show skipped tests in the output.
+      printSkippedTests : bool
       /// Verbosity level (default: Info).
       verbosity : LogLevel
       /// Process name to log under (default: "Expecto")
@@ -536,6 +538,7 @@ module Impl =
             TestPrinters.defaultPrinter
           else
             TestPrinters.teamCityPrinter TestPrinters.defaultPrinter
+        printSkippedTests = true
         verbosity = Info
         logName = None
         locate = fun _ -> SourceLocation.empty
@@ -648,13 +651,18 @@ module Impl =
 
         let beforeEach (test:FlatTest) =
           let name = config.joinWith.format test.name
-          config.printer.beforeEach name
+          if config.printSkippedTests || test.shouldSkipEvaluation.IsNone then
+            config.printer.beforeEach name
+          else
+            async.Zero()
 
         async {
           let! beforeAsync = beforeEach test |> Async.StartChild
           let! result = execTestAsync ct config test
           do! beforeAsync
-          do! TestPrinters.printResult config test result
+
+          if config.printSkippedTests || test.shouldSkipEvaluation.IsNone then
+            do! TestPrinters.printResult config test result
 
           if progressStarted && Option.isNone test.shouldSkipEvaluation then
             Fraction (Interlocked.Increment testsCompleted, testLength)

Expecto/Expecto.fs

@@ -259,7 +259,7 @@
     member inline __.TryFinally(p, cf) = task.TryFinally(p, cf)
     member inline __.TryWith(p, cf) = task.TryWith(p, cf)
     member __.Run f =
       let deferred () = task.Run f
       match focusState with
       | Normal -> testCaseTask name deferred
       | Focused -> ftestCaseTask name deferred
@@ -449,6 +449,8 @@
     | Colours of int
     /// Adds a test printer.
     | Printer of TestPrinters
+    /// Whether to show skipped tests in the output.
+    | PrintSkippedTests of bool
     /// Sets the verbosity level.
     | Verbosity of LogLevel
     /// Append a summary handler.
@@ -540,6 +542,7 @@
                                       | _ -> JoinWith.Dot
                   }
     | Printer p -> fun o -> { o with printer = p }
+    | PrintSkippedTests b -> fun o -> { o with printSkippedTests = b }
     | Verbosity l -> fun o -> { o with verbosity = l }
     | Append_Summary_Handler (SummaryHandler h) -> fun o -> o.appendSummaryHandler h
 

Expecto/Model.fs

@@ -16,7 +16,7 @@ type FsCheckConfig =
     /// The size to use for the last test, when all the tests are passing. The size increases linearly between Start- and EndSize.
     endSize: int
     /// If set, the seed to use to start testing. Allows reproduction of previous runs.
-    replay: (uint64 * uint64) option
+    replay: (uint64 * uint64 * int) option
     /// The Arbitrary instances on this class will be merged in back to front order, i.e. instances for the same generated type at the front
     /// of the list will override those at the back. The instances on Arb.Default are always known, and are at the back (so they can always be
     /// overridden)

README.md

@@ -1372,5 +1372,5 @@ This might be due to how terminals/the locking thereof work: try running your te
 ## Migration notes
 
 ### 11.0.0
-- Any usages of the `replay` (a.k.a `stdGen` with `etestProperty*` functions) config with FsCheck tests will need to be updated to use `uint64` by appending `UL` to the literals, e.g. from `(1865288075, 296281834)` to `(1865288075UL, 296281834UL)`.
+- Any usages of the `replay` (a.k.a `stdGen` with `etestProperty*` functions) config with FsCheck tests will need to be updated to use `uint64` by appending `UL` to the literals. They will also now require a third item indicating the size. E.g. from `(1865288075, 296281834)` to `(1865288075UL, 296281834UL, 3)`.
 - FsCheck 2 is no longer supported, so we're switching Expecto.FsCheck to use FsCheck 3 by default, even though FsCheck 3 is still in release candidate state. If you still want FsCheck2, we will continue to release FsCheck2 support for the time being using a version suffix, e.g. [11.0.0-fscheck2](https://www.nuget.org/packages/Expecto.FsCheck/11.0.0-alpha1-fscheck2)  

RELEASE_NOTES.md

@@ -1,3 +1,10 @@
+### 11.0.0-alpha4 - 2024-10-15
+* Add option to not print skipped tests
+* Breaking change: Add third item to `FsCheckConfig.replay` indicating the size
+  * Fixes issue where the replay seed without the size was playing all tests leading up to the failure, making debugging 
+    more difficult
+  * Existing FsCheck 2 users can enter any number for the size, because it is ignored.
+
 ### 11.0.0-alpha3 - 2024-10-13
 * Add testParamAsync and testParamTask (#512), thanks @1eyewonder