diff --git a/Expecto.Tests/Expecto.Tests.fsproj b/Expecto.Tests/Expecto.Tests.fsproj index f9595e24..33e53e9c 100644 --- a/Expecto.Tests/Expecto.Tests.fsproj +++ b/Expecto.Tests/Expecto.Tests.fsproj @@ -1,12 +1,14 @@ - + Expecto.Tests Exe net6.0 + false + @@ -20,4 +22,4 @@ - + \ No newline at end of file diff --git a/Expecto.Tests/Main.fs b/Expecto.Tests/Main.fs index fae29267..fb043e7b 100644 --- a/Expecto.Tests/Main.fs +++ b/Expecto.Tests/Main.fs @@ -2,13 +2,26 @@ module Main open Expecto open Expecto.Logging +open OpenTelemetry.Resources +open OpenTelemetry +open OpenTelemetry.Trace +open System.Threading +open System.Diagnostics +open System + +let serviceName = "Expecto.Tests" + +let logger = Log.create serviceName -let logger = Log.create "Expecto.Tests" [] let main args = + let test = Impl.testFromThisAssembly() |> Option.orDefault (TestList ([], Normal)) |> Test.shuffle "." - runTestsWithCLIArgs [NUnit_Summary "bin/Expecto.Tests.TestResults.xml"] args test + runTestsWithCLIArgs [NUnit_Summary "bin/Expecto.Tests.TestResults.xml";] args test + + + diff --git a/Expecto.Tests/OpenTelemetry.fs b/Expecto.Tests/OpenTelemetry.fs new file mode 100644 index 00000000..57f9603c --- /dev/null +++ b/Expecto.Tests/OpenTelemetry.fs @@ -0,0 +1,172 @@ +namespace Expecto + +module OpenTelemetry = + open System + open System.Diagnostics + open System.Collections.Generic + open System.Threading + open Impl + + + module internal Activity = + let inline isNotNull x = isNull x |> not + + let inline setStatus (status : ActivityStatusCode) (span : Activity) = + if isNotNull span then + span.SetStatus(status) |> ignore + + let inline setExn (e : exn) (span : Activity) = + if isNotNull span|> not then + let tags = + ActivityTagsCollection( + seq { + KeyValuePair("exception.type", box (e.GetType().Name)) + KeyValuePair("exception.stacktrace", box (e.ToString())) + if not <| String.IsNullOrEmpty(e.Message) then + KeyValuePair("exception.message", box e.Message) + } + ) + + ActivityEvent("exception", tags = tags) + |> span.AddEvent + |> ignore + + let inline setExnMarkFailed (e : exn) (span : Activity) = + if isNotNull span then + setExn e span + span |> setStatus ActivityStatusCode.Error + + let setSourceLocation (sourceLoc : SourceLocation) (span : Activity) = + if isNotNull span && sourceLoc <> SourceLocation.empty then + span.SetTag("code.lineno", sourceLoc.lineNumber) |> ignore + span.SetTag("code.filepath", sourceLoc.sourcePath) |> ignore + + let inline addOutcome (result : TestResult) (span : Activity) = + if isNotNull span then + span.SetTag("test.result.status", result.tag) |> ignore + span.SetTag("test.result.message", result) |> ignore + + let inline start (span : Activity) = + if isNotNull span then + span.Start() |> ignore + span + + let inline stop (span : Activity) = + if isNotNull span then + span.Stop() |> ignore + + let inline setEndTimeNow (span : Activity) = + if isNotNull span then + span.SetEndTime(DateTime.UtcNow) |> ignore + + let inline createActivity (name : string) (source : ActivitySource) = + match source with + | source when not(isNull source) -> source.CreateActivity(name, ActivityKind.Internal) + | _ -> null + + open Activity + open System.Runtime.ExceptionServices + open System.IO + + let inline internal reraiseAnywhere<'a> (e: exn) : 'a = + ExceptionDispatchInfo.Capture(e).Throw() + Unchecked.defaultof<'a> + + module TestResult = + let ofException (e:Exception) : TestResult = + match e with + | :? AssertException as e -> + let msg = + "\n" + e.Message + "\n" + + (e.StackTrace.Split('\n') + |> Seq.skipWhile (fun l -> l.StartsWith(" at Expecto.Expect.")) + |> Seq.truncate 5 + |> String.concat "\n") + Failed msg + + | :? FailedException as e -> + Failed ("\n"+e.Message) + | :? IgnoreException as e -> + Ignored e.Message + | :? AggregateException as e when e.InnerExceptions.Count = 1 -> + if e.InnerException :? IgnoreException then + Ignored e.InnerException.Message + else + Error e.InnerException + | e -> + Error e + + + let addExceptionOutcomeToSpan (span: Activity) (e: Exception) = + let testResult = TestResult.ofException e + + addOutcome testResult span + match testResult with + | Ignored _ -> + setExn e span + | _ -> + setExnMarkFailed e span + + let wrapCodeWithSpan (span: Activity) (test: TestCode) = + let inline handleSuccess span = + setEndTimeNow span + addOutcome Passed span + setStatus ActivityStatusCode.Ok span + let inline handleFailure span e = + setEndTimeNow span + addExceptionOutcomeToSpan span e + reraiseAnywhere e + + match test with + | Sync test -> + TestCode.Sync (fun () -> + use span = start span + try + test () + handleSuccess span + with + | e -> + handleFailure span e + ) + + | Async test -> + TestCode.Async (async { + use span = start span + try + do! test + handleSuccess span + with + | e -> + handleFailure span e + }) + | AsyncFsCheck (testConfig, stressConfig, test) -> + TestCode.AsyncFsCheck (testConfig, stressConfig, fun fsCheckConfig -> async { + use span = start span + try + do! test fsCheckConfig + handleSuccess span + with + | e -> + handleFailure span e + }) + | SyncWithCancel test-> + TestCode.SyncWithCancel (fun ct -> + use span = start span + try + test ct + handleSuccess span + with + | e -> + handleFailure span e + ) + + let addOpenTelemetry_SpanPerTest (config: ExpectoConfig) (activitySource: ActivitySource) (rootTest: Test) : Test = + rootTest + |> Test.toTestCodeList + |> List.map (fun test -> + let span = activitySource |> createActivity (config.joinWith.format test.name) + span |> setSourceLocation (config.locate test.test) + {test with test = wrapCodeWithSpan span test.test} + ) + |> Test.fromFlatTests config.joinWith.asString + diff --git a/Expecto.Tests/Tests.fs b/Expecto.Tests/Tests.fs index 7fc608ac..2f3ce2e2 100644 --- a/Expecto.Tests/Tests.fs +++ b/Expecto.Tests/Tests.fs @@ -10,6 +10,31 @@ open Expecto open Expecto.Impl open Expecto.Logging open System.Globalization +open OpenTelemetry.Resources +open OpenTelemetry.Trace +open System.Diagnostics +open OpenTelemetry + +let serviceName = "Expecto.Tests" + +let source = new ActivitySource(serviceName) + +let resourceBuilder () = + ResourceBuilder + .CreateDefault() + .AddService(serviceName = serviceName) + +let traceProvider () = + Sdk + .CreateTracerProviderBuilder() + .AddSource(serviceName) + .SetResourceBuilder(resourceBuilder ()) + .AddOtlpExporter() + .Build() +do + let provider = traceProvider() + AppDomain.CurrentDomain.ProcessExit.Add(fun _ -> provider.Dispose()) + module Dummy = @@ -1380,6 +1405,8 @@ let asyncTests = ] open System.Threading.Tasks +open OpenTelemetry +open System.Diagnostics [] let taskTests = @@ -1828,6 +1855,7 @@ let cancel = ) ] + [] let theory = testList "theory testing" [ @@ -1855,3 +1883,21 @@ let theory = } ] ] + |> addOpenTelemetry_SpanPerTest ExpectoConfig.defaultConfig source + + + +[] +let fixtures = + let rng = Random() + let tests = [ + for i in 1..(Environment.ProcessorCount * 2) do + testCaseAsync (sprintf "test %d" i) <| async { + printfn "Running test %d" i + do! Async.Sleep(rng.Next(1, 5000)) + printfn "Finished Running test %d" i + } + ] + + testList "MyTests" tests + |> addOpenTelemetry_SpanPerTest ExpectoConfig.defaultConfig source diff --git a/Expecto.Tests/paket.references b/Expecto.Tests/paket.references index 36cbbdb4..4c4484e0 100644 --- a/Expecto.Tests/paket.references +++ b/Expecto.Tests/paket.references @@ -1 +1,4 @@ -FsCheck \ No newline at end of file +FsCheck +OpenTelemetry.Exporter.OpenTelemetryProtocol +YoloDev.Expecto.TestSdk +Microsoft.NET.Test.Sdk \ No newline at end of file diff --git a/Expecto/Expecto.Impl.fs b/Expecto/Expecto.Impl.fs index db4b0145..83d33665 100644 --- a/Expecto/Expecto.Impl.fs +++ b/Expecto/Expecto.Impl.fs @@ -1,6 +1,7 @@ namespace Expecto open System +open System.Collections.Generic open System.Diagnostics open System.Reflection open System.Threading diff --git a/Expecto/Expecto.fs b/Expecto/Expecto.fs index cae655c4..4dd3c7dd 100644 --- a/Expecto/Expecto.fs +++ b/Expecto/Expecto.fs @@ -11,6 +11,7 @@ module Tests = open Impl open Helpers open Expecto.Logging + open System.Diagnostics let mutable private afterRunTestsList = [] let private afterRunTestsListLock = obj() @@ -445,6 +446,7 @@ module Tests = /// Specify test names join character. | JoinWith of split: string + let options = [ "--sequenced", "Don't run the tests in parallel.", Args.none Sequenced "--parallel", "Run all tests in parallel (default).", Args.none Parallel diff --git a/paket.dependencies b/paket.dependencies index 4b937e21..ce6e1219 100644 --- a/paket.dependencies +++ b/paket.dependencies @@ -9,6 +9,9 @@ nuget Hopac ~> 0.4 nuget DiffPlex ~> 1.5 nuget Mono.Cecil ~> 0.11 nuget BenchmarkDotNet ~> 0.13.5 +nuget OpenTelemetry.Exporter.OpenTelemetryProtocol +nuget YoloDev.Expecto.TestSdk +nuget Microsoft.NET.Test.Sdk group FsCheck3 source https://api.nuget.org/v3/index.json diff --git a/paket.lock b/paket.lock index 914c436f..0713eaa3 100644 --- a/paket.lock +++ b/paket.lock @@ -21,10 +21,23 @@ NUGET BenchmarkDotNet.Annotations (0.13.5) CommandLineParser (2.7.82) DiffPlex (1.7.1) + Expecto (10.2.1) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net6.0)) + FSharp.Core (>= 7.0.200) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net6.0)) + Mono.Cecil (>= 0.11.4 < 1.0) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net6.0)) FsCheck (2.16.5) FSharp.Core (>= 4.2.3) FSharp.Core (7.0.200) Gee.External.Capstone (2.3) + Google.Protobuf (3.26.1) + System.Memory (>= 4.5.3) - restriction: || (&& (== net6.0) (>= net45)) (&& (== net6.0) (< net5.0)) (&& (== net6.0) (< netstandard2.0)) (== netstandard2.1) + System.Runtime.CompilerServices.Unsafe (>= 4.5.2) - restriction: || (&& (== net6.0) (< net5.0)) (== netstandard2.1) + Grpc.Core.Api (2.62) + Grpc.Net.Client (2.62) + Grpc.Net.Common (>= 2.62) + Microsoft.Extensions.Logging.Abstractions (>= 6.0) + System.Diagnostics.DiagnosticSource (>= 6.0.1) - restriction: || (&& (== net6.0) (>= net462)) (&& (== net6.0) (< netstandard2.1)) (== netstandard2.1) + Grpc.Net.Common (2.62) + Grpc.Core.Api (>= 2.62) Hopac (0.5.1) FSharp.Core (>= 4.5.2) Iced (1.18) @@ -40,6 +53,7 @@ NUGET System.Threading.Tasks.Extensions (>= 4.5.3) Microsoft.CodeAnalysis.CSharp (3.5) Microsoft.CodeAnalysis.Common (3.5) + Microsoft.CodeCoverage (17.9) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net462)) (&& (== netstandard2.1) (>= netcoreapp3.1)) Microsoft.Diagnostics.NETCore.Client (0.2.410101) Microsoft.Bcl.AsyncInterfaces (>= 1.1) Microsoft.Extensions.Logging (>= 2.1.1) @@ -50,39 +64,90 @@ NUGET Microsoft.Diagnostics.Tracing.TraceEvent (3.0.8) System.Runtime.CompilerServices.Unsafe (>= 5.0) Microsoft.DotNet.PlatformAbstractions (3.1.6) - Microsoft.Extensions.DependencyInjection (7.0) - Microsoft.Extensions.DependencyInjection.Abstractions (>= 7.0) - Microsoft.Extensions.DependencyInjection.Abstractions (7.0) - Microsoft.Extensions.Logging (7.0) - Microsoft.Extensions.DependencyInjection (>= 7.0) - Microsoft.Extensions.DependencyInjection.Abstractions (>= 7.0) - Microsoft.Extensions.Logging.Abstractions (>= 7.0) - Microsoft.Extensions.Options (>= 7.0) - System.Diagnostics.DiagnosticSource (>= 7.0) - restriction: || (&& (== net6.0) (>= net462)) (&& (== net6.0) (< netstandard2.1)) (== netstandard2.1) - Microsoft.Extensions.Logging.Abstractions (7.0) + Microsoft.Extensions.Configuration (8.0) + Microsoft.Extensions.Configuration.Abstractions (>= 8.0) + Microsoft.Extensions.Primitives (>= 8.0) + Microsoft.Extensions.Configuration.Abstractions (8.0) + Microsoft.Extensions.Primitives (>= 8.0) + Microsoft.Extensions.Configuration.Binder (8.0.1) + Microsoft.Extensions.Configuration.Abstractions (>= 8.0) + Microsoft.Extensions.DependencyInjection (8.0) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 8.0) + Microsoft.Extensions.DependencyInjection.Abstractions (8.0.1) + Microsoft.Extensions.Diagnostics.Abstractions (8.0) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 8.0) + Microsoft.Extensions.Options (>= 8.0) System.Buffers (>= 4.5.1) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) + System.Diagnostics.DiagnosticSource (>= 8.0) System.Memory (>= 4.5.5) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) - Microsoft.Extensions.Options (7.0.1) - Microsoft.Extensions.DependencyInjection.Abstractions (>= 7.0) - Microsoft.Extensions.Primitives (>= 7.0) - Microsoft.Extensions.Primitives (7.0) + Microsoft.Extensions.Logging (8.0) + Microsoft.Extensions.DependencyInjection (>= 8.0) + Microsoft.Extensions.Logging.Abstractions (>= 8.0) + Microsoft.Extensions.Options (>= 8.0) + System.Diagnostics.DiagnosticSource (>= 8.0) - restriction: || (&& (== net6.0) (>= net462)) (&& (== net6.0) (< netstandard2.1)) (== netstandard2.1) + Microsoft.Extensions.Logging.Abstractions (8.0.1) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 8.0.1) + System.Buffers (>= 4.5.1) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) + System.Memory (>= 4.5.5) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) + Microsoft.Extensions.Logging.Configuration (8.0) + Microsoft.Extensions.Configuration (>= 8.0) + Microsoft.Extensions.Configuration.Abstractions (>= 8.0) + Microsoft.Extensions.Configuration.Binder (>= 8.0) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 8.0) + Microsoft.Extensions.Logging (>= 8.0) + Microsoft.Extensions.Logging.Abstractions (>= 8.0) + Microsoft.Extensions.Options (>= 8.0) + Microsoft.Extensions.Options.ConfigurationExtensions (>= 8.0) + Microsoft.Extensions.Options (8.0.2) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 8.0) + Microsoft.Extensions.Primitives (>= 8.0) + System.ComponentModel.Annotations (>= 5.0) - restriction: || (&& (== net6.0) (< netstandard2.1)) (== netstandard2.1) + Microsoft.Extensions.Options.ConfigurationExtensions (8.0) + Microsoft.Extensions.Configuration.Abstractions (>= 8.0) + Microsoft.Extensions.Configuration.Binder (>= 8.0) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 8.0) + Microsoft.Extensions.Options (>= 8.0) + Microsoft.Extensions.Primitives (>= 8.0) + Microsoft.Extensions.Primitives (8.0) System.Memory (>= 4.5.5) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) System.Runtime.CompilerServices.Unsafe (>= 6.0) + Microsoft.NET.Test.Sdk (17.9) + Microsoft.CodeCoverage (>= 17.9) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net462)) (&& (== netstandard2.1) (>= netcoreapp3.1)) + Microsoft.TestPlatform.TestHost (>= 17.9) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp3.1)) Microsoft.NETCore.Platforms (3.1) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp2.0)) (&& (== netstandard2.1) (>= netcoreapp3.1)) + Microsoft.TestPlatform.ObjectModel (17.9) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp3.1)) + System.Reflection.Metadata (>= 1.6) + Microsoft.TestPlatform.TestHost (17.9) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp3.1)) + Microsoft.TestPlatform.ObjectModel (>= 17.9) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp3.1)) + Newtonsoft.Json (>= 13.0.1) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp3.1)) Microsoft.Win32.Registry (5.0) - restriction: == netstandard2.1 System.Buffers (>= 4.5.1) - restriction: || (&& (== net6.0) (>= monoandroid) (< netstandard1.3)) (&& (== net6.0) (>= monotouch)) (&& (== net6.0) (< netcoreapp2.0)) (&& (== net6.0) (>= xamarinios)) (&& (== net6.0) (>= xamarinmac)) (&& (== net6.0) (>= xamarintvos)) (&& (== net6.0) (>= xamarinwatchos)) (== netstandard2.1) System.Memory (>= 4.5.4) - restriction: || (&& (== net6.0) (< netcoreapp2.0)) (&& (== net6.0) (< netcoreapp2.1)) (&& (== net6.0) (>= uap10.1)) (== netstandard2.1) System.Security.AccessControl (>= 5.0) System.Security.Principal.Windows (>= 5.0) Mono.Cecil (0.11.4) + Newtonsoft.Json (13.0.3) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp3.1)) + OpenTelemetry (1.8) + Microsoft.Extensions.Diagnostics.Abstractions (>= 8.0) + Microsoft.Extensions.Logging.Configuration (>= 8.0) + OpenTelemetry.Api.ProviderBuilderExtensions (>= 1.8) + OpenTelemetry.Api (1.8) + System.Diagnostics.DiagnosticSource (>= 8.0) + OpenTelemetry.Api.ProviderBuilderExtensions (1.8) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 8.0) + OpenTelemetry.Api (>= 1.8) + OpenTelemetry.Exporter.OpenTelemetryProtocol (1.8) + Google.Protobuf (>= 3.22.5 < 4.0) + Grpc.Net.Client (>= 2.52 < 3.0) + OpenTelemetry (>= 1.8) Perfolizer (0.2.1) System.Memory (>= 4.5.3) System.Buffers (4.5.1) - restriction: == netstandard2.1 System.CodeDom (7.0) System.Collections.Immutable (7.0) - System.Memory (>= 4.5.5) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) System.Runtime.CompilerServices.Unsafe (>= 6.0) - System.Diagnostics.DiagnosticSource (7.0.2) - restriction: || (&& (== net6.0) (>= net462)) (&& (== net6.0) (< netstandard2.1)) (== netstandard2.1) + System.ComponentModel.Annotations (5.0) - restriction: || (&& (== net6.0) (< netstandard2.1)) (== netstandard2.1) + System.Diagnostics.DiagnosticSource (8.0) System.Memory (>= 4.5.5) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) System.Runtime.CompilerServices.Unsafe (>= 6.0) System.Management (7.0) @@ -105,6 +170,10 @@ NUGET System.Runtime.CompilerServices.Unsafe (>= 4.7) - restriction: || (&& (== net6.0) (>= net461)) (&& (== net6.0) (< netcoreapp2.0)) (&& (== net6.0) (< netcoreapp3.1)) (== netstandard2.1) System.Threading.Tasks.Extensions (4.5.4) System.Runtime.CompilerServices.Unsafe (>= 4.5.3) - restriction: || (&& (== net6.0) (>= net461)) (&& (== net6.0) (< netcoreapp2.1)) (&& (== net6.0) (< netstandard1.0)) (&& (== net6.0) (< netstandard2.0)) (&& (== net6.0) (>= wp8)) (== netstandard2.1) + YoloDev.Expecto.TestSdk (0.14.3) + Expecto (>= 10.0 < 11.0) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net6.0)) + FSharp.Core (>= 7.0.200) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net6.0)) + System.Collections.Immutable (>= 6.0) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net6.0)) GROUP Build STORAGE: NONE