Add threshold argument to ZeroMeasurementHelper.

dotnet · Jul 10, 2023 · ef16a87 · ef16a87
1 parent a6a4625
commit ef16a87
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 6 deletions.
diff --git a/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs b/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs
@@ -1,4 +1,5 @@
 using Perfolizer.Mathematics.SignificanceTesting;
+using Perfolizer.Mathematics.Thresholds;
 
 namespace BenchmarkDotNet.Analysers
 {
@@ -19,11 +20,11 @@ public static bool CheckZeroMeasurementOneSample(double[] results, double thresh
         /// Checks distribution against Zero Measurement hypothesis in case of two samples
         /// </summary>
         /// <returns>True if measurement is ZeroMeasurement</returns>
-        public static bool CheckZeroMeasurementTwoSamples(double[] workload, double[] overhead)
+        public static bool CheckZeroMeasurementTwoSamples(double[] workload, double[] overhead, Threshold threshold = null)
         {
             if (workload.Length < 3 || overhead.Length < 3)
                 return false;
-            return !WelchTest.Instance.IsGreater(workload, overhead).NullHypothesisIsRejected;
+            return !WelchTest.Instance.IsGreater(workload, overhead, threshold).NullHypothesisIsRejected;
         }
     }
 }
diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs
@@ -9,9 +9,12 @@
 using BenchmarkDotNet.Engines;
 using BenchmarkDotNet.Extensions;
 using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Reports;
 using BenchmarkDotNet.Tests.XUnit;
 using BenchmarkDotNet.Toolchains.InProcess.Emit;
+using Perfolizer.Horology;
+using Perfolizer.Mathematics.Thresholds;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -22,6 +25,8 @@ public class ExpectedBenchmarkResultsTests : BenchmarkTestExecutor
         // NativeAot takes a long time to build, so not including it in these tests.
         // We also don't test InProcessNoEmitToolchain because it is known to be less accurate than code-gen toolchains.
 
+        private static readonly TimeInterval FallbackCpuResolutionValue = TimeInterval.FromNanoseconds(0.2d);
+
         public ExpectedBenchmarkResultsTests(ITestOutputHelper output) : base(output) { }
 
         private static IEnumerable<Type> EmptyBenchmarkTypes() =>
@@ -106,15 +111,18 @@ private void AssertZeroResults(Type benchmarkType, IConfig config)
                 .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false)))
             );
 
+            var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue;
+            var threshold = Threshold.Create(ThresholdUnit.Nanoseconds, cpuResolution.Nanoseconds);
+
             foreach (var report in summary.Reports)
             {
                 var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers();
                 var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().WithoutOutliers();
 
-                bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements);
+                bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements, threshold);
                 Assert.True(isZero, $"Actual time was not 0.");
 
-                isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements);
+                isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements, threshold);
                 Assert.True(isZero, "Overhead took more time than workload.");
 
                 Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0.");
@@ -159,15 +167,18 @@ private void AssertDifferentSizedStructsResults(IConfig config)
                 .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false)))
             );
 
+            var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue;
+            var threshold = Threshold.Create(ThresholdUnit.Nanoseconds, cpuResolution.Nanoseconds);
+
             foreach (var report in summary.Reports)
             {
                 var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers();
                 var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().WithoutOutliers();
 
-                bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements);
+                bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements, threshold);
                 Assert.False(isZero, $"Actual time was 0.");
 
-                isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements);
+                isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements, threshold);
                 Assert.True(isZero, "Overhead took more time than workload.");
 
                 Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0.");