Memory diagnoser fix for Tiered Compilation

src/BenchmarkDotNet/Engines/Engine.cs

@@ -183,6 +183,13 @@ public Measurement RunIteration(IterationData data)
             // it does not matter, because we have already obtained the results!
             EnableMonitoring();
 
+            if (RuntimeInformation.IsNetCore)
+            {
+                // we put the current thread to sleep so Tiered Compiler can kick in, compile it's stuff
+                // and NOT allocate anything on the background thread when we are measuring allocations
+                System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(250));
+            }
+
             IterationSetupAction(); // we run iteration setup first, so even if it allocates, it is not included in the results
 
             var initialThreadingStats = ThreadingStats.ReadInitial(); // this method might allocate

src/BenchmarkDotNet/Portability/RuntimeInformation.cs

@@ -26,13 +26,13 @@ internal static class RuntimeInformation
 
         public static bool IsMono { get; } = Type.GetType("Mono.Runtime") != null; // it allocates a lot of memory, we need to check it once in order to keep Engine non-allocating!
 
-        public static bool IsFullFramework => FrameworkDescription.StartsWith(".NET Framework", StringComparison.OrdinalIgnoreCase);
+        public static bool IsFullFramework { get; } = FrameworkDescription.StartsWith(".NET Framework", StringComparison.OrdinalIgnoreCase);
 
         [PublicAPI]
         public static bool IsNetNative => FrameworkDescription.StartsWith(".NET Native", StringComparison.OrdinalIgnoreCase);
 
-        public static bool IsNetCore
-            => ((Environment.Version.Major >= 5) || FrameworkDescription.StartsWith(".NET Core", StringComparison.OrdinalIgnoreCase))
+        public static bool IsNetCore { get; }
+            = ((Environment.Version.Major >= 5) || FrameworkDescription.StartsWith(".NET Core", StringComparison.OrdinalIgnoreCase))
                 && !string.IsNullOrEmpty(typeof(object).Assembly.Location);
 
         /// <summary>

tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs

@@ -105,6 +105,16 @@ public void MemoryDiagnoserDoesNotIncludeAllocationsFromSetupAndCleanup(IToolcha
         public class NoAllocationsAtAll
         {
             [Benchmark] public void EmptyMethod() { }
+
+            [Benchmark] public ulong TimeConsuming()
+            {
+                var r = 1ul;
+                for (var i = 0; i < 50_000_000; i++)
+                {
+                    r /= 1;
+                }
+                return r;
+            }
         }
 
         [Theory, MemberData(nameof(GetToolchains))]
@@ -117,6 +127,19 @@ public void EngineShouldNotInterfereAllocationResults(IToolchain toolchain)
             });
         }
 
+        [Theory, MemberData(nameof(GetToolchains))]
+        [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
+        public void TieredJitShouldNotInterfereAllocationResults(IToolchain toolchain)
+        {
+            // see https://github.com/dotnet/BenchmarkDotNet/issues/1542 for details
+
+            AssertAllocations(toolchain, typeof(NoAllocationsAtAll), new Dictionary<string, long>
+            {
+                { nameof(NoAllocationsAtAll.TimeConsuming), 0 }
+            },
+            iterationCount: 10); // 1 iteration is not enough to repro the problem
+        }
+
         public class NoBoxing
         {
             [Benchmark] public ValueTuple<int> ReturnsValueType() => new ValueTuple<int>(0);
@@ -255,9 +278,9 @@ public void MemoryDiagnoserIsAccurateForMultiThreadedBenchmarks(IToolchain toolc
             });
         }
 
-        private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Dictionary<string, long> benchmarksAllocationsValidators)
+        private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Dictionary<string, long> benchmarksAllocationsValidators, int iterationCount = 1)
         {
-            var config = CreateConfig(toolchain);
+            var config = CreateConfig(toolchain, iterationCount);
             var benchmarks = BenchmarkConverter.TypeToBenchmarks(benchmarkType, config);
 
             var summary = BenchmarkRunner.Run(benchmarks);
@@ -285,12 +308,12 @@ private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Diction
             }
         }
 
-        private IConfig CreateConfig(IToolchain toolchain)
+        private IConfig CreateConfig(IToolchain toolchain, int iterationCount = 1) // single iteration is enough for most of the tests
             => ManualConfig.CreateEmpty()
                 .AddJob(Job.ShortRun
                     .WithEvaluateOverhead(false) // no need to run idle for this test
                     .WithWarmupCount(0) // don't run warmup to save some time for our CI runs
-                    .WithIterationCount(1) // single iteration is enough for us
+                    .WithIterationCount(iterationCount)
                     .WithGcForce(false)
                     .WithToolchain(toolchain))
                 .AddColumnProvider(DefaultColumnProviders.Instance)