Listen to JIT events

src/BenchmarkDotNet/Code/DeclarationsProvider.cs

@@ -34,6 +34,8 @@ public SmartStringBuilder ReplaceTemplate(SmartStringBuilder smartStringBuilder)
             return ReplaceCore(smartStringBuilder)
                 .Replace("$DisassemblerEntryMethodImpl$", GetWorkloadMethodCall(GetPassArgumentsDirect()))
                 .Replace("$OperationsPerInvoke$", Descriptor.OperationsPerInvoke.ToString())
+                .Replace("$WorkloadMethodName$", Descriptor.WorkloadMethod.Name)
+                .Replace("$WorkloadMethodParameterTypes$", GetWorkloadMethodParameterTypes())
                 .Replace("$WorkloadTypeName$", Descriptor.Type.GetCorrectCSharpTypeName());
         }
 
@@ -108,6 +110,26 @@ protected string GetPassArguments()
                     .Select((parameter, index) => $"{CodeGenerator.GetParameterModifier(parameter)} arg{index}")
             );
 
+        // Renders the benchmark method's parameter types as a Type[] for __ResolveWorkloadMethods to match overloads
+        // exactly. Each is a typeof(...) of the element type, re-wrapping by-ref/pointer via reflection (typeof can't
+        // express `T&`), so resolution never has to name the method's (possibly unspellable) return type.
+        private string GetWorkloadMethodParameterTypes()
+        {
+            var parameters = Descriptor.WorkloadMethod.GetParameters();
+            if (parameters.Length == 0)
+                return "global::System.Array.Empty<global::System.Type>()";
+            return $"new global::System.Type[] {{ {string.Join(", ", parameters.Select(p => GetTypeOfExpression(p.ParameterType)))} }}";
+        }
+
+        private static string GetTypeOfExpression(System.Type type)
+        {
+            if (type.IsByRef)
+                return $"{GetTypeOfExpression(type.GetElementType()!)}.MakeByRefType()";
+            if (type.IsPointer)
+                return $"{GetTypeOfExpression(type.GetElementType()!)}.MakePointerType()";
+            return $"typeof({type.GetCorrectCSharpTypeName()})";
+        }
+
         protected string GetPassArgumentsDirect()
             => string.Join(
                 ", ",

src/BenchmarkDotNet/Engines/Engine.cs

@@ -34,6 +34,7 @@ internal Engine(EngineParameters engineParameters)
             var job = engineParameters.TargetJob ?? throw new ArgumentNullException(nameof(EngineParameters.TargetJob));
             Parameters = new()
             {
+                WorkloadMethods = engineParameters.WorkloadMethods ?? throw new ArgumentNullException(nameof(EngineParameters.WorkloadMethods)),
                 WorkloadActionNoUnroll = engineParameters.WorkloadActionNoUnroll ?? throw new ArgumentNullException(nameof(EngineParameters.WorkloadActionNoUnroll)),
                 WorkloadActionUnroll = engineParameters.WorkloadActionUnroll ?? throw new ArgumentNullException(nameof(EngineParameters.WorkloadActionUnroll)),
                 OverheadActionNoUnroll = engineParameters.OverheadActionNoUnroll ?? throw new ArgumentNullException(nameof(EngineParameters.OverheadActionNoUnroll)),

src/BenchmarkDotNet/Engines/EngineJitStage.cs

@@ -1,3 +1,4 @@
+using BenchmarkDotNet.Attributes.CompilerServices;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Reports;
@@ -9,29 +10,49 @@ namespace BenchmarkDotNet.Engines;
 // and we purposefully don't spend too much time in this stage, so we can't guarantee it.
 // This should succeed for 99%+ of microbenchmarks. For any sufficiently short benchmarks where this fails,
 // the following stages (Pilot and Warmup) will likely take it the rest of the way. Long-running benchmarks may never fully reach tier1.
+[AggressivelyOptimizeMethods] // Reduce JIT event noise from the jit stage itself.
 internal sealed class EngineJitStage : EngineStage
 {
-    // Jit call counting delay is only for when the app starts up. We don't need to wait for every benchmark if multiple benchmarks are ran in-process.
-    private static TimeSpan s_tieredDelay = JitInfo.TieredDelay;
+    // After a tier's single burst fails to tier-up, we nudge one invocation at a time, giving the background worker a
+    // short window (~10ms) to pick each nudge up before trying the next — so we stop the instant the tier-up lands
+    // instead of overshooting by re-bursting the whole budget. Passed to WaitForTierUp as its busy-wait timeout.
+    private static readonly TimeSpan EventDeliveryLag = TimeSpan.FromMilliseconds(10);
 
     internal bool didStopEarly = false;
     internal Measurement lastMeasurement;
 
     private readonly IEnumerator<IterationData> enumerator;
     private readonly bool evaluateOverhead;
+    private readonly bool skipDelays;
+    // Watches the benchmark method(s)' background tier-up via JIT events so we can proceed once the JIT goes quiet after
+    // each tier (the whole call tree warmed), instead of waiting a fixed delay. Null when there's nothing to watch or
+    // EventSource is disabled, in which case we fall back to the fixed delay.
+    private readonly JitListener? listener;
+    // True when this stage created the listener and must dispose it; false when a caller (a test) injected one it owns.
+    private readonly bool disposeListener;
+
+    internal EngineJitStage(bool evaluateOverhead, EngineParameters parameters, bool skipDelays)
+        : this(evaluateOverhead, parameters, JitListener.Create(parameters.WorkloadMethods), disposeListener: true, skipDelays: skipDelays)
+    {
+    }
 
-    internal EngineJitStage(bool evaluateOverhead, EngineParameters parameters) : base(IterationStage.Jitting, IterationMode.Workload, parameters)
+    internal EngineJitStage(bool evaluateOverhead, EngineParameters parameters, JitListener? listener, bool disposeListener = false, bool skipDelays = false)
+        : base(IterationStage.Jitting, IterationMode.Workload, parameters)
     {
+        this.listener = listener;
+        this.disposeListener = disposeListener;
         enumerator = EnumerateIterations();
         this.evaluateOverhead = evaluateOverhead;
+        this.skipDelays = skipDelays;
     }
 
     internal override List<Measurement> GetMeasurementList() => new(GetMaxMeasurementCount());
 
     private int GetMaxMeasurementCount()
     {
+        int nudgeMultiplier = JitInfo.TieredDelay > TimeSpan.Zero ? 2 : 1;
         int count = JitInfo.IsTiered
-            ? JitInfo.MaxTierPromotions * JitInfo.TieredCallCountThreshold + 2
+            ? JitInfo.MaxTierPromotions * JitInfo.TieredCallCountThreshold * nudgeMultiplier + 2
             : 1;
         if (evaluateOverhead)
         {
@@ -44,7 +65,7 @@ internal override bool GetShouldRunIteration(List<Measurement> measurements, out
     {
         if (measurements.Count > 0)
         {
-            var measurement = measurements[measurements.Count - 1];
+            var measurement = measurements[^1];
             if (measurement.IterationMode == IterationMode.Workload)
             {
                 lastMeasurement = measurement;
@@ -55,6 +76,10 @@ internal override bool GetShouldRunIteration(List<Measurement> measurements, out
             iterationData = enumerator.Current;
             return true;
         }
+        if (disposeListener)
+        {
+            listener?.Dispose();
+        }
         enumerator.Dispose();
         iterationData = default;
         return false;
@@ -81,16 +106,23 @@ private IEnumerator<IterationData> EnumerateIterations()
             yield break;
         }
 
-        // Wait enough time for jit call counting to begin.
-        Engine.SleepIfPositive(s_tieredDelay);
-        // Don't make the next jit stage wait if it's ran in the same process.
-        s_tieredDelay = TimeSpan.Zero;
+        bool observeMethod = listener != null;
+        if (observeMethod)
+        {
+            // Before the tier loop, wait until the call-counting delay is inactive so the first burst is counted —
+            // or, if tiering is quiet because the method was pre-warmed past tier0, fake it and proceed. The first
+            // invoke above already fired the watched method's Pause if it was tier0. See WaitForInitialTieringActive.
+            listener!.WaitForInitialTieringActive(parameters.Host.CancellationToken);
+        }
+        else if (!skipDelays && JitInfo.TieredDelay > TimeSpan.Zero)
+        {
+            // Fall back to a fixed wait for the call-counting delay to elapse.
+            Thread.Sleep(JitInfo.TieredDelay + TimeSpan.FromMilliseconds(10));
+        }
 
-        // If the first iteration suggests a long-running benchmark (a single invocation already
-        // takes ~2/3 of IterationTime or more), run one confirmation iteration and bail out if
-        // it agrees. Same cutoff value that pilot stage uses.
-        // We do not bail out immediately if the first iteration is long-running because it could
-        // be due to cctors or other lazy initialization that won't be hit in steady-state. #2004
+        // Long-running early-exit: if a single invocation already takes ~2/3 of IterationTime, this is a long-running
+        // benchmark — bail and let the Pilot/Warmup stages finish tiering. The first invoke can be inflated by JIT or
+        // cctors, so confirm with one more iteration before bailing (it could be a one-time cost). #2004
         // JitTieringMode.Force opts out of this heuristic and always promotes through every tier.
         TimeInterval iterationTime = parameters.TargetJob.ResolveValue(RunMode.IterationTimeCharacteristic, parameters.Resolver);
         long remainingCalls = JitInfo.TieredCallCountThreshold;
@@ -104,12 +136,18 @@ private IEnumerator<IterationData> EnumerateIterations()
                 didStopEarly = true;
                 yield break;
             }
-            remainingCalls -= userInvokeCount;
         }
 
         // Promote methods to tier1.
-        for (int remainingTiers = JitInfo.MaxTierPromotions; remainingTiers > 0; --remainingTiers)
+        for (int tierCount = 0; tierCount < JitInfo.MaxTierPromotions; ++tierCount, remainingCalls = JitInfo.TieredCallCountThreshold)
         {
+            // Run ONE full burst of this tier's call budget, gated so it's counted rather than wasted into a
+            // deferred window. After it, wait for the background JIT to go QUIET (WaitForQuiescentTierUp): once the
+            // worker is idle, this tier's compiles — the watched method(s) AND their untracked callees — have all
+            // landed, so the next burst / the following stage won't race them. The per-tier counter persists, so if
+            // the burst didn't tier the watched method(s) up we nudge the rest one at a time below rather than
+            // re-bursting the whole budget.
+            listener?.WaitForTieringActive(parameters.Host.CancellationToken);
             while (remainingCalls > 0)
             {
                 // Run the whole tier's call budget in a single iteration unless the user pinned InvocationCount.
@@ -120,8 +158,71 @@ private IEnumerator<IterationData> EnumerateIterations()
                 yield return GetWorkloadIterationData(invokeCount);
             }
 
-            Engine.SleepIfPositive(JitInfo.BackgroundCompilationDelay);
-            remainingCalls = JitInfo.TieredCallCountThreshold;
+            if (listener != null)
+            {
+                // Wait for the background JIT to go quiet (the watched method(s) and their callees settle), then read
+                // whether the watched method(s) actually advanced this burst. Once we've stopped observing them the
+                // advanced result is ignored, but this still drains untracked-callee tier-ups before the next burst.
+                bool advanced = listener.WaitForQuiescentTierUp(tierCount, parameters.Host.CancellationToken);
+                if (observeMethod)
+                {
+                    if (!advanced)
+                    {
+                        // The burst didn't tier the watched method(s) up. With NO call-counting delay, the burst's whole
+                        // budget was counted, so a miss means they were pre-warmed past this tier (or are otherwise
+                        // unobservable) — nudging can't help, so stop consulting the listener for them. We don't bail out
+                        // entirely because the benchmark may call other (un-pre-warmed) methods via different control
+                        // flow (e.g. an InProcess toolchain with arguments/params); the remaining bursts warm those + callees.
+                        if (JitInfo.TieredDelay <= TimeSpan.Zero)
+                        {
+                            observeMethod = false;
+                            continue;
+                        }
+
+                        // Otherwise the call-counting delay was probably active for the first ~10ms of the burst due to event
+                        // delivery lag, so some invocations didn't count and we just need a few more. Re-bursting the whole
+                        // budget would overshoot wastefully (up to threshold * call-time), so nudge one invocation at a time,
+                        // detecting the tier-up cheaply (WaitForTierUp, no full quiescence settle per nudge), then
+                        // settle once at the end so this tier's callees are warm.
+                        listener.WaitForTieringActive(parameters.Host.CancellationToken);
+                        long nudgeCalls = hasUserInvocationCount ? userInvokeCount : 1;
+                        for (long nudged = 0; nudged < JitInfo.TieredCallCountThreshold && !advanced; nudged += nudgeCalls)
+                        {
+                            ++iterationIndex;
+                            yield return GetWorkloadIterationData(nudgeCalls);
+                            advanced = listener.WaitForTierUp(tierCount, EventDeliveryLag, parameters.Host.CancellationToken);
+                        }
+                        // Settle the callees pushed by the nudges (and re-read the tier state race-free); this also
+                        // catches a tier-up whose publication arrived just after the last cheap WaitForTierUp window.
+                        advanced = listener.WaitForQuiescentTierUp(tierCount, parameters.Host.CancellationToken);
+                        if (!advanced)
+                        {
+                            // Even nudging didn't tier them up — most likely pre-warmed to their final tier before the
+                            // stage started (e.g. via InProcess toolchains). Stop consulting the listener (same as the
+                            // no-delay case above). We already spent ~2 tiers' worth here, so skip a tier (the extra
+                            // ++tierCount on top of the loop's) so we don't overspend the budget.
+                            observeMethod = false;
+                            ++tierCount;
+                            continue;
+                        }
+                    }
+
+                    if (listener.ReachedFinalTier)
+                    {
+                        // ReachedFinalTier is the aggregate: every watched method is fully warmed, so we will not
+                        // receive any more tier-up JIT events for the method(s) we track. (OSR adds a quirk: a runtime
+                        // bug double-instruments an OSR'd callee, which JitInfo.MaxTierPromotions already budgets for.)
+                        // Keep bursting to push any untracked callees through their tiers — the quiescence wait above
+                        // handles them — but stop consulting the listener for the watched method(s).
+                        observeMethod = false;
+                    }
+                }
+            }
+            else if (!skipDelays)
+            {
+                // No listener (nothing to watch, or EventSource unavailable), fall back to the fixed delay.
+                Engine.SleepIfPositive(JitInfo.BackgroundCompilationDelay);
+            }
         }
 
         // Empirical evidence shows that the first call after the method is tiered up may take longer,

src/BenchmarkDotNet/Engines/EngineParameters.cs

@@ -1,3 +1,4 @@
+using System.Reflection;
 using BenchmarkDotNet.Characteristics;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Running;
@@ -16,6 +17,13 @@ public class EngineParameters
         public required Func<long, IClock, ValueTask<ClockSpan>> OverheadActionNoUnroll { get; set; }
         public required Func<long, IClock, ValueTask<ClockSpan>> OverheadActionUnroll { get; set; }
         public Job TargetJob { get; set; } = Job.Default;
+
+        /// <summary>
+        /// The benchmark method(s), used by the jit stage to watch for their tier-up via JIT events.
+        /// When empty (nothing to watch, or resolution failed), the jit stage falls back to a fixed delay.
+        /// </summary>
+        public required IEnumerable<MethodInfo> WorkloadMethods { get; set; }
+
         public long OperationsPerInvoke { get; set; } = 1;
         public required Func<ValueTask> GlobalSetupAction { get; set; }
         public required Func<ValueTask> GlobalCleanupAction { get; set; }

src/BenchmarkDotNet/Engines/EngineStage.cs

@@ -15,7 +15,9 @@ internal abstract class EngineStage(IterationStage stage, IterationMode mode, En
         internal abstract bool GetShouldRunIteration(List<Measurement> measurements, out IterationData iterationData);
 
         [MethodImpl(MethodImplOptions.NoInlining)]
-        internal static IEnumerable<EngineStage> EnumerateStages(EngineParameters parameters)
+        // skipJitDelays is used by EnumerateStagesTests to skip waiting when it's only testing the stage logic, not real JIT compilation.
+        // Real JIT compilation is tested in JitListenerTests.
+        internal static IEnumerable<EngineStage> EnumerateStages(EngineParameters parameters, bool skipJitDelays = false)
         {
             var strategy = parameters.TargetJob.ResolveValue(RunMode.RunStrategyCharacteristic, parameters.Resolver);
             var invokeCount = parameters.TargetJob.ResolveValue(RunMode.InvocationCountCharacteristic, parameters.Resolver, 1);
@@ -31,7 +33,7 @@ internal static IEnumerable<EngineStage> EnumerateStages(EngineParameters parame
                     int minInvokeCount = parameters.TargetJob.ResolveValue(AccuracyMode.MinInvokeCountCharacteristic, parameters.Resolver);
 
                     // AOT technically doesn't have a JIT, but we run jit stage regardless because of static constructors. #2004
-                    var jitStage = new EngineJitStage(evaluateOverhead, parameters);
+                    var jitStage = new EngineJitStage(evaluateOverhead, parameters, skipJitDelays);
                     yield return jitStage;
 
                     bool hasUnrollFactor = parameters.TargetJob.HasValue(RunMode.UnrollFactorCharacteristic);