From 869ba8d074ed9264c5a500afff25f93e67047bd8 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 13 Jul 2026 12:40:13 +0200 Subject: [PATCH 1/2] test: coarsen ASan sampling rate/workload to stop nightly heap OOMs Fixed-interval CPU/wall sampling (100us/1ms) over an iteration-bounded workload scales unboundedly under ASan (signal handling itself is instrumented), and each test materializes a String per sample, OOMing the ASan test-runner heap. Sample coarser and shrink the workload under isAsan(), matching the pattern already used in BoundMethodHandleProfilerTest. --- .../profiler/cpu/CTimerSamplerTest.java | 6 +++++- .../profiler/cpu/LightweightContextCpuTest.java | 5 ++++- .../profiler/cpu/VtableReceiverFrameTest.java | 11 +++++++++-- .../metadata/MetadataNormalisationTest.java | 12 ++++++++---- .../profiler/wallclock/MegamorphicCallTest.java | 17 +++++++++++++---- 5 files changed, 39 insertions(+), 12 deletions(-) diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/CTimerSamplerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/CTimerSamplerTest.java index 812bedc7f8..1dabea65b1 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/CTimerSamplerTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/CTimerSamplerTest.java @@ -72,6 +72,10 @@ protected void after() throws Exception { @Override protected String getProfilerCommand() { - return "cpu=100us,event=ctimer"; + // cpu=100us signal-based sampling is much more expensive under ASAN (the signal + // handler itself is ASAN-instrumented), which can inflate the sample count and, + // via the per-sample stack-trace materialization above, the test heap. Sample + // coarser under ASAN. + return isAsan() ? "cpu=1ms,event=ctimer" : "cpu=100us,event=ctimer"; } } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/LightweightContextCpuTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/LightweightContextCpuTest.java index 8d92967284..8498738fa8 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/LightweightContextCpuTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/LightweightContextCpuTest.java @@ -79,6 +79,9 @@ protected void after() throws Exception { @Override protected String getProfilerCommand() { - return "cpu=100us,lightweight=yes"; + // cpu=100us signal-based sampling is much more expensive under ASAN (the signal + // handler itself is ASAN-instrumented), which can inflate the sample count and, + // via the per-sample accessor calls above, the test heap. Sample coarser under ASAN. + return isAsan() ? "cpu=1ms,lightweight=yes" : "cpu=100us,lightweight=yes"; } } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/VtableReceiverFrameTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/VtableReceiverFrameTest.java index e3c4dfd1f9..34420cee7d 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/VtableReceiverFrameTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/VtableReceiverFrameTest.java @@ -18,7 +18,10 @@ public class VtableReceiverFrameTest extends AbstractProfilerTest { @Override protected String getProfilerCommand() { - return "cpu=1ms"; + // cpu=1ms sampling over the fixed workload below still produces an ASAN-scaled + // number of samples (signal handling is slower under ASAN), and every sample's + // stack trace is materialized below, so sample coarser under ASAN. + return isAsan() ? "cpu=10ms" : "cpu=1ms"; } abstract static class Shape { @@ -41,7 +44,11 @@ static class Triangle extends Shape { private int profiledWork(Shape... shapes) { int result = 0; - for (int i = 0; i < 10_000_000; i++) { + // Reduce workload under ASAN: combined with the coarser sampling rate above, this + // bounds the number of samples (and thus the stack-trace strings materialized + // below) regardless of how much ASAN slows down signal handling. + int iterations = isAsan() ? 1_000_000 : 10_000_000; + for (int i = 0; i < iterations; i++) { for (Shape shape : shapes) { result += shape.area(); } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/metadata/MetadataNormalisationTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/metadata/MetadataNormalisationTest.java index 44ac524eb8..08b6eafa3e 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/metadata/MetadataNormalisationTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/metadata/MetadataNormalisationTest.java @@ -27,14 +27,18 @@ public void test() throws Exception { Method arrayListAdd = ArrayList.class.getDeclaredMethod("add", Object.class); Constructor linkedListConstructor = LinkedList.class.getConstructor(); Method linkedListAdd = LinkedList.class.getDeclaredMethod("add", Object.class); - // need to invoke enough times to result in generation of accessors and to record some cpu time + // need to invoke enough times to result in generation of accessors and to record some cpu time. + // Under ASAN, cpu=100us sampling combined with this fixed iteration count produces a much + // larger sample set (stack-trace signal handling is slower under ASAN), and every sample's + // stack trace is materialized below, so reduce the iteration count to bound the sample count. + int iterations = isAsan() ? 10_000 : 100_000; int count = 0; - for (int i = 0; i < 100_000; i++) { + for (int i = 0; i < iterations; i++) { Object list = arrayListConstructor.newInstance(); arrayListAdd.invoke(list, "element"); count += ((List) list).size(); } - for (int i = 0; i < 100_000; i++) { + for (int i = 0; i < iterations; i++) { Object list = linkedListConstructor.newInstance(); linkedListAdd.invoke(list, "element"); count += ((List) list).size(); @@ -63,6 +67,6 @@ public void test() throws Exception { @Override protected String getProfilerCommand() { - return "cpu=100us"; + return isAsan() ? "cpu=1ms" : "cpu=100us"; } } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/MegamorphicCallTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/MegamorphicCallTest.java index 4c0c776bf5..763994ebad 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/MegamorphicCallTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/MegamorphicCallTest.java @@ -20,7 +20,12 @@ public class MegamorphicCallTest extends AbstractProfilerTest { @Override protected String getProfilerCommand() { - return "wall=100us"; + // wall=100us over the fixed workload below is ~10k samples/s. Under ASAN each + // invocation is much slower, so the same fixed-rate sampling over a much longer + // wall-clock duration produces hundreds of thousands of samples, and stack-trace + // stringification of all of them (below) OOMs the test-runner heap. Sample 10x + // coarser under ASAN, combined with a smaller workload, to bound the sample count. + return isAsan() ? "wall=1ms" : "wall=100us"; } private static int calculation() { @@ -60,9 +65,9 @@ public int calculate() { } } - private int profiledWork(Calculator... calculators) { + private int profiledWork(int iterations, Calculator... calculators) { int result = 0; - for (int i = 0; i < 1_000_000; i++) { + for (int i = 0; i < iterations; i++) { for (Calculator calculator : calculators) { result += calculator.calculate(); } @@ -74,7 +79,11 @@ private int profiledWork(Calculator... calculators) { public void testITableStubs() { Assumptions.assumeFalse(Platform.isZing() || Platform.isJ9()); registerCurrentThreadForWallClockProfiling(); - int result = profiledWork(new Calculator1(), new Calculator2(), new Calculator3()); + // Reduce workload under ASAN: combined with the coarser wall rate above, this + // bounds the number of samples (and thus the stack-trace strings materialized + // below) regardless of how much ASAN slows down each invocation. + int iterations = isAsan() ? 100_000 : 1_000_000; + int result = profiledWork(iterations, new Calculator1(), new Calculator2(), new Calculator3()); System.err.println(result); stopProfiler(); IItemCollection events = verifyEvents("datadog.MethodSample"); From 1f3b41efee20a4547d19f0d69d6fe356b941f594 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 13 Jul 2026 13:03:46 +0200 Subject: [PATCH 2/2] test: revert VtableReceiverFrameTest ASan throttling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unlike the other tests here, this one was never observed failing in CI, already samples at a coarser cpu=1ms, and drives pure-Java virtual dispatch that ASan doesn't slow down — stacking a 10x coarser interval with a 10x smaller workload risked starving the narrow vtable-receiver stack-pattern search instead of fixing a real problem. --- .../profiler/cpu/VtableReceiverFrameTest.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/VtableReceiverFrameTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/VtableReceiverFrameTest.java index 34420cee7d..e3c4dfd1f9 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/VtableReceiverFrameTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/VtableReceiverFrameTest.java @@ -18,10 +18,7 @@ public class VtableReceiverFrameTest extends AbstractProfilerTest { @Override protected String getProfilerCommand() { - // cpu=1ms sampling over the fixed workload below still produces an ASAN-scaled - // number of samples (signal handling is slower under ASAN), and every sample's - // stack trace is materialized below, so sample coarser under ASAN. - return isAsan() ? "cpu=10ms" : "cpu=1ms"; + return "cpu=1ms"; } abstract static class Shape { @@ -44,11 +41,7 @@ static class Triangle extends Shape { private int profiledWork(Shape... shapes) { int result = 0; - // Reduce workload under ASAN: combined with the coarser sampling rate above, this - // bounds the number of samples (and thus the stack-trace strings materialized - // below) regardless of how much ASAN slows down signal handling. - int iterations = isAsan() ? 1_000_000 : 10_000_000; - for (int i = 0; i < iterations; i++) { + for (int i = 0; i < 10_000_000; i++) { for (Shape shape : shapes) { result += shape.area(); }