From cbe304578d8f65f7c4d155fbef5bc68fb988051a Mon Sep 17 00:00:00 2001 From: Robert Pickering Date: Thu, 28 May 2026 15:26:49 +0000 Subject: [PATCH 01/12] Add JMH benchmark CI Visibility instrumentation (SDTEST-930) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instruments JMH's Runner constructor to wrap its OutputFormat with a DDOutputFormat decorator. The decorator fires once per benchmark method (after all forks and iterations complete) to emit CI Visibility test spans — zero overhead on the benchmark hot path. Each benchmark method produces a suite span + test span with benchmark metrics (score, error, unit, percentiles, run config) attached as tags. Parameterised @Param benchmarks follow the same test.parameters convention as JUnit 5 parameterized tests. Changes: - New module: dd-java-agent/instrumentation/jmh/jmh-1.0 - Tags.java: add benchmark.* tag constants - TestFrameworkInstrumentation: add JMH enum value - TestDecorator: add TEST_TYPE_BENCHMARK constant - Design spec: docs/design/jmh-ci-visibility.md Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../civisibility/decorator/TestDecorator.java | 1 + .../instrumentation/jmh/jmh-1.0/build.gradle | 16 ++ .../instrumentation/jmh/DDOutputFormat.java | 206 +++++++++++++++ .../jmh/JmhInstrumentation.java | 58 +++++ .../trace/instrumentation/jmh/JmhUtils.java | 52 ++++ .../instrumentation/jmh/JmhUtilsTest.java | 53 ++++ docs/design/jmh-ci-visibility.md | 235 ++++++++++++++++++ .../tag/TestFrameworkInstrumentation.java | 1 + .../bootstrap/instrumentation/api/Tags.java | 17 ++ settings.gradle.kts | 1 + 10 files changed, 640 insertions(+) create mode 100644 dd-java-agent/instrumentation/jmh/jmh-1.0/build.gradle create mode 100644 dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DDOutputFormat.java create mode 100644 dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhInstrumentation.java create mode 100644 dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhUtils.java create mode 100644 dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/JmhUtilsTest.java create mode 100644 docs/design/jmh-ci-visibility.md diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecorator.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecorator.java index 91f3c5e8101..7236e8c823f 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecorator.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecorator.java @@ -4,6 +4,7 @@ public interface TestDecorator { String TEST_TYPE = "test"; + String TEST_TYPE_BENCHMARK = "benchmark"; AgentSpan afterStart(final AgentSpan span); diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/build.gradle b/dd-java-agent/instrumentation/jmh/jmh-1.0/build.gradle new file mode 100644 index 00000000000..a0b0b64393c --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/build.gradle @@ -0,0 +1,16 @@ +apply from: "$rootDir/gradle/java.gradle" + +muzzle { + pass { + group = 'org.openjdk.jmh' + module = 'jmh-core' + versions = '[1.0,)' + } +} + +dependencies { + compileOnly group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.0' + + testImplementation group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.37' + testImplementation project(':dd-java-agent:agent-ci-visibility:civisibility-instrumentation-test-fixtures') +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DDOutputFormat.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DDOutputFormat.java new file mode 100644 index 00000000000..a5639710c57 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DDOutputFormat.java @@ -0,0 +1,206 @@ +package datadog.trace.instrumentation.jmh; + +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_ERROR; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_FORKS; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_ITERATIONS; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_MAX; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_MIN; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_MODE; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_P50; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_P90; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_P95; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_P99; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_SAMPLE_COUNT; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_THREADS; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_TIME_UNIT; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_UNIT; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_VALUE; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_WARMUP_ITERATIONS; + +import datadog.trace.api.civisibility.InstrumentationBridge; +import datadog.trace.api.civisibility.config.TestSourceData; +import datadog.trace.api.civisibility.events.TestEventsHandler; +import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import java.util.Collections; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.results.BenchmarkResult; +import org.openjdk.jmh.results.Result; +import org.openjdk.jmh.results.RunResult; +import org.openjdk.jmh.runner.format.OutputFormat; +import org.openjdk.jmh.util.Statistics; + +/** + * Wraps a JMH {@link OutputFormat} to emit CI Visibility spans for each benchmark method. + * + *

Hooks only fire once per benchmark method (after all forks and iterations complete), so there + * is zero overhead on the benchmark hot path. + */ +public class DDOutputFormat implements OutputFormat { + + private final OutputFormat delegate; + private final TestEventsHandler handler; + private final String frameworkVersion; + + // Keys used as suite/test descriptors in the handler — just the full benchmark name strings. + // We keep suite and test keys separate so the handler can manage their lifetimes independently. + private volatile String currentSuiteKey; + private volatile String currentTestKey; + + public DDOutputFormat(OutputFormat delegate, String frameworkVersion) { + this.delegate = delegate; + this.frameworkVersion = frameworkVersion; + this.handler = + InstrumentationBridge.createTestEventsHandler( + JmhUtils.FRAMEWORK_NAME, null, null, Collections.emptyList()); + } + + @Override + public void startBenchmark(BenchmarkParams benchParams) { + delegate.startBenchmark(benchParams); + + String fullName = benchParams.getBenchmark(); + String[] parts = JmhUtils.splitBenchmarkName(fullName); + String suiteName = parts[0]; + String testName = parts[1]; + String testParameters = JmhUtils.testParameters(fullName); + + currentSuiteKey = suiteName + "#" + fullName; + currentTestKey = fullName; + + handler.onTestSuiteStart( + currentSuiteKey, + suiteName, + JmhUtils.FRAMEWORK_NAME, + frameworkVersion, + null, + Collections.emptyList(), + false, + TestFrameworkInstrumentation.JMH, + null); + + handler.onTestStart( + currentSuiteKey, + currentTestKey, + testName, + JmhUtils.FRAMEWORK_NAME, + frameworkVersion, + testParameters, + Collections.emptyList(), + TestSourceData.UNKNOWN, + null, + null); + } + + @Override + public void endBenchmark(BenchmarkResult result) { + String suiteKey = currentSuiteKey; + String testKey = currentTestKey; + + tagBenchmarkMetrics(result); + + handler.onTestFinish(testKey, null, null); + handler.onTestSuiteFinish(suiteKey, null); + + delegate.endBenchmark(result); + } + + private void tagBenchmarkMetrics(BenchmarkResult result) { + AgentSpan span = AgentTracer.activeSpan(); + if (span == null) { + return; + } + + BenchmarkParams params = result.getParams(); + span.setTag(BENCHMARK_MODE, params.getMode().shortLabel()); + span.setTag(BENCHMARK_ITERATIONS, params.getMeasurement().getCount()); + span.setTag(BENCHMARK_WARMUP_ITERATIONS, params.getWarmup().getCount()); + span.setTag(BENCHMARK_FORKS, params.getForks()); + span.setTag(BENCHMARK_THREADS, params.getThreads()); + span.setTag(BENCHMARK_TIME_UNIT, params.getTimeUnit().name()); + + Result primary = result.getPrimaryResult(); + span.setMetric(BENCHMARK_VALUE, primary.getScore()); + span.setTag(BENCHMARK_UNIT, primary.getScoreUnit()); + + double error = primary.getScoreError(); + if (!Double.isNaN(error)) { + span.setMetric(BENCHMARK_ERROR, error); + } + + Statistics stats = primary.getStatistics(); + if (stats.getN() > 1) { + span.setMetric(BENCHMARK_P50, stats.getPercentile(50)); + span.setMetric(BENCHMARK_P90, stats.getPercentile(90)); + span.setMetric(BENCHMARK_P95, stats.getPercentile(95)); + span.setMetric(BENCHMARK_P99, stats.getPercentile(99)); + span.setMetric(BENCHMARK_MIN, stats.getMin()); + span.setMetric(BENCHMARK_MAX, stats.getMax()); + span.setMetric(BENCHMARK_SAMPLE_COUNT, stats.getN()); + } + } + + // ---- Delegation-only methods ---- + + @Override + public void iteration( + BenchmarkParams benchParams, org.openjdk.jmh.infra.IterationParams params, int iteration) { + delegate.iteration(benchParams, params, iteration); + } + + @Override + public void iterationResult( + BenchmarkParams benchParams, + org.openjdk.jmh.infra.IterationParams params, + int iteration, + org.openjdk.jmh.results.IterationResult data) { + delegate.iterationResult(benchParams, params, iteration, data); + } + + @Override + public void startRun() { + delegate.startRun(); + } + + @Override + public void endRun(java.util.Collection result) { + handler.close(); + delegate.endRun(result); + } + + @Override + public void print(String s) { + delegate.print(s); + } + + @Override + public void println(String s) { + delegate.println(s); + } + + @Override + public void verbosePrintln(String s) { + delegate.verbosePrintln(s); + } + + @Override + public void write(int b) { + delegate.write(b); + } + + @Override + public void write(byte[] b) throws java.io.IOException { + delegate.write(b); + } + + @Override + public void flush() { + delegate.flush(); + } + + @Override + public void close() { + delegate.close(); + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhInstrumentation.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhInstrumentation.java new file mode 100644 index 00000000000..432cd61207b --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhInstrumentation.java @@ -0,0 +1,58 @@ +package datadog.trace.instrumentation.jmh; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isConstructor; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import net.bytebuddy.asm.Advice; +import org.openjdk.jmh.runner.format.OutputFormat; + +@AutoService(InstrumenterModule.class) +public class JmhInstrumentation extends InstrumenterModule.CiVisibility + implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { + + public JmhInstrumentation() { + super("ci-visibility", "jmh"); + } + + @Override + public String instrumentedType() { + return "org.openjdk.jmh.runner.Runner"; + } + + @Override + public String[] helperClassNames() { + return new String[] { + packageName + ".JmhUtils", packageName + ".DDOutputFormat", + }; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + isConstructor() + .and(takesArgument(0, named("org.openjdk.jmh.runner.options.Options"))) + .and(takesArgument(1, named("org.openjdk.jmh.runner.format.OutputFormat"))), + JmhInstrumentation.class.getName() + "$RunnerConstructorAdvice"); + } + + public static class RunnerConstructorAdvice { + @Advice.OnMethodExit(suppress = Throwable.class) + public static void onExit( + @Advice.FieldValue(value = "out", readOnly = false) OutputFormat out) { + if (out instanceof DDOutputFormat) { + return; + } + String version; + try { + version = org.openjdk.jmh.Main.class.getPackage().getImplementationVersion(); + } catch (Throwable t) { + version = null; + } + out = new DDOutputFormat(out, version != null ? version : "unknown"); + } + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhUtils.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhUtils.java new file mode 100644 index 00000000000..2788b32b7ec --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhUtils.java @@ -0,0 +1,52 @@ +package datadog.trace.instrumentation.jmh; + +import javax.annotation.Nullable; + +public final class JmhUtils { + + static final String FRAMEWORK_NAME = "jmh"; + + /** + * Splits a JMH benchmark name into suite (class) and method parts. + * + *

JMH names have the form {@code "com.example.MyBenchmark.myMethod"} or, when {@code @Param} + * combinations are present, {@code "com.example.MyBenchmark.myMethod:size=1000,threads=4"}. + */ + public static String[] splitBenchmarkName(String fullName) { + // Strip any @Param suffix before splitting on the class/method boundary + int colonIdx = fullName.indexOf(':'); + String baseName = colonIdx >= 0 ? fullName.substring(0, colonIdx) : fullName; + + int lastDot = baseName.lastIndexOf('.'); + if (lastDot < 0) { + return new String[] {"", fullName}; + } + return new String[] {baseName.substring(0, lastDot), fullName.substring(lastDot + 1)}; + } + + /** + * Returns the {@code test.parameters} JSON string for a parameterized benchmark, or {@code null} + * for an unparameterized one. + * + *

Follows the same convention as JUnit 5 parameterized tests: {@code + * {"metadata":{"test_name":""}}}. + */ + @Nullable + public static String testParameters(String fullName) { + int colonIdx = fullName.indexOf(':'); + if (colonIdx < 0) { + return null; + } + // fullName after last dot includes the param suffix, e.g. "myMethod:size=1000" + int lastDot = fullName.lastIndexOf('.', colonIdx); + String displayName = lastDot >= 0 ? fullName.substring(lastDot + 1) : fullName; + return "{\"metadata\":{\"test_name\":\"" + escapeJson(displayName) + "\"}}"; + } + + /** Minimal JSON string escaping for benchmark names (no unicode escaping needed). */ + private static String escapeJson(String s) { + return s.replace("\\", "\\\\").replace("\"", "\\\""); + } + + private JmhUtils() {} +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/JmhUtilsTest.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/JmhUtilsTest.java new file mode 100644 index 00000000000..8d8696ba58e --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/JmhUtilsTest.java @@ -0,0 +1,53 @@ +package datadog.trace.instrumentation.jmh; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.junit.jupiter.api.Test; + +class JmhUtilsTest { + + @Test + void splitBenchmarkName_simple() { + String[] parts = JmhUtils.splitBenchmarkName("com.example.MyBenchmark.myMethod"); + assertArrayEquals(new String[] {"com.example.MyBenchmark", "myMethod"}, parts); + } + + @Test + void splitBenchmarkName_withParams() { + String[] parts = + JmhUtils.splitBenchmarkName("com.example.MyBenchmark.myMethod:size=1000,threads=4"); + assertArrayEquals( + new String[] {"com.example.MyBenchmark", "myMethod:size=1000,threads=4"}, parts); + } + + @Test + void splitBenchmarkName_noPackage() { + String[] parts = JmhUtils.splitBenchmarkName("MyBenchmark.myMethod"); + assertArrayEquals(new String[] {"MyBenchmark", "myMethod"}, parts); + } + + @Test + void splitBenchmarkName_noDot() { + String[] parts = JmhUtils.splitBenchmarkName("noDot"); + assertArrayEquals(new String[] {"", "noDot"}, parts); + } + + @Test + void testParameters_noParams() { + assertNull(JmhUtils.testParameters("com.example.MyBenchmark.myMethod")); + } + + @Test + void testParameters_withParams() { + String result = JmhUtils.testParameters("com.example.MyBenchmark.myMethod:size=1000,threads=4"); + assertEquals("{\"metadata\":{\"test_name\":\"myMethod:size=1000,threads=4\"}}", result); + } + + @Test + void testParameters_escapesQuotes() { + String result = JmhUtils.testParameters("com.example.MyBenchmark.myMethod:key=\"value\""); + assertEquals("{\"metadata\":{\"test_name\":\"myMethod:key=\\\"value\\\"\"}}", result); + } +} diff --git a/docs/design/jmh-ci-visibility.md b/docs/design/jmh-ci-visibility.md new file mode 100644 index 00000000000..dc62f345021 --- /dev/null +++ b/docs/design/jmh-ci-visibility.md @@ -0,0 +1,235 @@ +# Design: JMH Benchmark CI Visibility Instrumentation (SDTEST-930) + +## Problem + +JMH (Java Microbenchmark Harness, `org.openjdk.jmh`, version 1.37) is the dominant Java +benchmarking framework. Benchmark runs are not currently reported to CI Visibility, so +performance regressions are invisible in the Datadog test explorer. + +## Goals + +- Report each JMH benchmark method as a CI Visibility **test span** (`test.type = "benchmark"`) +- Attach aggregated performance metrics (score, error, unit, percentiles) as span tags +- **Zero overhead on the benchmark hot path** — hook only fires once per benchmark method, not per invocation +- Disabled by default (like Renaissance); opt-in via `DD_TRACE_JMH_ENABLED=true` + +## JMH Lifecycle and Hook Point + +JMH execution flow (per benchmark method): + +``` +Runner.run() + └─ for each benchmark method: + OutputFormat.startBenchmark(BenchmarkParams) ← span start + for each fork: + for each warmup iteration: + OutputFormat.iteration(...) + for each measurement iteration: + OutputFormat.iterationResult(...) + OutputFormat.iteration(...) + OutputFormat.endBenchmark(BenchmarkResult) ← span finish + attach metrics + └─ OutputFormat.endRun(Collection) +``` + +`OutputFormat` is an interface that JMH calls for all lifecycle events. Critically: +- `startBenchmark(BenchmarkParams)` fires **once** per benchmark method before any invocations +- `endBenchmark(BenchmarkResult)` fires **once** per benchmark method after all forks and iterations + +These are the only two hooks needed. No hot-path instrumentation is required. + +### Why not instrument `@Benchmark`-annotated methods directly? + +Those methods are called millions of times during warmup and measurement. Advice on the +hot path would perturb the benchmark results and add massive instrumentation overhead. + +## Instrumentation Strategy + +### Hook: `OutputFormat` injection + +JMH's `Runner` is constructed by user code: + +```java +Runner runner = new Runner(options); +runner.run(); +``` + +The `Runner` constructor accepts an `Options` object, which includes an `OutputFormat`. We +instrument `Runner.` to wrap the user-supplied `OutputFormat` with our own +`DDOutputFormat` decorator before the field is stored. + +**Bytecode advice on `Runner.`:** + +```java +@Advice.OnMethodExit +public static void onExit(@Advice.FieldValue(value = "out", readOnly = false) OutputFormat out) { + out = new DDOutputFormat(out); +} +``` + +`Runner.out` is the `OutputFormat` field. Wrapping it at construction time means our +decorator receives all lifecycle callbacks without any per-invocation cost. + +### Alternative: instrument `Runner.run()` return value + +If wrapping the constructor is fragile due to JMH refactors, a fallback is to instrument +`Runner.run()` / `Runner.runBenchmarks()` exit and iterate the returned +`Collection` to emit spans retroactively. This loses wall-clock timing fidelity +(span duration reflects only the post-run callback time) but is simpler and requires no +field access. + +The constructor approach is preferred; the `run()` return approach is the fallback. + +## Data Model + +### Span structure + +Each JMH benchmark method produces **two spans**: + +| Span | Mapping | Notes | +|------|---------|-------| +| Test suite span | Benchmark class (e.g., `com.example.MyBenchmark`) | One per class | +| Test span | Benchmark method (e.g., `myMethod`) | One per `@Benchmark` method per parameter set | + +`BenchmarkParams.getBenchmark()` returns the fully-qualified name +`"com.example.MyBenchmark.myMethod"` — split on the last `.` to get class and method. + +### Standard CI Visibility tags + +| Tag | Source | Value | +|-----|--------|-------| +| `test.type` | constant | `"benchmark"` | +| `test.framework` | constant | `"jmh"` | +| `test.framework_version` | `Version.getVersion(Runner.class)` | e.g. `"1.37"` | +| `test.name` | `BenchmarkParams.getBenchmark()` last segment | e.g. `"myMethod"` | +| `test.suite` | `BenchmarkParams.getBenchmark()` prefix | e.g. `"com.example.MyBenchmark"` | +| `test.status` | always `"pass"` (JMH throws on error) | `"fail"` if exception from `endBenchmark` | +| `test.parameters` | `BenchmarkParams.getParamsKeys()` + `getParam(key)` | JSON object, omit if empty | +| `test.source.class` | derived from suite name | class name | +| `test.source.method` | derived from test name | method name | + +### Benchmark-specific metric tags + +These are numeric tags added on the test span, not on suite spans: + +| Tag | Source | Notes | +|-----|--------|-------| +| `benchmark.run.iterations` | `BenchmarkParams.getMeasurement().getCount()` | Measurement iteration count | +| `benchmark.run.forks` | `BenchmarkParams.getForks()` | Fork count | +| `benchmark.run.threads` | `BenchmarkParams.getThreads()` | Thread count | +| `benchmark.run.warmup_iterations` | `BenchmarkParams.getWarmup().getCount()` | Warmup iteration count | +| `benchmark.run.time_unit` | `BenchmarkParams.getTimeUnit().name()` | e.g. `"NANOSECONDS"` | +| `benchmark.run.mode` | `BenchmarkParams.getMode().shortLabel()` | e.g. `"thrpt"`, `"avgt"` | +| `benchmark.value` | `Result.getScore()` | Primary metric score | +| `benchmark.error` | `Result.getScoreError()` | 99.9% CI half-width; `NaN` for single-shot | +| `benchmark.unit` | `Result.getScoreUnit()` | e.g. `"ops/ms"`, `"ns/op"` | +| `benchmark.p50` | `Statistics.getPercentile(50)` | Median | +| `benchmark.p90` | `Statistics.getPercentile(90)` | | +| `benchmark.p95` | `Statistics.getPercentile(95)` | | +| `benchmark.p99` | `Statistics.getPercentile(99)` | | +| `benchmark.min` | `Statistics.getMin()` | | +| `benchmark.max` | `Statistics.getMax()` | | +| `benchmark.sample_count` | `Statistics.getN()` | Total sample count | + +`Statistics` is available via `BenchmarkResult.getPrimaryResult().getStatistics()`. + +For `SingleShotTime` mode, only `benchmark.value` and `benchmark.unit` are populated +(no iterations → no distribution). + +### Access path summary (zero hot-path calls) + +``` +endBenchmark(BenchmarkResult result) +├── result.getParams() → BenchmarkParams (all config) +│ ├── .getBenchmark() → "com.example.MyBenchmark.myMethod" +│ ├── .getMode().shortLabel() → "thrpt" +│ ├── .getThreads() → 4 +│ ├── .getForks() → 5 +│ ├── .getMeasurement().getCount() → 5 +│ ├── .getWarmup().getCount() → 5 +│ ├── .getTimeUnit() → NANOSECONDS +│ └── .getParamsKeys() + .getParam(key) → {"size": "1000"} +└── result.getPrimaryResult() → Result + ├── .getScore() → 1234.56 + ├── .getScoreError() → 12.34 + ├── .getScoreUnit() → "ns/op" + └── .getStatistics() → Statistics + ├── .getPercentile(50/90/95/99) → distribution + ├── .getMin() / .getMax() → bounds + └── .getN() → sample count +``` + +## Module Layout + +``` +dd-java-agent/instrumentation/jmh/ +└── jmh-1.0/ # JMH's OutputFormat API is stable since 1.0 + ├── build.gradle + ├── gradle.lockfile + └── src/ + └── main/java/datadog/trace/instrumentation/jmh/ + ├── JmhInstrumentation.java # InstrumenterModule targeting Runner. + ├── DDOutputFormat.java # OutputFormat decorator + └── JmhUtils.java # Parsing helpers (benchmark name split, etc.) +``` + +### `build.gradle` + +```groovy +apply from: "$rootDir/gradle/java.gradle" + +dependencies { + compileOnly group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.0' + testImplementation group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.37' +} +``` + +## Changes Required Outside the New Module + +| File | Change | +|------|--------| +| `internal-api/.../telemetry/tag/TestFrameworkInstrumentation.java` | Add `JMH` enum constant | +| `internal-api/.../bootstrap/instrumentation/api/Tags.java` | Add `benchmark.*` tag constants | +| `internal-api/.../decorator/TestDecorator.java` | Add `TEST_TYPE_BENCHMARK = "benchmark"` constant | +| `dd-java-agent/agent-ci-visibility/.../decorator/TestDecoratorImpl.java` | Handle benchmark type | + +## Resolved Design Decisions + +### Suite span scope +Flat: each benchmark method (including each `@Param` combination) gets its own suite span +and test span pair. No grouping by class. + +### Parameterized benchmarks +Follow the same pattern as JUnit 5 parameterized tests: set `test.parameters` to +`{"metadata":{"test_name":""}}` where the display name is the parameterized +suffix of the benchmark name. + +JMH encodes `@Param` combinations by appending them after a colon: +`"com.example.MyBenchmark.myMethod:size=1000,threads=4"` + +Parsing: +- `test.suite` = everything before the last `.` before the colon: `"com.example.MyBenchmark"` +- `test.name` = method name segment without params: `"myMethod"` +- `test.parameters` = `{"metadata":{"test_name":"myMethod:size=1000,threads=4"}}` (non-null only when a colon is present) + +The parameterized variant is a distinct test identity (unique `test.name` + `test.parameters` +combination), matching how JUnit 5 handles `@ParameterizedTest`. + +## Open Questions + +1. **CI Visibility opt-out for ITR**: JMH benchmarks should likely be excluded from + Intelligent Test Runner (skip logic) since skipping a benchmark run defeats its purpose. + Mark them as `@ITRUnskippable` equivalent or configure the handler to always-run. + +2. **Forked JVM mode**: When `@Fork(1+)` is used, each fork is a separate JVM process. + The tracer in the forked process needs to propagate the session/module/suite IDs from + the parent. This is the same challenge as Gradle worker forks — check if the existing + IPC mechanism in `ProcessHierarchy` covers it. + +## Implementation Order + +1. Add `benchmark.*` tag constants to `Tags.java` +2. Add `JMH` to `TestFrameworkInstrumentation` +3. Add `TEST_TYPE_BENCHMARK` to `TestDecorator` +4. Implement `DDOutputFormat` + `JmhInstrumentation` + `JmhUtils` +5. Wire module into the agent's instrumentation list +6. Add JUnit 5 integration tests (use JMH `Runner` programmatically in test; verify spans) diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/TestFrameworkInstrumentation.java b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/TestFrameworkInstrumentation.java index eedbaed7e80..20604f54739 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/TestFrameworkInstrumentation.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/TestFrameworkInstrumentation.java @@ -13,6 +13,7 @@ public enum TestFrameworkInstrumentation implements TagValue { SCALATEST, KARATE, WEAVER, + JMH, OTHER; private final String s; diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java index 14496e8b243..66018e35cec 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java @@ -86,6 +86,23 @@ public class Tags { public static final String TEST_BROWSER_VERSION = "test.browser.version"; public static final String TEST_CALLBACK = "test.callback"; + public static final String BENCHMARK_VALUE = "benchmark.value"; + public static final String BENCHMARK_ERROR = "benchmark.error"; + public static final String BENCHMARK_UNIT = "benchmark.unit"; + public static final String BENCHMARK_MODE = "benchmark.run.mode"; + public static final String BENCHMARK_ITERATIONS = "benchmark.run.iterations"; + public static final String BENCHMARK_WARMUP_ITERATIONS = "benchmark.run.warmup_iterations"; + public static final String BENCHMARK_FORKS = "benchmark.run.forks"; + public static final String BENCHMARK_THREADS = "benchmark.run.threads"; + public static final String BENCHMARK_TIME_UNIT = "benchmark.run.time_unit"; + public static final String BENCHMARK_P50 = "benchmark.p50"; + public static final String BENCHMARK_P90 = "benchmark.p90"; + public static final String BENCHMARK_P95 = "benchmark.p95"; + public static final String BENCHMARK_P99 = "benchmark.p99"; + public static final String BENCHMARK_MIN = "benchmark.min"; + public static final String BENCHMARK_MAX = "benchmark.max"; + public static final String BENCHMARK_SAMPLE_COUNT = "benchmark.sample_count"; + public static final String TEST_SESSION_ID = "test_session_id"; public static final String TEST_MODULE_ID = "test_module_id"; public static final String TEST_SUITE_ID = "test_suite_id"; diff --git a/settings.gradle.kts b/settings.gradle.kts index dda1f432e6f..e0eee98140a 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -445,6 +445,7 @@ include( ":dd-java-agent:instrumentation:jetty:jetty-util-9.4.31", ":dd-java-agent:instrumentation:jms:jakarta-jms-3.0", ":dd-java-agent:instrumentation:jms:javax-jms-1.1", + ":dd-java-agent:instrumentation:jmh:jmh-1.0", ":dd-java-agent:instrumentation:jose-jwt-4.0", ":dd-java-agent:instrumentation:jsp-2.3", ":dd-java-agent:instrumentation:junit:junit-4:junit-4-cucumber-5.4", From 64814c7db8f6d04b55cb19d90d9cfae37a0e9a5f Mon Sep 17 00:00:00 2001 From: Robert Pickering Date: Thu, 28 May 2026 16:00:27 +0000 Subject: [PATCH 02/12] Add integration tests for JMH CI Visibility instrumentation Groovy/Spock integration tests extending CiVisibilityInstrumentationTest that run JMH benchmarks in-process (forks=0) and verify the emitted CI Visibility spans against FTL fixture templates. Covers: - Simple (unparameterized) benchmark: suite + test spans with benchmark run config metrics (mode, unit, iterations, forks, threads, time_unit) - Parameterised benchmark (@Param): two test spans with test.parameters set following the JUnit 5 convention Also fixes: - BaseRunner instrumented instead of Runner (JDK 17+ rejects PUTFIELD on a final field of a superclass from advice injected into the subclass) - JMH annotation processor added to testAnnotationProcessor so that META-INF/BenchmarkList is generated at test compile time - DD_TRACE_JMH_ENABLED registered in supported-configurations.json Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../instrumentation/jmh/jmh-1.0/build.gradle | 7 + .../jmh/JmhInstrumentation.java | 12 +- .../test/groovy/JmhInstrumentationTest.groovy | 51 ++++ .../benchmarks/ParameterizedBenchmark.java | 30 +++ .../jmh/benchmarks/SimpleBenchmark.java | 22 ++ .../coverages.ftl | 1 + .../test-benchmark-parameterized/events.ftl | 225 ++++++++++++++++++ .../test-benchmark-simple/coverages.ftl | 1 + .../test-benchmark-simple/events.ftl | 145 +++++++++++ metadata/supported-configurations.json | 8 + 10 files changed, 497 insertions(+), 5 deletions(-) create mode 100644 dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/groovy/JmhInstrumentationTest.groovy create mode 100644 dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/ParameterizedBenchmark.java create mode 100644 dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/SimpleBenchmark.java create mode 100644 dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/coverages.ftl create mode 100644 dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/events.ftl create mode 100644 dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/coverages.ftl create mode 100644 dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/events.ftl diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/build.gradle b/dd-java-agent/instrumentation/jmh/jmh-1.0/build.gradle index a0b0b64393c..a5256396007 100644 --- a/dd-java-agent/instrumentation/jmh/jmh-1.0/build.gradle +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/build.gradle @@ -1,3 +1,7 @@ +plugins { + id 'dd-trace-java.instrumentation.testing-framework-tests' +} + apply from: "$rootDir/gradle/java.gradle" muzzle { @@ -13,4 +17,7 @@ dependencies { testImplementation group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.37' testImplementation project(':dd-java-agent:agent-ci-visibility:civisibility-instrumentation-test-fixtures') + + // JMH annotation processor generates META-INF/BenchmarkList from @Benchmark annotations + testAnnotationProcessor group: 'org.openjdk.jmh', name: 'jmh-generator-annprocess', version: '1.37' } diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhInstrumentation.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhInstrumentation.java index 432cd61207b..5bfb0013ff1 100644 --- a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhInstrumentation.java +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhInstrumentation.java @@ -20,7 +20,10 @@ public JmhInstrumentation() { @Override public String instrumentedType() { - return "org.openjdk.jmh.runner.Runner"; + // Instrument BaseRunner (where the 'out' field is declared) so that the final-field write + // in RunnerConstructorAdvice is legal: JDK 17+ rejects writing a final field declared in a + // superclass from advice injected into the subclass (Runner). + return "org.openjdk.jmh.runner.BaseRunner"; } @Override @@ -46,13 +49,12 @@ public static void onExit( if (out instanceof DDOutputFormat) { return; } - String version; + String version = null; try { version = org.openjdk.jmh.Main.class.getPackage().getImplementationVersion(); - } catch (Throwable t) { - version = null; + } catch (Throwable ignored) { } - out = new DDOutputFormat(out, version != null ? version : "unknown"); + out = new DDOutputFormat(out, version); } } } diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/groovy/JmhInstrumentationTest.groovy b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/groovy/JmhInstrumentationTest.groovy new file mode 100644 index 00000000000..4b4e6a7ee5f --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/groovy/JmhInstrumentationTest.groovy @@ -0,0 +1,51 @@ +import datadog.trace.api.DisableTestTrace +import datadog.trace.civisibility.CiVisibilityInstrumentationTest +import datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark +import datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark +import org.openjdk.jmh.runner.Runner +import org.openjdk.jmh.runner.options.OptionsBuilder + +@DisableTestTrace(reason = "avoid self-tracing") +class JmhInstrumentationTest extends CiVisibilityInstrumentationTest { + + // Benchmark numeric metrics vary each run — they are verified structurally in the smoke test + static final List BENCHMARK_METRIC_TAGS = [ + "content.metrics.['benchmark.value']", + "content.metrics.['benchmark.error']", + "content.metrics.['benchmark.p50']", + "content.metrics.['benchmark.p90']", + "content.metrics.['benchmark.p95']", + "content.metrics.['benchmark.p99']", + "content.metrics.['benchmark.min']", + "content.metrics.['benchmark.max']", + "content.metrics.['benchmark.sample_count']", + ] + + def "test #testcaseName"() { + runBenchmark(benchmarkClass) + assertSpansData(testcaseName, [:], BENCHMARK_METRIC_TAGS) + + where: + testcaseName | benchmarkClass + "test-benchmark-simple" | SimpleBenchmark + "test-benchmark-parameterized" | ParameterizedBenchmark + } + + private void runBenchmark(Class benchmarkClass) { + def options = new OptionsBuilder() + .include(benchmarkClass.getName()) + .jvmArgsAppend("-Djmh.ignoreLock=true") + .build() + new Runner(options).run() + } + + @Override + String instrumentedLibraryName() { + "jmh" + } + + @Override + String instrumentedLibraryVersion() { + Runner.class.getPackage().getImplementationVersion() + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/ParameterizedBenchmark.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/ParameterizedBenchmark.java new file mode 100644 index 00000000000..2a752e58cab --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/ParameterizedBenchmark.java @@ -0,0 +1,30 @@ +package datadog.trace.instrumentation.jmh.benchmarks; + +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Fork(0) +@State(Scope.Benchmark) +public class ParameterizedBenchmark { + + @Param({"1", "2"}) + int size; + + @Benchmark + public int measure() { + return size * 2; + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/SimpleBenchmark.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/SimpleBenchmark.java new file mode 100644 index 00000000000..2b2b5b6ef8a --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/SimpleBenchmark.java @@ -0,0 +1,22 @@ +package datadog.trace.instrumentation.jmh.benchmarks; + +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Warmup; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Fork(0) +public class SimpleBenchmark { + @Benchmark + public int measure() { + return 42; + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/coverages.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/events.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/events.ftl new file mode 100644 index 00000000000..39dd9782269 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/events.ftl @@ -0,0 +1,225 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "jmh", + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "jmh.test_suite", + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "jmh", + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2} + }, + "name" : "jmh.test_suite", + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id_2} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "benchmark.run.mode" : "avgt", + "benchmark.run.time_unit" : "NANOSECONDS", + "benchmark.unit" : "ns/op", + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "jmh", + "test.module" : "jmh-1.0", + "test.name" : "measure", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "benchmark.run.forks" : 0, + "benchmark.run.iterations" : 1, + "benchmark.run.threads" : 1, + "benchmark.run.warmup_iterations" : 1, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test", + "parent_id" : ${content_parent_id}, + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark.measure", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "benchmark.run.mode" : "avgt", + "benchmark.run.time_unit" : "NANOSECONDS", + "benchmark.unit" : "ns/op", + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "jmh", + "test.module" : "jmh-1.0", + "test.name" : "measure", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "benchmark.run.forks" : 0, + "benchmark.run.iterations" : 1, + "benchmark.run.threads" : 1, + "benchmark.run.warmup_iterations" : 1, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test", + "parent_id" : ${content_parent_id}, + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark.measure", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id_2}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "jmh-1.0", + "test.framework" : "jmh", + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test_session", + "resource" : "jmh-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_5}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_4}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "jmh", + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_6} + }, + "name" : "jmh.test_module", + "resource" : "jmh-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_6}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/coverages.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/events.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/events.ftl new file mode 100644 index 00000000000..677f9ae486d --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/events.ftl @@ -0,0 +1,145 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "jmh", + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "jmh.test_suite", + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "benchmark.run.mode" : "avgt", + "benchmark.run.time_unit" : "NANOSECONDS", + "benchmark.unit" : "ns/op", + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "jmh", + "test.module" : "jmh-1.0", + "test.name" : "measure", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "benchmark.run.forks" : 0, + "benchmark.run.iterations" : 1, + "benchmark.run.threads" : 1, + "benchmark.run.warmup_iterations" : 1, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test", + "parent_id" : ${content_parent_id}, + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark.measure", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "jmh-1.0", + "test.framework" : "jmh", + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test_session", + "resource" : "jmh-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "jmh", + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4} + }, + "name" : "jmh.test_module", + "resource" : "jmh-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 32e40412662..6cf109ac1d9 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -7281,6 +7281,14 @@ "aliases": ["DD_TRACE_INTEGRATION_JETTY_WEBSOCKET_ENABLED", "DD_INTEGRATION_JETTY_WEBSOCKET_ENABLED"] } ], + "DD_TRACE_JMH_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "true", + "aliases": ["DD_TRACE_INTEGRATION_JMH_ENABLED", "DD_INTEGRATION_JMH_ENABLED"] + } + ], "DD_TRACE_JMS_1_ENABLED": [ { "version": "A", From 90f18e337d9f17483911e5ec645aa19bde367ec5 Mon Sep 17 00:00:00 2001 From: Robert Pickering Date: Fri, 29 May 2026 06:04:20 +0000 Subject: [PATCH 03/12] Add JMH CI Visibility smoke test Java JUnit 5 smoke test that forks a real JVM subprocess with the dd-java-agent attached, runs a JMH benchmark in-process (forks=0) against a MockBackend, and verifies that the expected CI Visibility spans arrive with correct tags: - test.framework = "jmh" - test.name, test.suite, test.status - benchmark.run.mode, benchmark.unit - benchmark.value > 0 (measured score actually present) The benchmark class (SmokeTestBenchmark) lives in src/main/java so the JMH annotation processor can generate META-INF/BenchmarkList at compile time, making it available on the classpath that is passed to the subprocess. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- dd-smoke-tests/jmh/build.gradle | 30 ++++ .../datadog/smoketest/SmokeTestBenchmark.java | 16 ++ .../java/datadog/smoketest/JmhSmokeTest.java | 138 ++++++++++++++++++ .../jmh/src/test/resources/logback.xml | 3 + settings.gradle.kts | 1 + 5 files changed, 188 insertions(+) create mode 100644 dd-smoke-tests/jmh/build.gradle create mode 100644 dd-smoke-tests/jmh/src/main/java/datadog/smoketest/SmokeTestBenchmark.java create mode 100644 dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java create mode 100644 dd-smoke-tests/jmh/src/test/resources/logback.xml diff --git a/dd-smoke-tests/jmh/build.gradle b/dd-smoke-tests/jmh/build.gradle new file mode 100644 index 00000000000..2bbe5485741 --- /dev/null +++ b/dd-smoke-tests/jmh/build.gradle @@ -0,0 +1,30 @@ +apply from: "$rootDir/gradle/java.gradle" +description = 'JMH CI Visibility Smoke Tests.' + +dependencies { + implementation group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.37' + annotationProcessor group: 'org.openjdk.jmh', name: 'jmh-generator-annprocess', version: '1.37' + + testImplementation project(':dd-smoke-tests:backend-mock') +} + +tasks.withType(Test).configureEach { + jvmArgumentProviders.add(new CommandLineArgumentProvider() { + @Override + Iterable asArguments() { + def jmhJar = configurations.named("runtimeClasspath") + .get() + .find { it.name.contains("jmh-core") } + return ["-Ddatadog.smoketest.jmh.core.jar.path=${jmhJar}"] + } + }) + jvmArgumentProviders.add(new CommandLineArgumentProvider() { + @Override + Iterable asArguments() { + def annprocJar = configurations.named("annotationProcessor") + .get() + .find { it.name.contains("jmh-generator-annprocess") } + return ["-Ddatadog.smoketest.jmh.annproc.jar.path=${annprocJar}"] + } + }) +} diff --git a/dd-smoke-tests/jmh/src/main/java/datadog/smoketest/SmokeTestBenchmark.java b/dd-smoke-tests/jmh/src/main/java/datadog/smoketest/SmokeTestBenchmark.java new file mode 100644 index 00000000000..d9a22def1ff --- /dev/null +++ b/dd-smoke-tests/jmh/src/main/java/datadog/smoketest/SmokeTestBenchmark.java @@ -0,0 +1,16 @@ +package datadog.smoketest; + +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +public class SmokeTestBenchmark { + @Benchmark + public int measure() { + return 42; + } +} diff --git a/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java b/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java new file mode 100644 index 00000000000..d7e15cacf0e --- /dev/null +++ b/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java @@ -0,0 +1,138 @@ +package datadog.smoketest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.api.config.CiVisibilityConfig; +import datadog.trace.api.config.GeneralConfig; +import datadog.trace.civisibility.CiVisibilitySmokeTest; +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class JmhSmokeTest extends CiVisibilitySmokeTest { + + private static final String TEST_SERVICE_NAME = "test-jmh-service"; + private static final int PROCESS_TIMEOUT_SECS = 120; + + private static final String JMH_CORE_JAR = + System.getProperty("datadog.smoketest.jmh.core.jar.path"); + + static final MockBackend mockBackend = new MockBackend(); + + @BeforeEach + void resetMockBackend() { + mockBackend.reset(); + } + + @AfterAll + static void closeMockBackend() throws Exception { + mockBackend.close(); + } + + @Test + void testBenchmarkSpansAreEmitted() throws Exception { + Map agentArgs = new HashMap<>(); + agentArgs.put(CiVisibilityConfig.CIVISIBILITY_BUILD_INSTRUMENTATION_ENABLED, "false"); + agentArgs.put(GeneralConfig.AGENTLESS_LOG_SUBMISSION_URL, mockBackend.getIntakeUrl()); + + int exitCode = runBenchmark(agentArgs); + assertEquals(0, exitCode, "JMH process should exit cleanly"); + + // 4 events: test_session_end, test_module_end, test_suite_end, test (benchmark method) + List> events = mockBackend.waitForEvents(4); + assertEquals(4, events.size()); + + Map testEvent = findEvent(events, "test"); + assertNotNull(testEvent, "Expected a test span for the benchmark method"); + + @SuppressWarnings("unchecked") + Map meta = + (Map) ((Map) testEvent.get("content")).get("meta"); + @SuppressWarnings("unchecked") + Map metrics = + (Map) ((Map) testEvent.get("content")).get("metrics"); + + assertEquals("jmh", meta.get("test.framework")); + assertEquals("measure", meta.get("test.name")); + assertEquals("datadog.smoketest.SmokeTestBenchmark", meta.get("test.suite")); + assertEquals("pass", meta.get("test.status")); + assertEquals("avgt", meta.get("benchmark.run.mode")); + assertEquals("ns/op", meta.get("benchmark.unit")); + + assertNotNull(metrics.get("benchmark.value"), "benchmark.value should be present"); + assertTrue( + ((Number) metrics.get("benchmark.value")).doubleValue() > 0, + "benchmark.value should be positive"); + } + + private int runBenchmark(Map additionalAgentArgs) throws Exception { + assertTrue(new File(JMH_CORE_JAR).isFile(), "JMH core jar not found: " + JMH_CORE_JAR); + + String classpath = buildClasspath(); + + List command = new ArrayList<>(); + command.add(javaPath()); + command.addAll( + buildJvmArguments(mockBackend.getIntakeUrl(), TEST_SERVICE_NAME, additionalAgentArgs)); + Collections.addAll(command, "-cp", classpath); + command.add("org.openjdk.jmh.Main"); + command.add("datadog.smoketest.SmokeTestBenchmark.*"); + Collections.addAll(command, "-f", "0"); // run in-process (no forking) + Collections.addAll(command, "-wi", "1"); // 1 warmup iteration + Collections.addAll(command, "-i", "1"); // 1 measurement iteration + Collections.addAll(command, "-w", "1ms"); // warmup duration + Collections.addAll(command, "-r", "1ms"); // measurement duration + Collections.addAll(command, "-jvmArgs", "-Djmh.ignoreLock=true"); + + ProcessBuilder processBuilder = new ProcessBuilder(command); + processBuilder.redirectErrorStream(true); + processBuilder.environment().put("DD_API_KEY", "01234567890abcdef123456789ABCDEF"); + Process p = processBuilder.start(); + + // consume output to avoid blocking + final java.io.InputStream stdout = p.getInputStream(); + Thread outputConsumer = + new Thread() { + @Override + public void run() { + try { + byte[] buf = new byte[1024]; + int n; + while ((n = stdout.read(buf)) != -1) { + System.out.write(buf, 0, n); + } + } catch (Exception ignored) { + } + } + }; + outputConsumer.setDaemon(true); + outputConsumer.start(); + + if (!p.waitFor(PROCESS_TIMEOUT_SECS, TimeUnit.SECONDS)) { + p.destroyForcibly(); + throw new TimeoutException("JMH process timed out after " + PROCESS_TIMEOUT_SECS + "s"); + } + return p.exitValue(); + } + + private static String buildClasspath() { + // Use the current test process classpath — it includes jmh-core, SmokeTestBenchmark, and its + // META-INF/BenchmarkList (generated by the annotation processor at compile time) + return System.getProperty("java.class.path"); + } + + @SuppressWarnings("unchecked") + private static Map findEvent(List> events, String type) { + return events.stream().filter(e -> type.equals(e.get("type"))).findFirst().orElse(null); + } +} diff --git a/dd-smoke-tests/jmh/src/test/resources/logback.xml b/dd-smoke-tests/jmh/src/test/resources/logback.xml new file mode 100644 index 00000000000..24d6bcd768a --- /dev/null +++ b/dd-smoke-tests/jmh/src/test/resources/logback.xml @@ -0,0 +1,3 @@ + + + diff --git a/settings.gradle.kts b/settings.gradle.kts index e0eee98140a..fe18d3f349c 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -201,6 +201,7 @@ include( ":dd-smoke-tests:jersey-2", ":dd-smoke-tests:jersey-3", ":dd-smoke-tests:jboss-modules", + ":dd-smoke-tests:jmh", ":dd-smoke-tests:junit-console", ":dd-smoke-tests:kafka-2", ":dd-smoke-tests:kafka-3", From f02138899013a6d7015b9b4b3c6e04704d02cd5b Mon Sep 17 00:00:00 2001 From: Robert Pickering Date: Fri, 29 May 2026 06:29:58 +0000 Subject: [PATCH 04/12] Fix three bugs found in code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. splitBenchmarkName returned the full parameterised suffix as the test name (e.g. "myMethod:size=1000") instead of just the method name ("myMethod"). Fix: use baseName (param-stripped) for the method slice. 2. endBenchmark had no null guard — if called without a prior startBenchmark the handler would receive null keys. Fix: early-return when suiteKey/testKey are null. 3. handler.close() in endRun was not in a finally block, so a crash in close() would swallow delegate.endRun(); and an exception in endBenchmark could bypass close() entirely. Fix: try/finally in both endBenchmark and endRun. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../instrumentation/jmh/DDOutputFormat.java | 24 ++++++++++++------- .../trace/instrumentation/jmh/JmhUtils.java | 2 +- .../instrumentation/jmh/JmhUtilsTest.java | 3 +-- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DDOutputFormat.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DDOutputFormat.java index a5639710c57..9b94cb69d87 100644 --- a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DDOutputFormat.java +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DDOutputFormat.java @@ -97,13 +97,18 @@ public void startBenchmark(BenchmarkParams benchParams) { public void endBenchmark(BenchmarkResult result) { String suiteKey = currentSuiteKey; String testKey = currentTestKey; + if (suiteKey == null || testKey == null) { + delegate.endBenchmark(result); + return; + } - tagBenchmarkMetrics(result); - - handler.onTestFinish(testKey, null, null); - handler.onTestSuiteFinish(suiteKey, null); - - delegate.endBenchmark(result); + try { + tagBenchmarkMetrics(result); + handler.onTestFinish(testKey, null, null); + handler.onTestSuiteFinish(suiteKey, null); + } finally { + delegate.endBenchmark(result); + } } private void tagBenchmarkMetrics(BenchmarkResult result) { @@ -165,8 +170,11 @@ public void startRun() { @Override public void endRun(java.util.Collection result) { - handler.close(); - delegate.endRun(result); + try { + handler.close(); + } finally { + delegate.endRun(result); + } } @Override diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhUtils.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhUtils.java index 2788b32b7ec..b5bfea0e2ef 100644 --- a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhUtils.java +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhUtils.java @@ -21,7 +21,7 @@ public static String[] splitBenchmarkName(String fullName) { if (lastDot < 0) { return new String[] {"", fullName}; } - return new String[] {baseName.substring(0, lastDot), fullName.substring(lastDot + 1)}; + return new String[] {baseName.substring(0, lastDot), baseName.substring(lastDot + 1)}; } /** diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/JmhUtilsTest.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/JmhUtilsTest.java index 8d8696ba58e..4da75dfc901 100644 --- a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/JmhUtilsTest.java +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/JmhUtilsTest.java @@ -18,8 +18,7 @@ void splitBenchmarkName_simple() { void splitBenchmarkName_withParams() { String[] parts = JmhUtils.splitBenchmarkName("com.example.MyBenchmark.myMethod:size=1000,threads=4"); - assertArrayEquals( - new String[] {"com.example.MyBenchmark", "myMethod:size=1000,threads=4"}, parts); + assertArrayEquals(new String[] {"com.example.MyBenchmark", "myMethod"}, parts); } @Test From ded841bec651a630352db3fa0b78b3d0ae5e82b2 Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Tue, 2 Jun 2026 16:39:31 +0200 Subject: [PATCH 05/12] fix: generate lockfiles --- .../jmh/jmh-1.0/gradle.lockfile | 144 ++++++++++++++++++ dd-smoke-tests/jmh/build.gradle | 14 +- dd-smoke-tests/jmh/gradle.lockfile | 131 ++++++++++++++++ 3 files changed, 280 insertions(+), 9 deletions(-) create mode 100644 dd-java-agent/instrumentation/jmh/jmh-1.0/gradle.lockfile create mode 100644 dd-smoke-tests/jmh/gradle.lockfile diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/gradle.lockfile b/dd-java-agent/instrumentation/jmh/jmh-1.0/gradle.lockfile new file mode 100644 index 00000000000..010265b6f6f --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/gradle.lockfile @@ -0,0 +1,144 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +cafe.cryptography:curve25519-elisabeth:0.1.0=testRuntimeClasspath +cafe.cryptography:ed25519-elisabeth:0.1.0=testRuntimeClasspath +ch.qos.logback:logback-classic:1.2.13=testCompileClasspath,testRuntimeClasspath +ch.qos.logback:logback-core:1.2.13=testCompileClasspath,testRuntimeClasspath +com.blogspot.mydailyjava:weak-lock-free:0.17=buildTimeInstrumentationPlugin,compileClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.datadoghq.okhttp3:okhttp:3.12.15=testCompileClasspath,testRuntimeClasspath +com.datadoghq.okio:okio:1.17.6=testCompileClasspath,testRuntimeClasspath +com.datadoghq:dd-instrument-java:0.0.3=buildTimeInstrumentationPlugin,compileClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.datadoghq:dd-javac-plugin-client:0.2.2=buildTimeInstrumentationPlugin,compileClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.datadoghq:java-dogstatsd-client:4.4.5=testRuntimeClasspath +com.datadoghq:sketches-java:0.8.3=testRuntimeClasspath +com.fasterxml.jackson.core:jackson-annotations:2.20=testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-core:2.20.0=testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-databind:2.20.0=testCompileClasspath,testRuntimeClasspath +com.github.javaparser:javaparser-core:3.25.6=codenarc +com.github.jnr:jffi:1.3.14=testRuntimeClasspath +com.github.jnr:jnr-a64asm:1.0.0=testRuntimeClasspath +com.github.jnr:jnr-constants:0.10.4=testRuntimeClasspath +com.github.jnr:jnr-enxio:0.32.19=testRuntimeClasspath +com.github.jnr:jnr-ffi:2.2.18=testRuntimeClasspath +com.github.jnr:jnr-posix:3.1.21=testRuntimeClasspath +com.github.jnr:jnr-unixsocket:0.38.24=testRuntimeClasspath +com.github.jnr:jnr-x86asm:1.0.2=testRuntimeClasspath +com.github.spotbugs:spotbugs-annotations:4.9.8=compileClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath +com.github.spotbugs:spotbugs:4.9.8=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.1.1=annotationProcessor,compileClasspath,testAnnotationProcessor,testCompileClasspath +com.google.auto.service:auto-service:1.1.1=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.1=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=annotationProcessor,compileClasspath,spotbugs,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotations:2.18.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.guava:failureaccess:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:20.0=testCompileClasspath,testRuntimeClasspath +com.google.guava:guava:32.0.1-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:2.8=annotationProcessor,testAnnotationProcessor +com.google.re2j:re2j:1.7=testRuntimeClasspath +com.jayway.jsonpath:json-path:2.8.0=testCompileClasspath,testRuntimeClasspath +com.squareup.moshi:moshi:1.11.0=testCompileClasspath,testRuntimeClasspath +com.squareup.okhttp3:logging-interceptor:3.12.12=testCompileClasspath,testRuntimeClasspath +com.squareup.okhttp3:okhttp:3.12.12=testCompileClasspath,testRuntimeClasspath +com.squareup.okio:okio:1.17.5=testCompileClasspath,testRuntimeClasspath +com.thoughtworks.qdox:qdox:1.12.1=codenarc +com.vaadin.external.google:android-json:0.0.20131108.vaadin1=testCompileClasspath,testRuntimeClasspath +commons-fileupload:commons-fileupload:1.5=testCompileClasspath,testRuntimeClasspath +commons-io:commons-io:2.11.0=testCompileClasspath,testRuntimeClasspath +commons-io:commons-io:2.20.0=spotbugs +de.thetaphi:forbiddenapis:3.10=compileClasspath,testCompileClasspath,testRuntimeClasspath +io.leangen.geantyref:geantyref:1.3.16=testRuntimeClasspath +io.sqreen:libsqreen:17.3.0=testRuntimeClasspath +javax.servlet:javax.servlet-api:3.1.0=testCompileClasspath,testRuntimeClasspath +jaxen:jaxen:2.0.0=spotbugs +junit:junit:4.13.2=testRuntimeClasspath +net.bytebuddy:byte-buddy-agent:1.18.8=buildTimeInstrumentationPlugin,compileClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +net.bytebuddy:byte-buddy:1.18.8=buildTimeInstrumentationPlugin,compileClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +net.java.dev.jna:jna-platform:5.8.0=testRuntimeClasspath +net.java.dev.jna:jna:5.8.0=testRuntimeClasspath +net.minidev:accessors-smart:2.4.9=testRuntimeClasspath +net.minidev:json-smart:2.4.10=testRuntimeClasspath +net.sf.jopt-simple:jopt-simple:4.6=compileClasspath +net.sf.jopt-simple:jopt-simple:5.0.4=testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=spotbugs +org.apache.ant:ant-antlr:1.10.14=codenarc +org.apache.ant:ant-junit:1.10.14=codenarc +org.apache.bcel:bcel:6.11.0=spotbugs +org.apache.commons:commons-lang3:3.19.0=spotbugs +org.apache.commons:commons-math3:3.2=compileClasspath +org.apache.commons:commons-math3:3.6.1=testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +org.apache.commons:commons-text:1.14.0=spotbugs +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.checkerframework:checker-qual:3.33.0=annotationProcessor,testAnnotationProcessor +org.codehaus.groovy:groovy-ant:3.0.23=codenarc +org.codehaus.groovy:groovy-docgenerator:3.0.23=codenarc +org.codehaus.groovy:groovy-groovydoc:3.0.23=codenarc +org.codehaus.groovy:groovy-json:3.0.23=codenarc +org.codehaus.groovy:groovy-json:3.0.25=testCompileClasspath,testRuntimeClasspath +org.codehaus.groovy:groovy-templates:3.0.23=codenarc +org.codehaus.groovy:groovy-xml:3.0.23=codenarc +org.codehaus.groovy:groovy:3.0.23=codenarc +org.codehaus.groovy:groovy:3.0.25=testCompileClasspath,testRuntimeClasspath +org.codenarc:CodeNarc:3.7.0=codenarc +org.dom4j:dom4j:2.2.0=spotbugs +org.freemarker:freemarker:2.3.31=testCompileClasspath,testRuntimeClasspath +org.gmetrics:GMetrics:2.1.0=codenarc +org.hamcrest:hamcrest-core:1.3=testRuntimeClasspath +org.hamcrest:hamcrest:3.0=testCompileClasspath,testRuntimeClasspath +org.jacoco:org.jacoco.core:0.8.14=testRuntimeClasspath +org.jacoco:org.jacoco.report:0.8.14=testRuntimeClasspath +org.jctools:jctools-core-jdk11:4.0.6=testRuntimeClasspath +org.jctools:jctools-core:4.0.6=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:5.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:5.14.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:5.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:5.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:1.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:1.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-launcher:1.14.1=testRuntimeClasspath +org.junit.platform:junit-platform-runner:1.14.1=testRuntimeClasspath +org.junit.platform:junit-platform-suite-api:1.14.1=testRuntimeClasspath +org.junit.platform:junit-platform-suite-commons:1.14.1=testRuntimeClasspath +org.junit:junit-bom:5.14.0=spotbugs +org.junit:junit-bom:5.14.1=testCompileClasspath,testRuntimeClasspath +org.mockito:mockito-core:4.4.0=testRuntimeClasspath +org.msgpack:jackson-dataformat-msgpack:0.9.6=testCompileClasspath,testRuntimeClasspath +org.msgpack:msgpack-core:0.9.6=testCompileClasspath,testRuntimeClasspath +org.objenesis:objenesis:3.3=testCompileClasspath,testRuntimeClasspath +org.openjdk.jmh:jmh-core:1.0=compileClasspath +org.openjdk.jmh:jmh-core:1.37=testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +org.openjdk.jmh:jmh-generator-annprocess:1.37=testAnnotationProcessor +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.7.1=testRuntimeClasspath +org.ow2.asm:asm-analysis:9.9=spotbugs +org.ow2.asm:asm-commons:9.9=spotbugs +org.ow2.asm:asm-commons:9.9.1=testRuntimeClasspath +org.ow2.asm:asm-tree:9.9=spotbugs +org.ow2.asm:asm-tree:9.9.1=testRuntimeClasspath +org.ow2.asm:asm-util:9.7.1=testRuntimeClasspath +org.ow2.asm:asm-util:9.9=spotbugs +org.ow2.asm:asm:9.9=spotbugs +org.ow2.asm:asm:9.9.1=buildTimeInstrumentationPlugin,compileClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.skyscreamer:jsonassert:1.5.1=testCompileClasspath,testRuntimeClasspath +org.slf4j:jcl-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath +org.slf4j:jul-to-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath +org.slf4j:log4j-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-api:1.7.30=buildTimeInstrumentationPlugin,compileClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath +org.slf4j:slf4j-api:1.7.32=testCompileClasspath +org.slf4j:slf4j-api:1.7.36=testRuntimeClasspath +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j +org.snakeyaml:snakeyaml-engine:2.9=buildTimeInstrumentationPlugin,muzzleTooling,runtimeClasspath,testRuntimeClasspath +org.spockframework:spock-bom:2.4-groovy-3.0=testCompileClasspath,testRuntimeClasspath +org.spockframework:spock-core:2.4-groovy-3.0=testCompileClasspath,testRuntimeClasspath +org.tabletest:tabletest-junit:1.2.1=testCompileClasspath,testRuntimeClasspath +org.tabletest:tabletest-parser:1.2.0=testCompileClasspath,testRuntimeClasspath +org.xmlresolver:xmlresolver:5.3.3=spotbugs +org.xmlunit:xmlunit-core:2.10.3=testCompileClasspath,testRuntimeClasspath +empty=spotbugsPlugins diff --git a/dd-smoke-tests/jmh/build.gradle b/dd-smoke-tests/jmh/build.gradle index 2bbe5485741..2da20c85b07 100644 --- a/dd-smoke-tests/jmh/build.gradle +++ b/dd-smoke-tests/jmh/build.gradle @@ -18,13 +18,9 @@ tasks.withType(Test).configureEach { return ["-Ddatadog.smoketest.jmh.core.jar.path=${jmhJar}"] } }) - jvmArgumentProviders.add(new CommandLineArgumentProvider() { - @Override - Iterable asArguments() { - def annprocJar = configurations.named("annotationProcessor") - .get() - .find { it.name.contains("jmh-generator-annprocess") } - return ["-Ddatadog.smoketest.jmh.annproc.jar.path=${annprocJar}"] - } - }) + + if (project.hasProperty("mavenRepositoryProxy")) { + // propagate proxy URL to tests, to then propagate it to nested Gradle builds + environment "MAVEN_REPOSITORY_PROXY", project.property("mavenRepositoryProxy") + } } diff --git a/dd-smoke-tests/jmh/gradle.lockfile b/dd-smoke-tests/jmh/gradle.lockfile new file mode 100644 index 00000000000..cff54e73834 --- /dev/null +++ b/dd-smoke-tests/jmh/gradle.lockfile @@ -0,0 +1,131 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +cafe.cryptography:curve25519-elisabeth:0.1.0=testRuntimeClasspath +cafe.cryptography:ed25519-elisabeth:0.1.0=testRuntimeClasspath +ch.qos.logback:logback-classic:1.2.13=testCompileClasspath,testRuntimeClasspath +ch.qos.logback:logback-core:1.2.13=testCompileClasspath,testRuntimeClasspath +com.blogspot.mydailyjava:weak-lock-free:0.17=testCompileClasspath,testRuntimeClasspath +com.datadoghq.okhttp3:okhttp:3.12.15=testCompileClasspath,testRuntimeClasspath +com.datadoghq.okio:okio:1.17.6=testCompileClasspath,testRuntimeClasspath +com.datadoghq:dd-instrument-java:0.0.3=testCompileClasspath,testRuntimeClasspath +com.datadoghq:dd-javac-plugin-client:0.2.2=testCompileClasspath,testRuntimeClasspath +com.datadoghq:java-dogstatsd-client:4.4.5=testRuntimeClasspath +com.datadoghq:sketches-java:0.8.3=testRuntimeClasspath +com.fasterxml.jackson.core:jackson-annotations:2.20=testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-core:2.20.0=testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-databind:2.20.0=testCompileClasspath,testRuntimeClasspath +com.github.javaparser:javaparser-core:3.25.6=codenarc +com.github.jnr:jffi:1.3.14=testRuntimeClasspath +com.github.jnr:jnr-a64asm:1.0.0=testRuntimeClasspath +com.github.jnr:jnr-constants:0.10.4=testRuntimeClasspath +com.github.jnr:jnr-enxio:0.32.19=testRuntimeClasspath +com.github.jnr:jnr-ffi:2.2.18=testRuntimeClasspath +com.github.jnr:jnr-posix:3.1.21=testRuntimeClasspath +com.github.jnr:jnr-unixsocket:0.38.24=testRuntimeClasspath +com.github.jnr:jnr-x86asm:1.0.2=testRuntimeClasspath +com.github.spotbugs:spotbugs-annotations:4.9.8=compileClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath +com.github.spotbugs:spotbugs:4.9.8=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.code.findbugs:jsr305:3.0.2=compileClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.guava:guava:20.0=testCompileClasspath,testRuntimeClasspath +com.google.re2j:re2j:1.7=testRuntimeClasspath +com.jayway.jsonpath:json-path:2.8.0=testCompileClasspath,testRuntimeClasspath +com.squareup.moshi:moshi:1.11.0=testCompileClasspath,testRuntimeClasspath +com.squareup.okhttp3:logging-interceptor:3.12.12=testCompileClasspath,testRuntimeClasspath +com.squareup.okhttp3:okhttp:3.12.12=testCompileClasspath,testRuntimeClasspath +com.squareup.okio:okio:1.17.5=testCompileClasspath,testRuntimeClasspath +com.thoughtworks.qdox:qdox:1.12.1=codenarc +com.vaadin.external.google:android-json:0.0.20131108.vaadin1=testCompileClasspath,testRuntimeClasspath +commons-fileupload:commons-fileupload:1.5=testCompileClasspath,testRuntimeClasspath +commons-io:commons-io:2.11.0=testCompileClasspath,testRuntimeClasspath +commons-io:commons-io:2.20.0=spotbugs +de.thetaphi:forbiddenapis:3.10=compileClasspath,testCompileClasspath,testRuntimeClasspath +io.leangen.geantyref:geantyref:1.3.16=testRuntimeClasspath +io.sqreen:libsqreen:17.3.0=testRuntimeClasspath +javax.servlet:javax.servlet-api:3.1.0=testCompileClasspath,testRuntimeClasspath +jaxen:jaxen:2.0.0=spotbugs +junit:junit:4.13.2=testRuntimeClasspath +net.bytebuddy:byte-buddy-agent:1.18.8=testCompileClasspath,testRuntimeClasspath +net.bytebuddy:byte-buddy:1.18.8=testCompileClasspath,testRuntimeClasspath +net.java.dev.jna:jna-platform:5.8.0=testRuntimeClasspath +net.java.dev.jna:jna:5.8.0=testRuntimeClasspath +net.minidev:accessors-smart:2.4.9=testRuntimeClasspath +net.minidev:json-smart:2.4.10=testRuntimeClasspath +net.sf.jopt-simple:jopt-simple:5.0.4=annotationProcessor,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=spotbugs +org.apache.ant:ant-antlr:1.10.14=codenarc +org.apache.ant:ant-junit:1.10.14=codenarc +org.apache.bcel:bcel:6.11.0=spotbugs +org.apache.commons:commons-lang3:3.19.0=spotbugs +org.apache.commons:commons-math3:3.6.1=annotationProcessor,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.apache.commons:commons-text:1.14.0=spotbugs +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.codehaus.groovy:groovy-ant:3.0.23=codenarc +org.codehaus.groovy:groovy-docgenerator:3.0.23=codenarc +org.codehaus.groovy:groovy-groovydoc:3.0.23=codenarc +org.codehaus.groovy:groovy-json:3.0.23=codenarc +org.codehaus.groovy:groovy-json:3.0.25=testCompileClasspath,testRuntimeClasspath +org.codehaus.groovy:groovy-templates:3.0.23=codenarc +org.codehaus.groovy:groovy-xml:3.0.23=codenarc +org.codehaus.groovy:groovy:3.0.23=codenarc +org.codehaus.groovy:groovy:3.0.25=testCompileClasspath,testRuntimeClasspath +org.codenarc:CodeNarc:3.7.0=codenarc +org.dom4j:dom4j:2.2.0=spotbugs +org.freemarker:freemarker:2.3.31=testCompileClasspath,testRuntimeClasspath +org.gmetrics:GMetrics:2.1.0=codenarc +org.hamcrest:hamcrest-core:1.3=testRuntimeClasspath +org.hamcrest:hamcrest:3.0=testCompileClasspath,testRuntimeClasspath +org.jacoco:org.jacoco.core:0.8.14=testRuntimeClasspath +org.jacoco:org.jacoco.report:0.8.14=testRuntimeClasspath +org.jctools:jctools-core-jdk11:4.0.6=testRuntimeClasspath +org.jctools:jctools-core:4.0.6=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:5.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:5.14.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:5.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:5.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:1.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:1.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-launcher:1.14.1=testRuntimeClasspath +org.junit.platform:junit-platform-runner:1.14.1=testRuntimeClasspath +org.junit.platform:junit-platform-suite-api:1.14.1=testRuntimeClasspath +org.junit.platform:junit-platform-suite-commons:1.14.1=testRuntimeClasspath +org.junit:junit-bom:5.14.0=spotbugs +org.junit:junit-bom:5.14.1=testCompileClasspath,testRuntimeClasspath +org.mockito:mockito-core:4.4.0=testRuntimeClasspath +org.msgpack:jackson-dataformat-msgpack:0.9.6=testCompileClasspath,testRuntimeClasspath +org.msgpack:msgpack-core:0.9.6=testCompileClasspath,testRuntimeClasspath +org.objenesis:objenesis:3.3=testCompileClasspath,testRuntimeClasspath +org.openjdk.jmh:jmh-core:1.37=annotationProcessor,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.openjdk.jmh:jmh-generator-annprocess:1.37=annotationProcessor +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.7.1=testRuntimeClasspath +org.ow2.asm:asm-analysis:9.9=spotbugs +org.ow2.asm:asm-commons:9.9=spotbugs +org.ow2.asm:asm-commons:9.9.1=testRuntimeClasspath +org.ow2.asm:asm-tree:9.9=spotbugs +org.ow2.asm:asm-tree:9.9.1=testRuntimeClasspath +org.ow2.asm:asm-util:9.7.1=testRuntimeClasspath +org.ow2.asm:asm-util:9.9=spotbugs +org.ow2.asm:asm:9.9=spotbugs +org.ow2.asm:asm:9.9.1=testCompileClasspath,testRuntimeClasspath +org.skyscreamer:jsonassert:1.5.1=testCompileClasspath,testRuntimeClasspath +org.slf4j:jcl-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath +org.slf4j:jul-to-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath +org.slf4j:log4j-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-api:1.7.32=testCompileClasspath +org.slf4j:slf4j-api:1.7.36=testRuntimeClasspath +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j +org.snakeyaml:snakeyaml-engine:2.9=testRuntimeClasspath +org.spockframework:spock-bom:2.4-groovy-3.0=testCompileClasspath,testRuntimeClasspath +org.spockframework:spock-core:2.4-groovy-3.0=testCompileClasspath,testRuntimeClasspath +org.tabletest:tabletest-junit:1.2.1=testCompileClasspath,testRuntimeClasspath +org.tabletest:tabletest-parser:1.2.0=testCompileClasspath,testRuntimeClasspath +org.xmlresolver:xmlresolver:5.3.3=spotbugs +org.xmlunit:xmlunit-core:2.10.3=testCompileClasspath,testRuntimeClasspath +empty=spotbugsPlugins,testAnnotationProcessor From 98bc960bc3d5a25a57cb268880bb991548f03366 Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Tue, 2 Jun 2026 17:46:22 +0200 Subject: [PATCH 06/12] fix: tag JMH benchmarks as test.type benchmark --- .../trace/civisibility/decorator/TestDecoratorImpl.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecoratorImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecoratorImpl.java index de2ab526728..d6b97384059 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecoratorImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecoratorImpl.java @@ -6,12 +6,17 @@ import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.util.Strings; +import java.util.Collections; import java.util.Map; +import java.util.Set; public class TestDecoratorImpl implements TestDecorator { private static final UTF8BytesString CIAPP_TEST_ORIGIN = UTF8BytesString.create("ciapp-test"); + // components whose tests represent benchmarks (test.type = "benchmark") + private static final Set BENCHMARK_COMPONENTS = Collections.singleton("jmh"); + private final String component; private final String sessionName; private final Map ciTags; @@ -32,7 +37,7 @@ public TestDecoratorImpl( } protected String testType() { - return TEST_TYPE; + return BENCHMARK_COMPONENTS.contains(component) ? TEST_TYPE_BENCHMARK : TEST_TYPE; } protected UTF8BytesString origin() { From 459cfc9079638f9921206e656d064b4c8b018011 Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Tue, 2 Jun 2026 17:47:58 +0200 Subject: [PATCH 07/12] feat: rework jmh reporter - Extracts span lifecycle into DatadogJmhReporter - Group same-class benchmarks under one suite and marks them as parallelized - Build test.parameters from BenchmarkParams.getParamsKeys()/getParam() - Resolve framework version - Finish open test/suite on swallowed failures --- .../instrumentation/jmh/DDOutputFormat.java | 140 ++---------- .../jmh/DatadogJmhReporter.java | 215 ++++++++++++++++++ .../jmh/JmhInstrumentation.java | 11 +- .../trace/instrumentation/jmh/JmhUtils.java | 88 +++++-- 4 files changed, 299 insertions(+), 155 deletions(-) create mode 100644 dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DatadogJmhReporter.java diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DDOutputFormat.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DDOutputFormat.java index 9b94cb69d87..097f4de2c41 100644 --- a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DDOutputFormat.java +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DDOutputFormat.java @@ -1,35 +1,11 @@ package datadog.trace.instrumentation.jmh; -import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_ERROR; -import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_FORKS; -import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_ITERATIONS; -import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_MAX; -import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_MIN; -import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_MODE; -import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_P50; -import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_P90; -import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_P95; -import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_P99; -import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_SAMPLE_COUNT; -import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_THREADS; -import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_TIME_UNIT; -import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_UNIT; -import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_VALUE; -import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_WARMUP_ITERATIONS; - -import datadog.trace.api.civisibility.InstrumentationBridge; -import datadog.trace.api.civisibility.config.TestSourceData; -import datadog.trace.api.civisibility.events.TestEventsHandler; -import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation; -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import datadog.trace.bootstrap.instrumentation.api.AgentTracer; -import java.util.Collections; import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.IterationParams; import org.openjdk.jmh.results.BenchmarkResult; -import org.openjdk.jmh.results.Result; +import org.openjdk.jmh.results.IterationResult; import org.openjdk.jmh.results.RunResult; import org.openjdk.jmh.runner.format.OutputFormat; -import org.openjdk.jmh.util.Statistics; /** * Wraps a JMH {@link OutputFormat} to emit CI Visibility spans for each benchmark method. @@ -40,126 +16,45 @@ public class DDOutputFormat implements OutputFormat { private final OutputFormat delegate; - private final TestEventsHandler handler; - private final String frameworkVersion; - // Keys used as suite/test descriptors in the handler — just the full benchmark name strings. - // We keep suite and test keys separate so the handler can manage their lifetimes independently. - private volatile String currentSuiteKey; - private volatile String currentTestKey; - - public DDOutputFormat(OutputFormat delegate, String frameworkVersion) { + public DDOutputFormat(OutputFormat delegate) { this.delegate = delegate; - this.frameworkVersion = frameworkVersion; - this.handler = - InstrumentationBridge.createTestEventsHandler( - JmhUtils.FRAMEWORK_NAME, null, null, Collections.emptyList()); } @Override public void startBenchmark(BenchmarkParams benchParams) { delegate.startBenchmark(benchParams); - - String fullName = benchParams.getBenchmark(); - String[] parts = JmhUtils.splitBenchmarkName(fullName); - String suiteName = parts[0]; - String testName = parts[1]; - String testParameters = JmhUtils.testParameters(fullName); - - currentSuiteKey = suiteName + "#" + fullName; - currentTestKey = fullName; - - handler.onTestSuiteStart( - currentSuiteKey, - suiteName, - JmhUtils.FRAMEWORK_NAME, - frameworkVersion, - null, - Collections.emptyList(), - false, - TestFrameworkInstrumentation.JMH, - null); - - handler.onTestStart( - currentSuiteKey, - currentTestKey, - testName, - JmhUtils.FRAMEWORK_NAME, - frameworkVersion, - testParameters, - Collections.emptyList(), - TestSourceData.UNKNOWN, - null, - null); + DatadogJmhReporter.onBenchmarkStart(benchParams); } @Override public void endBenchmark(BenchmarkResult result) { - String suiteKey = currentSuiteKey; - String testKey = currentTestKey; - if (suiteKey == null || testKey == null) { - delegate.endBenchmark(result); - return; - } - try { - tagBenchmarkMetrics(result); - handler.onTestFinish(testKey, null, null); - handler.onTestSuiteFinish(suiteKey, null); + DatadogJmhReporter.onBenchmarkEnd(result); } finally { delegate.endBenchmark(result); } } - private void tagBenchmarkMetrics(BenchmarkResult result) { - AgentSpan span = AgentTracer.activeSpan(); - if (span == null) { - return; - } - - BenchmarkParams params = result.getParams(); - span.setTag(BENCHMARK_MODE, params.getMode().shortLabel()); - span.setTag(BENCHMARK_ITERATIONS, params.getMeasurement().getCount()); - span.setTag(BENCHMARK_WARMUP_ITERATIONS, params.getWarmup().getCount()); - span.setTag(BENCHMARK_FORKS, params.getForks()); - span.setTag(BENCHMARK_THREADS, params.getThreads()); - span.setTag(BENCHMARK_TIME_UNIT, params.getTimeUnit().name()); - - Result primary = result.getPrimaryResult(); - span.setMetric(BENCHMARK_VALUE, primary.getScore()); - span.setTag(BENCHMARK_UNIT, primary.getScoreUnit()); - - double error = primary.getScoreError(); - if (!Double.isNaN(error)) { - span.setMetric(BENCHMARK_ERROR, error); - } - - Statistics stats = primary.getStatistics(); - if (stats.getN() > 1) { - span.setMetric(BENCHMARK_P50, stats.getPercentile(50)); - span.setMetric(BENCHMARK_P90, stats.getPercentile(90)); - span.setMetric(BENCHMARK_P95, stats.getPercentile(95)); - span.setMetric(BENCHMARK_P99, stats.getPercentile(99)); - span.setMetric(BENCHMARK_MIN, stats.getMin()); - span.setMetric(BENCHMARK_MAX, stats.getMax()); - span.setMetric(BENCHMARK_SAMPLE_COUNT, stats.getN()); + @Override + public void endRun(java.util.Collection result) { + try { + DatadogJmhReporter.onRunEnd(); + } finally { + delegate.endRun(result); } } // ---- Delegation-only methods ---- @Override - public void iteration( - BenchmarkParams benchParams, org.openjdk.jmh.infra.IterationParams params, int iteration) { + public void iteration(BenchmarkParams benchParams, IterationParams params, int iteration) { delegate.iteration(benchParams, params, iteration); } @Override public void iterationResult( - BenchmarkParams benchParams, - org.openjdk.jmh.infra.IterationParams params, - int iteration, - org.openjdk.jmh.results.IterationResult data) { + BenchmarkParams benchParams, IterationParams params, int iteration, IterationResult data) { delegate.iterationResult(benchParams, params, iteration, data); } @@ -168,15 +63,6 @@ public void startRun() { delegate.startRun(); } - @Override - public void endRun(java.util.Collection result) { - try { - handler.close(); - } finally { - delegate.endRun(result); - } - } - @Override public void print(String s) { delegate.print(s); diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DatadogJmhReporter.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DatadogJmhReporter.java new file mode 100644 index 00000000000..f1df49a2015 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DatadogJmhReporter.java @@ -0,0 +1,215 @@ +package datadog.trace.instrumentation.jmh; + +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_ERROR; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_FORKS; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_ITERATIONS; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_MAX; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_MIN; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_MODE; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_P50; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_P90; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_P95; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_P99; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_SAMPLE_COUNT; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_THREADS; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_TIME_UNIT; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_UNIT; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_VALUE; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_WARMUP_ITERATIONS; + +import datadog.trace.api.civisibility.InstrumentationBridge; +import datadog.trace.api.civisibility.config.TestSourceData; +import datadog.trace.api.civisibility.events.TestDescriptor; +import datadog.trace.api.civisibility.events.TestEventsHandler; +import datadog.trace.api.civisibility.events.TestSuiteDescriptor; +import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.results.BenchmarkResult; +import org.openjdk.jmh.results.Result; +import org.openjdk.jmh.util.Statistics; + +public final class DatadogJmhReporter { + + private static final String TEST_FRAMEWORK = "jmh"; + private static final String TEST_FRAMEWORK_VERSION = JmhUtils.frameworkVersion(); + + private static volatile TestEventsHandler + TEST_EVENTS_HANDLER; + + // Suites opened so far in this run, keyed by suite (class) name. JMH has no class-level + // start/finish callbacks, so we open a suite the first time we see a benchmark of that class, + // keep it open while every benchmark of the class runs under it, and finish all suites together + // in onRunEnd. + private static final Map OPEN_SUITES = new ConcurrentHashMap<>(); + + // The single benchmark (test) currently in flight. startBenchmark/endBenchmark only ever fire in + // the launcher JVM, strictly sequentially (forked JVMs never call them), so a single reference is + // enough. If it is still set when the next benchmark starts (or the run ends) JMH skipped + // endBenchmark for a failed benchmark, and we finish it as a failure rather than leaking it. + private static volatile TestDescriptor currentTest; + + private DatadogJmhReporter() {} + + public static synchronized void start() { + if (TEST_EVENTS_HANDLER == null) { + TEST_EVENTS_HANDLER = + InstrumentationBridge.createTestEventsHandler( + TEST_FRAMEWORK, null, null, JmhUtils.CAPABILITIES); + } + } + + public static synchronized void stop() { + if (TEST_EVENTS_HANDLER != null) { + TEST_EVENTS_HANDLER.close(); + TEST_EVENTS_HANDLER = null; + } + } + + public static void onBenchmarkStart(BenchmarkParams benchParams) { + // lazy load handler + start(); + + // A previous benchmark whose endBenchmark was skipped (JMH swallows benchmark errors when + // fail-on-error is off) is still open. Finish it as a failure before starting the next one. + finishOpenTest(true); + + String fullName = benchParams.getBenchmark(); + String[] parts = JmhUtils.splitBenchmarkName(fullName); + String suiteName = parts[0]; + String testName = parts[1]; + String testParameters = JmhUtils.testParameters(benchParams); + + // JMH gives us only fully-qualified names, not Class objects, so testClass is null. The + // TestDescriptor includes testParameters in its identity, so each @Param variant is distinct. + TestSuiteDescriptor suite = new TestSuiteDescriptor(suiteName, null); + TestDescriptor test = new TestDescriptor(suiteName, null, testName, testParameters, null); + currentTest = test; + + // Open the suite the first time we see this class; later benchmarks of the class reuse it. + if (OPEN_SUITES.putIfAbsent(suiteName, suite) == null) { + // A run can have benchmarks from several classes, so multiple suites stay open at once and + // are finished together in onRunEnd (in arbitrary map order). Mark them parallelized so the + // suite spans are not pushed onto the active-span stack — otherwise finishing an outer suite + // while an inner one is still active throws IllegalStateException (TestSuiteImpl.end). Test + // spans still self-activate, so tagBenchmarkMetrics' activeSpan() is unaffected. + boolean parallelized = true; + TEST_EVENTS_HANDLER.onTestSuiteStart( + suite, + suiteName, + TEST_FRAMEWORK, + TEST_FRAMEWORK_VERSION, + null, + Collections.emptyList(), + parallelized, + TestFrameworkInstrumentation.JMH, + null); + } + + TEST_EVENTS_HANDLER.onTestStart( + suite, + test, + testName, + TEST_FRAMEWORK, + TEST_FRAMEWORK_VERSION, + testParameters, + Collections.emptyList(), + TestSourceData.UNKNOWN, + null, + null); + } + + public static void onBenchmarkEnd(BenchmarkResult result) { + if (currentTest != null) { + tagBenchmarkMetrics(result); + finishOpenTest(false); + } + } + + /** + * Finishes the in-flight benchmark's test span, if any, and clears it so a spurious second call + * is a no-op. When {@code failed} is true the test is marked failed first (the failure propagates + * to its suite when both are finished), so a benchmark whose {@code endBenchmark} was skipped + * surfaces as {@code test.status = fail} instead of being silently leaked. Suites are left open + * and finished together in {@link #onRunEnd()}. + */ + private static void finishOpenTest(boolean failed) { + TestDescriptor test = currentTest; + if (test == null) { + return; + } + currentTest = null; + + TestEventsHandler handler = TEST_EVENTS_HANDLER; + if (handler == null) { + return; + } + if (failed) { + handler.onTestFailure(test, null); + } + handler.onTestFinish(test, null, null); + } + + private static void tagBenchmarkMetrics(BenchmarkResult result) { + AgentSpan span = AgentTracer.activeSpan(); + if (span == null) { + return; + } + + BenchmarkParams params = result.getParams(); + span.setTag(BENCHMARK_MODE, params.getMode().shortLabel()); + span.setTag(BENCHMARK_ITERATIONS, params.getMeasurement().getCount()); + span.setTag(BENCHMARK_WARMUP_ITERATIONS, params.getWarmup().getCount()); + span.setTag(BENCHMARK_FORKS, params.getForks()); + span.setTag(BENCHMARK_THREADS, params.getThreads()); + span.setTag(BENCHMARK_TIME_UNIT, params.getTimeUnit().name()); + + Result primary = result.getPrimaryResult(); + span.setMetric(BENCHMARK_VALUE, primary.getScore()); + span.setTag(BENCHMARK_UNIT, primary.getScoreUnit()); + + double error = primary.getScoreError(); + if (!Double.isNaN(error)) { + span.setMetric(BENCHMARK_ERROR, error); + } + + // Single-shot mode has no per-invocation distribution: any spread is across forks, not + // samples, so the percentiles would be misleading. Only emit them when there is a real + // sample distribution. + Statistics stats = primary.getStatistics(); + if (params.getMode() != Mode.SingleShotTime && stats.getN() > 1) { + span.setMetric(BENCHMARK_P50, stats.getPercentile(50)); + span.setMetric(BENCHMARK_P90, stats.getPercentile(90)); + span.setMetric(BENCHMARK_P95, stats.getPercentile(95)); + span.setMetric(BENCHMARK_P99, stats.getPercentile(99)); + span.setMetric(BENCHMARK_MIN, stats.getMin()); + span.setMetric(BENCHMARK_MAX, stats.getMax()); + span.setMetric(BENCHMARK_SAMPLE_COUNT, stats.getN()); + } + } + + public static void onRunEnd() { + // Flush a test left open by a swallowed failure, finish every suite opened during the run (JMH + // gives us no per-class finish callback), then close the session/module. + // + // Note: onRunEnd is only invoked on the normal path (Runner.runBenchmarks -> out.endRun). With + // fail-on-error enabled (non-default) a benchmark exception aborts the run before endRun, so + // the open suites/tests for that run are not flushed — same data loss as a JVM crash. The + // common + // path (fail-on-error off, the JMH default) always reaches endRun and closes cleanly. + finishOpenTest(true); + TestEventsHandler handler = TEST_EVENTS_HANDLER; + if (handler != null) { + for (TestSuiteDescriptor suite : OPEN_SUITES.values()) { + handler.onTestSuiteFinish(suite, null); + } + } + OPEN_SUITES.clear(); + stop(); + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhInstrumentation.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhInstrumentation.java index 5bfb0013ff1..98ac021132e 100644 --- a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhInstrumentation.java +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhInstrumentation.java @@ -29,7 +29,9 @@ public String instrumentedType() { @Override public String[] helperClassNames() { return new String[] { - packageName + ".JmhUtils", packageName + ".DDOutputFormat", + packageName + ".JmhUtils", + packageName + ".DDOutputFormat", + packageName + ".DatadogJmhReporter", }; } @@ -49,12 +51,7 @@ public static void onExit( if (out instanceof DDOutputFormat) { return; } - String version = null; - try { - version = org.openjdk.jmh.Main.class.getPackage().getImplementationVersion(); - } catch (Throwable ignored) { - } - out = new DDOutputFormat(out, version); + out = new DDOutputFormat(out); } } } diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhUtils.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhUtils.java index b5bfea0e2ef..e668f525e38 100644 --- a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhUtils.java +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhUtils.java @@ -1,46 +1,92 @@ package datadog.trace.instrumentation.jmh; +import datadog.trace.api.civisibility.config.LibraryCapability; +import java.io.InputStream; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; import javax.annotation.Nullable; +import org.openjdk.jmh.infra.BenchmarkParams; public final class JmhUtils { - static final String FRAMEWORK_NAME = "jmh"; + public static final List CAPABILITIES = Collections.emptyList(); + + /** + * Resolves the running JMH version. JMH does not publish an {@code Implementation-Version} in its + * jar manifest, so {@code Package.getImplementationVersion()} returns {@code null}; the version + * actually lives in the {@code jmh.properties} classpath resource (the same source JMH's own + * {@code org.openjdk.jmh.util.Version} reads, which we cannot reference because it is absent in + * older versions within the supported range). Falls back to the manifest, then {@code null}. + */ + @Nullable + public static String frameworkVersion() { + try (InputStream is = org.openjdk.jmh.Main.class.getResourceAsStream("/jmh.properties")) { + if (is != null) { + Properties props = new Properties(); + props.load(is); + String version = props.getProperty("jmh.version"); + if (version != null && !version.isEmpty()) { + return version; + } + } + } catch (Throwable ignored) { + // fall through to the manifest lookup + } + try { + return org.openjdk.jmh.Main.class.getPackage().getImplementationVersion(); + } catch (Throwable ignored) { + return null; + } + } /** * Splits a JMH benchmark name into suite (class) and method parts. * - *

JMH names have the form {@code "com.example.MyBenchmark.myMethod"} or, when {@code @Param} - * combinations are present, {@code "com.example.MyBenchmark.myMethod:size=1000,threads=4"}. + *

JMH benchmark names have the form {@code "com.example.MyBenchmark.myMethod"}. {@code @Param} + * values are not part of the name — they are exposed separately via {@link + * BenchmarkParams#getParamsKeys()} (see {@link #testParameters(BenchmarkParams)}). */ public static String[] splitBenchmarkName(String fullName) { - // Strip any @Param suffix before splitting on the class/method boundary - int colonIdx = fullName.indexOf(':'); - String baseName = colonIdx >= 0 ? fullName.substring(0, colonIdx) : fullName; - - int lastDot = baseName.lastIndexOf('.'); + int lastDot = fullName.lastIndexOf('.'); if (lastDot < 0) { return new String[] {"", fullName}; } - return new String[] {baseName.substring(0, lastDot), baseName.substring(lastDot + 1)}; + return new String[] {fullName.substring(0, lastDot), fullName.substring(lastDot + 1)}; } /** - * Returns the {@code test.parameters} JSON string for a parameterized benchmark, or {@code null} - * for an unparameterized one. - * - *

Follows the same convention as JUnit 5 parameterized tests: {@code - * {"metadata":{"test_name":""}}}. + * Returns the {@code test.parameters} JSON string for a parameterized benchmark (one declaring + * {@code @Param} fields), or {@code null} for an unparameterized one. */ @Nullable - public static String testParameters(String fullName) { - int colonIdx = fullName.indexOf(':'); - if (colonIdx < 0) { + public static String testParameters(BenchmarkParams params) { + // getParamsKeys() is a raw Collection in older JMH versions, so iterate as Object and cast to + // avoid an unchecked-conversion warning when compiling against the minimum supported version. + Map values = new LinkedHashMap<>(); + for (Object key : params.getParamsKeys()) { + String name = (String) key; + values.put(name, params.getParam(name)); + } + if (values.isEmpty()) { return null; } - // fullName after last dot includes the param suffix, e.g. "myMethod:size=1000" - int lastDot = fullName.lastIndexOf('.', colonIdx); - String displayName = lastDot >= 0 ? fullName.substring(lastDot + 1) : fullName; - return "{\"metadata\":{\"test_name\":\"" + escapeJson(displayName) + "\"}}"; + return paramsToJson(values); + } + + static String paramsToJson(Map params) { + StringBuilder displayName = new StringBuilder(); + boolean first = true; + for (Map.Entry entry : params.entrySet()) { + if (!first) { + displayName.append(", "); + } + first = false; + displayName.append(entry.getKey()).append('=').append(entry.getValue()); + } + return "{\"metadata\":{\"test_name\":\"" + escapeJson(displayName.toString()) + "\"}}"; } /** Minimal JSON string escaping for benchmark names (no unicode escaping needed). */ From 4a9f8d75b1be76a02a20e659bd070aeac636cc0d Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Tue, 2 Jun 2026 17:51:11 +0200 Subject: [PATCH 08/12] test: update test cases and fixtures --- .../test/groovy/JmhInstrumentationTest.groovy | 24 +- .../instrumentation/jmh/JmhUtilsTest.java | 46 +-- .../test-benchmark-multi-class/coverages.ftl | 1 + .../test-benchmark-multi-class/events.ftl | 283 ++++++++++++++++++ .../test-benchmark-parameterized/events.ftl | 76 ++--- .../test-benchmark-simple/events.ftl | 12 +- 6 files changed, 356 insertions(+), 86 deletions(-) create mode 100644 dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-multi-class/coverages.ftl create mode 100644 dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-multi-class/events.ftl diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/groovy/JmhInstrumentationTest.groovy b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/groovy/JmhInstrumentationTest.groovy index 4b4e6a7ee5f..a360facd3d5 100644 --- a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/groovy/JmhInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/groovy/JmhInstrumentationTest.groovy @@ -1,5 +1,6 @@ import datadog.trace.api.DisableTestTrace import datadog.trace.civisibility.CiVisibilityInstrumentationTest +import datadog.trace.instrumentation.jmh.JmhUtils import datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark import datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark import org.openjdk.jmh.runner.Runner @@ -22,21 +23,24 @@ class JmhInstrumentationTest extends CiVisibilityInstrumentationTest { ] def "test #testcaseName"() { - runBenchmark(benchmarkClass) + runBenchmarks(*benchmarkClasses) assertSpansData(testcaseName, [:], BENCHMARK_METRIC_TAGS) where: - testcaseName | benchmarkClass - "test-benchmark-simple" | SimpleBenchmark - "test-benchmark-parameterized" | ParameterizedBenchmark + testcaseName | benchmarkClasses + "test-benchmark-simple" | [SimpleBenchmark] + "test-benchmark-parameterized" | [ParameterizedBenchmark] + // Multiple classes in one run keep several suites open at once (all finished in onRunEnd); + // guards against suite spans being activated on the active-span stack, which would make + // finishing them out of order throw IllegalStateException. + "test-benchmark-multi-class" | [SimpleBenchmark, ParameterizedBenchmark] } - private void runBenchmark(Class benchmarkClass) { - def options = new OptionsBuilder() - .include(benchmarkClass.getName()) + private void runBenchmarks(Class... benchmarkClasses) { + def builder = new OptionsBuilder() .jvmArgsAppend("-Djmh.ignoreLock=true") - .build() - new Runner(options).run() + benchmarkClasses.each { builder.include(it.getName()) } + new Runner(builder.build()).run() } @Override @@ -46,6 +50,6 @@ class JmhInstrumentationTest extends CiVisibilityInstrumentationTest { @Override String instrumentedLibraryVersion() { - Runner.class.getPackage().getImplementationVersion() + JmhUtils.frameworkVersion() } } diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/JmhUtilsTest.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/JmhUtilsTest.java index 4da75dfc901..7efbd4b19ea 100644 --- a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/JmhUtilsTest.java +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/JmhUtilsTest.java @@ -2,51 +2,53 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; +import java.util.LinkedHashMap; +import java.util.Map; import org.junit.jupiter.api.Test; class JmhUtilsTest { @Test void splitBenchmarkName_simple() { - String[] parts = JmhUtils.splitBenchmarkName("com.example.MyBenchmark.myMethod"); - assertArrayEquals(new String[] {"com.example.MyBenchmark", "myMethod"}, parts); - } - - @Test - void splitBenchmarkName_withParams() { - String[] parts = - JmhUtils.splitBenchmarkName("com.example.MyBenchmark.myMethod:size=1000,threads=4"); - assertArrayEquals(new String[] {"com.example.MyBenchmark", "myMethod"}, parts); + assertArrayEquals( + new String[] {"com.example.MyBenchmark", "myMethod"}, + JmhUtils.splitBenchmarkName("com.example.MyBenchmark.myMethod")); } @Test void splitBenchmarkName_noPackage() { - String[] parts = JmhUtils.splitBenchmarkName("MyBenchmark.myMethod"); - assertArrayEquals(new String[] {"MyBenchmark", "myMethod"}, parts); + assertArrayEquals( + new String[] {"MyBenchmark", "myMethod"}, + JmhUtils.splitBenchmarkName("MyBenchmark.myMethod")); } @Test void splitBenchmarkName_noDot() { - String[] parts = JmhUtils.splitBenchmarkName("noDot"); - assertArrayEquals(new String[] {"", "noDot"}, parts); + assertArrayEquals(new String[] {"", "noDot"}, JmhUtils.splitBenchmarkName("noDot")); } @Test - void testParameters_noParams() { - assertNull(JmhUtils.testParameters("com.example.MyBenchmark.myMethod")); + void paramsToJson_singleParam() { + Map params = new LinkedHashMap<>(); + params.put("size", "1000"); + assertEquals("{\"metadata\":{\"test_name\":\"size=1000\"}}", JmhUtils.paramsToJson(params)); } @Test - void testParameters_withParams() { - String result = JmhUtils.testParameters("com.example.MyBenchmark.myMethod:size=1000,threads=4"); - assertEquals("{\"metadata\":{\"test_name\":\"myMethod:size=1000,threads=4\"}}", result); + void paramsToJson_multipleParams() { + Map params = new LinkedHashMap<>(); + params.put("size", "1000"); + params.put("threads", "4"); + assertEquals( + "{\"metadata\":{\"test_name\":\"size=1000, threads=4\"}}", JmhUtils.paramsToJson(params)); } @Test - void testParameters_escapesQuotes() { - String result = JmhUtils.testParameters("com.example.MyBenchmark.myMethod:key=\"value\""); - assertEquals("{\"metadata\":{\"test_name\":\"myMethod:key=\\\"value\\\"\"}}", result); + void paramsToJson_escapesQuotes() { + Map params = new LinkedHashMap<>(); + params.put("key", "\"value\""); + assertEquals( + "{\"metadata\":{\"test_name\":\"key=\\\"value\\\"\"}}", JmhUtils.paramsToJson(params)); } } diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-multi-class/coverages.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-multi-class/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-multi-class/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-multi-class/events.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-multi-class/events.ftl new file mode 100644 index 00000000000..a989c19f17c --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-multi-class/events.ftl @@ -0,0 +1,283 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "jmh.test_suite", + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "benchmark.run.mode" : "avgt", + "benchmark.run.time_unit" : "NANOSECONDS", + "benchmark.unit" : "ns/op", + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.name" : "measure", + "test.parameters" : "{\"metadata\":{\"test_name\":\"size=1\"}}", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "benchmark.run.forks" : 0, + "benchmark.run.iterations" : 1, + "benchmark.run.threads" : 1, + "benchmark.run.warmup_iterations" : 1, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test", + "parent_id" : ${content_parent_id}, + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark.measure", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "benchmark.run.mode" : "avgt", + "benchmark.run.time_unit" : "NANOSECONDS", + "benchmark.unit" : "ns/op", + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.name" : "measure", + "test.parameters" : "{\"metadata\":{\"test_name\":\"size=2\"}}", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "benchmark.run.forks" : 0, + "benchmark.run.iterations" : 1, + "benchmark.run.threads" : 1, + "benchmark.run.warmup_iterations" : 1, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test", + "parent_id" : ${content_parent_id}, + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark.measure", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4} + }, + "name" : "jmh.test_suite", + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id_2} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "benchmark.run.mode" : "avgt", + "benchmark.run.time_unit" : "NANOSECONDS", + "benchmark.unit" : "ns/op", + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.name" : "measure", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "benchmark.run.forks" : 0, + "benchmark.run.iterations" : 1, + "benchmark.run.threads" : 1, + "benchmark.run.warmup_iterations" : 1, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test", + "parent_id" : ${content_parent_id}, + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark.measure", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_3}, + "start" : ${content_start_5}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id_2}, + "trace_id" : ${content_trace_id_3} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "jmh-1.0", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "pass", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_6}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test_session", + "resource" : "jmh-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_6}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_7}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_4}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_7} + }, + "name" : "jmh.test_module", + "resource" : "jmh-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_7}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/events.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/events.ftl index 39dd9782269..b7603a72bdc 100644 --- a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/events.ftl +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/events.ftl @@ -10,10 +10,11 @@ "library_version" : ${content_meta_library_version}, "span.kind" : "test_suite_end", "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "jmh-1.0", "test.status" : "pass", "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", - "test.type" : "test", + "test.type" : "benchmark", "test_session.name" : "session-name" }, "metrics" : { @@ -33,37 +34,6 @@ "content" : { "duration" : ${content_duration_2}, "error" : 0, - "meta" : { - "_dd.p.tid" : ${content_meta__dd_p_tid_2}, - "component" : "jmh", - "dummy_ci_tag" : "dummy_ci_tag_value", - "env" : "none", - "library_version" : ${content_meta_library_version}, - "span.kind" : "test_suite_end", - "test.framework" : "jmh", - "test.module" : "jmh-1.0", - "test.status" : "pass", - "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", - "test.type" : "test", - "test_session.name" : "session-name" - }, - "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2} - }, - "name" : "jmh.test_suite", - "resource" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", - "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", - "start" : ${content_start_2}, - "test_module_id" : ${content_test_module_id}, - "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id_2} - }, - "type" : "test_suite_end", - "version" : 1 -}, { - "content" : { - "duration" : ${content_duration_3}, - "error" : 0, "meta" : { "_dd.profiling.ctx" : "test", "_dd.tracer_host" : ${content_meta__dd_tracer_host}, @@ -79,15 +49,17 @@ "span.kind" : "test", "test.final_status" : "pass", "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "jmh-1.0", "test.name" : "measure", + "test.parameters" : "{\"metadata\":{\"test_name\":\"size=1\"}}", "test.status" : "pass", "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", - "test.type" : "test", + "test.type" : "benchmark", "test_session.name" : "session-name" }, "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, "_dd.profiling.enabled" : 0, "_dd.trace_span_attribute_schema" : 0, "benchmark.run.forks" : 0, @@ -101,7 +73,7 @@ "resource" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark.measure", "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", "span_id" : ${content_span_id}, - "start" : ${content_start_3}, + "start" : ${content_start_2}, "test_module_id" : ${content_test_module_id}, "test_session_id" : ${content_test_session_id}, "test_suite_id" : ${content_test_suite_id}, @@ -111,7 +83,7 @@ "version" : 2 }, { "content" : { - "duration" : ${content_duration_4}, + "duration" : ${content_duration_3}, "error" : 0, "meta" : { "_dd.profiling.ctx" : "test", @@ -128,15 +100,17 @@ "span.kind" : "test", "test.final_status" : "pass", "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "jmh-1.0", "test.name" : "measure", + "test.parameters" : "{\"metadata\":{\"test_name\":\"size=2\"}}", "test.status" : "pass", "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", - "test.type" : "test", + "test.type" : "benchmark", "test_session.name" : "session-name" }, "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, "_dd.profiling.enabled" : 0, "_dd.trace_span_attribute_schema" : 0, "benchmark.run.forks" : 0, @@ -150,20 +124,20 @@ "resource" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark.measure", "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", "span_id" : ${content_span_id_2}, - "start" : ${content_start_4}, + "start" : ${content_start_3}, "test_module_id" : ${content_test_module_id}, "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id_2}, + "test_suite_id" : ${content_test_suite_id}, "trace_id" : ${content_trace_id_2} }, "type" : "test", "version" : 2 }, { "content" : { - "duration" : ${content_duration_5}, + "duration" : ${content_duration_4}, "error" : 0, "meta" : { - "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, "_dd.profiling.ctx" : "test", "_dd.tracer_host" : ${content_meta__dd_tracer_host}, "component" : "jmh", @@ -175,12 +149,13 @@ "span.kind" : "test_session_end", "test.command" : "jmh-1.0", "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, "test.status" : "pass", - "test.type" : "test", + "test.type" : "benchmark", "test_session.name" : "session-name" }, "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, "_dd.profiling.enabled" : 0, "_dd.trace_span_attribute_schema" : 0, "process_id" : ${content_metrics_process_id} @@ -188,35 +163,36 @@ "name" : "jmh.test_session", "resource" : "jmh-1.0", "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", - "start" : ${content_start_5}, + "start" : ${content_start_4}, "test_session_id" : ${content_test_session_id} }, "type" : "test_session_end", "version" : 1 }, { "content" : { - "duration" : ${content_duration_6}, + "duration" : ${content_duration_5}, "error" : 0, "meta" : { - "_dd.p.tid" : ${content_meta__dd_p_tid_4}, + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, "component" : "jmh", "dummy_ci_tag" : "dummy_ci_tag_value", "env" : "none", "library_version" : ${content_meta_library_version}, "span.kind" : "test_module_end", "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "jmh-1.0", "test.status" : "pass", - "test.type" : "test", + "test.type" : "benchmark", "test_session.name" : "session-name" }, "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_6} + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5} }, "name" : "jmh.test_module", "resource" : "jmh-1.0", "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", - "start" : ${content_start_6}, + "start" : ${content_start_5}, "test_module_id" : ${content_test_module_id}, "test_session_id" : ${content_test_session_id} }, diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/events.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/events.ftl index 677f9ae486d..03df540ec33 100644 --- a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/events.ftl +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/events.ftl @@ -10,10 +10,11 @@ "library_version" : ${content_meta_library_version}, "span.kind" : "test_suite_end", "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "jmh-1.0", "test.status" : "pass", "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark", - "test.type" : "test", + "test.type" : "benchmark", "test_session.name" : "session-name" }, "metrics" : { @@ -48,11 +49,12 @@ "span.kind" : "test", "test.final_status" : "pass", "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "jmh-1.0", "test.name" : "measure", "test.status" : "pass", "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark", - "test.type" : "test", + "test.type" : "benchmark", "test_session.name" : "session-name" }, "metrics" : { @@ -95,8 +97,9 @@ "span.kind" : "test_session_end", "test.command" : "jmh-1.0", "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, "test.status" : "pass", - "test.type" : "test", + "test.type" : "benchmark", "test_session.name" : "session-name" }, "metrics" : { @@ -125,9 +128,10 @@ "library_version" : ${content_meta_library_version}, "span.kind" : "test_module_end", "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "jmh-1.0", "test.status" : "pass", - "test.type" : "test", + "test.type" : "benchmark", "test_session.name" : "session-name" }, "metrics" : { From a50e4f33247d7bb467271eac99217db3318baaf8 Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Wed, 3 Jun 2026 11:22:19 +0200 Subject: [PATCH 09/12] test: migrate JmhSmokeTest to a standalone project with fixtures - It's now compiled during test execution, closer to what JUnitConsole does - It also now verifies the spans generated through fixtures instead of manual validation --- dd-smoke-tests/jmh/build.gradle | 14 ++ .../java/datadog/smoketest/JmhSmokeTest.java | 181 ++++++++++++++--- .../test-jmh-benchmark/coverages.ftl | 1 + .../resources/test-jmh-benchmark/events.ftl | 188 ++++++++++++++++++ .../java/com/example}/SmokeTestBenchmark.java | 2 +- 5 files changed, 355 insertions(+), 31 deletions(-) create mode 100644 dd-smoke-tests/jmh/src/test/resources/test-jmh-benchmark/coverages.ftl create mode 100644 dd-smoke-tests/jmh/src/test/resources/test-jmh-benchmark/events.ftl rename dd-smoke-tests/jmh/src/{main/java/datadog/smoketest => test/resources/test-jmh-benchmark/src/main/java/com/example}/SmokeTestBenchmark.java (93%) diff --git a/dd-smoke-tests/jmh/build.gradle b/dd-smoke-tests/jmh/build.gradle index 2da20c85b07..e66d7588f54 100644 --- a/dd-smoke-tests/jmh/build.gradle +++ b/dd-smoke-tests/jmh/build.gradle @@ -2,7 +2,12 @@ apply from: "$rootDir/gradle/java.gradle" description = 'JMH CI Visibility Smoke Tests.' dependencies { + // jmh-core is on the test classpath so org.openjdk.jmh.Main (and its runtime deps) are available + // to the spawned benchmark process; its jar path is also passed to the test-time javac below. implementation group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.37' + // Not used to process this module's sources (it has none) — resolved only so its jar path can be + // passed to the test-time javac, which compiles the example benchmark project under + // src/test/resources and runs the JMH annotation processor to generate META-INF/BenchmarkList. annotationProcessor group: 'org.openjdk.jmh', name: 'jmh-generator-annprocess', version: '1.37' testImplementation project(':dd-smoke-tests:backend-mock') @@ -18,6 +23,15 @@ tasks.withType(Test).configureEach { return ["-Ddatadog.smoketest.jmh.core.jar.path=${jmhJar}"] } }) + jvmArgumentProviders.add(new CommandLineArgumentProvider() { + @Override + Iterable asArguments() { + def annprocJar = configurations.named("annotationProcessor") + .get() + .find { it.name.contains("jmh-generator-annprocess") } + return ["-Ddatadog.smoketest.jmh.annproc.jar.path=${annprocJar}"] + } + }) if (project.hasProperty("mavenRepositoryProxy")) { // propagate proxy URL to tests, to then propagate it to nested Gradle builds diff --git a/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java b/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java index d7e15cacf0e..84de70e56dd 100644 --- a/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java +++ b/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java @@ -8,24 +8,58 @@ import datadog.trace.api.config.GeneralConfig; import datadog.trace.civisibility.CiVisibilitySmokeTest; import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; class JmhSmokeTest extends CiVisibilitySmokeTest { private static final String TEST_SERVICE_NAME = "test-jmh-service"; private static final int PROCESS_TIMEOUT_SECS = 120; + private static final String PROJECT_NAME = "test-jmh-benchmark"; + private static final String BENCHMARK_CLASS = "com.example.SmokeTestBenchmark"; + private static final String JMH_CORE_JAR = System.getProperty("datadog.smoketest.jmh.core.jar.path"); + private static final String JMH_ANNPROC_JAR = + System.getProperty("datadog.smoketest.jmh.annproc.jar.path"); + + private static final Set CI_VISIBILITY_EVENT_TYPES = + new HashSet<>(Arrays.asList("test", "test_suite_end", "test_module_end", "test_session_end")); + + private static final List BENCHMARK_DYNAMIC_TAGS = + Arrays.asList( + "content.metrics.['benchmark.value']", + "content.metrics.['benchmark.error']", + "content.metrics.['benchmark.p50']", + "content.metrics.['benchmark.p90']", + "content.metrics.['benchmark.p95']", + "content.metrics.['benchmark.p99']", + "content.metrics.['benchmark.min']", + "content.metrics.['benchmark.max']", + "content.metrics.['benchmark.sample_count']"); + + @TempDir Path projectHome; static final MockBackend mockBackend = new MockBackend(); @@ -41,44 +75,87 @@ static void closeMockBackend() throws Exception { @Test void testBenchmarkSpansAreEmitted() throws Exception { + // only the 4 CI Visibility events are emitted. + assertBenchmarkSpansAreEmitted(0, 4); + } + + @Test + void testBenchmarkSpansAreEmittedWhenForked() throws Exception { + // when forking, the parent records one APM "command_execution" span for the forked java process, + // hence 5 total events (4 CI Visibility + 1 process span) + assertBenchmarkSpansAreEmitted(1, 5); + } + + private void assertBenchmarkSpansAreEmitted(int forks, int expectedEventCount) throws Exception { + givenBenchmarkProject(); + assertEquals(0, compileBenchmark(), "benchmark project should compile"); + Map agentArgs = new HashMap<>(); agentArgs.put(CiVisibilityConfig.CIVISIBILITY_BUILD_INSTRUMENTATION_ENABLED, "false"); agentArgs.put(GeneralConfig.AGENTLESS_LOG_SUBMISSION_URL, mockBackend.getIntakeUrl()); + agentArgs.put(CiVisibilityConfig.CIVISIBILITY_CODE_COVERAGE_ENABLED, "false"); - int exitCode = runBenchmark(agentArgs); + int exitCode = runBenchmark(agentArgs, forks); assertEquals(0, exitCode, "JMH process should exit cleanly"); - // 4 events: test_session_end, test_module_end, test_suite_end, test (benchmark method) - List> events = mockBackend.waitForEvents(4); - assertEquals(4, events.size()); + // filter out APM spans in forked-mode + List> events = + ciVisibilityEvents(mockBackend.waitForEvents(expectedEventCount)); + // validate benchmark.value is a real measure before verifyEventsAndCoverages rewrites the template Map testEvent = findEvent(events, "test"); assertNotNull(testEvent, "Expected a test span for the benchmark method"); - - @SuppressWarnings("unchecked") - Map meta = - (Map) ((Map) testEvent.get("content")).get("meta"); @SuppressWarnings("unchecked") Map metrics = (Map) ((Map) testEvent.get("content")).get("metrics"); + Object benchmarkValue = metrics.get("benchmark.value"); + assertNotNull(benchmarkValue, "benchmark.value should be present"); + assertTrue(((Number) benchmarkValue).doubleValue() > 0, "benchmark.value should be positive"); - assertEquals("jmh", meta.get("test.framework")); - assertEquals("measure", meta.get("test.name")); - assertEquals("datadog.smoketest.SmokeTestBenchmark", meta.get("test.suite")); - assertEquals("pass", meta.get("test.status")); - assertEquals("avgt", meta.get("benchmark.run.mode")); - assertEquals("ns/op", meta.get("benchmark.unit")); + // drop to reuse the same fixture between both test scenarios + metrics.remove("benchmark.run.forks"); - assertNotNull(metrics.get("benchmark.value"), "benchmark.value should be present"); - assertTrue( - ((Number) metrics.get("benchmark.value")).doubleValue() > 0, - "benchmark.value should be positive"); + verifyEventsAndCoverages( + PROJECT_NAME, + "jmh", + "headless", + events, + mockBackend.waitForCoverages(0), + BENCHMARK_DYNAMIC_TAGS); + } + + private void givenBenchmarkProject() throws Exception { + Path projectResources = + Paths.get(getClass().getClassLoader().getResource(PROJECT_NAME).toURI()); + copyFolder(projectResources, projectHome); + // empty .git so the tracer detects projectHome (not the build's repo) as the project root. + Files.createDirectories(projectHome.resolve(".git")); } - private int runBenchmark(Map additionalAgentArgs) throws Exception { + private int compileBenchmark() throws Exception { + // TODO: extract to common util for JUnitConsole and JMH assertTrue(new File(JMH_CORE_JAR).isFile(), "JMH core jar not found: " + JMH_CORE_JAR); + assertTrue( + new File(JMH_ANNPROC_JAR).isFile(), + "JMH annotation processor jar not found: " + JMH_ANNPROC_JAR); + + Path classesDir = projectHome.resolve("target/classes"); + Files.createDirectories(classesDir); + + List command = new ArrayList<>(); + command.add(javacPath()); + command.addAll(Arrays.asList("-cp", JMH_CORE_JAR + File.pathSeparator + JMH_ANNPROC_JAR)); + command.addAll(Arrays.asList("-d", classesDir.toString())); + command.addAll(findJavaFiles(projectHome.resolve("src/main/java"))); - String classpath = buildClasspath(); + return runProcess(new ProcessBuilder(command), "javac"); + } + + private int runBenchmark(Map additionalAgentArgs, int forks) throws Exception { + String classpath = + projectHome.resolve("target/classes") + + File.pathSeparator + + System.getProperty("java.class.path"); List command = new ArrayList<>(); command.add(javaPath()); @@ -86,8 +163,8 @@ private int runBenchmark(Map additionalAgentArgs) throws Excepti buildJvmArguments(mockBackend.getIntakeUrl(), TEST_SERVICE_NAME, additionalAgentArgs)); Collections.addAll(command, "-cp", classpath); command.add("org.openjdk.jmh.Main"); - command.add("datadog.smoketest.SmokeTestBenchmark.*"); - Collections.addAll(command, "-f", "0"); // run in-process (no forking) + command.add(BENCHMARK_CLASS + ".*"); + Collections.addAll(command, "-f", Integer.toString(forks)); // fork count Collections.addAll(command, "-wi", "1"); // 1 warmup iteration Collections.addAll(command, "-i", "1"); // 1 measurement iteration Collections.addAll(command, "-w", "1ms"); // warmup duration @@ -95,12 +172,16 @@ private int runBenchmark(Map additionalAgentArgs) throws Excepti Collections.addAll(command, "-jvmArgs", "-Djmh.ignoreLock=true"); ProcessBuilder processBuilder = new ProcessBuilder(command); - processBuilder.redirectErrorStream(true); processBuilder.environment().put("DD_API_KEY", "01234567890abcdef123456789ABCDEF"); + return runProcess(processBuilder, "jmh"); + } + + private int runProcess(ProcessBuilder processBuilder, String name) throws Exception { + processBuilder.directory(projectHome.toFile()); + processBuilder.redirectErrorStream(true); Process p = processBuilder.start(); - // consume output to avoid blocking - final java.io.InputStream stdout = p.getInputStream(); + final InputStream stdout = p.getInputStream(); Thread outputConsumer = new Thread() { @Override @@ -120,19 +201,59 @@ public void run() { if (!p.waitFor(PROCESS_TIMEOUT_SECS, TimeUnit.SECONDS)) { p.destroyForcibly(); - throw new TimeoutException("JMH process timed out after " + PROCESS_TIMEOUT_SECS + "s"); + throw new TimeoutException(name + " process timed out after " + PROCESS_TIMEOUT_SECS + "s"); } return p.exitValue(); } - private static String buildClasspath() { - // Use the current test process classpath — it includes jmh-core, SmokeTestBenchmark, and its - // META-INF/BenchmarkList (generated by the annotation processor at compile time) - return System.getProperty("java.class.path"); + private static List findJavaFiles(Path directory) throws IOException { + List javaFiles = new ArrayList<>(); + Files.walkFileTree( + directory, + new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + if (file.toString().endsWith(".java")) { + javaFiles.add(file.toString()); + } + return FileVisitResult.CONTINUE; + } + }); + return javaFiles; + } + + private static void copyFolder(Path src, Path dest) throws IOException { + Files.walkFileTree( + src, + new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) + throws IOException { + Files.createDirectories(dest.resolve(src.relativize(dir))); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException { + Files.copy(file, dest.resolve(src.relativize(file))); + return FileVisitResult.CONTINUE; + } + }); } @SuppressWarnings("unchecked") private static Map findEvent(List> events, String type) { return events.stream().filter(e -> type.equals(e.get("type"))).findFirst().orElse(null); } + + private static List> ciVisibilityEvents(List> events) { + List> filtered = new ArrayList<>(); + for (Map event : events) { + if (CI_VISIBILITY_EVENT_TYPES.contains(event.get("type"))) { + filtered.add(event); + } + } + return filtered; + } } diff --git a/dd-smoke-tests/jmh/src/test/resources/test-jmh-benchmark/coverages.ftl b/dd-smoke-tests/jmh/src/test/resources/test-jmh-benchmark/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-smoke-tests/jmh/src/test/resources/test-jmh-benchmark/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-smoke-tests/jmh/src/test/resources/test-jmh-benchmark/events.ftl b/dd-smoke-tests/jmh/src/test/resources/test-jmh-benchmark/events.ftl new file mode 100644 index 00000000000..b883a10fb86 --- /dev/null +++ b/dd-smoke-tests/jmh/src/test/resources/test-jmh-benchmark/events.ftl @@ -0,0 +1,188 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "_dd.test.is_user_provided_service" : "true", + "ci.workspace_path" : ${content_meta_ci_workspace_path}, + "component" : "jmh", + "env" : "integration-test", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "os.architecture" : ${content_meta_os_architecture}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "runtime-id" : ${content_meta_runtime_id}, + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "runtime.version" : ${content_meta_runtime_version}, + "span.kind" : "test_suite_end", + "test.framework" : "jmh", + "test.framework_version" : "1.37", + "test.itr.tests_skipping.enabled" : "true", + "test.module" : "test-jmh-service", + "test.status" : "pass", + "test.suite" : "com.example.SmokeTestBenchmark", + "test.type" : "benchmark", + "test_session.name" : "test-jmh-service" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "jmh.test_suite", + "resource" : "com.example.SmokeTestBenchmark", + "service" : "test-jmh-service", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.test.is_user_provided_service" : "true", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "benchmark.run.mode" : "avgt", + "benchmark.run.time_unit" : "NANOSECONDS", + "benchmark.unit" : "ns/op", + "ci.workspace_path" : ${content_meta_ci_workspace_path}, + "component" : "jmh", + "env" : "integration-test", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "os.architecture" : ${content_meta_os_architecture}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "runtime-id" : ${content_meta_runtime_id}, + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "runtime.version" : ${content_meta_runtime_version}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "jmh", + "test.framework_version" : "1.37", + "test.itr.tests_skipping.enabled" : "true", + "test.module" : "test-jmh-service", + "test.name" : "measure", + "test.status" : "pass", + "test.suite" : "com.example.SmokeTestBenchmark", + "test.type" : "benchmark", + "test_session.name" : "test-jmh-service" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "benchmark.run.iterations" : 1, + "benchmark.run.threads" : 1, + "benchmark.run.warmup_iterations" : 1, + "benchmark.value" : ${content_metrics_benchmark_value}, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test", + "parent_id" : ${content_parent_id}, + "resource" : "com.example.SmokeTestBenchmark.measure", + "service" : "test-jmh-service", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "_dd.test.is_user_provided_service" : "true", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "ci.workspace_path" : ${content_meta_ci_workspace_path}, + "component" : "jmh", + "env" : "integration-test", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "os.architecture" : ${content_meta_os_architecture}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "runtime-id" : ${content_meta_runtime_id}, + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "runtime.version" : ${content_meta_runtime_version}, + "span.kind" : "test_session_end", + "test.command" : "test-jmh-service", + "test.framework" : "jmh", + "test.framework_version" : "1.37", + "test.itr.tests_skipping.enabled" : "true", + "test.itr.tests_skipping.type" : "test", + "test.status" : "pass", + "test.type" : "benchmark", + "test_session.name" : "test-jmh-service" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.itr.tests_skipping.count" : 0 + }, + "name" : "jmh.test_session", + "resource" : "test-jmh-service", + "service" : "test-jmh-service", + "start" : ${content_start_3}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_4}, + "_dd.test.is_user_provided_service" : "true", + "ci.workspace_path" : ${content_meta_ci_workspace_path}, + "component" : "jmh", + "env" : "integration-test", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "os.architecture" : ${content_meta_os_architecture}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "runtime-id" : ${content_meta_runtime_id}, + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "runtime.version" : ${content_meta_runtime_version}, + "span.kind" : "test_module_end", + "test.framework" : "jmh", + "test.framework_version" : "1.37", + "test.itr.tests_skipping.enabled" : "true", + "test.itr.tests_skipping.type" : "test", + "test.module" : "test-jmh-service", + "test.status" : "pass", + "test.type" : "benchmark", + "test_session.name" : "test-jmh-service" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "test.itr.tests_skipping.count" : 0 + }, + "name" : "jmh.test_module", + "resource" : "test-jmh-service", + "service" : "test-jmh-service", + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-smoke-tests/jmh/src/main/java/datadog/smoketest/SmokeTestBenchmark.java b/dd-smoke-tests/jmh/src/test/resources/test-jmh-benchmark/src/main/java/com/example/SmokeTestBenchmark.java similarity index 93% rename from dd-smoke-tests/jmh/src/main/java/datadog/smoketest/SmokeTestBenchmark.java rename to dd-smoke-tests/jmh/src/test/resources/test-jmh-benchmark/src/main/java/com/example/SmokeTestBenchmark.java index d9a22def1ff..3fc686d6e74 100644 --- a/dd-smoke-tests/jmh/src/main/java/datadog/smoketest/SmokeTestBenchmark.java +++ b/dd-smoke-tests/jmh/src/test/resources/test-jmh-benchmark/src/main/java/com/example/SmokeTestBenchmark.java @@ -1,4 +1,4 @@ -package datadog.smoketest; +package com.example; import java.util.concurrent.TimeUnit; import org.openjdk.jmh.annotations.Benchmark; From 7ccb20f0eae500a7f2cdc9c0b33a78585c0819b4 Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Wed, 3 Jun 2026 12:01:17 +0200 Subject: [PATCH 10/12] test: add distribution test to instrumentation --- .../CiVisibilityInstrumentationTest.groovy | 10 +- .../civisibility/CiVisibilityTestUtils.java | 44 +++++ .../test/groovy/JmhInstrumentationTest.groovy | 8 +- .../jmh/benchmarks/DistributionBenchmark.java | 28 ++++ .../test-benchmark-distribution/coverages.ftl | 1 + .../test-benchmark-distribution/events.ftl | 157 ++++++++++++++++++ .../test-benchmark-multi-class/events.ftl | 3 + .../test-benchmark-parameterized/events.ftl | 2 + .../test-benchmark-simple/events.ftl | 1 + 9 files changed, 244 insertions(+), 10 deletions(-) create mode 100644 dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/DistributionBenchmark.java create mode 100644 dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-distribution/coverages.ftl create mode 100644 dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-distribution/events.ftl diff --git a/dd-java-agent/agent-ci-visibility/civisibility-instrumentation-test-fixtures/src/main/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy b/dd-java-agent/agent-ci-visibility/civisibility-instrumentation-test-fixtures/src/main/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy index e71a8a5d8c9..930f53da4ac 100644 --- a/dd-java-agent/agent-ci-visibility/civisibility-instrumentation-test-fixtures/src/main/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy +++ b/dd-java-agent/agent-ci-visibility/civisibility-instrumentation-test-fixtures/src/main/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy @@ -353,7 +353,7 @@ abstract class CiVisibilityInstrumentationTest extends InstrumentationSpecificat settings.impactedTestsDetectionEnabled = impactedTestsDetectionEnabled } - def assertSpansData(String testcaseName, Map replacements = [:], List ignoredTags = []) { + def assertSpansData(String testcaseName, Map replacements = [:], List ignoredTags = [], List dynamicTags = []) { Predicate sessionSpan = span -> span.spanType == "test_session_end" spanFilter.waitForSpan(sessionSpan, TimeUnit.SECONDS.toMillis(20)) @@ -369,13 +369,13 @@ abstract class CiVisibilityInstrumentationTest extends InstrumentationSpecificat def additionalIgnoredTags = CiVisibilityTestUtils.IGNORED_TAGS + ignoredTags if (System.getenv().get("GENERATE_TEST_FIXTURES") != null) { - return generateTestFixtures(testcaseName, events, coverages, additionalReplacements, additionalIgnoredTags) + return generateTestFixtures(testcaseName, events, coverages, additionalReplacements, additionalIgnoredTags, dynamicTags) } - return CiVisibilityTestUtils.assertData(testcaseName, events, coverages, additionalReplacements, additionalIgnoredTags) + return CiVisibilityTestUtils.assertData(testcaseName, events, coverages, additionalReplacements, additionalIgnoredTags, [], dynamicTags) } - def generateTestFixtures(String testcaseName, List events, List coverages, Map additionalReplacements, List additionalIgnoredTags) { + def generateTestFixtures(String testcaseName, List events, List coverages, Map additionalReplacements, List additionalIgnoredTags, List dynamicTags = []) { def clazz = this.getClass() def resourceName = "/" + clazz.name.replace('.', '/') + ".class" def classfilePath = clazz.getResource(resourceName).toURI().schemeSpecificPart @@ -387,7 +387,7 @@ abstract class CiVisibilityInstrumentationTest extends InstrumentationSpecificat submoduleName = "test" } def baseTemplatesPath = modulePath + "/src/" + submoduleName + "/resources/" + testcaseName - CiVisibilityTestUtils.generateTemplates(baseTemplatesPath, events, coverages, additionalReplacements.keySet(), additionalIgnoredTags) + CiVisibilityTestUtils.generateTemplates(baseTemplatesPath, events, coverages, additionalReplacements.keySet(), additionalIgnoredTags, dynamicTags) return [:] } diff --git a/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/java/datadog/trace/civisibility/CiVisibilityTestUtils.java b/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/java/datadog/trace/civisibility/CiVisibilityTestUtils.java index fb9ff951a88..b71de7356e0 100644 --- a/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/java/datadog/trace/civisibility/CiVisibilityTestUtils.java +++ b/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/java/datadog/trace/civisibility/CiVisibilityTestUtils.java @@ -120,6 +120,22 @@ public static void generateTemplates( List> coverages, Collection additionalDynamicPaths, List ignoredTags) { + generateTemplates( + baseTemplatesPath, + events, + coverages, + additionalDynamicPaths, + ignoredTags, + Collections.emptyList()); + } + + public static void generateTemplates( + String baseTemplatesPath, + List> events, + List> coverages, + Collection additionalDynamicPaths, + List ignoredTags, + Collection additionalNonUniqueDynamicPaths) { List> mutableEvents = new ArrayList<>(events); if (!ignoredTags.isEmpty()) { mutableEvents = removeTags(mutableEvents, ignoredTags); @@ -128,6 +144,7 @@ public static void generateTemplates( TemplateGenerator templateGenerator = new TemplateGenerator(new LabelGenerator()); List compiledAdditionalReplacements = compile(additionalDynamicPaths); + compiledAdditionalReplacements.addAll(compileNonUnique(additionalNonUniqueDynamicPaths)); try { Files.createDirectories(Paths.get(baseTemplatesPath)); @@ -219,6 +236,24 @@ public static Map assertData( Map additionalReplacements, List ignoredTags, List additionalDynamicPaths) { + return assertData( + baseTemplatesPath, + events, + coverages, + additionalReplacements, + ignoredTags, + additionalDynamicPaths, + Collections.emptyList()); + } + + public static Map assertData( + String baseTemplatesPath, + List> events, + List> coverages, + Map additionalReplacements, + List ignoredTags, + List additionalDynamicPaths, + List additionalNonUniqueDynamicPaths) { List> mutableEvents = new ArrayList<>(events); mutableEvents.sort(EVENT_RESOURCE_COMPARATOR); @@ -227,6 +262,7 @@ public static Map assertData( List eventPaths = new ArrayList<>(EVENT_DYNAMIC_PATHS); eventPaths.addAll(compile(additionalDynamicPaths)); + eventPaths.addAll(compileNonUnique(additionalNonUniqueDynamicPaths)); templateGenerator.generateReplacementMap(mutableEvents, eventPaths); Map replacementMap = templateGenerator.generateReplacementMap(coverages, COVERAGE_DYNAMIC_PATHS); @@ -512,6 +548,14 @@ private static List compile(Iterable rawPaths) { return compiledPaths; } + private static List compileNonUnique(Iterable rawPaths) { + List compiledPaths = new ArrayList<>(); + for (String rawPath : rawPaths) { + compiledPaths.add(path(rawPath, false)); + } + return compiledPaths; + } + private static DynamicPath path(String rawPath) { return path(rawPath, true); } diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/groovy/JmhInstrumentationTest.groovy b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/groovy/JmhInstrumentationTest.groovy index a360facd3d5..72ca436715a 100644 --- a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/groovy/JmhInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/groovy/JmhInstrumentationTest.groovy @@ -3,13 +3,13 @@ import datadog.trace.civisibility.CiVisibilityInstrumentationTest import datadog.trace.instrumentation.jmh.JmhUtils import datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark import datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark +import datadog.trace.instrumentation.jmh.benchmarks.DistributionBenchmark import org.openjdk.jmh.runner.Runner import org.openjdk.jmh.runner.options.OptionsBuilder @DisableTestTrace(reason = "avoid self-tracing") class JmhInstrumentationTest extends CiVisibilityInstrumentationTest { - // Benchmark numeric metrics vary each run — they are verified structurally in the smoke test static final List BENCHMARK_METRIC_TAGS = [ "content.metrics.['benchmark.value']", "content.metrics.['benchmark.error']", @@ -24,16 +24,14 @@ class JmhInstrumentationTest extends CiVisibilityInstrumentationTest { def "test #testcaseName"() { runBenchmarks(*benchmarkClasses) - assertSpansData(testcaseName, [:], BENCHMARK_METRIC_TAGS) + assertSpansData(testcaseName, [:], [], BENCHMARK_METRIC_TAGS) where: testcaseName | benchmarkClasses "test-benchmark-simple" | [SimpleBenchmark] "test-benchmark-parameterized" | [ParameterizedBenchmark] - // Multiple classes in one run keep several suites open at once (all finished in onRunEnd); - // guards against suite spans being activated on the active-span stack, which would make - // finishing them out of order throw IllegalStateException. "test-benchmark-multi-class" | [SimpleBenchmark, ParameterizedBenchmark] + "test-benchmark-distribution" | [DistributionBenchmark] } private void runBenchmarks(Class... benchmarkClasses) { diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/DistributionBenchmark.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/DistributionBenchmark.java new file mode 100644 index 00000000000..5783f582e11 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/DistributionBenchmark.java @@ -0,0 +1,28 @@ +package datadog.trace.instrumentation.jmh.benchmarks; + +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Warmup; + +/** + * Runs two measurement iterations so the primary result's statistics hold more than one data point + * ({@code getN() > 1}) — the gate for emitting the distribution metrics (p50/p90/p95/p99/min/max/ + * sample_count). The single-iteration benchmarks cover the complementary path where they are + * omitted. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 2, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Fork(0) +public class DistributionBenchmark { + @Benchmark + public int measure() { + return 42; + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-distribution/coverages.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-distribution/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-distribution/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-distribution/events.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-distribution/events.ftl new file mode 100644 index 00000000000..040fb1e2b6a --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-distribution/events.ftl @@ -0,0 +1,157 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.DistributionBenchmark", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "jmh.test_suite", + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.DistributionBenchmark", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "benchmark.run.mode" : "avgt", + "benchmark.run.time_unit" : "NANOSECONDS", + "benchmark.unit" : "ns/op", + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.name" : "measure", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.DistributionBenchmark", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "benchmark.max" : ${content_metrics_benchmark_max}, + "benchmark.min" : ${content_metrics_benchmark_min}, + "benchmark.p50" : ${content_metrics_benchmark_p50}, + "benchmark.p90" : ${content_metrics_benchmark_p90}, + "benchmark.p95" : ${content_metrics_benchmark_p95}, + "benchmark.p99" : ${content_metrics_benchmark_p99}, + "benchmark.run.forks" : 0, + "benchmark.run.iterations" : 2, + "benchmark.run.threads" : 1, + "benchmark.run.warmup_iterations" : 1, + "benchmark.sample_count" : ${content_metrics_benchmark_sample_count}, + "benchmark.value" : ${content_metrics_benchmark_value}, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test", + "parent_id" : ${content_parent_id}, + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.DistributionBenchmark.measure", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "jmh-1.0", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "pass", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test_session", + "resource" : "jmh-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4} + }, + "name" : "jmh.test_module", + "resource" : "jmh-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-multi-class/events.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-multi-class/events.ftl index a989c19f17c..055ee09b26d 100644 --- a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-multi-class/events.ftl +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-multi-class/events.ftl @@ -66,6 +66,7 @@ "benchmark.run.iterations" : 1, "benchmark.run.threads" : 1, "benchmark.run.warmup_iterations" : 1, + "benchmark.value" : ${content_metrics_benchmark_value}, "process_id" : ${content_metrics_process_id} }, "name" : "jmh.test", @@ -117,6 +118,7 @@ "benchmark.run.iterations" : 1, "benchmark.run.threads" : 1, "benchmark.run.warmup_iterations" : 1, + "benchmark.value" : ${content_metrics_benchmark_value_2}, "process_id" : ${content_metrics_process_id} }, "name" : "jmh.test", @@ -199,6 +201,7 @@ "benchmark.run.iterations" : 1, "benchmark.run.threads" : 1, "benchmark.run.warmup_iterations" : 1, + "benchmark.value" : ${content_metrics_benchmark_value_3}, "process_id" : ${content_metrics_process_id} }, "name" : "jmh.test", diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/events.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/events.ftl index b7603a72bdc..1446a701e79 100644 --- a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/events.ftl +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/events.ftl @@ -66,6 +66,7 @@ "benchmark.run.iterations" : 1, "benchmark.run.threads" : 1, "benchmark.run.warmup_iterations" : 1, + "benchmark.value" : ${content_metrics_benchmark_value}, "process_id" : ${content_metrics_process_id} }, "name" : "jmh.test", @@ -117,6 +118,7 @@ "benchmark.run.iterations" : 1, "benchmark.run.threads" : 1, "benchmark.run.warmup_iterations" : 1, + "benchmark.value" : ${content_metrics_benchmark_value_2}, "process_id" : ${content_metrics_process_id} }, "name" : "jmh.test", diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/events.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/events.ftl index 03df540ec33..2864f3d89be 100644 --- a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/events.ftl +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/events.ftl @@ -65,6 +65,7 @@ "benchmark.run.iterations" : 1, "benchmark.run.threads" : 1, "benchmark.run.warmup_iterations" : 1, + "benchmark.value" : ${content_metrics_benchmark_value}, "process_id" : ${content_metrics_process_id} }, "name" : "jmh.test", From 30076b987438a4cff45ee1abf5eb11b49b855205 Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Wed, 3 Jun 2026 12:06:11 +0200 Subject: [PATCH 11/12] chore: spotless --- .../jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java b/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java index 84de70e56dd..72faec3bfb2 100644 --- a/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java +++ b/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java @@ -81,8 +81,8 @@ void testBenchmarkSpansAreEmitted() throws Exception { @Test void testBenchmarkSpansAreEmittedWhenForked() throws Exception { - // when forking, the parent records one APM "command_execution" span for the forked java process, - // hence 5 total events (4 CI Visibility + 1 process span) + // when forking, the parent records one APM "command_execution" span for the forked java + // process, hence 5 total events (4 CI Visibility + 1 process span) assertBenchmarkSpansAreEmitted(1, 5); } @@ -102,7 +102,7 @@ private void assertBenchmarkSpansAreEmitted(int forks, int expectedEventCount) t List> events = ciVisibilityEvents(mockBackend.waitForEvents(expectedEventCount)); - // validate benchmark.value is a real measure before verifyEventsAndCoverages rewrites the template + // validate benchmark.value is a real measure before verifyEventsAndCoverages rewrites it Map testEvent = findEvent(events, "test"); assertNotNull(testEvent, "Expected a test span for the benchmark method"); @SuppressWarnings("unchecked") From 2d82c9e35b4997ea798922d94b6daef40c3b1da8 Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Wed, 3 Jun 2026 13:06:03 +0200 Subject: [PATCH 12/12] fix: add annotation processor to smoke test compilation --- .../jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java b/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java index 72faec3bfb2..27393e3de82 100644 --- a/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java +++ b/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java @@ -145,6 +145,9 @@ private int compileBenchmark() throws Exception { List command = new ArrayList<>(); command.add(javacPath()); command.addAll(Arrays.asList("-cp", JMH_CORE_JAR + File.pathSeparator + JMH_ANNPROC_JAR)); + command.addAll( + Arrays.asList("-processorpath", JMH_ANNPROC_JAR + File.pathSeparator + JMH_CORE_JAR)); + command.addAll(Arrays.asList("-processor", "org.openjdk.jmh.generators.BenchmarkProcessor")); command.addAll(Arrays.asList("-d", classesDir.toString())); command.addAll(findJavaFiles(projectHome.resolve("src/main/java")));