Add CI Visibility instrumentation for JMH benchmarks

dd-java-agent/agent-ci-visibility/civisibility-instrumentation-test-fixtures/src/main/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy

@@ -353,7 +353,7 @@ abstract class CiVisibilityInstrumentationTest extends InstrumentationSpecificat
     settings.impactedTestsDetectionEnabled = impactedTestsDetectionEnabled
   }
 
-  def assertSpansData(String testcaseName, Map<String, String> replacements = [:], List<String> ignoredTags = []) {
+  def assertSpansData(String testcaseName, Map<String, String> replacements = [:], List<String> ignoredTags = [], List<String> dynamicTags = []) {
     Predicate<DDSpan> sessionSpan = span -> span.spanType == "test_session_end"
     spanFilter.waitForSpan(sessionSpan, TimeUnit.SECONDS.toMillis(20))
 
@@ -369,13 +369,13 @@ abstract class CiVisibilityInstrumentationTest extends InstrumentationSpecificat
     def additionalIgnoredTags = CiVisibilityTestUtils.IGNORED_TAGS + ignoredTags
 
     if (System.getenv().get("GENERATE_TEST_FIXTURES") != null) {
-      return generateTestFixtures(testcaseName, events, coverages, additionalReplacements, additionalIgnoredTags)
+      return generateTestFixtures(testcaseName, events, coverages, additionalReplacements, additionalIgnoredTags, dynamicTags)
     }
 
-    return CiVisibilityTestUtils.assertData(testcaseName, events, coverages, additionalReplacements, additionalIgnoredTags)
+    return CiVisibilityTestUtils.assertData(testcaseName, events, coverages, additionalReplacements, additionalIgnoredTags, [], dynamicTags)
   }
 
-  def generateTestFixtures(String testcaseName, List<Map> events, List<Map> coverages, Map<String, String> additionalReplacements, List<String> additionalIgnoredTags) {
+  def generateTestFixtures(String testcaseName, List<Map> events, List<Map> coverages, Map<String, String> additionalReplacements, List<String> additionalIgnoredTags, List<String> dynamicTags = []) {
     def clazz = this.getClass()
     def resourceName = "/" + clazz.name.replace('.', '/') + ".class"
     def classfilePath = clazz.getResource(resourceName).toURI().schemeSpecificPart
@@ -387,7 +387,7 @@ abstract class CiVisibilityInstrumentationTest extends InstrumentationSpecificat
       submoduleName = "test"
     }
     def baseTemplatesPath = modulePath + "/src/" + submoduleName + "/resources/" + testcaseName
-    CiVisibilityTestUtils.generateTemplates(baseTemplatesPath, events, coverages, additionalReplacements.keySet(), additionalIgnoredTags)
+    CiVisibilityTestUtils.generateTemplates(baseTemplatesPath, events, coverages, additionalReplacements.keySet(), additionalIgnoredTags, dynamicTags)
     return [:]
   }
 

dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/java/datadog/trace/civisibility/CiVisibilityTestUtils.java

@@ -120,6 +120,22 @@ public static void generateTemplates(
       List<? extends Map<?, ?>> coverages,
       Collection<String> additionalDynamicPaths,
       List<String> ignoredTags) {
+    generateTemplates(
+        baseTemplatesPath,
+        events,
+        coverages,
+        additionalDynamicPaths,
+        ignoredTags,
+        Collections.emptyList());
+  }
+
+  public static void generateTemplates(
+      String baseTemplatesPath,
+      List<? extends Map<?, ?>> events,
+      List<? extends Map<?, ?>> coverages,
+      Collection<String> additionalDynamicPaths,
+      List<String> ignoredTags,
+      Collection<String> additionalNonUniqueDynamicPaths) {
     List<Map<?, ?>> mutableEvents = new ArrayList<>(events);
     if (!ignoredTags.isEmpty()) {
       mutableEvents = removeTags(mutableEvents, ignoredTags);
@@ -128,6 +144,7 @@ public static void generateTemplates(
 
     TemplateGenerator templateGenerator = new TemplateGenerator(new LabelGenerator());
     List<DynamicPath> compiledAdditionalReplacements = compile(additionalDynamicPaths);
+    compiledAdditionalReplacements.addAll(compileNonUnique(additionalNonUniqueDynamicPaths));
 
     try {
       Files.createDirectories(Paths.get(baseTemplatesPath));
@@ -219,6 +236,24 @@ public static Map<String, String> assertData(
       Map<String, ?> additionalReplacements,
       List<String> ignoredTags,
       List<String> additionalDynamicPaths) {
+    return assertData(
+        baseTemplatesPath,
+        events,
+        coverages,
+        additionalReplacements,
+        ignoredTags,
+        additionalDynamicPaths,
+        Collections.emptyList());
+  }
+
+  public static Map<String, String> assertData(
+      String baseTemplatesPath,
+      List<? extends Map<?, ?>> events,
+      List<? extends Map<?, ?>> coverages,
+      Map<String, ?> additionalReplacements,
+      List<String> ignoredTags,
+      List<String> additionalDynamicPaths,
+      List<String> additionalNonUniqueDynamicPaths) {
     List<Map<?, ?>> mutableEvents = new ArrayList<>(events);
     mutableEvents.sort(EVENT_RESOURCE_COMPARATOR);
 
@@ -227,6 +262,7 @@ public static Map<String, String> assertData(
 
     List<DynamicPath> eventPaths = new ArrayList<>(EVENT_DYNAMIC_PATHS);
     eventPaths.addAll(compile(additionalDynamicPaths));
+    eventPaths.addAll(compileNonUnique(additionalNonUniqueDynamicPaths));
     templateGenerator.generateReplacementMap(mutableEvents, eventPaths);
     Map<String, String> replacementMap =
         templateGenerator.generateReplacementMap(coverages, COVERAGE_DYNAMIC_PATHS);
@@ -512,6 +548,14 @@ private static List<DynamicPath> compile(Iterable<String> rawPaths) {
     return compiledPaths;
   }
 
+  private static List<DynamicPath> compileNonUnique(Iterable<String> rawPaths) {
+    List<DynamicPath> compiledPaths = new ArrayList<>();
+    for (String rawPath : rawPaths) {
+      compiledPaths.add(path(rawPath, false));
+    }
+    return compiledPaths;
+  }
+
   private static DynamicPath path(String rawPath) {
     return path(rawPath, true);
   }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecorator.java

@@ -4,6 +4,7 @@
 
 public interface TestDecorator {
   String TEST_TYPE = "test";
+  String TEST_TYPE_BENCHMARK = "benchmark";
 
   AgentSpan afterStart(final AgentSpan span);
 

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecoratorImpl.java

@@ -6,12 +6,17 @@
 import datadog.trace.bootstrap.instrumentation.api.Tags;
 import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString;
 import datadog.trace.util.Strings;
+import java.util.Collections;
 import java.util.Map;
+import java.util.Set;
 
 public class TestDecoratorImpl implements TestDecorator {
 
   private static final UTF8BytesString CIAPP_TEST_ORIGIN = UTF8BytesString.create("ciapp-test");
 
+  // components whose tests represent benchmarks (test.type = "benchmark")
+  private static final Set<String> BENCHMARK_COMPONENTS = Collections.singleton("jmh");
+
   private final String component;
   private final String sessionName;
   private final Map<String, String> ciTags;
@@ -32,7 +37,7 @@ public TestDecoratorImpl(
   }
 
   protected String testType() {
-    return TEST_TYPE;
+    return BENCHMARK_COMPONENTS.contains(component) ? TEST_TYPE_BENCHMARK : TEST_TYPE;
   }
 
   protected UTF8BytesString origin() {

dd-java-agent/instrumentation/jmh/jmh-1.0/build.gradle

@@ -0,0 +1,23 @@
+plugins {
+  id 'dd-trace-java.instrumentation.testing-framework-tests'
+}
+
+apply from: "$rootDir/gradle/java.gradle"
+
+muzzle {
+  pass {
+    group = 'org.openjdk.jmh'
+    module = 'jmh-core'
+    versions = '[1.0,)'
+  }
+}
+
+dependencies {
+  compileOnly group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.0'
+
+  testImplementation group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.37'
+  testImplementation project(':dd-java-agent:agent-ci-visibility:civisibility-instrumentation-test-fixtures')
+
+  // JMH annotation processor generates META-INF/BenchmarkList from @Benchmark annotations
+  testAnnotationProcessor group: 'org.openjdk.jmh', name: 'jmh-generator-annprocess', version: '1.37'
+}

dd-java-agent/instrumentation/jmh/jmh-1.0/gradle.lockfile

@@ -0,0 +1,144 @@
+# This is a Gradle generated file for dependency locking.
+# Manual edits can break the build and are not advised.
+# This file is expected to be part of source control.
+cafe.cryptography:curve25519-elisabeth:0.1.0=testRuntimeClasspath
+cafe.cryptography:ed25519-elisabeth:0.1.0=testRuntimeClasspath
+ch.qos.logback:logback-classic:1.2.13=testCompileClasspath,testRuntimeClasspath
+ch.qos.logback:logback-core:1.2.13=testCompileClasspath,testRuntimeClasspath
+com.blogspot.mydailyjava:weak-lock-free:0.17=buildTimeInstrumentationPlugin,compileClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.datadoghq.okhttp3:okhttp:3.12.15=testCompileClasspath,testRuntimeClasspath
+com.datadoghq.okio:okio:1.17.6=testCompileClasspath,testRuntimeClasspath
+com.datadoghq:dd-instrument-java:0.0.3=buildTimeInstrumentationPlugin,compileClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.datadoghq:dd-javac-plugin-client:0.2.2=buildTimeInstrumentationPlugin,compileClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.datadoghq:java-dogstatsd-client:4.4.5=testRuntimeClasspath
+com.datadoghq:sketches-java:0.8.3=testRuntimeClasspath
+com.fasterxml.jackson.core:jackson-annotations:2.20=testCompileClasspath,testRuntimeClasspath
+com.fasterxml.jackson.core:jackson-core:2.20.0=testCompileClasspath,testRuntimeClasspath
+com.fasterxml.jackson.core:jackson-databind:2.20.0=testCompileClasspath,testRuntimeClasspath
+com.github.javaparser:javaparser-core:3.25.6=codenarc
+com.github.jnr:jffi:1.3.14=testRuntimeClasspath
+com.github.jnr:jnr-a64asm:1.0.0=testRuntimeClasspath
+com.github.jnr:jnr-constants:0.10.4=testRuntimeClasspath
+com.github.jnr:jnr-enxio:0.32.19=testRuntimeClasspath
+com.github.jnr:jnr-ffi:2.2.18=testRuntimeClasspath
+com.github.jnr:jnr-posix:3.1.21=testRuntimeClasspath
+com.github.jnr:jnr-unixsocket:0.38.24=testRuntimeClasspath
+com.github.jnr:jnr-x86asm:1.0.2=testRuntimeClasspath
+com.github.spotbugs:spotbugs-annotations:4.9.8=compileClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath
+com.github.spotbugs:spotbugs:4.9.8=spotbugs
+com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs
+com.google.auto.service:auto-service-annotations:1.1.1=annotationProcessor,compileClasspath,testAnnotationProcessor,testCompileClasspath
+com.google.auto.service:auto-service:1.1.1=annotationProcessor,testAnnotationProcessor
+com.google.auto:auto-common:1.2.1=annotationProcessor,testAnnotationProcessor
+com.google.code.findbugs:jsr305:3.0.2=annotationProcessor,compileClasspath,spotbugs,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath
+com.google.code.gson:gson:2.13.2=spotbugs
+com.google.errorprone:error_prone_annotations:2.18.0=annotationProcessor,testAnnotationProcessor
+com.google.errorprone:error_prone_annotations:2.41.0=spotbugs
+com.google.guava:failureaccess:1.0.1=annotationProcessor,testAnnotationProcessor
+com.google.guava:guava:20.0=testCompileClasspath,testRuntimeClasspath
+com.google.guava:guava:32.0.1-jre=annotationProcessor,testAnnotationProcessor
+com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,testAnnotationProcessor
+com.google.j2objc:j2objc-annotations:2.8=annotationProcessor,testAnnotationProcessor
+com.google.re2j:re2j:1.7=testRuntimeClasspath
+com.jayway.jsonpath:json-path:2.8.0=testCompileClasspath,testRuntimeClasspath
+com.squareup.moshi:moshi:1.11.0=testCompileClasspath,testRuntimeClasspath
+com.squareup.okhttp3:logging-interceptor:3.12.12=testCompileClasspath,testRuntimeClasspath
+com.squareup.okhttp3:okhttp:3.12.12=testCompileClasspath,testRuntimeClasspath
+com.squareup.okio:okio:1.17.5=testCompileClasspath,testRuntimeClasspath
+com.thoughtworks.qdox:qdox:1.12.1=codenarc
+com.vaadin.external.google:android-json:0.0.20131108.vaadin1=testCompileClasspath,testRuntimeClasspath
+commons-fileupload:commons-fileupload:1.5=testCompileClasspath,testRuntimeClasspath
+commons-io:commons-io:2.11.0=testCompileClasspath,testRuntimeClasspath
+commons-io:commons-io:2.20.0=spotbugs
+de.thetaphi:forbiddenapis:3.10=compileClasspath,testCompileClasspath,testRuntimeClasspath
+io.leangen.geantyref:geantyref:1.3.16=testRuntimeClasspath
+io.sqreen:libsqreen:17.3.0=testRuntimeClasspath
+javax.servlet:javax.servlet-api:3.1.0=testCompileClasspath,testRuntimeClasspath
+jaxen:jaxen:2.0.0=spotbugs
+junit:junit:4.13.2=testRuntimeClasspath
+net.bytebuddy:byte-buddy-agent:1.18.8=buildTimeInstrumentationPlugin,compileClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+net.bytebuddy:byte-buddy:1.18.8=buildTimeInstrumentationPlugin,compileClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+net.java.dev.jna:jna-platform:5.8.0=testRuntimeClasspath
+net.java.dev.jna:jna:5.8.0=testRuntimeClasspath
+net.minidev:accessors-smart:2.4.9=testRuntimeClasspath
+net.minidev:json-smart:2.4.10=testRuntimeClasspath
+net.sf.jopt-simple:jopt-simple:4.6=compileClasspath
+net.sf.jopt-simple:jopt-simple:5.0.4=testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath
+net.sf.saxon:Saxon-HE:12.9=spotbugs
+org.apache.ant:ant-antlr:1.10.14=codenarc
+org.apache.ant:ant-junit:1.10.14=codenarc
+org.apache.bcel:bcel:6.11.0=spotbugs
+org.apache.commons:commons-lang3:3.19.0=spotbugs
+org.apache.commons:commons-math3:3.2=compileClasspath
+org.apache.commons:commons-math3:3.6.1=testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath
+org.apache.commons:commons-text:1.14.0=spotbugs
+org.apache.logging.log4j:log4j-api:2.25.2=spotbugs
+org.apache.logging.log4j:log4j-core:2.25.2=spotbugs
+org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath
+org.checkerframework:checker-qual:3.33.0=annotationProcessor,testAnnotationProcessor
+org.codehaus.groovy:groovy-ant:3.0.23=codenarc
+org.codehaus.groovy:groovy-docgenerator:3.0.23=codenarc
+org.codehaus.groovy:groovy-groovydoc:3.0.23=codenarc
+org.codehaus.groovy:groovy-json:3.0.23=codenarc
+org.codehaus.groovy:groovy-json:3.0.25=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-templates:3.0.23=codenarc
+org.codehaus.groovy:groovy-xml:3.0.23=codenarc
+org.codehaus.groovy:groovy:3.0.23=codenarc
+org.codehaus.groovy:groovy:3.0.25=testCompileClasspath,testRuntimeClasspath
+org.codenarc:CodeNarc:3.7.0=codenarc
+org.dom4j:dom4j:2.2.0=spotbugs
+org.freemarker:freemarker:2.3.31=testCompileClasspath,testRuntimeClasspath
+org.gmetrics:GMetrics:2.1.0=codenarc
+org.hamcrest:hamcrest-core:1.3=testRuntimeClasspath
+org.hamcrest:hamcrest:3.0=testCompileClasspath,testRuntimeClasspath
+org.jacoco:org.jacoco.core:0.8.14=testRuntimeClasspath
+org.jacoco:org.jacoco.report:0.8.14=testRuntimeClasspath
+org.jctools:jctools-core-jdk11:4.0.6=testRuntimeClasspath
+org.jctools:jctools-core:4.0.6=testRuntimeClasspath
+org.junit.jupiter:junit-jupiter-api:5.14.1=testCompileClasspath,testRuntimeClasspath
+org.junit.jupiter:junit-jupiter-engine:5.14.1=testRuntimeClasspath
+org.junit.jupiter:junit-jupiter-params:5.14.1=testCompileClasspath,testRuntimeClasspath
+org.junit.jupiter:junit-jupiter:5.14.1=testCompileClasspath,testRuntimeClasspath
+org.junit.platform:junit-platform-commons:1.14.1=testCompileClasspath,testRuntimeClasspath
+org.junit.platform:junit-platform-engine:1.14.1=testCompileClasspath,testRuntimeClasspath
+org.junit.platform:junit-platform-launcher:1.14.1=testRuntimeClasspath
+org.junit.platform:junit-platform-runner:1.14.1=testRuntimeClasspath
+org.junit.platform:junit-platform-suite-api:1.14.1=testRuntimeClasspath
+org.junit.platform:junit-platform-suite-commons:1.14.1=testRuntimeClasspath
+org.junit:junit-bom:5.14.0=spotbugs
+org.junit:junit-bom:5.14.1=testCompileClasspath,testRuntimeClasspath
+org.mockito:mockito-core:4.4.0=testRuntimeClasspath
+org.msgpack:jackson-dataformat-msgpack:0.9.6=testCompileClasspath,testRuntimeClasspath
+org.msgpack:msgpack-core:0.9.6=testCompileClasspath,testRuntimeClasspath
+org.objenesis:objenesis:3.3=testCompileClasspath,testRuntimeClasspath
+org.openjdk.jmh:jmh-core:1.0=compileClasspath
+org.openjdk.jmh:jmh-core:1.37=testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath
+org.openjdk.jmh:jmh-generator-annprocess:1.37=testAnnotationProcessor
+org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath
+org.ow2.asm:asm-analysis:9.7.1=testRuntimeClasspath
+org.ow2.asm:asm-analysis:9.9=spotbugs
+org.ow2.asm:asm-commons:9.9=spotbugs
+org.ow2.asm:asm-commons:9.9.1=testRuntimeClasspath
+org.ow2.asm:asm-tree:9.9=spotbugs
+org.ow2.asm:asm-tree:9.9.1=testRuntimeClasspath
+org.ow2.asm:asm-util:9.7.1=testRuntimeClasspath
+org.ow2.asm:asm-util:9.9=spotbugs
+org.ow2.asm:asm:9.9=spotbugs
+org.ow2.asm:asm:9.9.1=buildTimeInstrumentationPlugin,compileClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.skyscreamer:jsonassert:1.5.1=testCompileClasspath,testRuntimeClasspath
+org.slf4j:jcl-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath
+org.slf4j:jul-to-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath
+org.slf4j:log4j-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath
+org.slf4j:slf4j-api:1.7.30=buildTimeInstrumentationPlugin,compileClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath
+org.slf4j:slf4j-api:1.7.32=testCompileClasspath
+org.slf4j:slf4j-api:1.7.36=testRuntimeClasspath
+org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j
+org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j
+org.snakeyaml:snakeyaml-engine:2.9=buildTimeInstrumentationPlugin,muzzleTooling,runtimeClasspath,testRuntimeClasspath
+org.spockframework:spock-bom:2.4-groovy-3.0=testCompileClasspath,testRuntimeClasspath
+org.spockframework:spock-core:2.4-groovy-3.0=testCompileClasspath,testRuntimeClasspath
+org.tabletest:tabletest-junit:1.2.1=testCompileClasspath,testRuntimeClasspath
+org.tabletest:tabletest-parser:1.2.0=testCompileClasspath,testRuntimeClasspath
+org.xmlresolver:xmlresolver:5.3.3=spotbugs
+org.xmlunit:xmlunit-core:2.10.3=testCompileClasspath,testRuntimeClasspath
+empty=spotbugsPlugins