From 09682f2c6a59fc9a42d962d51803282ca181f34c Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Thu, 23 Jul 2026 10:07:50 +0200 Subject: [PATCH 01/30] feat(ci): Add test agent sidecar to smoke tests --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index df44494734c..762b371523f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1115,7 +1115,7 @@ test_debugger_arm64: - !reference [.test_job_arm64, script] test_smoke: - extends: .test_job + extends: .test_job_with_test_agent # needs:parallel:matrix limits this job to a specific build_tests combination. # Keep matrix vars exact and in build_tests declaration order: # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix @@ -1135,7 +1135,7 @@ test_smoke: matrix: *test_matrix_8 test_smoke_arm64: - extends: .test_job_arm64 + extends: .test_job_arm64_with_test_agent variables: <<: *tier_l_variables GRADLE_TARGET: "stageMainDist :smokeTest" From d3f8153eba947da0661152d73570b32d13b3f96b Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Sun, 12 Jul 2026 18:49:32 +0200 Subject: [PATCH 02/30] feat(core): Add support for test session token header --- .../main/java/datadog/trace/api/config/TracerConfig.java | 1 + .../datadog/trace/common/writer/ddagent/DDAgentApi.java | 6 ++++++ internal-api/src/main/java/datadog/trace/api/Config.java | 7 +++++++ metadata/supported-configurations.json | 8 ++++++++ 4 files changed, 22 insertions(+) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/TracerConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/TracerConfig.java index 9faf4f4ea8e..0ab030189b5 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/TracerConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/TracerConfig.java @@ -32,6 +32,7 @@ public final class TracerConfig { public static final String PROXY_NO_PROXY = "proxy.no_proxy"; public static final String TRACE_AGENT_PATH = "trace.agent.path"; public static final String TRACE_AGENT_ARGS = "trace.agent.args"; + public static final String TEST_AGENT_SESSION_TOKEN = "test.agent.session.token"; public static final String PRIORITY_SAMPLING = "priority.sampling"; public static final String PRIORITY_SAMPLING_FORCE = "priority.sampling.force"; @Deprecated public static final String TRACE_RESOLVER_ENABLED = "trace.resolver.enabled"; diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/DDAgentApi.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/DDAgentApi.java index 8907e634cfc..3da99de173f 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/DDAgentApi.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/DDAgentApi.java @@ -38,6 +38,8 @@ public class DDAgentApi extends RemoteApi { private static final String DATADOG_CLIENT_COMPUTED_TOP_LEVEL = "Datadog-Client-Computed-Top-Level"; private static final String X_DATADOG_TRACE_COUNT = "X-Datadog-Trace-Count"; + // Optional test-agent session + private static final String X_DATADOG_TEST_SESSION_TOKEN = "X-Datadog-Test-Session-Token"; private static final String DATADOG_DROPPED_TRACE_COUNT = "Datadog-Client-Dropped-P0-Traces"; private static final String DATADOG_DROPPED_SPAN_COUNT = "Datadog-Client-Dropped-P0-Spans"; private static final String DATADOG_AGENT_STATE = "Datadog-Agent-State"; @@ -79,6 +81,10 @@ public DDAgentApi( this.headers = new HashMap<>(); this.headers.put(DATADOG_CLIENT_COMPUTED_TOP_LEVEL, "true"); this.headers.put(DATADOG_META_TRACER_VERSION, DDTraceCoreInfo.VERSION); + String testSessionToken = Config.get().getTestAgentSessionToken(); + if (testSessionToken != null) { + this.headers.put(X_DATADOG_TEST_SESSION_TOKEN, testSessionToken); + } } public void addResponseListener(final RemoteResponseListener listener) { diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 7d6df22df1c..ec51403b85b 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -669,6 +669,7 @@ import static datadog.trace.api.config.TracerConfig.SPAN_SAMPLING_RULES_FILE; import static datadog.trace.api.config.TracerConfig.SPAN_TAGS; import static datadog.trace.api.config.TracerConfig.SPLIT_BY_TAGS; +import static datadog.trace.api.config.TracerConfig.TEST_AGENT_SESSION_TOKEN; import static datadog.trace.api.config.TracerConfig.TRACE_128_BIT_TRACEID_GENERATION_ENABLED; import static datadog.trace.api.config.TracerConfig.TRACE_AGENT_ARGS; import static datadog.trace.api.config.TracerConfig.TRACE_AGENT_PATH; @@ -1365,6 +1366,7 @@ public static String getHostName() { private final boolean azureFunctions; private final boolean awsServerless; private final String traceAgentPath; + private final String testAgentSessionToken; private final List traceAgentArgs; private final String dogStatsDPath; private final List dogStatsDArgs; @@ -3105,6 +3107,7 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) azureAppServices = configProvider.getBoolean(AZURE_APP_SERVICES, false); traceAgentPath = configProvider.getString(TRACE_AGENT_PATH); + testAgentSessionToken = configProvider.getString(TEST_AGENT_SESSION_TOKEN); String traceAgentArgsString = configProvider.getString(TRACE_AGENT_ARGS); if (traceAgentArgsString == null) { traceAgentArgs = Collections.emptyList(); @@ -5028,6 +5031,10 @@ public String getTraceAgentPath() { return traceAgentPath; } + public String getTestAgentSessionToken() { + return testAgentSessionToken; + } + public List getTraceAgentArgs() { return traceAgentArgs; } diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index d6b02fbfa28..44ad3c64c75 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -4065,6 +4065,14 @@ "aliases": [] } ], + "DD_TEST_AGENT_SESSION_TOKEN": [ + { + "version": "A", + "type": "string", + "default": null, + "aliases": [] + } + ], "DD_TEST_FAILED_TEST_REPLAY_ENABLED": [ { "version": "A", From f497997c7cc2311b87f59feab72b41738caa14bc Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Thu, 23 Jul 2026 10:34:25 +0200 Subject: [PATCH 03/30] feat(telemetry): Add support for test session token header --- .../src/main/java/datadog/telemetry/TelemetryClient.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/telemetry/src/main/java/datadog/telemetry/TelemetryClient.java b/telemetry/src/main/java/datadog/telemetry/TelemetryClient.java index c13411e0e69..445c03f63dd 100644 --- a/telemetry/src/main/java/datadog/telemetry/TelemetryClient.java +++ b/telemetry/src/main/java/datadog/telemetry/TelemetryClient.java @@ -65,11 +65,13 @@ private static String buildIntakeTelemetryUrl(Config config) { private static final String AGENT_TELEMETRY_API_ENDPOINT = "telemetry/proxy/api/v2/apmtelemetry"; private static final String DD_TELEMETRY_REQUEST_TYPE = "DD-Telemetry-Request-Type"; + private static final String X_DATADOG_TEST_SESSION_TOKEN = "X-Datadog-Test-Session-Token"; private final OkHttpClient okHttpClient; private final HttpRetryPolicy.Factory httpRetryPolicy; private final HttpUrl url; private final String apiKey; + private final String testSessionToken; public TelemetryClient( OkHttpClient okHttpClient, @@ -80,6 +82,7 @@ public TelemetryClient( this.httpRetryPolicy = httpRetryPolicy; this.url = url; this.apiKey = apiKey; + this.testSessionToken = Config.get().getTestAgentSessionToken(); } public HttpUrl getUrl() { @@ -90,6 +93,8 @@ public Result sendHttpRequest(Request.Builder httpRequestBuilder) { httpRequestBuilder.url(url); if (apiKey != null) { httpRequestBuilder.addHeader("DD-API-KEY", apiKey); + } else if (testSessionToken != null && !testSessionToken.isEmpty()) { + httpRequestBuilder.addHeader(X_DATADOG_TEST_SESSION_TOKEN, testSessionToken); } Request httpRequest = httpRequestBuilder.build(); From d1c85f88d5ab472af938792c4c55bb7175718beb Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Sun, 12 Jul 2026 19:12:25 +0200 Subject: [PATCH 04/30] WIP --- .../backend/TestAgentTraceDecoder.java | 165 ++++++++++++++++ .../smoketest/trace/SmokeTraceAssertions.java | 35 ++++ .../datadog/smoketest/trace/SpanMatcher.java | 129 +++++++++++++ .../datadog/smoketest/trace/TraceMatcher.java | 40 ++++ .../backend/TestAgentTraceDecoderTest.java | 182 ++++++++++++++++++ 5 files changed, 551 insertions(+) create mode 100644 dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentTraceDecoder.java create mode 100644 dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java create mode 100644 dd-smoke-tests/src/main/java/datadog/smoketest/trace/SpanMatcher.java create mode 100644 dd-smoke-tests/src/main/java/datadog/smoketest/trace/TraceMatcher.java create mode 100644 dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentTraceDecoderTest.java diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentTraceDecoder.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentTraceDecoder.java new file mode 100644 index 00000000000..52b65fb1001 --- /dev/null +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentTraceDecoder.java @@ -0,0 +1,165 @@ +package datadog.smoketest.backend; + +import com.squareup.moshi.Json; +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.JsonDataException; +import com.squareup.moshi.Moshi; +import com.squareup.moshi.Types; +import datadog.trace.test.agent.decoder.DecodedSpan; +import datadog.trace.test.agent.decoder.DecodedTrace; +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Parses the dd-apm-test-agent {@code /test/traces} JSON response into the same {@link + * DecodedTrace} / {@link DecodedSpan} model the mock backend produces via the msgpack {@code + * Decoder}. This is what lets the thin smoke matcher ({@code datadog.smoketest.trace}) work + * uniformly across both backends (Q1). + * + *

The endpoint returns a JSON array of traces, each a JSON array of spans in the standard v0.4 + * shape. + */ +public final class TestAgentTraceDecoder { + private static final Type LIST_OF_TRACES = + Types.newParameterizedType(List.class, Types.newParameterizedType(List.class, RawSpan.class)); + private static final JsonAdapter>> ADAPTER = + new Moshi.Builder().build().adapter(LIST_OF_TRACES); + + private TestAgentTraceDecoder() {} + + /** Decodes a {@code /test/traces} JSON body into the shared decoded-trace model. */ + public static List decode(String json) { + List> rawTraces; + try { + // IOException covers malformed JSON (JsonEncodingException); JsonDataException (unchecked) + // covers a well-formed body of the wrong shape (e.g. an object where a trace array is + // expected). Wrap both so callers always get the offending body for diagnosis. + rawTraces = ADAPTER.fromJson(json); + } catch (IOException | JsonDataException e) { + throw new IllegalStateException("Failed to parse /test/traces response: " + json, e); + } + List traces = new ArrayList<>(); + if (rawTraces != null) { + for (List spans : rawTraces) { + List decodedSpans = + spans == null ? Collections.emptyList() : new ArrayList<>(spans); + traces.add(new RawTrace(decodedSpans)); + } + } + return traces; + } + + private static final class RawTrace implements DecodedTrace { + private final List spans; + + RawTrace(List spans) { + this.spans = Collections.unmodifiableList(spans); + } + + @Override + public List getSpans() { + return spans; + } + } + + // TODO verify these field names against a live ddapm-test-agent v1.44.0 /test/traces response. + // They follow the standard v0.4 trace shape (matching the msgpack Decoder's fields), but have + // not + // yet been validated end-to-end against the container — do so when S3/S8 run with Docker. + static final class RawSpan implements DecodedSpan { + String service; + String name; + String resource; + String type; + + @Json(name = "trace_id") + long traceId; + + @Json(name = "span_id") + long spanId; + + @Json(name = "parent_id") + long parentId; + + long start; + long duration; + int error; + Map meta; + + @Json(name = "meta_struct") + Map metaStruct; + + Map metrics; + + @Override + public String getService() { + return service; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getResource() { + return resource; + } + + @Override + public long getTraceId() { + return traceId; + } + + @Override + public long getSpanId() { + return spanId; + } + + @Override + public long getParentId() { + return parentId; + } + + @Override + public long getStart() { + return start; + } + + @Override + public long getDuration() { + return duration; + } + + @Override + public int getError() { + return error; + } + + @Override + public Map getMeta() { + return meta; + } + + @Override + public Map getMetaStruct() { + return metaStruct; + } + + @Override + public Map getMetrics() { + // The decoder model exposes Number; Moshi deserializes JSON numbers as Double. + return metrics == null ? null : new HashMap<>(metrics); + } + + @Override + public String getType() { + return type; + } + } +} diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java new file mode 100644 index 00000000000..2da717f4337 --- /dev/null +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java @@ -0,0 +1,35 @@ +package datadog.smoketest.trace; + +import datadog.trace.test.agent.decoder.DecodedTrace; +import java.util.List; + +/** + * Entry point for the thin smoke trace DSL. Assert a captured trace collection against one {@link + * TraceMatcher} per expected trace: + * + * 

{@code
+ * import static datadog.smoketest.trace.SmokeTraceAssertions.assertTraces;
+ * import static datadog.smoketest.trace.TraceMatcher.trace;
+ * import static datadog.smoketest.trace.SpanMatcher.span;
+ *
+ * assertTraces(traces, trace(span().operationName("servlet.request").resourceName("GET /greeting")));
+ * }
+ * + * The traces come from a {@code TraceBackend} (mock or test-agent), both decoded to {@link + * DecodedTrace}. + */ +public final class SmokeTraceAssertions { + private SmokeTraceAssertions() {} + + public static void assertTraces(List traces, TraceMatcher... matchers) { + // TODO thin first cut: traces are matched positionally, in the order received. Add sorting + // (e.g. + // by root-span start time) when a smoke test needs order-independence. + if (traces.size() != matchers.length) { + throw new AssertionError("Expected " + matchers.length + " traces but got " + traces.size()); + } + for (int i = 0; i < matchers.length; i++) { + matchers[i].assertTrace(traces.get(i)); + } + } +} diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SpanMatcher.java b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SpanMatcher.java new file mode 100644 index 00000000000..cd703bb3263 --- /dev/null +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SpanMatcher.java @@ -0,0 +1,129 @@ +package datadog.smoketest.trace; + +import static datadog.trace.junit.utils.assertions.Matchers.assertValue; +import static datadog.trace.junit.utils.assertions.Matchers.is; +import static datadog.trace.junit.utils.assertions.Matchers.isFalse; +import static datadog.trace.junit.utils.assertions.Matchers.isTrue; + +import datadog.trace.junit.utils.assertions.Matcher; +import datadog.trace.test.agent.decoder.DecodedSpan; +import java.util.HashMap; +import java.util.Map; + +/** + * Thin span matcher for smoke tests, operating on the serialized {@link DecodedSpan} model + * produced by both smoke backends (the in-process mock agent and the dd-apm-test-agent). + * + *

This is deliberately separate from the in-process instrumentation DSL ({@code + * datadog.trace.agent.test.assertions}, which matches on {@code DDSpan}); the two share only the + * generic value matchers in {@code datadog.trace.junit.utils.assertions}. Only what a span carries + * once serialized is matchable here: name/service/resource/type, error status, {@code meta} (string + * tags), {@code metrics} (numeric tags), and parent linkage. + * + *

Use the {@link #span()} factory as a fluent builder. Unset constraints are not checked (thin + * by design). + */ +public final class SpanMatcher { + private Matcher serviceMatcher; + private Matcher operationNameMatcher; + private Matcher resourceNameMatcher; + private Matcher typeMatcher; + private Matcher errorMatcher; + private Long parentId; // null => parent not checked + private final Map> metaMatchers = new HashMap<>(); + private final Map> metricMatchers = new HashMap<>(); + + private SpanMatcher() {} + + /** Creates a new span matcher; configure it with the fluent methods below. */ + public static SpanMatcher span() { + return new SpanMatcher(); + } + + public SpanMatcher service(String service) { + this.serviceMatcher = is(service); + return this; + } + + public SpanMatcher operationName(String operationName) { + this.operationNameMatcher = is(operationName); + return this; + } + + public SpanMatcher resourceName(String resourceName) { + this.resourceNameMatcher = is(resourceName); + return this; + } + + public SpanMatcher type(String type) { + this.typeMatcher = is(type); + return this; + } + + /** Matches a non-error span. */ + public SpanMatcher error(boolean errored) { + this.errorMatcher = errored ? isTrue() : isFalse(); + return this; + } + + /** Matches a root span (no parent). */ + public SpanMatcher root() { + this.parentId = 0L; + return this; + } + + /** Matches a span whose parent is the given span id. */ + public SpanMatcher childOf(long parentSpanId) { + this.parentId = parentSpanId; + return this; + } + + /** Matches a {@code meta} (string) tag against the given matcher. */ + public SpanMatcher tag(String name, Matcher matcher) { + this.metaMatchers.put(name, matcher); + return this; + } + + /** Matches a {@code meta} (string) tag against the given value. */ + public SpanMatcher tag(String name, String value) { + this.metaMatchers.put(name, is(value)); + return this; + } + + /** Matches a {@code metrics} (numeric) tag against the given matcher. */ + public SpanMatcher metric(String name, Matcher matcher) { + this.metricMatchers.put(name, matcher); + return this; + } + + // TODO thin first cut — add when a smoke test needs them: + // - operationName/resourceName by Pattern or Predicate, + // - childOf(SpanMatcher) resolving the parent within the trace, + // - exhaustive tag coverage (fail on unexpected meta/metrics), and span-links. + + @SuppressWarnings("unchecked") + void assertSpan(DecodedSpan span) { + assertValue(serviceMatcher, span.getService(), "Unexpected service name"); + assertValue(operationNameMatcher, span.getName(), "Unexpected operation name"); + assertValue(resourceNameMatcher, span.getResource(), "Unexpected resource name"); + assertValue(typeMatcher, span.getType(), "Unexpected span type"); + assertValue(errorMatcher, span.getError() != 0, "Unexpected error status"); + if (parentId != null) { + assertValue(is(parentId), span.getParentId(), "Unexpected parent id"); + } + Map meta = span.getMeta(); + for (Map.Entry> entry : metaMatchers.entrySet()) { + Object value = meta == null ? null : meta.get(entry.getKey()); + assertValue( + (Matcher) entry.getValue(), + value, + "Unexpected meta tag '" + entry.getKey() + "'"); + } + Map metrics = span.getMetrics(); + for (Map.Entry> entry : metricMatchers.entrySet()) { + Object value = metrics == null ? null : metrics.get(entry.getKey()); + assertValue( + (Matcher) entry.getValue(), value, "Unexpected metric '" + entry.getKey() + "'"); + } + } +} diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/TraceMatcher.java b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/TraceMatcher.java new file mode 100644 index 00000000000..05880b3b032 --- /dev/null +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/TraceMatcher.java @@ -0,0 +1,40 @@ +package datadog.smoketest.trace; + +import datadog.trace.test.agent.decoder.DecodedSpan; +import datadog.trace.test.agent.decoder.DecodedTrace; +import java.util.List; + +/** + * Thin trace matcher for smoke tests: one {@link SpanMatcher} per expected span in a {@link + * DecodedTrace}. Companion of {@link SpanMatcher}; see its class doc for the serialized-model + * rationale. + */ +public final class TraceMatcher { + private final SpanMatcher[] spanMatchers; + + private TraceMatcher(SpanMatcher[] spanMatchers) { + if (spanMatchers.length == 0) { + throw new IllegalArgumentException("No span matchers provided"); + } + this.spanMatchers = spanMatchers; + } + + /** Checks a trace structure, one matcher per expected span. */ + public static TraceMatcher trace(SpanMatcher... matchers) { + return new TraceMatcher(matchers); + } + + void assertTrace(DecodedTrace trace) { + List spans = trace.getSpans(); + // TODO thin first cut: spans are matched positionally, in the order received. Add sorting + // (e.g. by start time, mirroring the instrumentation DSL) and/or find-by-id matching when a + // smoke test needs order-independence. + if (spans.size() != spanMatchers.length) { + throw new AssertionError( + "Expected " + spanMatchers.length + " spans but got " + spans.size() + ": " + spans); + } + for (int i = 0; i < spanMatchers.length; i++) { + spanMatchers[i].assertSpan(spans.get(i)); + } + } +} diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentTraceDecoderTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentTraceDecoderTest.java new file mode 100644 index 00000000000..bbe30d7280f --- /dev/null +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentTraceDecoderTest.java @@ -0,0 +1,182 @@ +package datadog.smoketest.backend; + +import static datadog.smoketest.trace.SmokeTraceAssertions.assertTraces; +import static datadog.smoketest.trace.SpanMatcher.span; +import static datadog.smoketest.trace.TraceMatcher.trace; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.test.agent.decoder.DecodedSpan; +import datadog.trace.test.agent.decoder.DecodedTrace; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link TestAgentTraceDecoder}: the JSON {@code /test/traces} body must decode into + * the same {@link DecodedTrace}/{@link DecodedSpan} model the msgpack {@code Decoder} yields for + * the mock backend, so the thin smoke matcher ({@code datadog.smoketest.trace}) works uniformly + * across both backends (S1b / Q1). + */ +class TestAgentTraceDecoderTest { + + // A representative /test/traces body: an array of traces, each an array of v0.4-shaped spans. + private static final String TWO_TRACES = + "[" + + " [" + + " {" + + " \"service\": \"my-service\"," + + " \"name\": \"servlet.request\"," + + " \"resource\": \"GET /greeting\"," + + " \"type\": \"web\"," + + " \"trace_id\": 1234567890," + + " \"span_id\": 111," + + " \"parent_id\": 0," + + " \"start\": 1600000000000000000," + + " \"duration\": 500000," + + " \"error\": 0," + + " \"meta\": {\"http.method\": \"GET\", \"http.status_code\": \"200\"}," + + " \"metrics\": {\"_dd.top_level\": 1, \"_dd.agent_psr\": 0.75}" + + " }," + + " {" + + " \"service\": \"my-service\"," + + " \"name\": \"repository.query\"," + + " \"resource\": \"SELECT users\"," + + " \"type\": \"sql\"," + + " \"trace_id\": 1234567890," + + " \"span_id\": 222," + + " \"parent_id\": 111," + + " \"start\": 1600000000000100000," + + " \"duration\": 200000," + + " \"error\": 1," + + " \"meta\": {\"db.type\": \"postgres\"}," + + " \"metrics\": {}" + + " }" + + " ]," + + " [" + + " {" + + " \"service\": \"batch\"," + + " \"name\": \"scheduled.job\"," + + " \"resource\": \"nightly\"," + + " \"type\": \"custom\"," + + " \"trace_id\": 42," + + " \"span_id\": 7," + + " \"parent_id\": 0," + + " \"start\": 1," + + " \"duration\": 1," + + " \"error\": 0," + + " \"meta\": {}," + + " \"metrics\": {}" + + " }" + + " ]" + + "]"; + + @Test + void decodesTraceAndSpanStructure() { + List traces = TestAgentTraceDecoder.decode(TWO_TRACES); + + assertEquals(2, traces.size(), "trace count"); + assertEquals(2, traces.get(0).getSpans().size(), "spans in first trace"); + assertEquals(1, traces.get(1).getSpans().size(), "spans in second trace"); + + DecodedSpan root = traces.get(0).getSpans().get(0); + assertEquals("my-service", root.getService()); + assertEquals("servlet.request", root.getName()); + assertEquals("GET /greeting", root.getResource()); + assertEquals("web", root.getType()); + assertEquals(1234567890L, root.getTraceId()); + assertEquals(111L, root.getSpanId()); + assertEquals(0L, root.getParentId()); + assertEquals(1600000000000000000L, root.getStart()); + assertEquals(500000L, root.getDuration()); + assertEquals(0, root.getError()); + } + + @Test + void mapsMetaAsStringsAndMetricsAsNumbers() { + List spans = TestAgentTraceDecoder.decode(TWO_TRACES).get(0).getSpans(); + + Map meta = spans.get(0).getMeta(); + assertEquals("GET", meta.get("http.method")); + assertEquals("200", meta.get("http.status_code")); + + Map metrics = spans.get(0).getMetrics(); + // Moshi decodes every JSON number in the metrics map as a Double, matching the Number model. + assertEquals(1.0, metrics.get("_dd.top_level").doubleValue()); + assertEquals(0.75, metrics.get("_dd.agent_psr").doubleValue()); + + // A serialized error span carries error != 0; the parent link is the enclosing root span id. + DecodedSpan child = spans.get(1); + assertEquals(1, child.getError()); + assertEquals(111L, child.getParentId()); + assertEquals("postgres", child.getMeta().get("db.type")); + } + + @Test + void metaStructIsNullWhenAbsentAndAMapWhenPresent() { + // Absent meta_struct decodes to null (matching the msgpack Decoder's plain-span shape). + DecodedSpan withoutMetaStruct = + TestAgentTraceDecoder.decode(TWO_TRACES).get(0).getSpans().get(0); + assertNull(withoutMetaStruct.getMetaStruct()); + + String withMetaStruct = + "[[{" + + "\"service\": \"s\", \"name\": \"n\", \"resource\": \"r\", \"type\": \"web\"," + + "\"trace_id\": 1, \"span_id\": 1, \"parent_id\": 0, \"start\": 0, \"duration\": 0," + + "\"error\": 0, \"meta\": {}, \"metrics\": {}," + + "\"meta_struct\": {\"appsec\": {\"triggers\": 3}}" + + "}]]"; + Map metaStruct = + TestAgentTraceDecoder.decode(withMetaStruct).get(0).getSpans().get(0).getMetaStruct(); + assertNotNull(metaStruct); + assertTrue(metaStruct.containsKey("appsec")); + } + + @Test + void emptyAndNullBodiesYieldNoTraces() { + assertTrue(TestAgentTraceDecoder.decode("[]").isEmpty(), "empty array => no traces"); + // Moshi parses the JSON literal null as a null document; decode tolerates it. + assertTrue(TestAgentTraceDecoder.decode("null").isEmpty(), "null body => no traces"); + + List oneEmptyTrace = TestAgentTraceDecoder.decode("[[]]"); + assertEquals(1, oneEmptyTrace.size()); + assertTrue(oneEmptyTrace.get(0).getSpans().isEmpty(), "empty trace => no spans"); + } + + @Test + void decodedTracesFeedTheSmokeMatcher() { + // The point of S1b: the test-agent-decoded traces are matchable by the same thin smoke DSL the + // mock backend uses, since both produce DecodedTrace/DecodedSpan. + List traces = TestAgentTraceDecoder.decode(TWO_TRACES); + + assertTraces( + traces, + trace( + span() + .service("my-service") + .operationName("servlet.request") + .resourceName("GET /greeting") + .type("web") + .error(false) + .root() + .tag("http.method", "GET"), + span() + .service("my-service") + .operationName("repository.query") + .resourceName("SELECT users") + .error(true) + .childOf(111L)), + trace(span().service("batch").operationName("scheduled.job").root())); + } + + @Test + void malformedJsonThrows() { + IllegalStateException e = + assertThrows( + IllegalStateException.class, () -> TestAgentTraceDecoder.decode("{ not valid json")); + assertTrue(e.getMessage().contains("/test/traces"), "message should reference the endpoint"); + } +} From 19fc11444f7acd3808598d3b7355ce433c1e6fb6 Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Sun, 12 Jul 2026 19:28:08 +0200 Subject: [PATCH 05/30] WIP --- .../smoketest/backend/MockAgentBackend.java | 121 ++++++++++++++ .../smoketest/backend/TraceBackend.java | 43 +++++ .../datadog/smoketest/backend/Traces.java | 56 +++++++ .../backend/MockAgentBackendTest.java | 155 ++++++++++++++++++ .../smoketest/backend/webflux.04.msgpack | Bin 0 -> 1044 bytes 5 files changed, 375 insertions(+) create mode 100644 dd-smoke-tests/src/main/java/datadog/smoketest/backend/MockAgentBackend.java create mode 100644 dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java create mode 100644 dd-smoke-tests/src/main/java/datadog/smoketest/backend/Traces.java create mode 100644 dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java create mode 100644 dd-smoke-tests/src/test/resources/datadog/smoketest/backend/webflux.04.msgpack diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/MockAgentBackend.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/MockAgentBackend.java new file mode 100644 index 00000000000..ec8a32e62ac --- /dev/null +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/MockAgentBackend.java @@ -0,0 +1,121 @@ +package datadog.smoketest.backend; + +import datadog.trace.agent.test.server.http.JavaTestHttpServer; +import datadog.trace.agent.test.server.http.JavaTestHttpServer.HandlerApi; +import datadog.trace.test.agent.decoder.DecodedMessage; +import datadog.trace.test.agent.decoder.DecodedTrace; +import datadog.trace.test.agent.decoder.Decoder; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * In-process mock-agent {@link TraceBackend} wrapping the testing {@link JavaTestHttpServer}. It + * answers the tracer's {@code /info} probe, decodes the trace payloads the tracer submits (v0.4 / + * v0.5 / v1.0 msgpack) via the shared {@link Decoder}, and 200s everything else so the app's other + * agent calls (telemetry, remote-config, ...) don't error out. + * + *

Backend-specific capture surfaces (remote-config / EVP-proxy / DSM — Q10/S10) will hang off + * this concrete type rather than the common {@link TraceBackend} facade. + */ +public final class MockAgentBackend implements TraceBackend { + // Minimal agent-info payload advertising the trace endpoints, mirroring the real agent's /info + // (see SimpleAgentMock). The tracer negotiates its trace encoding from this list. + private static final String INFO_BODY = + "{\"version\":\"7.77.0\",\"endpoints\":[\"/v0.4/traces\",\"/v0.5/traces\",\"/v1.0/traces\"]}"; + private static final String JSON = "application/json"; + + private final List traces = new CopyOnWriteArrayList<>(); + private volatile JavaTestHttpServer server; + + MockAgentBackend() {} + + @Override + public void start() { + if (server != null) { + return; + } + server = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> { + // Trace endpoints are method-agnostic prefix handlers: the tracer submits + // traces with PUT (DDAgentApi), not POST. + h.prefix("/info", this::sendInfo); + h.prefix("/v1.0/traces", api -> collect(api, TraceFormat.V1)); + h.prefix("/v0.5/traces", api -> collect(api, TraceFormat.V05)); + h.prefix("/v0.4/traces", api -> collect(api, TraceFormat.V04)); + // Everything else (telemetry, remote-config, ...) just succeeds. + h.all(api -> api.getResponse().status(200).send()); + })); + } + + private void sendInfo(HandlerApi api) { + api.getResponse().status(200).sendWithType(JSON, INFO_BODY); + } + + private void collect(HandlerApi api, TraceFormat format) { + DecodedMessage message = format.decode(api.getRequest().getBody()); + traces.addAll(message.getTraces()); + api.getResponse().status(200).sendWithType(JSON, "{}"); + } + + @Override + public int port() { + return url().getPort(); + } + + @Override + public URI url() { + JavaTestHttpServer running = server; + if (running == null) { + throw new IllegalStateException("MockAgentBackend not started — call start() first"); + } + return running.getAddress(); + } + + @Override + public Traces traces() { + return new Traces(() -> new ArrayList<>(traces)); + } + + @Override + public void clear() { + traces.clear(); + } + + @Override + public void close() { + JavaTestHttpServer running = server; + if (running != null) { + server = null; + running.close(); + } + } + + /** The msgpack trace-payload formats the mock agent accepts, each decoded by {@link Decoder}. */ + private enum TraceFormat { + V04 { + @Override + DecodedMessage decode(byte[] body) { + return Decoder.decodeV04(body); + } + }, + V05 { + @Override + DecodedMessage decode(byte[] body) { + return Decoder.decodeV05(body); + } + }, + V1 { + @Override + DecodedMessage decode(byte[] body) { + return Decoder.decodeV1(body); + } + }; + + abstract DecodedMessage decode(byte[] body); + } +} diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java new file mode 100644 index 00000000000..3bf184d6580 --- /dev/null +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java @@ -0,0 +1,43 @@ +package datadog.smoketest.backend; + +import datadog.trace.test.agent.decoder.DecodedTrace; +import java.net.URI; + +/** + * A pluggable trace backend a smoke-test app sends its traces to. Two implementations are planned: + * the in-process {@link #mockAgent() mock agent} (this step, S2) and a Dockerized/external + * dd-apm-test-agent (S3). Both decode received traces into the shared {@link DecodedTrace} model + * (via the msgpack {@code Decoder}, or {@link TestAgentTraceDecoder} for the test agent), so a test + * body written against the common {@link Traces} surface runs unchanged on either backend (Q2). + * + *

The lifecycle mirrors the JUnit extension that will own the backend (S4/S6): {@link #start()} + * once per test class, {@link #clear()} between methods, {@link #close()} at teardown. + */ +public interface TraceBackend extends AutoCloseable { + /** Starts the backend and binds it to a port. Idempotent. */ + void start(); + + /** The agent port the app should send traces to (e.g. {@code -Ddd.trace.agent.port}). */ + int port(); + + /** The base URL of the backend. */ + URI url(); + + /** Query/assert facade over the traces this backend has received. */ + Traces traces(); + + /** Discards all traces received so far — call between test methods to isolate them. */ + void clear(); + + @Override + void close(); + + /** Creates an in-process mock-agent backend wrapping the testing {@code JavaTestHttpServer}. */ + static MockAgentBackend mockAgent() { + return new MockAgentBackend(); + } + + // TODO S3: testAgent() — a Testcontainers-managed .container() / external CI agent backend, plus + // the test-agent trace-invariant checks (ENABLED_CHECKS) and JSON decoding via + // TestAgentTraceDecoder. +} diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Traces.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Traces.java new file mode 100644 index 00000000000..57409ace8b8 --- /dev/null +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Traces.java @@ -0,0 +1,56 @@ +package datadog.smoketest.backend; + +import datadog.trace.test.agent.decoder.DecodedTrace; +import datadog.trace.test.util.PollingConditions; +import java.util.List; +import java.util.function.Supplier; + +/** + * Query facade over the traces a {@link TraceBackend} has received. It is backend-agnostic: it + * reads a live snapshot of decoded traces, so the same assertions run against either backend (Q2). + * + *

S2 provides trace retrieval and count-polling. The fluent matcher integration ({@code + * assertTraces(trace(span()...))} over the smoke matcher in {@code datadog.smoketest.trace}) and + * the test-agent trace-invariant checks land in S5. + */ +public final class Traces { + /** Default time to wait for traces to arrive from a separately-launched app before giving up. */ + private static final double DEFAULT_TIMEOUT_SECONDS = 10; + + private final Supplier> source; + + Traces(Supplier> source) { + this.source = source; + } + + /** A snapshot of the traces received so far. */ + public List getTraces() { + return source.get(); + } + + /** + * Waits (up to the default timeout) until at least {@code count} traces have been + * received, then returns for chaining. Traces arrive asynchronously from the app, so callers wait + * for the expected count before asserting structure. + */ + public Traces waitForTraceCount(int count) { + return waitForTraceCount(count, DEFAULT_TIMEOUT_SECONDS); + } + + /** As {@link #waitForTraceCount(int)}, but overriding the timeout for this call. */ + public Traces waitForTraceCount(int count, double timeoutSeconds) { + new PollingConditions(timeoutSeconds) + .eventually( + () -> { + int actual = getTraces().size(); + if (actual < count) { + throw new AssertionError( + "Expected at least " + count + " trace(s) but got " + actual); + } + }); + return this; + } + + // TODO S5: fluent assertTraces(TraceMatcher...) delegating to the smoke matcher, and (test-agent + // backends only) trace-invariant check assertions over /test/trace_check/failures. +} diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java new file mode 100644 index 00000000000..d7ba5f51cb1 --- /dev/null +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java @@ -0,0 +1,155 @@ +package datadog.smoketest.backend; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.test.agent.decoder.DecodedSpan; +import datadog.trace.test.agent.decoder.DecodedTrace; +import datadog.trace.test.agent.decoder.Decoder; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Tests the in-process {@link MockAgentBackend}: it must answer the tracer's {@code /info} probe, + * accept the trace payloads the tracer PUTs, and decode them into the shared {@link DecodedTrace} + * model. Drives the backend with a recorded v0.4 msgpack payload over real HTTP (okhttp), the same + * way a launched app's tracer would (S2). + * + *

Uses a per-class backend cleared per method, mirroring the intended smoke-test lifecycle (Q3: + * backend started once per class, reset between methods). + */ +class MockAgentBackendTest { + private static final MediaType MSGPACK = MediaType.parse("application/msgpack"); + private static final OkHttpClient CLIENT = new OkHttpClient(); + + // Recorded /v0.4/traces payload: 1 trace, 2 spans (netty.request -> WebController.hello), + // service "smoke-test-java-app". Same fixture the decoder module's DecoderTest uses. + private static byte[] v04Payload; + + private static MockAgentBackend backend; + + @BeforeAll + static void setUp() throws IOException { + v04Payload = readResource("/datadog/smoketest/backend/webflux.04.msgpack"); + backend = TraceBackend.mockAgent(); + backend.start(); + } + + @AfterAll + static void tearDown() { + if (backend != null) { + backend.close(); + } + } + + @BeforeEach + void resetTraces() { + backend.clear(); + } + + @Test + void receivesAndDecodesSubmittedTraces() throws IOException { + putTraces("/v0.4/traces", v04Payload); + + Traces traces = backend.traces(); + traces.waitForTraceCount(1); + + List decoded = traces.getTraces(); + assertEquals(1, decoded.size(), "trace count"); + + // sortByStart makes the assertion independent of the received span order (thin matcher is + // positional — see TraceMatcher's TODO). + List spans = Decoder.sortByStart(decoded.get(0).getSpans()); + assertEquals(2, spans.size(), "span count"); + + DecodedSpan root = spans.get(0); + assertEquals("smoke-test-java-app", root.getService()); + assertEquals("netty.request", root.getName()); + assertEquals("GET /hello", root.getResource()); + assertEquals(0L, root.getParentId(), "root has no parent"); + assertEquals("netty", root.getMeta().get("component")); + + DecodedSpan child = spans.get(1); + assertEquals("WebController.hello", child.getName()); + assertEquals(root.getSpanId(), child.getParentId(), "child parents the root span"); + } + + @Test + void accumulatesTracesAcrossSubmissions() throws IOException { + putTraces("/v0.4/traces", v04Payload); + putTraces("/v0.4/traces", v04Payload); + + backend.traces().waitForTraceCount(2); + assertEquals(2, backend.traces().getTraces().size()); + } + + @Test + void clearDiscardsReceivedTraces() throws IOException { + putTraces("/v0.4/traces", v04Payload); + backend.traces().waitForTraceCount(1); + + backend.clear(); + + assertTrue(backend.traces().getTraces().isEmpty(), "clear() drops collected traces"); + } + + @Test + void infoAdvertisesTraceEndpoints() throws IOException { + Request request = new Request.Builder().url(backend.url() + "/info").get().build(); + try (Response response = CLIENT.newCall(request).execute()) { + assertEquals(200, response.code()); + String body = response.body().string(); + assertTrue(body.contains("/v0.4/traces"), body); + } + } + + @Test + void waitForTraceCountTimesOutWhenTooFew() { + // Nothing submitted, so polling for a trace with a short timeout must fail rather than hang. + assertThrows( + AssertionError.class, () -> backend.traces().waitForTraceCount(1, 0.2 /* seconds */)); + } + + @Test + void exposesBoundPort() { + assertTrue(backend.port() > 0, "port is bound"); + assertEquals(backend.url().getPort(), backend.port(), "port matches url"); + } + + private static void putTraces(String path, byte[] payload) throws IOException { + Request request = + new Request.Builder() + .url(backend.url() + path) + .put(RequestBody.create(MSGPACK, payload)) + .build(); + try (Response response = CLIENT.newCall(request).execute()) { + assertEquals(200, response.code(), "mock agent should accept trace submissions"); + } + } + + private static byte[] readResource(String name) throws IOException { + try (InputStream in = MockAgentBackendTest.class.getResourceAsStream(name)) { + assertNotNull(in, "missing test resource " + name); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int read; + while ((read = in.read(buffer)) != -1) { + out.write(buffer, 0, read); + } + return out.toByteArray(); + } + } +} diff --git a/dd-smoke-tests/src/test/resources/datadog/smoketest/backend/webflux.04.msgpack b/dd-smoke-tests/src/test/resources/datadog/smoketest/backend/webflux.04.msgpack new file mode 100644 index 0000000000000000000000000000000000000000..6d4863a3b0f648a57e5a126e9541bf6fda59d6a1 GIT binary patch literal 1044 zcma)*J#5r46vtaeCZGa@)S(DQk|x*VKDtz>f(|eO1qLD~es^)3*skrQ6d?vAJ_dvY zVrJml-lZQqVl7LBXps;S5i=u#r8{<7?yiCg;w5@7ejoqeKhIx0ZA%a-2_PSmF`ETT zfmGHEN|A*)U!*A3nQ8DR8C8Ns5ePekC}JG}k|o02`&(8C6o5zY_Hb=t=he5PV{OS1 zH34|IefsNV{wCu{097VLHa6a!BVTGuQX!$XPi)@a`f>l`_YO`3QiRd1!;_!IqFUh4 zng@RggkWNz9fJ}ikPp^8jO`eZOa$P;Ooajvwh2D7sJ}puSN8(NCdYIxP)t~%Qrx11 zS!10D(Po00(3(l(EMPHbRL_;4l3Wnlv~2x!@fhgiu-O=T+vDII`tkE@0XY-u!_Jce>9hQ#DZ=f9OQnIv-kybJJ`$AN z6qT+iJazppo^8r9GAVt#q(UiiHSI%b8oM+KxW%deOktGd9h-)q8$iDj5Iau$AJX zV2Og|-62*)LKTU@A~-(_PQ!P=FIz6uYgQR5$f}i5+44~ZJK)yq4Zpftl+gc5$Z$(r F<`-)4vxNWv literal 0 HcmV?d00001 From 79e0cd67d10faba6b8bb0dfbbf851c1722b1781d Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Sun, 12 Jul 2026 19:55:34 +0200 Subject: [PATCH 06/30] WIP --- dd-smoke-tests/build.gradle | 12 + dd-smoke-tests/gradle.lockfile | 20 +- .../smoketest/backend/TestAgentBackend.java | 244 ++++++++++++++++++ .../backend/TestAgentTraceDecoder.java | 10 +- .../smoketest/backend/TraceBackend.java | 31 ++- .../TestAgentBackendContainerTest.java | 147 +++++++++++ .../backend/TestAgentBackendTest.java | 47 ++++ 7 files changed, 499 insertions(+), 12 deletions(-) create mode 100644 dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentBackend.java create mode 100644 dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendContainerTest.java create mode 100644 dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendTest.java diff --git a/dd-smoke-tests/build.gradle b/dd-smoke-tests/build.gradle index 15e070ea84d..4e72ff6a609 100644 --- a/dd-smoke-tests/build.gradle +++ b/dd-smoke-tests/build.gradle @@ -11,8 +11,20 @@ dependencies { compileOnly(libs.junit.jupiter) + // Testcontainers backs the test-agent .container() backend (TestAgentBackend). Kept compileOnly + // so it is NOT forced onto the ~79 modules that depend on this base for the mock/.external() + // backends; modules using .container() opt in with their own testImplementation dependency. + compileOnly(libs.testcontainers) + compileOnly(libs.bundles.groovy) compileOnly(libs.bundles.spock) + + testImplementation(libs.testcontainers) +} + +tasks.withType(Test).configureEach { + // Cap concurrent Testcontainers usage across the build (TestAgentBackend container tests). + usesService(testcontainersLimit) } tasks.withType(GroovyCompile).configureEach { diff --git a/dd-smoke-tests/gradle.lockfile b/dd-smoke-tests/gradle.lockfile index 5be43003ede..13823505b1d 100644 --- a/dd-smoke-tests/gradle.lockfile +++ b/dd-smoke-tests/gradle.lockfile @@ -13,6 +13,10 @@ com.datadoghq:dd-instrument-java:0.0.4=compileClasspath,runtimeClasspath,testCom com.datadoghq:dd-javac-plugin-client:0.2.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.datadoghq:java-dogstatsd-client:4.4.5=runtimeClasspath,testRuntimeClasspath com.datadoghq:sketches-java:0.8.3=runtimeClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-annotations:2.10.3=compileClasspath,testCompileClasspath,testRuntimeClasspath +com.github.docker-java:docker-java-api:3.4.2=compileClasspath,testCompileClasspath,testRuntimeClasspath +com.github.docker-java:docker-java-transport-zerodep:3.4.2=compileClasspath,testCompileClasspath,testRuntimeClasspath +com.github.docker-java:docker-java-transport:3.4.2=compileClasspath,testCompileClasspath,testRuntimeClasspath com.github.javaparser:javaparser-core:3.25.6=codenarc com.github.jnr:jffi:1.3.15=runtimeClasspath,testRuntimeClasspath com.github.jnr:jnr-a64asm:1.0.0=runtimeClasspath,testRuntimeClasspath @@ -47,20 +51,22 @@ io.leangen.geantyref:geantyref:1.3.16=testRuntimeClasspath io.sqreen:libsqreen:17.4.0=runtimeClasspath,testRuntimeClasspath javax.servlet:javax.servlet-api:3.1.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath jaxen:jaxen:2.0.0=spotbugs -junit:junit:4.13.2=runtimeClasspath,testRuntimeClasspath +junit:junit:4.13.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath net.bytebuddy:byte-buddy-agent:1.18.10=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath net.bytebuddy:byte-buddy:1.18.10=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath net.java.dev.jna:jna-platform:5.8.0=runtimeClasspath,testRuntimeClasspath -net.java.dev.jna:jna:5.8.0=runtimeClasspath,testRuntimeClasspath +net.java.dev.jna:jna:5.13.0=compileClasspath,testCompileClasspath,testRuntimeClasspath +net.java.dev.jna:jna:5.8.0=runtimeClasspath net.sf.saxon:Saxon-HE:12.9=spotbugs org.apache.ant:ant-antlr:1.10.14=codenarc org.apache.ant:ant-junit:1.10.14=codenarc org.apache.bcel:bcel:6.11.0=spotbugs +org.apache.commons:commons-compress:1.24.0=compileClasspath,testCompileClasspath,testRuntimeClasspath org.apache.commons:commons-lang3:3.19.0=spotbugs org.apache.commons:commons-text:1.14.0=spotbugs org.apache.logging.log4j:log4j-api:2.25.2=spotbugs org.apache.logging.log4j:log4j-core:2.25.2=spotbugs -org.apiguardian:apiguardian-api:1.1.2=compileClasspath,testCompileClasspath +org.apiguardian:apiguardian-api:1.1.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.codehaus.groovy:groovy-ant:3.0.23=codenarc org.codehaus.groovy:groovy-docgenerator:3.0.23=codenarc org.codehaus.groovy:groovy-groovydoc:3.0.23=codenarc @@ -73,10 +79,11 @@ org.codehaus.groovy:groovy:3.0.25=compileClasspath,testCompileClasspath,testRunt org.codenarc:CodeNarc:3.7.0=codenarc org.dom4j:dom4j:2.2.0=spotbugs org.gmetrics:GMetrics:2.1.0=codenarc -org.hamcrest:hamcrest-core:1.3=runtimeClasspath,testRuntimeClasspath +org.hamcrest:hamcrest-core:1.3=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.hamcrest:hamcrest:3.0=compileClasspath,testCompileClasspath,testRuntimeClasspath org.jctools:jctools-core-jdk11:4.0.6=runtimeClasspath,testRuntimeClasspath org.jctools:jctools-core:4.0.6=runtimeClasspath,testRuntimeClasspath +org.jetbrains:annotations:17.0.0=compileClasspath,testCompileClasspath,testRuntimeClasspath org.jspecify:jspecify:1.0.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.junit.jupiter:junit-jupiter-api:5.14.1=compileClasspath,testCompileClasspath,testRuntimeClasspath org.junit.jupiter:junit-jupiter-engine:5.14.1=testRuntimeClasspath @@ -104,15 +111,18 @@ org.ow2.asm:asm-util:9.7.1=runtimeClasspath,testRuntimeClasspath org.ow2.asm:asm-util:9.9=spotbugs org.ow2.asm:asm:9.10.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.ow2.asm:asm:9.9=spotbugs +org.rnorth.duct-tape:duct-tape:1.0.8=compileClasspath,testCompileClasspath,testRuntimeClasspath org.slf4j:jcl-over-slf4j:1.7.30=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.slf4j:jul-to-slf4j:1.7.30=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.slf4j:log4j-over-slf4j:1.7.30=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -org.slf4j:slf4j-api:1.7.32=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-api:1.7.32=runtimeClasspath +org.slf4j:slf4j-api:1.7.36=compileClasspath,testCompileClasspath,testRuntimeClasspath org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j org.spockframework:spock-bom:2.4-groovy-3.0=compileClasspath,testCompileClasspath,testRuntimeClasspath org.spockframework:spock-core:2.4-groovy-3.0=compileClasspath,testCompileClasspath,testRuntimeClasspath org.tabletest:tabletest-junit:1.2.1=testCompileClasspath,testRuntimeClasspath org.tabletest:tabletest-parser:1.2.0=testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers:1.21.4=compileClasspath,testCompileClasspath,testRuntimeClasspath org.xmlresolver:xmlresolver:5.3.3=spotbugs empty=annotationProcessor,spotbugsPlugins,testAnnotationProcessor diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentBackend.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentBackend.java new file mode 100644 index 00000000000..4cba222d2c7 --- /dev/null +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentBackend.java @@ -0,0 +1,244 @@ +package datadog.smoketest.backend; + +import datadog.trace.test.agent.decoder.DecodedTrace; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +/** + * {@link TraceBackend} backed by a + * dd-apm-test-agent — either a Testcontainers-managed container ({@link Builder#build() + * .container()}, the local-dev default) or an already-running external agent ({@link + * Builder#external(String, int)}, the CI sidecar). It reads received traces from the agent's {@code + * /test/session/traces} JSON endpoint and decodes them into the shared {@link DecodedTrace} model + * via {@link TestAgentTraceDecoder} (S1b), so a test body written against the common {@link Traces} + * surface runs unchanged against this backend or the {@link MockAgentBackend} (Q2). + * + *

Per-test isolation (Q4a, Option B). A shared external agent serves every test + * in a job, so traces are scoped by an {@code X-Datadog-Test-Session-Token}: the backend owns a + * token (see {@link #sessionToken()}), the launched app emits it via {@code + * dd.trace.agent.test.session.token} (S3a tracer change, wired by the S4 app launcher), {@link + * #clear()} opens a fresh session with it, and {@link #traces()} reads only that session's traces. + * + *

Testcontainers is a {@code compileOnly} dependency of the smoke base, so this class only loads + * when a test actually selects a test-agent backend (mock-only tests stay Testcontainers-free). + */ +public final class TestAgentBackend implements TraceBackend { + /** The dd-apm-test-agent trace port inside the container. */ + private static final int AGENT_PORT = 8126; + + /** + * Default image — the CI mirror used by {@code TracerConnectionReliabilityTest}. Override with + * {@link Builder#image(String)} or the {@code datadog.smoketest.testagent.image} system property + * (e.g. the public {@code ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent} for local runs + * without registry access). + */ + private static final String DEFAULT_IMAGE = + System.getProperty( + "datadog.smoketest.testagent.image", + "registry.ddbuild.io/images/mirror/dd-apm-test-agent/ddapm-test-agent:v1.44.0"); + + /** + * Trace-invariant checks enabled by default, mirroring {@code TracerConnectionReliabilityTest}. + */ + private static final List DEFAULT_ENABLED_CHECKS = + Arrays.asList("trace_count_header", "meta_tracer_version_header", "trace_content_length"); + + private final String image; + private final String externalHost; // null => Testcontainers-managed container + private final int externalPort; + private final List enabledChecks; + private final boolean shared; + private final String sessionToken; + + private final OkHttpClient client = new OkHttpClient(); + private volatile GenericContainer container; + private volatile HttpUrl baseUrl; + + private TestAgentBackend(Builder builder) { + this.image = builder.image; + this.externalHost = builder.externalHost; + this.externalPort = builder.externalPort; + this.enabledChecks = new ArrayList<>(builder.enabledChecks); + this.shared = builder.shared; + this.sessionToken = + builder.sessionToken != null ? builder.sessionToken : "smoke-" + UUID.randomUUID(); + } + + static Builder builder() { + return new Builder(); + } + + /** + * The session token this backend scopes its traces by. The launched app must emit it via {@code + * -Ddd.trace.agent.test.session.token=} so its traces land in this backend's session. + */ + public String sessionToken() { + return sessionToken; + } + + /** Whether this backend is meant to be shared across multiple apps (consumed by S6). */ + public boolean isShared() { + return shared; + } + + @Override + public void start() { + if (baseUrl != null) { + return; + } + if (externalHost != null) { + baseUrl = new HttpUrl.Builder().scheme("http").host(externalHost).port(externalPort).build(); + } else { + GenericContainer started = new GenericContainer<>(DockerImageName.parse(image)); + started.withExposedPorts(AGENT_PORT); + started.withEnv("ENABLED_CHECKS", String.join(",", enabledChecks)); + started.setWaitStrategy(Wait.forHttp("/test/traces")); + started.start(); + container = started; + baseUrl = + new HttpUrl.Builder() + .scheme("http") + .host(started.getHost()) + .port(started.getMappedPort(AGENT_PORT)) + .build(); + } + // Open a fresh session so the very first test method starts clean. + clear(); + } + + @Override + public int port() { + return requireStarted().port(); + } + + @Override + public URI url() { + HttpUrl url = requireStarted(); + // Clean base URI without HttpUrl's trailing "/", so callers can append their own path. + return URI.create(url.scheme() + "://" + url.host() + ":" + url.port()); + } + + @Override + public Traces traces() { + return new Traces(this::fetchTraces); + } + + @Override + public void clear() { + // GET /test/session/start begins (and clears) a session identified by the token. The + // dd-apm-test-agent session endpoints are GET (verified against v1.44.0: POST returns 405). + HttpUrl url = + requireStarted() + .newBuilder() + .addPathSegments("test/session/start") + .addQueryParameter("test_session_token", sessionToken) + .build(); + Request request = new Request.Builder().url(url).get().build(); + execute(request, "start test-agent session"); + } + + @Override + public void close() { + GenericContainer running = container; + if (running != null) { + container = null; + running.stop(); + } + baseUrl = null; + // TODO S5: on teardown of a container backend, query /test/trace_check/failures and fail the + // class if any enabled trace-invariant check failed (Q5). External CI agents are validated by + // the job-final .gitlab/check_test_agent_results.sh instead. + } + + private List fetchTraces() { + HttpUrl url = + requireStarted() + .newBuilder() + .addPathSegments("test/session/traces") + .addQueryParameter("test_session_token", sessionToken) + .build(); + Request request = new Request.Builder().url(url).get().build(); + return TestAgentTraceDecoder.decode(execute(request, "read test-agent session traces")); + } + + private String execute(Request request, String action) { + try (Response response = client.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new IllegalStateException( + "Failed to " + action + ": HTTP " + response.code() + " from " + request.url()); + } + return response.body() == null ? "" : response.body().string(); + } catch (IOException e) { + throw new IllegalStateException("Failed to " + action + " at " + request.url(), e); + } + } + + private HttpUrl requireStarted() { + HttpUrl url = baseUrl; + if (url == null) { + throw new IllegalStateException("TestAgentBackend not started — call start() first"); + } + return url; + } + + /** Fluent builder for a {@link TestAgentBackend}; obtain via {@code TraceBackend.testAgent()}. */ + public static final class Builder { + private String image = DEFAULT_IMAGE; + private String externalHost; + private int externalPort = AGENT_PORT; + private final List enabledChecks = new ArrayList<>(DEFAULT_ENABLED_CHECKS); + private boolean shared; + private String sessionToken; + + private Builder() {} + + /** + * Uses a Testcontainers-managed container of the given image (default {@link #DEFAULT_IMAGE}). + */ + public Builder image(String image) { + this.image = image; + return this; + } + + /** Overrides the enabled trace-invariant checks ({@code ENABLED_CHECKS}). */ + public Builder enabledChecks(String... checks) { + this.enabledChecks.clear(); + this.enabledChecks.addAll(Arrays.asList(checks)); + return this; + } + + /** Talks to an already-running external agent (e.g. the CI sidecar) instead of a container. */ + public Builder external(String host, int port) { + this.externalHost = host; + this.externalPort = port; + return this; + } + + /** Marks this backend as shared across multiple apps (multi-app wiring lands in S6). */ + public Builder shared() { + this.shared = true; + return this; + } + + /** Overrides the auto-generated session token (mainly for deterministic tests). */ + public Builder sessionToken(String token) { + this.sessionToken = token; + return this; + } + + public TestAgentBackend build() { + return new TestAgentBackend(this); + } + } +} diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentTraceDecoder.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentTraceDecoder.java index 52b65fb1001..006a457d9e3 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentTraceDecoder.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentTraceDecoder.java @@ -67,10 +67,12 @@ public List getSpans() { } } - // TODO verify these field names against a live ddapm-test-agent v1.44.0 /test/traces response. - // They follow the standard v0.4 trace shape (matching the msgpack Decoder's fields), but have - // not - // yet been validated end-to-end against the container — do so when S3/S8 run with Docker. + // Field names follow the standard v0.4 trace shape and are verified end-to-end against a live + // ddapm-test-agent v1.44.0 /test/session/traces response (S3 TestAgentBackendContainerTest): + // service/name/resource/type, trace_id/span_id/parent_id, start/duration/error, meta, metrics. + // TODO the exact meta_struct wire shape over JSON is not asserted yet (no meta_struct-bearing + // span + // in the fixture) — confirm when a smoke test needs meta_struct. static final class RawSpan implements DecodedSpan { String service; String name; diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java index 3bf184d6580..acc73a5fe59 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java @@ -2,6 +2,8 @@ import datadog.trace.test.agent.decoder.DecodedTrace; import java.net.URI; +import org.opentest4j.TestAbortedException; +import org.testcontainers.DockerClientFactory; /** * A pluggable trace backend a smoke-test app sends its traces to. Two implementations are planned: @@ -37,7 +39,30 @@ static MockAgentBackend mockAgent() { return new MockAgentBackend(); } - // TODO S3: testAgent() — a Testcontainers-managed .container() / external CI agent backend, plus - // the test-agent trace-invariant checks (ENABLED_CHECKS) and JSON decoding via - // TestAgentTraceDecoder. + /** Fluent builder for a {@link TestAgentBackend} (dd-apm-test-agent container or external). */ + static TestAgentBackend.Builder testAgent() { + return TestAgentBackend.builder(); + } + + /** + * Resolves the environment's default test-agent backend (Q4): reuse the external CI sidecar when + * {@code CI_AGENT_HOST} is set, else a Testcontainers-managed container when Docker is available, + * else skip loudly (a {@link TestAbortedException} marks the test as aborted rather than failed — + * a dev not running smoke tests isn't blocked). The mock backend is never an automatic fallback; + * select it explicitly with {@link #mockAgent()}. + * + *

The reusable conditional-execution annotation over this policy is S7. + */ + static TraceBackend testAgentFromEnv() { + String ciAgentHost = System.getenv("CI_AGENT_HOST"); + if (ciAgentHost != null && !ciAgentHost.isEmpty()) { + return testAgent().external(ciAgentHost, 8126).build(); + } + if (DockerClientFactory.instance().isDockerAvailable()) { + return testAgent().build(); + } + throw new TestAbortedException( + "No test-agent backend available: set CI_AGENT_HOST for an external agent, or start Docker " + + "for a container. Use TraceBackend.mockAgent() to opt into the in-process mock."); + } } diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendContainerTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendContainerTest.java new file mode 100644 index 00000000000..56f5c219d06 --- /dev/null +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendContainerTest.java @@ -0,0 +1,147 @@ +package datadog.smoketest.backend; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import datadog.trace.test.agent.decoder.DecodedSpan; +import datadog.trace.test.agent.decoder.DecodedTrace; +import datadog.trace.test.agent.decoder.Decoder; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.util.List; +import okhttp3.HttpUrl; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.DockerClientFactory; + +/** + * End-to-end tests for {@link TestAgentBackend} against a real dd-apm-test-agent container. Skipped + * (aborted) when Docker is unavailable. Simulates a launched app by submitting the recorded v0.4 + * msgpack payload to {@code /v0.4/traces} with the backend's session-token header, then verifies + * the backend reads and decodes exactly that session's traces via {@code /test/session/*} (S3a / + * Q4a). + * + *

Uses the public {@code ghcr.io} image so it runs without internal-registry access; real smoke + * tests default to the CI mirror (see {@link TestAgentBackend}). + */ +class TestAgentBackendContainerTest { + private static final String PUBLIC_IMAGE = + "ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.44.0"; + private static final MediaType MSGPACK = MediaType.parse("application/msgpack"); + private static final OkHttpClient CLIENT = new OkHttpClient(); + + private static byte[] v04Payload; + private static TestAgentBackend backend; + + @BeforeAll + static void setUp() throws IOException { + assumeTrue( + DockerClientFactory.instance().isDockerAvailable(), + "Docker is required for the test-agent container backend"); + v04Payload = readResource("/datadog/smoketest/backend/webflux.04.msgpack"); + backend = TraceBackend.testAgent().image(PUBLIC_IMAGE).build(); + backend.start(); + } + + @AfterAll + static void tearDown() { + if (backend != null) { + backend.close(); + } + } + + @BeforeEach + void freshSession() { + if (backend != null) { + backend.clear(); + } + } + + @Test + void capturesSessionScopedTraces() throws IOException { + submitAppTraces(backend.url(), backend.sessionToken(), v04Payload); + + Traces traces = backend.traces(); + traces.waitForTraceCount(1); + + List decoded = traces.getTraces(); + assertEquals(1, decoded.size(), "trace count"); + + List spans = Decoder.sortByStart(decoded.get(0).getSpans()); + assertEquals(2, spans.size(), "span count"); + DecodedSpan root = spans.get(0); + assertEquals("smoke-test-java-app", root.getService()); + assertEquals("netty.request", root.getName()); + assertEquals("GET /hello", root.getResource()); + assertEquals(0L, root.getParentId(), "root has no parent"); + assertEquals(root.getSpanId(), spans.get(1).getParentId(), "child parents the root"); + } + + @Test + void clearStartsAFreshSession() throws IOException { + submitAppTraces(backend.url(), backend.sessionToken(), v04Payload); + backend.traces().waitForTraceCount(1); + + backend.clear(); + + assertTrue(backend.traces().getTraces().isEmpty(), "clear() opens an empty session"); + } + + @Test + void externalBackendReadsFromRunningAgent() throws IOException { + // Point an .external() backend at the same running container: exercises the external code path + // (no container of its own) and, via its own fresh token, that sessions are isolated. + TestAgentBackend external = + TraceBackend.testAgent().external(backend.url().getHost(), backend.port()).build(); + external.start(); + try { + submitAppTraces(external.url(), external.sessionToken(), v04Payload); + + external.traces().waitForTraceCount(1); + assertEquals(1, external.traces().getTraces().size(), "external reads only its own session"); + } finally { + external.close(); + } + } + + private static void submitAppTraces(URI agentUrl, String token, byte[] payload) + throws IOException { + HttpUrl url = HttpUrl.get(agentUrl).newBuilder().addPathSegments("v0.4/traces").build(); + Request request = + new Request.Builder() + .url(url) + .header("X-Datadog-Trace-Count", "1") + .header("Datadog-Meta-Tracer-Version", "0.0.0-smoke-test") + .header("X-Datadog-Test-Session-Token", token) + .put(RequestBody.create(MSGPACK, payload)) + .build(); + try (Response response = CLIENT.newCall(request).execute()) { + assertTrue( + response.isSuccessful(), "test agent accepts trace submission: HTTP " + response.code()); + } + } + + private static byte[] readResource(String name) throws IOException { + try (InputStream in = TestAgentBackendContainerTest.class.getResourceAsStream(name)) { + assertNotNull(in, "missing test resource " + name); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int read; + while ((read = in.read(buffer)) != -1) { + out.write(buffer, 0, read); + } + return out.toByteArray(); + } + } +} diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendTest.java new file mode 100644 index 00000000000..82ab49a136a --- /dev/null +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendTest.java @@ -0,0 +1,47 @@ +package datadog.smoketest.backend; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +/** + * Docker-free unit tests for {@link TestAgentBackend} configuration/lifecycle guards. The container + * and session-capture behavior is exercised end-to-end against a real agent in {@link + * TestAgentBackendContainerTest}. + */ +class TestAgentBackendTest { + + @Test + void sessionTokenIsStableAndNonEmpty() { + TestAgentBackend backend = TraceBackend.testAgent().build(); + String token = backend.sessionToken(); + assertNotNull(token); + assertFalse(token.isEmpty()); + assertEquals(token, backend.sessionToken(), "token is stable across calls"); + } + + @Test + void sessionTokenCanBeOverridden() { + assertEquals( + "fixed-token", + TraceBackend.testAgent().sessionToken("fixed-token").build().sessionToken(), + "explicit token wins over the auto-generated one"); + } + + @Test + void sharedFlagDefaultsFalseAndIsSettable() { + assertFalse(TraceBackend.testAgent().build().isShared(), "not shared by default"); + assertTrue(TraceBackend.testAgent().shared().build().isShared(), "shared() opts in"); + } + + @Test + void accessBeforeStartFails() { + TestAgentBackend backend = TraceBackend.testAgent().build(); + assertThrows(IllegalStateException.class, backend::url, "url() before start()"); + assertThrows(IllegalStateException.class, backend::port, "port() before start()"); + } +} From 4c0459de5f6504e76ba5c516de5cefc0ba42465c Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Mon, 13 Jul 2026 17:56:54 +0200 Subject: [PATCH 07/30] WIP --- .../main/java/datadog/smoketest/SmokeApp.java | 429 ++++++++++++++++++ .../smoketest/backend/TestAgentBackend.java | 2 + .../smoketest/backend/TraceBackend.java | 17 + .../java/datadog/smoketest/SmokeAppTest.java | 56 +++ .../java/datadog/smoketest/TestServerApp.java | 45 ++ 5 files changed, 549 insertions(+) create mode 100644 dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java create mode 100644 dd-smoke-tests/src/test/java/datadog/smoketest/SmokeAppTest.java create mode 100644 dd-smoke-tests/src/test/java/datadog/smoketest/TestServerApp.java diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java b/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java new file mode 100644 index 00000000000..fd100889264 --- /dev/null +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java @@ -0,0 +1,429 @@ +package datadog.smoketest; + +import datadog.smoketest.backend.TraceBackend; +import datadog.smoketest.backend.Traces; +import datadog.trace.agent.test.utils.PortUtils; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Function; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolutionException; +import org.junit.jupiter.api.extension.ParameterResolver; + +/** + * A smoke-test application launched in its own JVM, managed as a JUnit 5 extension. Declare it as a + * {@code static @RegisterExtension} field: its {@link #beforeAll} launches the app (and its owned + * {@link TraceBackend}) once per class, {@link #beforeEach} resets per method, and {@link + * #afterAll} tears everything down — no {@code @TestInstance(PER_CLASS)} required (Q7). Access the + * handle either through the field ({@code app.get(...)}, {@code app.traces()}) or via {@link + * ParameterResolver} injection of {@link SmokeApp}/{@link Traces} into a test method (Q7). + * + * 

{@code
+ * @RegisterExtension
+ * static final SmokeApp app = SmokeApp.named("springboot")
+ *     .jar(System.getProperty("datadog.smoketest.springboot.shadowJar.path"))
+ *     .args("--server.port=${app.httpPort}")
+ *     .backend(TraceBackend.mockAgent())
+ *     .build();
+ * }
+ * + *

Core launch capabilities are reproduced here (Q6); several are deliberately deferred and + * marked with explicit {@code // TODO}s below so the gaps are discoverable rather than silent. + */ +public final class SmokeApp + implements BeforeAllCallback, AfterAllCallback, BeforeEachCallback, ParameterResolver { + + // Defaults mirroring the Groovy ProcessManager base so ported tests behave the same. + private static final String SERVICE_NAME = "smoke-test-java-app"; + private static final String ENV = "smoketest"; + private static final String VERSION = "99"; + private static final String API_KEY = "01234567890abcdef123456789ABCDEF"; + private static final Set NOISY_ENVIRONMENT_VARIABLES = + new HashSet<>(Arrays.asList("CI_COMMIT_TITLE", "CI_COMMIT_MESSAGE", "CI_COMMIT_DESCRIPTION")); + private static final String AGENT_JAR_PROPERTY = "datadog.smoketest.agent.shadowJar.path"; + private static final String BUILD_DIR_PROPERTY = "datadog.smoketest.builddir"; + private static final String HTTP_PORT_PLACEHOLDER = "${app.httpPort}"; + + private final String name; + private final String jar; + private final String mainClass; + private final String classpath; + private final List jvmArgs; + private final List programArgs; + private final Map extraEnv; + private final File workingDirectory; + private final TraceBackend backend; + private final boolean ownsBackend; + private final String agentJar; // null => launch without -javaagent + private final boolean server; // wait for the HTTP port to open on start + private final long startupTimeoutSeconds; + private final int httpPort; + + private final OkHttpClient httpClient = new OkHttpClient(); + private final OutputThreads outputThreads = new OutputThreads(); + private Process process; + + private SmokeApp(Builder builder) { + this.name = builder.name; + this.jar = builder.jar; + this.mainClass = builder.mainClass; + this.classpath = + builder.classpath != null ? builder.classpath : System.getProperty("java.class.path"); + this.jvmArgs = new ArrayList<>(builder.jvmArgs); + this.programArgs = new ArrayList<>(builder.programArgs); + this.extraEnv = new HashMap<>(builder.extraEnv); + this.workingDirectory = builder.workingDirectory; + this.backend = builder.backend; + this.ownsBackend = !builder.backend.isShared(); + this.agentJar = builder.resolveAgentJar(); + this.server = builder.server; + this.startupTimeoutSeconds = builder.startupTimeoutSeconds; + this.httpPort = PortUtils.randomOpenPort(); + } + + /** Starts a fluent builder for an app with the given (log/diagnostic) name. */ + public static Builder named(String name) { + return new Builder(name); + } + + // --- Handle API (field access) --- + + /** + * The randomly-allocated port a server app should bind (substituted for {@value + * #HTTP_PORT_PLACEHOLDER}). + */ + public int httpPort() { + return httpPort; + } + + /** Base URL of the app's HTTP server. */ + public URI url() { + return URI.create("http://localhost:" + httpPort); + } + + /** + * Issues a GET to the app and returns the HTTP status code (the response is drained and closed). + */ + public int get(String path) { + String full = url() + (path.startsWith("/") ? path : "/" + path); + Request request = new Request.Builder().url(full).get().build(); + try (Response response = httpClient.newCall(request).execute()) { + return response.code(); + } catch (IOException e) { + throw new IllegalStateException("GET " + full + " failed", e); + } + } + + /** The trace query/assert facade of this app's backend. */ + public Traces traces() { + return backend.traces(); + } + + /** The backend this app sends traces to. */ + public TraceBackend backend() { + return backend; + } + + /** + * Waits (up to the log helper's timeout) for a captured stdout/stderr line matching {@code + * predicate}; returns {@code false} on timeout. Lines are reset per test method. + */ + public boolean awaitLogLine(Function predicate) { + try { + return outputThreads.processTestLogLines(predicate); + } catch (TimeoutException e) { + return false; + } + } + + // --- Lifecycle (per-class start, per-method reset, teardown) --- + + @Override + public void beforeAll(ExtensionContext context) throws Exception { + if (ownsBackend) { + backend.start(); + } + launch(); + if (server) { + PortUtils.waitForPortToOpen(httpPort, startupTimeoutSeconds, TimeUnit.SECONDS, process); + } else if (!process.isAlive() && process.exitValue() != 0) { + throw new IllegalStateException( + "App '" + name + "' exited abnormally on start (exit " + process.exitValue() + ")"); + } + } + + @Override + public void beforeEach(ExtensionContext context) { + if (process == null || !process.isAlive()) { + throw new IllegalStateException("App '" + name + "' is not alive at the start of a test"); + } + outputThreads.clearMessages(); + if (ownsBackend) { + backend.clear(); + } + } + + @Override + public void afterAll(ExtensionContext context) { + try { + stopProcess(); + } finally { + outputThreads.close(); + if (ownsBackend) { + backend.close(); + } + } + } + + private void launch() throws IOException { + List command = new ArrayList<>(); + command.add(javaExecutable()); + + if (agentJar != null) { + command.add("-javaagent:" + agentJar); + command.add("-Ddd.trace.agent.host=" + backend.url().getHost()); + command.add("-Ddd.trace.agent.port=" + backend.port()); + command.add("-Ddd.service.name=" + SERVICE_NAME); + command.add("-Ddd.env=" + ENV); + command.add("-Ddd.version=" + VERSION); + String sessionToken = backend.sessionToken(); + if (sessionToken != null) { + command.add("-Ddd.trace.agent.test.session.token=" + sessionToken); + } + } + for (String jvmArg : jvmArgs) { + command.add(substitute(jvmArg)); + } + if (jar != null) { + command.add("-jar"); + command.add(jar); + } else { + command.add("-cp"); + command.add(classpath); + command.add(mainClass); + } + for (String programArg : programArgs) { + command.add(substitute(programArg)); + } + + ProcessBuilder processBuilder = new ProcessBuilder(command); + if (workingDirectory != null) { + processBuilder.directory(workingDirectory); + } + Map env = processBuilder.environment(); + env.put("JAVA_HOME", System.getProperty("java.home")); + env.put("DD_API_KEY", API_KEY); + env.keySet().removeAll(NOISY_ENVIRONMENT_VARIABLES); + env.putAll(extraEnv); + processBuilder.redirectErrorStream(true); + + process = processBuilder.start(); + outputThreads.captureOutput(process, logFile()); + } + + private void stopProcess() { + if (process == null) { + return; + } + if (!process.isAlive()) { + return; + } + process.destroy(); + try { + if (!process.waitFor(5, TimeUnit.SECONDS)) { + process.destroyForcibly(); + process.waitFor(10, TimeUnit.SECONDS); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + process.destroyForcibly(); + } + } + + private String substitute(String value) { + return value.replace(HTTP_PORT_PLACEHOLDER, Integer.toString(httpPort)); + } + + private File logFile() { + String buildDir = System.getProperty(BUILD_DIR_PROPERTY); + File dir = + buildDir != null + ? new File(buildDir, "reports") + : new File(System.getProperty("java.io.tmpdir")); + dir.mkdirs(); + // TODO Q6 (deferred): retry-safe timestamped log file names so retries don't clobber prior + // logs. + return new File(dir, "smoke-app." + name + ".log"); + } + + private static String javaExecutable() { + return System.getProperty("java.home") + File.separator + "bin" + File.separator + "java"; + } + + // --- ParameterResolver (injection by type; qualifier for multi-instance is deferred) --- + + @Override + public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext context) { + Class type = parameterContext.getParameter().getType(); + return type == SmokeApp.class || type == Traces.class || type == TraceBackend.class; + } + + @Override + public Object resolveParameter(ParameterContext parameterContext, ExtensionContext context) { + Class type = parameterContext.getParameter().getType(); + if (type == SmokeApp.class) { + return this; + } + if (type == Traces.class) { + return traces(); + } + if (type == TraceBackend.class) { + return backend; + } + throw new ParameterResolutionException("Cannot resolve parameter of type " + type); + // TODO Q7: with multiple same-type apps/backends injection is ambiguous — add a qualifier + // annotation (e.g. @App("producer")) when a multi-app test needs parameter injection (S6). + } + + /** Fluent builder for a {@link SmokeApp}. */ + public static final class Builder { + private final String name; + private String jar; + private String mainClass; + private String classpath; + private final List jvmArgs = new ArrayList<>(); + private final List programArgs = new ArrayList<>(); + private final Map extraEnv = new HashMap<>(); + private File workingDirectory; + private TraceBackend backend; + private String explicitAgentJar; + private boolean noAgent; + private boolean server = true; + private long startupTimeoutSeconds = 120; + + private Builder(String name) { + this.name = name; + } + + /** Runs {@code java -jar }. Mutually exclusive with {@link #mainClass(String)}. */ + public Builder jar(String jarPath) { + this.jar = jarPath; + return this; + } + + /** Runs {@code java -cp } (classpath defaults to the current one). */ + public Builder mainClass(String mainClass) { + this.mainClass = mainClass; + return this; + } + + /** Classpath for {@link #mainClass(String)} mode; defaults to the launching JVM's classpath. */ + public Builder classpath(String classpath) { + this.classpath = classpath; + return this; + } + + /** + * Program arguments (after the jar/main class). Supports the {@value #HTTP_PORT_PLACEHOLDER} + * placeholder. + */ + public Builder args(String... args) { + this.programArgs.addAll(Arrays.asList(args)); + return this; + } + + /** + * Extra JVM arguments (before the jar/main class). Supports the {@value #HTTP_PORT_PLACEHOLDER} + * placeholder. + */ + public Builder jvmArgs(String... jvmArgs) { + this.jvmArgs.addAll(Arrays.asList(jvmArgs)); + return this; + } + + /** Sets an environment variable for the launched process (applied after the defaults). */ + public Builder env(String key, String value) { + this.extraEnv.put(key, value); + return this; + } + + /** Working directory for the launched process. */ + public Builder workingDirectory(File workingDirectory) { + this.workingDirectory = workingDirectory; + return this; + } + + /** + * The backend the app sends traces to; started/stopped by this app unless it is shared (S6). + */ + public Builder backend(TraceBackend backend) { + this.backend = backend; + return this; + } + + /** + * Overrides the agent jar (default: the {@code datadog.smoketest.agent.shadowJar.path} + * property). + */ + public Builder javaAgent(String agentJarPath) { + this.explicitAgentJar = agentJarPath; + return this; + } + + /** Launches the app without {@code -javaagent} (e.g. for launch-mechanics tests). */ + public Builder noAgent() { + this.noAgent = true; + return this; + } + + /** Declares the app is not an HTTP server, so start-up won't wait for a port to open. */ + public Builder notAServer() { + this.server = false; + return this; + } + + /** How long start-up waits for a server app's port to open (default 120s). */ + public Builder startupTimeoutSeconds(long seconds) { + this.startupTimeoutSeconds = seconds; + return this; + } + + private String resolveAgentJar() { + if (noAgent) { + return null; + } + return explicitAgentJar != null ? explicitAgentJar : System.getProperty(AGENT_JAR_PROPERTY); + } + + public SmokeApp build() { + if (backend == null) { + throw new IllegalStateException("A TraceBackend is required — call backend(...)"); + } + if ((jar == null) == (mainClass == null)) { + throw new IllegalStateException("Exactly one of jar(...) or mainClass(...) must be set"); + } + // TODO Q6 (deferred, opt-in mixins / .jvmArgs(...) escape hatch — not baked into the base): + // profiling args; crash-tracking args (-XX:OnError=...dd_crash_uploader.sh); memory tuning + // (ForkedTestUtils); error-log assertion helpers (assertNoErrorLogs + FIXME allowlist); + // retry log-file timestamping. Add as explicit opt-ins when a ported test needs them. + return new SmokeApp(this); + } + } +} diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentBackend.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentBackend.java index 4cba222d2c7..81ec8eb86da 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentBackend.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentBackend.java @@ -83,11 +83,13 @@ static Builder builder() { * The session token this backend scopes its traces by. The launched app must emit it via {@code * -Ddd.trace.agent.test.session.token=} so its traces land in this backend's session. */ + @Override public String sessionToken() { return sessionToken; } /** Whether this backend is meant to be shared across multiple apps (consumed by S6). */ + @Override public boolean isShared() { return shared; } diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java index acc73a5fe59..89ab753958d 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java @@ -34,6 +34,23 @@ public interface TraceBackend extends AutoCloseable { @Override void close(); + /** + * The session token the launched app must emit (via {@code dd.trace.agent.test.session.token}) + * for its traces to be attributed to this backend, or {@code null} if the backend does not scope + * by session (e.g. the in-process mock, which owns its own server). Overridden by the test agent. + */ + default String sessionToken() { + return null; + } + + /** + * Whether this backend manages its own lifecycle as a separate {@code @RegisterExtension} shared + * across apps (S6). When {@code false} (default), the owning {@code SmokeApp} starts/stops it. + */ + default boolean isShared() { + return false; + } + /** Creates an in-process mock-agent backend wrapping the testing {@code JavaTestHttpServer}. */ static MockAgentBackend mockAgent() { return new MockAgentBackend(); diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/SmokeAppTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/SmokeAppTest.java new file mode 100644 index 00000000000..dc75df5151a --- /dev/null +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/SmokeAppTest.java @@ -0,0 +1,56 @@ +package datadog.smoketest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.smoketest.backend.TraceBackend; +import datadog.smoketest.backend.Traces; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Exercises {@link SmokeApp}'s launch mechanics end-to-end against a trivial JVM app ({@link + * TestServerApp}): port allocation + {@code ${app.httpPort}} substitution, process launch, HTTP + * reachability, stdout capture, owned-backend lifecycle, and parameter injection. Runs without the + * agent (mechanics only); a real agent + instrumented app + trace assertions land in the S8 pilot. + */ +class SmokeAppTest { + + @RegisterExtension + static final SmokeApp app = + SmokeApp.named("test-server") + .mainClass("datadog.smoketest.TestServerApp") + .args("--server.port=${app.httpPort}") + .backend(TraceBackend.mockAgent()) + .noAgent() + .build(); + + @Test + void respondsOnTheAllocatedPort() { + assertTrue(app.httpPort() > 0, "a port was allocated"); + // Reaching the app proves ${app.httpPort} was substituted into the launch args. + assertEquals(200, app.get("/hello"), "app serves HTTP on the substituted port"); + } + + @Test + void capturesApplicationLogOutput() { + app.get("/ping"); + assertTrue( + app.awaitLogLine(line -> line.contains("REQUEST GET /ping")), + "app stdout is captured during the test"); + } + + @Test + void ownsAndStartsItsBackend() { + assertNotNull(app.backend().url(), "the owned backend was started before the app"); + assertTrue(app.traces().getTraces().isEmpty(), "no traces arrive without an agent"); + } + + @Test + void injectsHandlesByType(SmokeApp injected, Traces traces) { + assertSame(app, injected, "SmokeApp resolved by type"); + assertNotNull(traces, "Traces resolved by type"); + } +} diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/TestServerApp.java b/dd-smoke-tests/src/test/java/datadog/smoketest/TestServerApp.java new file mode 100644 index 00000000000..e53e9989890 --- /dev/null +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/TestServerApp.java @@ -0,0 +1,45 @@ +package datadog.smoketest; + +import com.sun.net.httpserver.HttpServer; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CountDownLatch; + +/** + * Minimal HTTP server launched by {@link SmokeAppTest} to exercise {@link SmokeApp}'s + * launch/log-capture/lifecycle mechanics without needing the agent or a full sample app. Reads + * {@code --server.port=} (substituted by {@code SmokeApp} from {@code ${app.httpPort}}), + * echoes each request to stdout so log capture can be asserted, and blocks until the process is + * destroyed. + */ +public final class TestServerApp { + private TestServerApp() {} + + public static void main(String[] args) throws Exception { + int port = 0; + for (String arg : args) { + if (arg.startsWith("--server.port=")) { + port = Integer.parseInt(arg.substring("--server.port=".length())); + } + } + + HttpServer server = HttpServer.create(new InetSocketAddress("localhost", port), 0); + server.createContext( + "/", + exchange -> { + System.out.println( + "REQUEST " + exchange.getRequestMethod() + " " + exchange.getRequestURI().getPath()); + byte[] body = "ok".getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(body); + } + }); + server.start(); + System.out.println("TestServerApp listening on " + port); + + // Block forever; SmokeApp destroys the process at teardown. + new CountDownLatch(1).await(); + } +} From 7b4d3093771ce7c4f1892723cc08525673f04847 Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Mon, 13 Jul 2026 18:23:44 +0200 Subject: [PATCH 08/30] WIP --- .../smoketest/backend/TestAgentBackend.java | 48 ++++++++++++--- .../datadog/smoketest/backend/Traces.java | 20 ++++++- .../backend/MockAgentBackendTest.java | 15 +++++ .../backend/TestAgentBackendTest.java | 58 +++++++++++++++++++ 4 files changed, 132 insertions(+), 9 deletions(-) diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentBackend.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentBackend.java index 81ec8eb86da..4f87083f8df 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentBackend.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentBackend.java @@ -153,14 +153,48 @@ public void clear() { @Override public void close() { GenericContainer running = container; - if (running != null) { - container = null; - running.stop(); + try { + // Container backends auto-validate their trace-invariant checks at teardown (Q5). External CI + // agents are validated by the job-final .gitlab/check_test_agent_results.sh instead, so we + // don't check them here. Run before stopping so the agent is still reachable. + if (running != null) { + assertNoInvariantFailures(); + } + } finally { + if (running != null) { + container = null; + running.stop(); + } + baseUrl = null; + } + } + + /** + * Fails (with an {@link AssertionError}) if the test agent recorded any trace-invariant check + * failure ({@code ENABLED_CHECKS}) for this backend's session. Auto-invoked at container teardown + * (Q5); may also be called explicitly against an external agent mid-test. + */ + public void assertNoInvariantFailures() { + HttpUrl url = + requireStarted() + .newBuilder() + .addPathSegments("test/trace_check/failures") + .addQueryParameter("test_session_token", sessionToken) + .build(); + Request request = new Request.Builder().url(url).get().build(); + try (Response response = client.newCall(request).execute()) { + int code = response.code(); + // 200 => all checks passed; 404 => a real agent is running (no checks). Anything else is a + // recorded failure, whose body describes the failing check(s) (see check_test_agent_results). + if (code == 200 || code == 404) { + return; + } + String body = response.body() == null ? "" : response.body().string(); + throw new AssertionError( + "Test-agent trace-invariant checks failed (HTTP " + code + "):\n" + body); + } catch (IOException e) { + throw new IllegalStateException("Failed to query trace-invariant checks at " + url, e); } - baseUrl = null; - // TODO S5: on teardown of a container backend, query /test/trace_check/failures and fail the - // class if any enabled trace-invariant check failed (Q5). External CI agents are validated by - // the job-final .gitlab/check_test_agent_results.sh instead. } private List fetchTraces() { diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Traces.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Traces.java index 57409ace8b8..a047a8dd7c8 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Traces.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Traces.java @@ -1,5 +1,7 @@ package datadog.smoketest.backend; +import datadog.smoketest.trace.SmokeTraceAssertions; +import datadog.smoketest.trace.TraceMatcher; import datadog.trace.test.agent.decoder.DecodedTrace; import datadog.trace.test.util.PollingConditions; import java.util.List; @@ -51,6 +53,20 @@ public Traces waitForTraceCount(int count, double timeoutSeconds) { return this; } - // TODO S5: fluent assertTraces(TraceMatcher...) delegating to the smoke matcher, and (test-agent - // backends only) trace-invariant check assertions over /test/trace_check/failures. + /** + * Asserts the received traces against the thin smoke matcher — one {@link TraceMatcher} per + * expected trace (matched positionally). Sugar over {@link SmokeTraceAssertions#assertTraces}: + * + * 

{@code
+   * app.traces().waitForTraceCount(1)
+   *    .assertTraces(trace(span().operationName("servlet.request").resourceName("GET /greeting")));
+   * }
+ */ + public void assertTraces(TraceMatcher... matchers) { + SmokeTraceAssertions.assertTraces(getTraces(), matchers); + } + + // Trace-invariant checks (ENABLED_CHECKS) are a test-agent-specific concern, validated by that + // backend itself (TestAgentBackend#assertNoInvariantFailures, auto-run at container teardown per + // Q5) rather than on this common facade, which stays portable across both backends (Q2). } diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java index d7ba5f51cb1..f2ef6fcc1e8 100644 --- a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java @@ -1,5 +1,7 @@ package datadog.smoketest.backend; +import static datadog.smoketest.trace.SpanMatcher.span; +import static datadog.smoketest.trace.TraceMatcher.trace; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -87,6 +89,19 @@ void receivesAndDecodesSubmittedTraces() throws IOException { assertEquals(root.getSpanId(), child.getParentId(), "child parents the root span"); } + @Test + void assertTracesFacadeMatchesDecodedTraces() throws IOException { + putTraces("/v0.4/traces", v04Payload); + + // The fluent facade (S5) chains count-polling into the thin smoke matcher (S1). Both fixture + // spans share the service, so this holds regardless of the received span order. + backend + .traces() + .waitForTraceCount(1) + .assertTraces( + trace(span().service("smoke-test-java-app"), span().service("smoke-test-java-app"))); + } + @Test void accumulatesTracesAcrossSubmissions() throws IOException { putTraces("/v0.4/traces", v04Payload); diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendTest.java index 82ab49a136a..446d7a0453a 100644 --- a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendTest.java +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendTest.java @@ -6,6 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import datadog.trace.agent.test.server.http.JavaTestHttpServer; import org.junit.jupiter.api.Test; /** @@ -44,4 +45,61 @@ void accessBeforeStartFails() { assertThrows(IllegalStateException.class, backend::url, "url() before start()"); assertThrows(IllegalStateException.class, backend::port, "port() before start()"); } + + @Test + void assertNoInvariantFailuresPassesWhenAgentReportsNoFailures() { + // A stub agent for /test/session/* and /test/trace_check/failures verifies the check logic + // without Docker; HTTP 200 from the failures endpoint means all checks passed. + try (JavaTestHttpServer agent = stubAgent(200, "")) { + TestAgentBackend backend = + TraceBackend.testAgent() + .external(agent.getAddress().getHost(), agent.getAddress().getPort()) + .build(); + backend.start(); + try { + backend.assertNoInvariantFailures(); // HTTP 200 => no failures => no throw + } finally { + backend.close(); + } + } + } + + @Test + void assertNoInvariantFailuresThrowsWhenAgentReportsFailures() { + try (JavaTestHttpServer agent = stubAgent(400, "span_count check failed")) { + TestAgentBackend backend = + TraceBackend.testAgent() + .external(agent.getAddress().getHost(), agent.getAddress().getPort()) + .build(); + backend.start(); + try { + AssertionError error = + assertThrows(AssertionError.class, backend::assertNoInvariantFailures); + assertTrue(error.getMessage().contains("span_count check failed"), error.getMessage()); + } finally { + backend.close(); + } + } + } + + /** A stub test agent: 200 on {@code /test/session/start}, {@code failuresStatus} on failures. */ + private static JavaTestHttpServer stubAgent(int failuresStatus, String failuresBody) { + return JavaTestHttpServer.httpServer( + server -> + server.handlers( + handlers -> { + handlers.prefix( + "/test/session/start", api -> api.getResponse().status(200).send()); + handlers.prefix( + "/test/trace_check/failures", + api -> { + if (failuresBody.isEmpty()) { + api.getResponse().status(failuresStatus).send(); + } else { + api.getResponse().status(failuresStatus).send(failuresBody); + } + }); + handlers.all(api -> api.getResponse().status(200).send()); + })); + } } From 794b30b1a2358df4d8d7a0498516668471b848e2 Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Mon, 13 Jul 2026 18:40:36 +0200 Subject: [PATCH 09/30] WIP --- .../main/java/datadog/smoketest/SmokeApp.java | 8 ++- .../smoketest/backend/MockAgentBackend.java | 16 +++++ .../smoketest/backend/TraceBackend.java | 27 ++++++- .../smoketest/SharedBackendMultiAppTest.java | 70 +++++++++++++++++++ 4 files changed, 117 insertions(+), 4 deletions(-) create mode 100644 dd-smoke-tests/src/test/java/datadog/smoketest/SharedBackendMultiAppTest.java diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java b/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java index fd100889264..86a6446b936 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java @@ -157,9 +157,11 @@ public boolean awaitLogLine(Function predicate) { @Override public void beforeAll(ExtensionContext context) throws Exception { - if (ownsBackend) { - backend.start(); - } + // Always start the backend before launching (start() is idempotent). This makes the app robust + // to @RegisterExtension callback ordering: a shared backend also self-starts via its own + // beforeAll, and an inline (owned) backend is started only here. Ownership governs + // clear()/close. + backend.start(); launch(); if (server) { PortUtils.waitForPortToOpen(httpPort, startupTimeoutSeconds, TimeUnit.SECONDS, process); diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/MockAgentBackend.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/MockAgentBackend.java index ec8a32e62ac..935e7a9ecef 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/MockAgentBackend.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/MockAgentBackend.java @@ -28,9 +28,25 @@ public final class MockAgentBackend implements TraceBackend { private final List traces = new CopyOnWriteArrayList<>(); private volatile JavaTestHttpServer server; + private boolean shared; MockAgentBackend() {} + /** + * Marks this backend as shared across multiple apps — declare it as its own + * {@code @RegisterExtension} field so it is started/reset/closed once rather than by each app + * (S6/Q8). + */ + public MockAgentBackend shared() { + this.shared = true; + return this; + } + + @Override + public boolean isShared() { + return shared; + } + @Override public void start() { if (server != null) { diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java index 89ab753958d..a43578c3a91 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java @@ -2,6 +2,10 @@ import datadog.trace.test.agent.decoder.DecodedTrace; import java.net.URI; +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; import org.opentest4j.TestAbortedException; import org.testcontainers.DockerClientFactory; @@ -15,7 +19,8 @@ *

The lifecycle mirrors the JUnit extension that will own the backend (S4/S6): {@link #start()} * once per test class, {@link #clear()} between methods, {@link #close()} at teardown. */ -public interface TraceBackend extends AutoCloseable { +public interface TraceBackend + extends AutoCloseable, BeforeAllCallback, BeforeEachCallback, AfterAllCallback { /** Starts the backend and binds it to a port. Idempotent. */ void start(); @@ -51,6 +56,26 @@ default boolean isShared() { return false; } + // JUnit lifecycle: a backend declared as its own `@RegisterExtension` field (shared across apps, + // S6/Q8) drives its own start/clear/close. An inline backend passed to `SmokeApp.backend(...)` is + // not registered as an extension, so `SmokeApp` drives it instead. start() is idempotent, so a + // shared backend that an app also (defensively) starts is unaffected. + + @Override + default void beforeAll(ExtensionContext context) { + start(); + } + + @Override + default void beforeEach(ExtensionContext context) { + clear(); + } + + @Override + default void afterAll(ExtensionContext context) { + close(); + } + /** Creates an in-process mock-agent backend wrapping the testing {@code JavaTestHttpServer}. */ static MockAgentBackend mockAgent() { return new MockAgentBackend(); diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/SharedBackendMultiAppTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/SharedBackendMultiAppTest.java new file mode 100644 index 00000000000..8f2b8a9a5cd --- /dev/null +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/SharedBackendMultiAppTest.java @@ -0,0 +1,70 @@ +package datadog.smoketest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.smoketest.backend.MockAgentBackend; +import datadog.smoketest.backend.TraceBackend; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Multi-app composition with a shared backend (Q8): a single {@code @RegisterExtension} backend + * declared before two {@link SmokeApp} fields, each launching its own JVM. The shared backend is + * started/reset/closed by its own extension — the apps reference it (via field access) but + * don't own its lifecycle. Start-up is order-independent because {@code SmokeApp} starts the + * backend idempotently. + * + *

Runs without the agent, so it asserts the composition wiring (distinct app ports, one shared + * backend instance) rather than trace flow — cross-app trace assertions against a shared agent land + * in the S8 pilot. + */ +class SharedBackendMultiAppTest { + + @RegisterExtension static final MockAgentBackend agent = TraceBackend.mockAgent().shared(); + + @RegisterExtension + static final SmokeApp producer = + SmokeApp.named("producer") + .mainClass("datadog.smoketest.TestServerApp") + .args("--server.port=${app.httpPort}") + .backend(agent) + .noAgent() + .build(); + + @RegisterExtension + static final SmokeApp consumer = + SmokeApp.named("consumer") + .mainClass("datadog.smoketest.TestServerApp") + .args("--server.port=${app.httpPort}") + .backend(agent) + .noAgent() + .build(); + + @Test + void bothAppsRunOnDistinctPorts() { + assertNotEquals(producer.httpPort(), consumer.httpPort(), "each app gets its own port"); + assertEquals(200, producer.get("/"), "producer serves HTTP"); + assertEquals(200, consumer.get("/"), "consumer serves HTTP"); + } + + @Test + void appsShareTheSameBackend() { + assertTrue(agent.isShared(), "backend is marked shared"); + assertSame(agent, producer.backend(), "producer uses the shared backend"); + assertSame(agent, consumer.backend(), "consumer uses the shared backend"); + assertEquals( + producer.backend().port(), + consumer.backend().port(), + "one shared backend => one agent port for both apps"); + } + + @Test + void sharedBackendIsStartedByItsOwnExtension() { + assertNotNull(agent.url(), "shared backend was started"); + assertTrue(agent.traces().getTraces().isEmpty(), "no traces arrive without an agent"); + } +} From 69aa5d367deeb2aa85e04002772a9ba4505ed1c3 Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Mon, 13 Jul 2026 19:06:53 +0200 Subject: [PATCH 10/30] WIP --- .../backend/DockerAvailableCondition.java | 47 +++++++++++++++++++ .../backend/EnabledIfDockerAvailable.java | 24 ++++++++++ .../backend/DockerAvailableConditionTest.java | 25 ++++++++++ .../TestAgentBackendContainerTest.java | 6 +-- 4 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 dd-smoke-tests/src/main/java/datadog/smoketest/backend/DockerAvailableCondition.java create mode 100644 dd-smoke-tests/src/main/java/datadog/smoketest/backend/EnabledIfDockerAvailable.java create mode 100644 dd-smoke-tests/src/test/java/datadog/smoketest/backend/DockerAvailableConditionTest.java diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/DockerAvailableCondition.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/DockerAvailableCondition.java new file mode 100644 index 00000000000..b1ef0484406 --- /dev/null +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/DockerAvailableCondition.java @@ -0,0 +1,47 @@ +package datadog.smoketest.backend; + +import static org.junit.jupiter.api.extension.ConditionEvaluationResult.disabled; +import static org.junit.jupiter.api.extension.ConditionEvaluationResult.enabled; + +import org.junit.jupiter.api.extension.ConditionEvaluationResult; +import org.junit.jupiter.api.extension.ExecutionCondition; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.testcontainers.DockerClientFactory; + +/** + * JUnit {@link ExecutionCondition} backing {@link EnabledIfDockerAvailable}. Docker availability is + * probed once per JVM and cached, since the probe can be slow. + */ +final class DockerAvailableCondition implements ExecutionCondition { + private static volatile Boolean dockerAvailable; + + @Override + public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { + if (isDockerAvailable()) { + return enabled("Docker is available"); + } + return disabled( + "Docker is not available — skipping test-agent container test. Start Docker to run it " + + "locally, or reuse a CI test agent via CI_AGENT_HOST."); + } + + private static boolean isDockerAvailable() { + Boolean cached = dockerAvailable; + if (cached == null) { + synchronized (DockerAvailableCondition.class) { + cached = dockerAvailable; + if (cached == null) { + try { + cached = DockerClientFactory.instance().isDockerAvailable(); + } catch (Throwable t) { + // Any failure probing the daemon (Testcontainers/docker-java errors) => treat as + // absent. + cached = false; + } + dockerAvailable = cached; + } + } + } + return cached; + } +} diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/EnabledIfDockerAvailable.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/EnabledIfDockerAvailable.java new file mode 100644 index 00000000000..865328d657d --- /dev/null +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/EnabledIfDockerAvailable.java @@ -0,0 +1,24 @@ +package datadog.smoketest.backend; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * Skips (loudly, as a JUnit conditional-execution result — not a failure) the annotated test class + * or method when Docker is unavailable, so a developer not running smoke tests locally isn't + * blocked (Q4/S7/G7). Apply it to tests that require a Testcontainers-managed test-agent {@code + * .container()} backend. + * + *

Tests that select their backend from the environment via {@code + * TraceBackend.testAgentFromEnv()} don't need this annotation — that resolver already reuses an + * external CI agent when {@code CI_AGENT_HOST} is set, and aborts loudly otherwise. + */ +@Target({ElementType.TYPE, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@ExtendWith(DockerAvailableCondition.class) +public @interface EnabledIfDockerAvailable {} diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/DockerAvailableConditionTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/DockerAvailableConditionTest.java new file mode 100644 index 00000000000..468e491ba48 --- /dev/null +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/DockerAvailableConditionTest.java @@ -0,0 +1,25 @@ +package datadog.smoketest.backend; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ConditionEvaluationResult; + +/** + * Verifies the {@link EnabledIfDockerAvailable} condition evaluates cleanly regardless of the + * Docker state — probing failures are swallowed and a reason is always attached. Whether it enables + * or disables is exercised end-to-end by {@link TestAgentBackendContainerTest} (which carries the + * annotation): it runs when Docker is present, and is skipped otherwise. + */ +class DockerAvailableConditionTest { + + @Test + void evaluatesToAResultWithAReason() { + ConditionEvaluationResult result = + new DockerAvailableCondition().evaluateExecutionCondition(null); + + assertNotNull(result, "condition returns a result without throwing"); + assertTrue(result.getReason().isPresent(), "both the enabled and disabled paths give a reason"); + } +} diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendContainerTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendContainerTest.java index 56f5c219d06..98dc423cae9 100644 --- a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendContainerTest.java +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendContainerTest.java @@ -3,7 +3,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assumptions.assumeTrue; import datadog.trace.test.agent.decoder.DecodedSpan; import datadog.trace.test.agent.decoder.DecodedTrace; @@ -23,7 +22,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.testcontainers.DockerClientFactory; /** * End-to-end tests for {@link TestAgentBackend} against a real dd-apm-test-agent container. Skipped @@ -35,6 +33,7 @@ *

Uses the public {@code ghcr.io} image so it runs without internal-registry access; real smoke * tests default to the CI mirror (see {@link TestAgentBackend}). */ +@EnabledIfDockerAvailable class TestAgentBackendContainerTest { private static final String PUBLIC_IMAGE = "ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.44.0"; @@ -46,9 +45,6 @@ class TestAgentBackendContainerTest { @BeforeAll static void setUp() throws IOException { - assumeTrue( - DockerClientFactory.instance().isDockerAvailable(), - "Docker is required for the test-agent container backend"); v04Payload = readResource("/datadog/smoketest/backend/webflux.04.msgpack"); backend = TraceBackend.testAgent().image(PUBLIC_IMAGE).build(); backend.start(); From aceb43804057e149e4f6a80369031d37961d1ffa Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Thu, 23 Jul 2026 10:34:09 +0200 Subject: [PATCH 11/30] WIP --- dd-smoke-tests/build.gradle | 7 ++ dd-smoke-tests/opentelemetry/build.gradle | 4 + dd-smoke-tests/opentelemetry/gradle.lockfile | 18 +++-- .../smoketest/OpenTelemetrySmokeTest.groovy | 27 ------- .../smoketest/OpenTelemetrySmokeTest.java | 44 +++++++++++ .../main/java/datadog/smoketest/SmokeApp.java | 42 +++++++++-- .../backend/EnabledIfDockerAvailable.java | 6 +- .../smoketest/backend/MockAgentBackend.java | 19 ++++- .../datadog/smoketest/backend/Telemetry.java | 73 +++++++++++++++++++ .../smoketest/backend/TelemetryDecoder.java | 50 +++++++++++++ .../smoketest/backend/TestAgentBackend.java | 22 +++++- .../smoketest/backend/TraceBackend.java | 24 +++--- .../backend/MockAgentBackendTest.java | 39 ++++++++++ .../TestAgentBackendContainerTest.java | 36 ++++++++- .../backend/TestAgentBackendTest.java | 14 ++-- 15 files changed, 363 insertions(+), 62 deletions(-) delete mode 100644 dd-smoke-tests/opentelemetry/src/test/groovy/datadog/smoketest/OpenTelemetrySmokeTest.groovy create mode 100644 dd-smoke-tests/opentelemetry/src/test/java/datadog/smoketest/OpenTelemetrySmokeTest.java create mode 100644 dd-smoke-tests/src/main/java/datadog/smoketest/backend/Telemetry.java create mode 100644 dd-smoke-tests/src/main/java/datadog/smoketest/backend/TelemetryDecoder.java diff --git a/dd-smoke-tests/build.gradle b/dd-smoke-tests/build.gradle index 4e72ff6a609..cec2a729cb0 100644 --- a/dd-smoke-tests/build.gradle +++ b/dd-smoke-tests/build.gradle @@ -54,6 +54,13 @@ subprojects { Project subProj -> // Tests depend on this to know where to run things and what agent jar to use jvmArgs "-Ddatadog.smoketest.builddir=${buildDir}" + // Forward an optional test-agent image override to the test JVM, so TraceBackend.testAgent() + // runs locally without internal-registry access, e.g. + // -Ddatadog.smoketest(.testagent.image=ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.44.0 + def testAgentImage = System.getProperty("datadog.smoketest.testagent.image") + if (testAgentImage != null) { + jvmArgs "-Ddatadog.smoketest.testagent.image=${testAgentImage}" + } jvmArgumentProviders.add(new CommandLineArgumentProvider() { @Override Iterable asArguments() { diff --git a/dd-smoke-tests/opentelemetry/build.gradle b/dd-smoke-tests/opentelemetry/build.gradle index 26b9c62c717..3e3104466f7 100644 --- a/dd-smoke-tests/opentelemetry/build.gradle +++ b/dd-smoke-tests/opentelemetry/build.gradle @@ -15,9 +15,13 @@ dependencies { implementation group: 'io.opentelemetry', name: 'opentelemetry-api', version: '1.4.0' implementation group: 'io.opentelemetry.instrumentation', name: 'opentelemetry-instrumentation-annotations', version: '1.20.0' testImplementation project(':dd-smoke-tests') + // Needed to use TraceBackend.testAgent(): Testcontainers is compileOnly on the smoke-test base + // (so it isn't forced on every consumer), so a module using the container backend opts in here. + testImplementation libs.testcontainers } tasks.withType(Test).configureEach { + usesService(testcontainersLimit) dependsOn 'shadowJar' def shadowJarTask = tasks.named('shadowJar', ShadowJar) jvmArgumentProviders.add(new CommandLineArgumentProvider() { diff --git a/dd-smoke-tests/opentelemetry/gradle.lockfile b/dd-smoke-tests/opentelemetry/gradle.lockfile index 4c2601b6c61..6be0b0c2930 100644 --- a/dd-smoke-tests/opentelemetry/gradle.lockfile +++ b/dd-smoke-tests/opentelemetry/gradle.lockfile @@ -13,6 +13,10 @@ com.datadoghq:dd-instrument-java:0.0.4=testCompileClasspath,testRuntimeClasspath com.datadoghq:dd-javac-plugin-client:0.2.2=testCompileClasspath,testRuntimeClasspath com.datadoghq:java-dogstatsd-client:4.4.5=testRuntimeClasspath com.datadoghq:sketches-java:0.8.3=testRuntimeClasspath +com.fasterxml.jackson.core:jackson-annotations:2.10.3=testCompileClasspath,testRuntimeClasspath +com.github.docker-java:docker-java-api:3.4.2=testCompileClasspath,testRuntimeClasspath +com.github.docker-java:docker-java-transport-zerodep:3.4.2=testCompileClasspath,testRuntimeClasspath +com.github.docker-java:docker-java-transport:3.4.2=testCompileClasspath,testRuntimeClasspath com.github.javaparser:javaparser-core:3.25.6=codenarc com.github.jnr:jffi:1.3.15=testRuntimeClasspath com.github.jnr:jnr-a64asm:1.0.0=testRuntimeClasspath @@ -50,20 +54,21 @@ io.opentelemetry:opentelemetry-context:1.19.0=compileClasspath,runtimeClasspath, io.sqreen:libsqreen:17.4.0=testRuntimeClasspath javax.servlet:javax.servlet-api:3.1.0=testCompileClasspath,testRuntimeClasspath jaxen:jaxen:2.0.0=spotbugs -junit:junit:4.13.2=testRuntimeClasspath +junit:junit:4.13.2=testCompileClasspath,testRuntimeClasspath net.bytebuddy:byte-buddy-agent:1.18.10=testCompileClasspath,testRuntimeClasspath net.bytebuddy:byte-buddy:1.18.10=testCompileClasspath,testRuntimeClasspath net.java.dev.jna:jna-platform:5.8.0=testRuntimeClasspath -net.java.dev.jna:jna:5.8.0=testRuntimeClasspath +net.java.dev.jna:jna:5.13.0=testCompileClasspath,testRuntimeClasspath net.sf.saxon:Saxon-HE:12.9=spotbugs org.apache.ant:ant-antlr:1.10.14=codenarc org.apache.ant:ant-junit:1.10.14=codenarc org.apache.bcel:bcel:6.11.0=spotbugs +org.apache.commons:commons-compress:1.24.0=testCompileClasspath,testRuntimeClasspath org.apache.commons:commons-lang3:3.19.0=spotbugs org.apache.commons:commons-text:1.14.0=spotbugs org.apache.logging.log4j:log4j-api:2.25.2=spotbugs org.apache.logging.log4j:log4j-core:2.25.2=spotbugs -org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath,testRuntimeClasspath org.codehaus.groovy:groovy-ant:3.0.23=codenarc org.codehaus.groovy:groovy-docgenerator:3.0.23=codenarc org.codehaus.groovy:groovy-groovydoc:3.0.23=codenarc @@ -76,10 +81,11 @@ org.codehaus.groovy:groovy:3.0.25=testCompileClasspath,testRuntimeClasspath org.codenarc:CodeNarc:3.7.0=codenarc org.dom4j:dom4j:2.2.0=spotbugs org.gmetrics:GMetrics:2.1.0=codenarc -org.hamcrest:hamcrest-core:1.3=testRuntimeClasspath +org.hamcrest:hamcrest-core:1.3=testCompileClasspath,testRuntimeClasspath org.hamcrest:hamcrest:3.0=testCompileClasspath,testRuntimeClasspath org.jctools:jctools-core-jdk11:4.0.6=testRuntimeClasspath org.jctools:jctools-core:4.0.6=testRuntimeClasspath +org.jetbrains:annotations:17.0.0=testCompileClasspath,testRuntimeClasspath org.jspecify:jspecify:1.0.0=testCompileClasspath,testRuntimeClasspath org.junit.jupiter:junit-jupiter-api:5.14.1=testCompileClasspath,testRuntimeClasspath org.junit.jupiter:junit-jupiter-engine:5.14.1=testRuntimeClasspath @@ -107,15 +113,17 @@ org.ow2.asm:asm-util:9.7.1=testRuntimeClasspath org.ow2.asm:asm-util:9.9=spotbugs org.ow2.asm:asm:9.10.1=testCompileClasspath,testRuntimeClasspath org.ow2.asm:asm:9.9=spotbugs +org.rnorth.duct-tape:duct-tape:1.0.8=testCompileClasspath,testRuntimeClasspath org.slf4j:jcl-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath org.slf4j:jul-to-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath org.slf4j:log4j-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath -org.slf4j:slf4j-api:1.7.32=testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-api:1.7.36=testCompileClasspath,testRuntimeClasspath org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j org.spockframework:spock-bom:2.4-groovy-3.0=testCompileClasspath,testRuntimeClasspath org.spockframework:spock-core:2.4-groovy-3.0=testCompileClasspath,testRuntimeClasspath org.tabletest:tabletest-junit:1.2.1=testCompileClasspath,testRuntimeClasspath org.tabletest:tabletest-parser:1.2.0=testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers:1.21.4=testCompileClasspath,testRuntimeClasspath org.xmlresolver:xmlresolver:5.3.3=spotbugs empty=annotationProcessor,shadow,spotbugsPlugins,testAnnotationProcessor diff --git a/dd-smoke-tests/opentelemetry/src/test/groovy/datadog/smoketest/OpenTelemetrySmokeTest.groovy b/dd-smoke-tests/opentelemetry/src/test/groovy/datadog/smoketest/OpenTelemetrySmokeTest.groovy deleted file mode 100644 index f9c85720b0b..00000000000 --- a/dd-smoke-tests/opentelemetry/src/test/groovy/datadog/smoketest/OpenTelemetrySmokeTest.groovy +++ /dev/null @@ -1,27 +0,0 @@ -package datadog.smoketest - -import static java.util.concurrent.TimeUnit.SECONDS - -class OpenTelemetrySmokeTest extends AbstractSmokeTest { - public static final int TIMEOUT_SECS = 30 - - @Override - ProcessBuilder createProcessBuilder() { - def jarPath = System.getProperty("datadog.smoketest.shadowJar.path") - def command = new ArrayList() - command.add(javaPath()) - command.addAll(defaultJavaProperties) - command.add("-Ddd.trace.otel.enabled=true") - command.addAll(["-jar", jarPath]) - - ProcessBuilder processBuilder = new ProcessBuilder(command) - processBuilder.directory(new File(buildDirectory)) - } - - def 'receive trace'() { - expect: - waitForTraceCount(11) // 1 annotated, 10 manual - assert testedProcess.waitFor(TIMEOUT_SECS, SECONDS) - assert testedProcess.exitValue() == 0 - } -} diff --git a/dd-smoke-tests/opentelemetry/src/test/java/datadog/smoketest/OpenTelemetrySmokeTest.java b/dd-smoke-tests/opentelemetry/src/test/java/datadog/smoketest/OpenTelemetrySmokeTest.java new file mode 100644 index 00000000000..78bf4a6684f --- /dev/null +++ b/dd-smoke-tests/opentelemetry/src/test/java/datadog/smoketest/OpenTelemetrySmokeTest.java @@ -0,0 +1,44 @@ +package datadog.smoketest; + +import static java.util.concurrent.TimeUnit.SECONDS; + +import datadog.smoketest.backend.EnabledIfDockerAvailable; +import datadog.smoketest.backend.TraceBackend; +import java.io.File; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * S8 pilot — ported from the Groovy {@code OpenTelemetrySmokeTest} onto the new Java smoke base. + * The OpenTelemetry sample app runs under the agent (OTel enabled), emits its traces, and exits; + * the test-agent backend must receive all of them. This is the first end-to-end exercise of the new + * base with a real agent and a non-server (batch) app. The {@code .testAgent()} container backend + * needs Docker, so the test is skipped when Docker is unavailable. + */ +@EnabledIfDockerAvailable +class OpenTelemetrySmokeTest { + private static final int TIMEOUT_SECONDS = 30; + + @RegisterExtension + static final SmokeApp app = + SmokeApp.named("opentelemetry") + .jar(System.getProperty("datadog.smoketest.shadowJar.path")) + .jvmArgs("-Ddd.trace.otel.enabled=true") + .backend(TraceBackend.testAgent()) + .notAServer() + .workingDirectory(new File(System.getProperty("datadog.smoketest.builddir"))) + .build(); + + @Test + void receivesTraces() { + // 1 @WithSpan-annotated span + 10 manual OpenTelemetry spans, each its own trace. + app.traces().waitForTraceCount(11, TIMEOUT_SECONDS); + + // Telemetry is captured too and attributed to this test's session (S9): the agent emits at + // least an app-started message, carrying the same session token as the traces. + app.backend().telemetry().waitForCount(1, TIMEOUT_SECONDS); + + // The app then runs to completion and exits cleanly. + app.assertCompletesWithValue(TIMEOUT_SECONDS, SECONDS, 0); + } +} diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java b/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java index 86a6446b936..54af836d1c1 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java @@ -153,6 +153,31 @@ public boolean awaitLogLine(Function predicate) { } } + /** + * Asserts a non-server (batch) app runs to completion within the timeout and exits with {@code + * expectedExitValue}. Fails with an {@link AssertionError} if it doesn't terminate in time or + * exits with a different code. Pass a non-zero value for apps expected to fail (e.g. a tool the + * agent aborts). + */ + public void assertCompletesWithValue(long timeout, TimeUnit unit, int expectedExitValue) { + boolean exited; + try { + exited = process.waitFor(timeout, unit); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("Interrupted while waiting for app '" + name + "' to complete", e); + } + if (!exited) { + throw new AssertionError( + "App '" + name + "' did not complete within " + timeout + " " + unit); + } + int actual = process.exitValue(); + if (actual != expectedExitValue) { + throw new AssertionError( + "App '" + name + "' exited with " + actual + " but expected " + expectedExitValue); + } + } + // --- Lifecycle (per-class start, per-method reset, teardown) --- @Override @@ -173,13 +198,20 @@ public void beforeAll(ExtensionContext context) throws Exception { @Override public void beforeEach(ExtensionContext context) { - if (process == null || !process.isAlive()) { - throw new IllegalStateException("App '" + name + "' is not alive at the start of a test"); + // A server app must stay up across methods, and its per-test traces are produced during the + // test body — so assert it's alive and reset the backend between methods. A batch app + // (notAServer) may have already run to completion and produced its traces at start-up, so + // neither applies: requiring it alive would spuriously fail, and clearing would wipe its + // traces. + if (server) { + if (process == null || !process.isAlive()) { + throw new IllegalStateException("App '" + name + "' is not alive at the start of a test"); + } + if (ownsBackend) { + backend.clear(); + } } outputThreads.clearMessages(); - if (ownsBackend) { - backend.clear(); - } } @Override diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/EnabledIfDockerAvailable.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/EnabledIfDockerAvailable.java index 865328d657d..e9c04ae4ced 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/EnabledIfDockerAvailable.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/EnabledIfDockerAvailable.java @@ -13,9 +13,9 @@ * blocked (Q4/S7/G7). Apply it to tests that require a Testcontainers-managed test-agent {@code * .container()} backend. * - *

Tests that select their backend from the environment via {@code - * TraceBackend.testAgentFromEnv()} don't need this annotation — that resolver already reuses an - * external CI agent when {@code CI_AGENT_HOST} is set, and aborts loudly otherwise. + *

Tests that select their backend from the environment via {@code TraceBackend.testAgent()} + * don't need this annotation — that resolver already reuses an external CI agent when {@code + * CI_AGENT_HOST} is set, and aborts loudly otherwise. */ @Target({ElementType.TYPE, ElementType.METHOD}) @Retention(RetentionPolicy.RUNTIME) diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/MockAgentBackend.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/MockAgentBackend.java index 935e7a9ecef..e603c98807a 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/MockAgentBackend.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/MockAgentBackend.java @@ -8,6 +8,7 @@ import java.net.URI; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; /** @@ -27,6 +28,7 @@ public final class MockAgentBackend implements TraceBackend { private static final String JSON = "application/json"; private final List traces = new CopyOnWriteArrayList<>(); + private final List> telemetry = new CopyOnWriteArrayList<>(); private volatile JavaTestHttpServer server; private boolean shared; @@ -63,7 +65,8 @@ public void start() { h.prefix("/v1.0/traces", api -> collect(api, TraceFormat.V1)); h.prefix("/v0.5/traces", api -> collect(api, TraceFormat.V05)); h.prefix("/v0.4/traces", api -> collect(api, TraceFormat.V04)); - // Everything else (telemetry, remote-config, ...) just succeeds. + h.prefix("/telemetry/proxy", this::collectTelemetry); + // Everything else (remote-config, ...) just succeeds. h.all(api -> api.getResponse().status(200).send()); })); } @@ -78,6 +81,14 @@ private void collect(HandlerApi api, TraceFormat format) { api.getResponse().status(200).sendWithType(JSON, "{}"); } + private void collectTelemetry(HandlerApi api) { + byte[] body = api.getRequest().getBody(); + if (body != null && body.length > 0) { + telemetry.add(TelemetryDecoder.decodeMessage(body)); + } + api.getResponse().status(202).send(); + } + @Override public int port() { return url().getPort(); @@ -97,9 +108,15 @@ public Traces traces() { return new Traces(() -> new ArrayList<>(traces)); } + @Override + public Telemetry telemetry() { + return new Telemetry(() -> new ArrayList<>(telemetry)); + } + @Override public void clear() { traces.clear(); + telemetry.clear(); } @Override diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Telemetry.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Telemetry.java new file mode 100644 index 00000000000..1b274596327 --- /dev/null +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Telemetry.java @@ -0,0 +1,73 @@ +package datadog.smoketest.backend; + +import datadog.trace.test.util.PollingConditions; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +/** + * Query facade over the app-telemetry messages a {@link TraceBackend} has received (S9). Common to + * both backends: the mock collects them in-process, the test agent exposes them at {@code + * /test/session/apmtelemetry}; each message is a decoded JSON map. + * + *

Telemetry is scoped to the test session (the tracer's telemetry client emits the same {@code + * X-Datadog-Test-Session-Token} as the trace writer), so it stays isolated per test even on a + * shared external agent. + */ +public final class Telemetry { + private static final double DEFAULT_TIMEOUT_SECONDS = 10; + + private final Supplier>> source; + + Telemetry(Supplier>> source) { + this.source = source; + } + + /** The raw telemetry messages received — one map per intake request. */ + public List> getMessages() { + return source.get(); + } + + /** + * Individual telemetry events, expanding each {@code message-batch} into its {@code payload} + * entries — the granularity most assertions want (e.g. locating {@code app-started} or {@code + * app-dependencies-loaded}). + */ + @SuppressWarnings("unchecked") + public List> getFlatMessages() { + List> flat = new ArrayList<>(); + for (Map message : getMessages()) { + Object payload = message.get("payload"); + if ("message-batch".equals(message.get("request_type")) && payload instanceof List) { + for (Object entry : (List) payload) { + if (entry instanceof Map) { + flat.add((Map) entry); + } + } + } else { + flat.add(message); + } + } + return flat; + } + + /** Waits (up to the default timeout) until at least {@code count} messages have been received. */ + public Telemetry waitForCount(int count) { + return waitForCount(count, DEFAULT_TIMEOUT_SECONDS); + } + + /** As {@link #waitForCount(int)}, but overriding the timeout for this call. */ + public Telemetry waitForCount(int count, double timeoutSeconds) { + new PollingConditions(timeoutSeconds) + .eventually( + () -> { + int actual = getMessages().size(); + if (actual < count) { + throw new AssertionError( + "Expected at least " + count + " telemetry message(s) but got " + actual); + } + }); + return this; + } +} diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TelemetryDecoder.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TelemetryDecoder.java new file mode 100644 index 00000000000..c2638a4047e --- /dev/null +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TelemetryDecoder.java @@ -0,0 +1,50 @@ +package datadog.smoketest.backend; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.JsonDataException; +import com.squareup.moshi.Moshi; +import com.squareup.moshi.Types; +import java.io.IOException; +import java.lang.reflect.Type; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Parses app-telemetry JSON into maps: a single intake body (what the mock backend collects + * per-request) or the test agent's {@code /test/session/apmtelemetry} array. Moshi decodes JSON + * numbers as {@code Double}, which is fine for the presence/string assertions telemetry tests do. + */ +final class TelemetryDecoder { + private static final Type MESSAGE = + Types.newParameterizedType(Map.class, String.class, Object.class); + private static final Moshi MOSHI = new Moshi.Builder().build(); + private static final JsonAdapter> MESSAGE_ADAPTER = MOSHI.adapter(MESSAGE); + private static final JsonAdapter>> MESSAGE_LIST_ADAPTER = + MOSHI.adapter(Types.newParameterizedType(List.class, MESSAGE)); + + private TelemetryDecoder() {} + + /** Decodes one telemetry intake body (a JSON object) into a message map. */ + static Map decodeMessage(byte[] json) { + try { + Map message = + MESSAGE_ADAPTER.fromJson(new String(json, StandardCharsets.UTF_8)); + return message == null ? Collections.emptyMap() : message; + } catch (IOException | JsonDataException e) { + throw new IllegalStateException("Failed to parse telemetry message", e); + } + } + + /** Decodes the test agent's {@code /test/apmtelemetry} response (a JSON array of messages). */ + static List> decodeMessages(String json) { + try { + List> messages = MESSAGE_LIST_ADAPTER.fromJson(json); + return messages == null ? Collections.emptyList() : messages; + } catch (IOException | JsonDataException e) { + throw new IllegalStateException( + "Failed to parse /test/session/apmtelemetry response: " + json, e); + } + } +} diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentBackend.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentBackend.java index 4f87083f8df..dc9ca23d1ff 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentBackend.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentBackend.java @@ -6,6 +6,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.UUID; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; @@ -136,6 +137,11 @@ public Traces traces() { return new Traces(this::fetchTraces); } + @Override + public Telemetry telemetry() { + return new Telemetry(this::fetchTelemetry); + } + @Override public void clear() { // GET /test/session/start begins (and clears) a session identified by the token. The @@ -208,6 +214,20 @@ private List fetchTraces() { return TestAgentTraceDecoder.decode(execute(request, "read test-agent session traces")); } + private List> fetchTelemetry() { + // Session-scoped, like traces: the tracer's telemetry client now emits the + // X-Datadog-Test-Session-Token header (TelemetryClient), so a shared external agent attributes + // telemetry to this test's session and it stays isolated per test. + HttpUrl url = + requireStarted() + .newBuilder() + .addPathSegments("test/session/apmtelemetry") + .addQueryParameter("test_session_token", sessionToken) + .build(); + Request request = new Request.Builder().url(url).get().build(); + return TelemetryDecoder.decodeMessages(execute(request, "read test-agent session telemetry")); + } + private String execute(Request request, String action) { try (Response response = client.newCall(request).execute()) { if (!response.isSuccessful()) { @@ -228,7 +248,7 @@ private HttpUrl requireStarted() { return url; } - /** Fluent builder for a {@link TestAgentBackend}; obtain via {@code TraceBackend.testAgent()}. */ + /** Fluent builder; obtain via {@code TraceBackend.testAgentBuilder()}. */ public static final class Builder { private String image = DEFAULT_IMAGE; private String externalHost; diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java index a43578c3a91..9a747e03e91 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java @@ -33,6 +33,9 @@ public interface TraceBackend /** Query/assert facade over the traces this backend has received. */ Traces traces(); + /** Query facade over the app-telemetry messages this backend has received (S9). */ + Telemetry telemetry(); + /** Discards all traces received so far — call between test methods to isolate them. */ void clear(); @@ -82,26 +85,25 @@ static MockAgentBackend mockAgent() { } /** Fluent builder for a {@link TestAgentBackend} (dd-apm-test-agent container or external). */ - static TestAgentBackend.Builder testAgent() { + static TestAgentBackend.Builder testAgentBuilder() { return TestAgentBackend.builder(); } /** - * Resolves the environment's default test-agent backend (Q4): reuse the external CI sidecar when - * {@code CI_AGENT_HOST} is set, else a Testcontainers-managed container when Docker is available, - * else skip loudly (a {@link TestAbortedException} marks the test as aborted rather than failed — - * a dev not running smoke tests isn't blocked). The mock backend is never an automatic fallback; - * select it explicitly with {@link #mockAgent()}. - * - *

The reusable conditional-execution annotation over this policy is S7. + * Resolves the environment's default test-agent backend (Q4): the external CI sidecar when {@code + * CI_AGENT_HOST} is set, else a Testcontainers-managed container when Docker is available, else a + * loud skip (a {@link TestAbortedException} marks the test aborted rather than failed, so a dev + * not running smoke tests isn't blocked). The mock backend is never an automatic fallback — + * select it explicitly with {@link #mockAgent()}. The reusable conditional-execution gate over + * this policy is {@link EnabledIfDockerAvailable} (S7). */ - static TraceBackend testAgentFromEnv() { + static TraceBackend testAgent() { String ciAgentHost = System.getenv("CI_AGENT_HOST"); if (ciAgentHost != null && !ciAgentHost.isEmpty()) { - return testAgent().external(ciAgentHost, 8126).build(); + return testAgentBuilder().external(ciAgentHost, 8126).build(); } if (DockerClientFactory.instance().isDockerAvailable()) { - return testAgent().build(); + return testAgentBuilder().build(); } throw new TestAbortedException( "No test-agent backend available: set CI_AGENT_HOST for an external agent, or start Docker " diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java index f2ef6fcc1e8..71f8fd9f91a 100644 --- a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java @@ -14,6 +14,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.List; +import java.util.Map; import okhttp3.MediaType; import okhttp3.OkHttpClient; import okhttp3.Request; @@ -35,6 +36,7 @@ */ class MockAgentBackendTest { private static final MediaType MSGPACK = MediaType.parse("application/msgpack"); + private static final MediaType JSON = MediaType.parse("application/json"); private static final OkHttpClient CLIENT = new OkHttpClient(); // Recorded /v0.4/traces payload: 1 trace, 2 spans (netty.request -> WebController.hello), @@ -121,6 +123,32 @@ void clearDiscardsReceivedTraces() throws IOException { assertTrue(backend.traces().getTraces().isEmpty(), "clear() drops collected traces"); } + @Test + void capturesTelemetry() throws IOException { + postTelemetry("{\"request_type\":\"app-started\",\"api_version\":\"v2\"}"); + postTelemetry( + "{\"request_type\":\"message-batch\",\"payload\":[" + + "{\"request_type\":\"app-dependencies-loaded\"}," + + "{\"request_type\":\"generate-metrics\"}]}"); + + Telemetry telemetry = backend.telemetry(); + telemetry.waitForCount(2); + assertEquals(2, telemetry.getMessages().size(), "raw messages: app-started + message-batch"); + + // getFlatMessages expands the batch into its two entries: 1 + 2 = 3. + List> flat = telemetry.getFlatMessages(); + assertEquals(3, flat.size(), "message-batch expanded into its entries"); + assertTrue( + flat.stream().anyMatch(m -> "app-started".equals(m.get("request_type"))), + "app-started present"); + assertTrue( + flat.stream().anyMatch(m -> "app-dependencies-loaded".equals(m.get("request_type"))), + "batch entry present"); + + backend.clear(); + assertTrue(backend.telemetry().getMessages().isEmpty(), "clear() drops telemetry too"); + } + @Test void infoAdvertisesTraceEndpoints() throws IOException { Request request = new Request.Builder().url(backend.url() + "/info").get().build(); @@ -144,6 +172,17 @@ void exposesBoundPort() { assertEquals(backend.url().getPort(), backend.port(), "port matches url"); } + private static void postTelemetry(String json) throws IOException { + Request request = + new Request.Builder() + .url(backend.url() + "/telemetry/proxy/api/v2/apmtelemetry") + .post(RequestBody.create(JSON, json)) + .build(); + try (Response response = CLIENT.newCall(request).execute()) { + assertTrue(response.isSuccessful(), "mock agent should accept telemetry: " + response.code()); + } + } + private static void putTraces(String path, byte[] payload) throws IOException { Request request = new Request.Builder() diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendContainerTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendContainerTest.java index 98dc423cae9..ab2c5c8dce4 100644 --- a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendContainerTest.java +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendContainerTest.java @@ -46,7 +46,7 @@ class TestAgentBackendContainerTest { @BeforeAll static void setUp() throws IOException { v04Payload = readResource("/datadog/smoketest/backend/webflux.04.msgpack"); - backend = TraceBackend.testAgent().image(PUBLIC_IMAGE).build(); + backend = TraceBackend.testAgentBuilder().image(PUBLIC_IMAGE).build(); backend.start(); } @@ -99,7 +99,7 @@ void externalBackendReadsFromRunningAgent() throws IOException { // Point an .external() backend at the same running container: exercises the external code path // (no container of its own) and, via its own fresh token, that sessions are isolated. TestAgentBackend external = - TraceBackend.testAgent().external(backend.url().getHost(), backend.port()).build(); + TraceBackend.testAgentBuilder().external(backend.url().getHost(), backend.port()).build(); external.start(); try { submitAppTraces(external.url(), external.sessionToken(), v04Payload); @@ -111,6 +111,38 @@ void externalBackendReadsFromRunningAgent() throws IOException { } } + @Test + void capturesTelemetry() throws IOException { + // Post a telemetry app-started message; the backend reads it back from /test/apmtelemetry. + HttpUrl url = + HttpUrl.get(backend.url()) + .newBuilder() + .addPathSegments("telemetry/proxy/api/v2/apmtelemetry") + .build(); + Request request = + new Request.Builder() + .url(url) + .header("DD-Telemetry-API-Version", "v2") + .header("DD-Telemetry-Request-Type", "app-started") + .header("X-Datadog-Test-Session-Token", backend.sessionToken()) + .post( + RequestBody.create( + MediaType.parse("application/json"), + "{\"request_type\":\"app-started\",\"api_version\":\"v2\"," + + "\"runtime_id\":\"r1\",\"seq_id\":1,\"payload\":{}}")) + .build(); + try (Response response = CLIENT.newCall(request).execute()) { + assertTrue(response.isSuccessful(), "telemetry accepted: HTTP " + response.code()); + } + + Telemetry telemetry = backend.telemetry(); + telemetry.waitForCount(1); + assertTrue( + telemetry.getFlatMessages().stream() + .anyMatch(message -> "app-started".equals(message.get("request_type"))), + "app-started telemetry captured"); + } + private static void submitAppTraces(URI agentUrl, String token, byte[] payload) throws IOException { HttpUrl url = HttpUrl.get(agentUrl).newBuilder().addPathSegments("v0.4/traces").build(); diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendTest.java index 446d7a0453a..c82f9d7018f 100644 --- a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendTest.java +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/TestAgentBackendTest.java @@ -18,7 +18,7 @@ class TestAgentBackendTest { @Test void sessionTokenIsStableAndNonEmpty() { - TestAgentBackend backend = TraceBackend.testAgent().build(); + TestAgentBackend backend = TraceBackend.testAgentBuilder().build(); String token = backend.sessionToken(); assertNotNull(token); assertFalse(token.isEmpty()); @@ -29,19 +29,19 @@ void sessionTokenIsStableAndNonEmpty() { void sessionTokenCanBeOverridden() { assertEquals( "fixed-token", - TraceBackend.testAgent().sessionToken("fixed-token").build().sessionToken(), + TraceBackend.testAgentBuilder().sessionToken("fixed-token").build().sessionToken(), "explicit token wins over the auto-generated one"); } @Test void sharedFlagDefaultsFalseAndIsSettable() { - assertFalse(TraceBackend.testAgent().build().isShared(), "not shared by default"); - assertTrue(TraceBackend.testAgent().shared().build().isShared(), "shared() opts in"); + assertFalse(TraceBackend.testAgentBuilder().build().isShared(), "not shared by default"); + assertTrue(TraceBackend.testAgentBuilder().shared().build().isShared(), "shared() opts in"); } @Test void accessBeforeStartFails() { - TestAgentBackend backend = TraceBackend.testAgent().build(); + TestAgentBackend backend = TraceBackend.testAgentBuilder().build(); assertThrows(IllegalStateException.class, backend::url, "url() before start()"); assertThrows(IllegalStateException.class, backend::port, "port() before start()"); } @@ -52,7 +52,7 @@ void assertNoInvariantFailuresPassesWhenAgentReportsNoFailures() { // without Docker; HTTP 200 from the failures endpoint means all checks passed. try (JavaTestHttpServer agent = stubAgent(200, "")) { TestAgentBackend backend = - TraceBackend.testAgent() + TraceBackend.testAgentBuilder() .external(agent.getAddress().getHost(), agent.getAddress().getPort()) .build(); backend.start(); @@ -68,7 +68,7 @@ void assertNoInvariantFailuresPassesWhenAgentReportsNoFailures() { void assertNoInvariantFailuresThrowsWhenAgentReportsFailures() { try (JavaTestHttpServer agent = stubAgent(400, "span_count check failed")) { TestAgentBackend backend = - TraceBackend.testAgent() + TraceBackend.testAgentBuilder() .external(agent.getAddress().getHost(), agent.getAddress().getPort()) .build(); backend.start(); From 51b715d637d1c249b4647bc6a5da89fe904668eb Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Tue, 14 Jul 2026 14:35:50 +0200 Subject: [PATCH 12/30] WIP --- .../smoketest/OpenTelemetrySmokeTest.java | 7 +- .../main/java/datadog/smoketest/SmokeApp.java | 147 +++++++++++++++++- .../datadog/smoketest/backend/Telemetry.java | 18 +++ .../datadog/smoketest/ErrorLogSmokeTest.java | 38 +++++ .../smoketest/SmokeAppErrorLogFilterTest.java | 33 ++++ .../java/datadog/smoketest/TestServerApp.java | 8 +- .../backend/MockAgentBackendTest.java | 15 ++ 7 files changed, 253 insertions(+), 13 deletions(-) create mode 100644 dd-smoke-tests/src/test/java/datadog/smoketest/ErrorLogSmokeTest.java create mode 100644 dd-smoke-tests/src/test/java/datadog/smoketest/SmokeAppErrorLogFilterTest.java diff --git a/dd-smoke-tests/opentelemetry/src/test/java/datadog/smoketest/OpenTelemetrySmokeTest.java b/dd-smoke-tests/opentelemetry/src/test/java/datadog/smoketest/OpenTelemetrySmokeTest.java index 78bf4a6684f..bac1474475a 100644 --- a/dd-smoke-tests/opentelemetry/src/test/java/datadog/smoketest/OpenTelemetrySmokeTest.java +++ b/dd-smoke-tests/opentelemetry/src/test/java/datadog/smoketest/OpenTelemetrySmokeTest.java @@ -34,11 +34,8 @@ void receivesTraces() { // 1 @WithSpan-annotated span + 10 manual OpenTelemetry spans, each its own trace. app.traces().waitForTraceCount(11, TIMEOUT_SECONDS); - // Telemetry is captured too and attributed to this test's session (S9): the agent emits at - // least an app-started message, carrying the same session token as the traces. - app.backend().telemetry().waitForCount(1, TIMEOUT_SECONDS); - - // The app then runs to completion and exits cleanly. + // The app then runs to completion and exits cleanly. (app-started telemetry is asserted + // automatically at teardown by SmokeApp's default check — the test body needn't check it.) app.assertCompletesWithValue(TIMEOUT_SECONDS, SECONDS, 0); } } diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java b/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java index 54af836d1c1..33a33039558 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java @@ -3,9 +3,13 @@ import datadog.smoketest.backend.TraceBackend; import datadog.smoketest.backend.Traces; import datadog.trace.agent.test.utils.PortUtils; +import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -16,6 +20,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.function.Function; +import java.util.function.Predicate; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; @@ -75,10 +80,14 @@ public final class SmokeApp private final boolean server; // wait for the HTTP port to open on start private final long startupTimeoutSeconds; private final int httpPort; + private final Predicate errorLogFilter; + private final boolean checkErrorLogs; + private final boolean checkTelemetry; private final OkHttpClient httpClient = new OkHttpClient(); private final OutputThreads outputThreads = new OutputThreads(); private Process process; + private File logFile; private SmokeApp(Builder builder) { this.name = builder.name; @@ -96,6 +105,12 @@ private SmokeApp(Builder builder) { this.server = builder.server; this.startupTimeoutSeconds = builder.startupTimeoutSeconds; this.httpPort = PortUtils.randomOpenPort(); + this.checkErrorLogs = builder.checkErrorLogs; + this.checkTelemetry = builder.checkTelemetry; + this.errorLogFilter = + builder.errorLogFilter != null + ? builder.errorLogFilter + : defaultErrorLogFilter(builder.allowedErrorLogs); } /** Starts a fluent builder for an app with the given (log/diagnostic) name. */ @@ -219,9 +234,24 @@ public void afterAll(ExtensionContext context) { try { stopProcess(); } finally { + // Join the output threads first so the log file is fully flushed before we scan it. outputThreads.close(); - if (ownsBackend) { - backend.close(); + try { + // Telemetry flushes on app shutdown; check it while the backend is still up, and only for + // agent-instrumented apps (a no-agent app emits none). + if (checkTelemetry && agentJar != null) { + assertTelemetryReceived(); + } + } finally { + try { + if (ownsBackend) { + backend.close(); + } + } finally { + if (checkErrorLogs) { + assertNoErrorLogs(); + } + } } } } @@ -268,8 +298,9 @@ private void launch() throws IOException { env.putAll(extraEnv); processBuilder.redirectErrorStream(true); + logFile = resolveLogFile(); process = processBuilder.start(); - outputThreads.captureOutput(process, logFile()); + outputThreads.captureOutput(process, logFile); } private void stopProcess() { @@ -295,7 +326,7 @@ private String substitute(String value) { return value.replace(HTTP_PORT_PLACEHOLDER, Integer.toString(httpPort)); } - private File logFile() { + private File resolveLogFile() { String buildDir = System.getProperty(BUILD_DIR_PROPERTY); File dir = buildDir != null @@ -307,6 +338,76 @@ private File logFile() { return new File(dir, "smoke-app." + name + ".log"); } + /** + * Asserts the app logged no error lines, per the configured filter. Reads the whole captured log + * (everything since launch), so it mirrors the Groovy base's universal no-error-logs check. + * Auto-invoked at teardown unless {@link Builder#skipErrorLogCheck()} was set; may also be called + * explicitly mid-run. + */ + public void assertNoErrorLogs() { + if (logFile == null) { + return; // never launched / nothing captured + } + List errors = new ArrayList<>(); + try (BufferedReader reader = + Files.newBufferedReader(logFile.toPath(), StandardCharsets.UTF_8)) { + String line; + while ((line = reader.readLine()) != null) { + if (errorLogFilter.test(line)) { + errors.add(line); + } + } + } catch (NoSuchFileException e) { + return; // no output file was produced + } catch (IOException e) { + throw new IllegalStateException("Failed to read app log " + logFile, e); + } + if (!errors.isEmpty()) { + StringBuilder message = + new StringBuilder("App '") + .append(name) + .append("' logged ") + .append(errors.size()) + .append(" error line(s):"); + for (String error : errors) { + message.append("\n ").append(error); + } + throw new AssertionError(message.toString()); + } + } + + /** + * Asserts the agent emitted {@code app-started} telemetry for this app. Auto-invoked at teardown + * for agent-instrumented apps (unless {@link Builder#skipTelemetryCheck()}); telemetry flushes on + * app shutdown, so it runs after the process stops but while the backend is still up. + */ + public void assertTelemetryReceived() { + backend + .telemetry() + .waitForFlat( + message -> "app-started".equals(message.get("request_type")), startupTimeoutSeconds); + } + + /** + * The default error-log predicate: a line is an error if it contains {@code ERROR}, {@code + * ASSERTION FAILED}, or {@code Failed to handle exception in instrumentation} — unless it + * contains one of the {@code allowed} substrings (the FIXME-allowlist escape hatch). + * Package-private for testing. + */ + static Predicate defaultErrorLogFilter(List allowed) { + List allowlist = new ArrayList<>(allowed); + return line -> { + for (String allowedSubstring : allowlist) { + if (line.contains(allowedSubstring)) { + return false; + } + } + return line.contains("ERROR") + || line.contains("ASSERTION FAILED") + || line.contains("Failed to handle exception in instrumentation"); + }; + } + private static String javaExecutable() { return System.getProperty("java.home") + File.separator + "bin" + File.separator + "java"; } @@ -351,6 +452,10 @@ public static final class Builder { private boolean noAgent; private boolean server = true; private long startupTimeoutSeconds = 120; + private Predicate errorLogFilter; + private final List allowedErrorLogs = new ArrayList<>(); + private boolean checkErrorLogs = true; + private boolean checkTelemetry = true; private Builder(String name) { this.name = name; @@ -439,6 +544,36 @@ public Builder startupTimeoutSeconds(long seconds) { return this; } + /** + * Overrides how a captured log line is judged an error (default: contains {@code ERROR} / + * {@code ASSERTION FAILED} / an instrumentation-exception marker). Replaces the allowlist. + */ + public Builder errorLogFilter(Predicate isError) { + this.errorLogFilter = isError; + return this; + } + + /** Allowlists log lines containing any of these substrings from the default error-log check. */ + public Builder allowedErrorLogs(String... substrings) { + this.allowedErrorLogs.addAll(Arrays.asList(substrings)); + return this; + } + + /** Disables the automatic no-error-logs check at teardown (e.g. for error-case tests). */ + public Builder skipErrorLogCheck() { + this.checkErrorLogs = false; + return this; + } + + /** + * Disables the automatic app-started telemetry check at teardown (for agent apps that run with + * telemetry disabled, e.g. {@code dd.instrumentation.telemetry.enabled=false}). + */ + public Builder skipTelemetryCheck() { + this.checkTelemetry = false; + return this; + } + private String resolveAgentJar() { if (noAgent) { return null; @@ -455,8 +590,8 @@ public SmokeApp build() { } // TODO Q6 (deferred, opt-in mixins / .jvmArgs(...) escape hatch — not baked into the base): // profiling args; crash-tracking args (-XX:OnError=...dd_crash_uploader.sh); memory tuning - // (ForkedTestUtils); error-log assertion helpers (assertNoErrorLogs + FIXME allowlist); - // retry log-file timestamping. Add as explicit opt-ins when a ported test needs them. + // (ForkedTestUtils); retry log-file timestamping. Add as explicit opt-ins when a ported test + // needs them. return new SmokeApp(this); } } diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Telemetry.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Telemetry.java index 1b274596327..626e53146e9 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Telemetry.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Telemetry.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import java.util.function.Supplier; /** @@ -70,4 +71,21 @@ public Telemetry waitForCount(int count, double timeoutSeconds) { }); return this; } + + /** Waits (default timeout) until a flattened telemetry event matches {@code predicate}. */ + public Telemetry waitForFlat(Predicate> predicate) { + return waitForFlat(predicate, DEFAULT_TIMEOUT_SECONDS); + } + + /** As {@link #waitForFlat(Predicate)}, but overriding the timeout for this call. */ + public Telemetry waitForFlat(Predicate> predicate, double timeoutSeconds) { + new PollingConditions(timeoutSeconds) + .eventually( + () -> { + if (getFlatMessages().stream().noneMatch(predicate)) { + throw new AssertionError("No telemetry event matched; received: " + getMessages()); + } + }); + return this; + } } diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/ErrorLogSmokeTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/ErrorLogSmokeTest.java new file mode 100644 index 00000000000..e75648bc61d --- /dev/null +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/ErrorLogSmokeTest.java @@ -0,0 +1,38 @@ +package datadog.smoketest; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.smoketest.backend.TraceBackend; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Verifies {@link SmokeApp#assertNoErrorLogs()} against a real launched app: it reads the captured + * log and flags error lines. {@code skipErrorLogCheck()} disables the automatic teardown check so + * the deliberately-logged error doesn't fail the class — the assertion is exercised explicitly. + */ +class ErrorLogSmokeTest { + + @RegisterExtension + static final SmokeApp app = + SmokeApp.named("error-logger") + .mainClass("datadog.smoketest.TestServerApp") + .args("--server.port=${app.httpPort}") + .backend(TraceBackend.mockAgent()) + .noAgent() + .skipErrorLogCheck() + .build(); + + @Test + void detectsErrorLinesInTheLog() { + app.get("/hello"); + app.assertNoErrorLogs(); // no error logged yet + + app.get("/error"); // the app logs "ERROR simulated application error" + app.awaitLogLine(line -> line.contains("ERROR simulated")); // ensure it reached the log file + + AssertionError failure = assertThrows(AssertionError.class, app::assertNoErrorLogs); + assertTrue(failure.getMessage().contains("ERROR simulated"), failure.getMessage()); + } +} diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/SmokeAppErrorLogFilterTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/SmokeAppErrorLogFilterTest.java new file mode 100644 index 00000000000..b37a5db3fe9 --- /dev/null +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/SmokeAppErrorLogFilterTest.java @@ -0,0 +1,33 @@ +package datadog.smoketest; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.function.Predicate; +import org.junit.jupiter.api.Test; + +/** Docker-free unit tests for {@link SmokeApp}'s default error-log predicate. */ +class SmokeAppErrorLogFilterTest { + + @Test + void flagsErrorAndAssertionLines() { + Predicate isError = SmokeApp.defaultErrorLogFilter(emptyList()); + + assertTrue(isError.test("2026-07-14 12:00:00 ERROR o.e.SomeClass - boom"), "ERROR line"); + assertTrue(isError.test("junit ASSERTION FAILED: expected X"), "assertion line"); + assertTrue( + isError.test("Failed to handle exception in instrumentation"), "instrumentation failure"); + assertFalse(isError.test("INFO all good"), "info line is not an error"); + assertFalse(isError.test("WARN heads up"), "warn line is not an error"); + } + + @Test + void respectsAllowlist() { + Predicate isError = SmokeApp.defaultErrorLogFilter(singletonList("known flaky ERROR")); + + assertFalse(isError.test("this is a known flaky ERROR we tolerate"), "allowlisted line"); + assertTrue(isError.test("a real ERROR here"), "non-allowlisted error still flagged"); + } +} diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/TestServerApp.java b/dd-smoke-tests/src/test/java/datadog/smoketest/TestServerApp.java index e53e9989890..94647356e32 100644 --- a/dd-smoke-tests/src/test/java/datadog/smoketest/TestServerApp.java +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/TestServerApp.java @@ -28,8 +28,12 @@ public static void main(String[] args) throws Exception { server.createContext( "/", exchange -> { - System.out.println( - "REQUEST " + exchange.getRequestMethod() + " " + exchange.getRequestURI().getPath()); + String path = exchange.getRequestURI().getPath(); + System.out.println("REQUEST " + exchange.getRequestMethod() + " " + path); + if ("/error".equals(path)) { + // Emit an error line so tests can exercise the no-error-logs check. + System.out.println("ERROR simulated application error"); + } byte[] body = "ok".getBytes(StandardCharsets.UTF_8); exchange.sendResponseHeaders(200, body.length); try (OutputStream os = exchange.getResponseBody()) { diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java index 71f8fd9f91a..1eb7ed6576f 100644 --- a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java @@ -149,6 +149,21 @@ void capturesTelemetry() throws IOException { assertTrue(backend.telemetry().getMessages().isEmpty(), "clear() drops telemetry too"); } + @Test + void waitForFlatMatchesTelemetryEvents() throws IOException { + postTelemetry("{\"request_type\":\"app-started\",\"api_version\":\"v2\"}"); + + // Matches a received event… + backend.telemetry().waitForFlat(message -> "app-started".equals(message.get("request_type"))); + // …and times out (short timeout) when nothing matches. + assertThrows( + AssertionError.class, + () -> + backend + .telemetry() + .waitForFlat(message -> "never-sent".equals(message.get("request_type")), 0.2)); + } + @Test void infoAdvertisesTraceEndpoints() throws IOException { Request request = new Request.Builder().url(backend.url() + "/info").get().build(); From eedb9bed687490c5c308c09ce778891e08b441dc Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Tue, 14 Jul 2026 18:04:46 +0200 Subject: [PATCH 13/30] WIP --- .../SpringBootRabbitIntegrationTest.groovy | 119 ------------ .../smoketest/SpringBootRabbitSmokeTest.java | 170 ++++++++++++++++++ .../main/java/datadog/smoketest/SmokeApp.java | 55 +++--- .../smoketest/backend/TestAgentBackend.java | 18 ++ .../backend/TestAgentTraceDecoder.java | 41 +++++ .../smoketest/backend/TraceBackend.java | 14 +- .../datadog/smoketest/backend/Traces.java | 58 ++++++ .../smoketest/trace/SmokeTraceAssertions.java | 168 +++++++++++++++-- .../datadog/smoketest/trace/SpanMatcher.java | 110 ++++++++++-- .../datadog/smoketest/trace/TraceMatcher.java | 68 +++++-- .../smoketest/trace/SmokeMatcherTest.java | 161 +++++++++++++++++ 11 files changed, 804 insertions(+), 178 deletions(-) delete mode 100644 dd-smoke-tests/spring-boot-rabbit/src/test/groovy/datadog/smoketest/SpringBootRabbitIntegrationTest.groovy create mode 100644 dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java create mode 100644 dd-smoke-tests/src/test/java/datadog/smoketest/trace/SmokeMatcherTest.java diff --git a/dd-smoke-tests/spring-boot-rabbit/src/test/groovy/datadog/smoketest/SpringBootRabbitIntegrationTest.groovy b/dd-smoke-tests/spring-boot-rabbit/src/test/groovy/datadog/smoketest/SpringBootRabbitIntegrationTest.groovy deleted file mode 100644 index 797e85a67ab..00000000000 --- a/dd-smoke-tests/spring-boot-rabbit/src/test/groovy/datadog/smoketest/SpringBootRabbitIntegrationTest.groovy +++ /dev/null @@ -1,119 +0,0 @@ -package datadog.smoketest - -import datadog.trace.agent.test.utils.PortUtils -import okhttp3.Request -import org.testcontainers.containers.RabbitMQContainer -import spock.lang.Shared - -import java.util.concurrent.TimeUnit - -class SpringBootRabbitIntegrationTest extends AbstractServerSmokeTest { - - @Shared - def rabbitMQContainer - - @Shared - String rabbitHost - - @Shared - Integer rabbitPort - - def cleanupSpec() { - if (rabbitMQContainer) { - rabbitMQContainer.stop() - } - } - - protected int numberOfProcesses() { - return 2 - } - - @Override - void beforeProcessBuilders() { - rabbitMQContainer = new RabbitMQContainer("rabbitmq:3.9.20-alpine") - rabbitMQContainer.start() - rabbitHost = rabbitMQContainer.getHost() - rabbitPort = rabbitMQContainer.getMappedPort(5672) - PortUtils.waitForPortToOpen(rabbitHost, rabbitPort, 5, TimeUnit.SECONDS) - } - - @Override - ProcessBuilder createProcessBuilder(int processIndex) { - String springBootShadowJar = System.getProperty("datadog.smoketest.springboot.shadowJar.path") - - List command = new ArrayList<>() - command.add(javaPath()) - command.addAll(defaultJavaProperties) - command.addAll((String[]) [ - "-Ddd.service.name=spring-rabbit-${processIndex}", - "-Ddd.rabbit.legacy.tracing.enabled=false", - "-Ddd.writer.type=TraceStructureWriter:${outputs[processIndex].getAbsolutePath()}:includeService:includeResource", - "-jar", - springBootShadowJar, - "--server.port=${httpPorts[processIndex]}", - "--rabbit.${processIndex == 0 ? "sender" : "receiver"}.queue=otherqueue", - ]) - if (processIndex > 0) { - command.add("--rabbit.receiver.forward=true") - } - if (rabbitHost) { - command.add("--spring.rabbitmq.host=$rabbitHost".toString()) - } - if (rabbitPort) { - command.add("--spring.rabbitmq.port=$rabbitPort".toString()) - } - ProcessBuilder processBuilder = new ProcessBuilder(command) - processBuilder.directory(new File(buildDirectory)) - } - - @Override - File createTemporaryFile(int processIndex) { - return new File("${buildDirectory}/tmp/trace-structure-rabbit.${processIndex}.out") - } - - @Override - protected Set expectedTraces(int processIndex) { - def service = "spring-rabbit-${processIndex}" - Set expected = [ - "[${service}:amqp.command:basic.qos]", - "[${service}:amqp.command:basic.consume]", - "[${service}:amqp.command:basic.ack]", - "[${service}:amqp.command:queue.declare]" - ] - if (processIndex == 0) { - expected.add("[${service}:servlet.request:GET /roundtrip/{message}[${service}:spring.handler:WebController.roundtrip[${service}:amqp.command:basic.publish -> otherqueue]]]") - expected.add("[rabbitmq:amqp.deliver:amqp.deliver queue[${service}:amqp.command:basic.deliver queue[${service}:amqp.consume:amqp.consume queue[${service}:spring.consume:Receiver.receiveMessage]]]]") - } else { - expected.add("[rabbitmq:amqp.deliver:amqp.deliver otherqueue[${service}:amqp.command:basic.deliver otherqueue[${service}:amqp.consume:amqp.consume otherqueue[${service}:spring.consume:Receiver.receiveMessage[${service}:amqp.command:basic.publish -> queue]]]]]") - } - return expected - } - - @Override - boolean isErrorLog(String log) { - if (log.contains('org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer - Failed to check/redeclare auto-delete queue(s).')) { - return false - } - if (log.contains('ERROR com.rabbitmq.client.impl.ForgivingExceptionHandler - An unexpected connection driver error occured')) { - return false - } - return super.isErrorLog(log) - } - - def "check message #message roundtrip"() { - setup: - String url = "http://localhost:${httpPort}/roundtrip/${message}" - - when: - def request = new Request.Builder().url(url).get().build() - def response = client.newCall(request).execute() - - then: - def responseBodyStr = response.body().string() - responseBodyStr == "Got: >${message}" - response.code() == 200 - - where: - message << ["foo", "bar", "baz"] - } -} diff --git a/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java b/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java new file mode 100644 index 00000000000..72b1321fb37 --- /dev/null +++ b/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java @@ -0,0 +1,170 @@ +package datadog.smoketest; + +import static datadog.smoketest.trace.SpanMatcher.span; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import datadog.smoketest.backend.EnabledIfDockerAvailable; +import datadog.smoketest.backend.TestAgentBackend; +import datadog.smoketest.backend.TraceBackend; +import datadog.smoketest.backend.Traces; +import datadog.smoketest.trace.SpanMatcher; +import datadog.trace.test.agent.decoder.DecodedSpan; +import java.io.IOException; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.testcontainers.containers.RabbitMQContainer; +import org.testcontainers.utility.DockerImageName; + +/** + * S8 multi-app pilot — ported from the Groovy {@code SpringBootRabbitIntegrationTest}. Two Spring + * Boot apps (a sender and a receiver) round-trip a message through RabbitMQ and both report to one + * shared test-agent backend (Q8), so a single {@code @RegisterExtension} agent captures + * the distributed traces from both JVMs. + * + *

Replaces the dropped {@code TraceStructureWriter} file-mode (Q10) with {@link DecodedSpan} + * assertions, and hardens the Groovy {@code expectedTraces}. With {@code + * dd.rabbit.legacy.tracing.enabled=false} the whole round-trip is a single context-propagated trace + * spanning both JVMs and the broker — a strict parent→child chain of 12 spans, rooted at the sender + * {@code servlet.request}: + * + * 

+ * spring-rabbit-0 servlet.request GET /roundtrip/{message}
+ *   spring-rabbit-0 spring.handler WebController.roundtrip
+ *     spring-rabbit-0 amqp.command  basic.publish -> otherqueue     (send)
+ *       rabbitmq        amqp.deliver amqp.deliver otherqueue
+ *         spring-rabbit-1 amqp.command basic.deliver otherqueue
+ *           spring-rabbit-1 amqp.consume amqp.consume otherqueue
+ *             spring-rabbit-1 spring.consume Receiver.receiveMessage  (receiver consumes)
+ *               spring-rabbit-1 amqp.command basic.publish -> queue    (receiver forwards reply)
+ *                 rabbitmq        amqp.deliver amqp.deliver queue
+ *                   spring-rabbit-0 amqp.command basic.deliver queue
+ *                     spring-rabbit-0 amqp.consume amqp.consume queue
+ *                       spring-rabbit-0 spring.consume Receiver.receiveMessage (sender consumes reply)
+ *
+ * + *

Asserting that whole chain in one shot ({@link Traces#assertContainsChain}) is stronger than + * the Groovy set-membership check: it verifies every AMQP operation (publish/deliver/consume, both + * directions) and its cross-service linkage. The connection setup commands ({@code + * basic.qos}/{@code basic.consume}/{@code queue.declare}) and {@code basic.ack} are separate + * single-span traces, asserted per service. + * + *

Two things the Groovy base did implicitly that this port makes explicit: + * + *

    + *
  • Accumulate, don't isolate — {@code .retainAcrossTests()} keeps traces from app + * startup onward, because {@code basic.qos}/{@code basic.consume}/{@code queue.declare} are + * emitted when the consumers start (before any test method), which a per-method session + * {@code clear()} would discard. The Groovy base verified once at {@code cleanupSpec} against + * everything written since launch; a single accumulating test method mirrors that. + *
  • Membership, not count-exact — {@code assertContainsChain} asserts the expected + * structure is present and ignores the rest (as the Groovy set-membership check did): the + * full distributed span set is large and timing-dependent, so a positional match over the + * whole collection is the wrong tool. + *
+ */ +@EnabledIfDockerAvailable +class SpringBootRabbitSmokeTest { + private static final int TIMEOUT_SECONDS = 60; + private static final int RABBIT_AMQP_PORT = 5672; + private static final OkHttpClient CLIENT = new OkHttpClient(); + private static final String[] MESSAGES = {"foo", "bar", "baz"}; + // AMQP connection-setup / ack commands each app emits as its own (single-span) trace. + private static final String[] ADMIN_COMMANDS = { + "basic.qos", "basic.consume", "basic.ack", "queue.declare" + }; + + // Started in a static initializer so the app fields below can read its mapped port when built + // (before the JUnit lifecycle). Stopped by Testcontainers' JVM-shutdown hook. + private static final RabbitMQContainer RABBIT = + new RabbitMQContainer(DockerImageName.parse("rabbitmq:3.9.20-alpine")); + + static { + RABBIT.start(); + } + + // @Order pins the lifecycle: the shared agent starts before the apps (so its session is open when + // the consumers emit their startup traces) and is torn down after them. retainAcrossTests() keeps + // those startup traces for the single verifying method. + @Order(1) + @RegisterExtension + static final TestAgentBackend agent = + TraceBackend.testAgentBuilder().shared().retainAcrossTests().build(); + + @Order(2) + @RegisterExtension + static final SmokeApp sender = rabbitApp(0).args("--rabbit.sender.queue=otherqueue").build(); + + @Order(3) + @RegisterExtension + static final SmokeApp receiver = + rabbitApp(1) + .args("--rabbit.receiver.queue=otherqueue", "--rabbit.receiver.forward=true") + .build(); + + @Test + void roundtripsProduceFullAmqpTraceStructure() throws IOException { + // Drive 3 round-trips through the sender (matches the Groovy `where:` foo/bar/baz); each + // travels sender -> otherqueue -> receiver -> queue -> sender. + for (String message : MESSAGES) { + Request request = + new Request.Builder().url(sender.url() + "/roundtrip/" + message).get().build(); + try (Response response = CLIENT.newCall(request).execute()) { + assertEquals(200, response.code(), "roundtrip " + message); + assertEquals("Got: >" + message, response.body().string(), "roundtrip " + message); + } + } + + Traces traces = agent.traces(); + + // Each round-trip is its own full distributed trace: a strict parent->child chain across both + // services and the broker, from the sender's HTTP entrypoint to the sender consuming the + // forwarded reply. Assert one such chain per message (MESSAGES.length), so all round-trips are + // verified, not just one. + traces.assertContainsChain( + MESSAGES.length, + TIMEOUT_SECONDS, + sp("spring-rabbit-0", "servlet.request", "GET /roundtrip/{message}").root(), + sp("spring-rabbit-0", "spring.handler", "WebController.roundtrip"), + sp("spring-rabbit-0", "amqp.command", "basic.publish -> otherqueue"), + sp("rabbitmq", "amqp.deliver", "amqp.deliver otherqueue"), + sp("spring-rabbit-1", "amqp.command", "basic.deliver otherqueue"), + sp("spring-rabbit-1", "amqp.consume", "amqp.consume otherqueue"), + sp("spring-rabbit-1", "spring.consume", "Receiver.receiveMessage"), + sp("spring-rabbit-1", "amqp.command", "basic.publish -> queue"), + sp("rabbitmq", "amqp.deliver", "amqp.deliver queue"), + sp("spring-rabbit-0", "amqp.command", "basic.deliver queue"), + sp("spring-rabbit-0", "amqp.consume", "amqp.consume queue"), + sp("spring-rabbit-0", "spring.consume", "Receiver.receiveMessage")); + + // The connection-setup / ack commands each app emits as its own single-span trace. + for (String service : new String[] {"spring-rabbit-0", "spring-rabbit-1"}) { + for (String command : ADMIN_COMMANDS) { + traces.assertContainsChain(TIMEOUT_SECONDS, sp(service, "amqp.command", command).root()); + } + } + } + + private static SpanMatcher sp(String service, String operation, String resource) { + return span().service(service).operationName(operation).resourceName(resource); + } + + private static SmokeApp.Builder rabbitApp(int index) { + return SmokeApp.named("spring-rabbit-" + index) + .jar(System.getProperty("datadog.smoketest.springboot.shadowJar.path")) + .backend(agent) + .jvmArgs( + "-Ddd.service.name=spring-rabbit-" + index, "-Ddd.rabbit.legacy.tracing.enabled=false") + .args( + "--server.port=${app.httpPort}", + "--spring.rabbitmq.host=" + RABBIT.getHost(), + "--spring.rabbitmq.port=" + RABBIT.getMappedPort(RABBIT_AMQP_PORT)) + // The broker connection is torn down noisily when the app is killed at teardown. + .allowedErrorLogs( + "Failed to check/redeclare auto-delete queue(s)", + "An unexpected connection driver error occurred"); + } +} diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java b/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java index 33a33039558..470e2247d87 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java @@ -25,6 +25,7 @@ import okhttp3.Request; import okhttp3.Response; import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.AfterEachCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; @@ -53,7 +54,11 @@ * marked with explicit {@code // TODO}s below so the gaps are discoverable rather than silent. */ public final class SmokeApp - implements BeforeAllCallback, AfterAllCallback, BeforeEachCallback, ParameterResolver { + implements BeforeAllCallback, + AfterAllCallback, + BeforeEachCallback, + AfterEachCallback, + ParameterResolver { // Defaults mirroring the Groovy ProcessManager base so ported tests behave the same. private static final String SERVICE_NAME = "smoke-test-java-app"; @@ -88,6 +93,7 @@ public final class SmokeApp private final OutputThreads outputThreads = new OutputThreads(); private Process process; private File logFile; + private boolean telemetryChecked; private SmokeApp(Builder builder) { this.name = builder.name; @@ -229,6 +235,18 @@ public void beforeEach(ExtensionContext context) { outputThreads.clearMessages(); } + @Override + public void afterEach(ExtensionContext context) { + // Check telemetry once, here, while the app and backend are still up — afterAll is too late + // (the + // app is killed, and the per-method session clear may have wiped a once-only app-started). Only + // for agent-instrumented apps (a no-agent app emits none). + if (checkTelemetry && agentJar != null && !telemetryChecked) { + telemetryChecked = true; + assertTelemetryReceived(); + } + } + @Override public void afterAll(ExtensionContext context) { try { @@ -237,20 +255,12 @@ public void afterAll(ExtensionContext context) { // Join the output threads first so the log file is fully flushed before we scan it. outputThreads.close(); try { - // Telemetry flushes on app shutdown; check it while the backend is still up, and only for - // agent-instrumented apps (a no-agent app emits none). - if (checkTelemetry && agentJar != null) { - assertTelemetryReceived(); + if (ownsBackend) { + backend.close(); } } finally { - try { - if (ownsBackend) { - backend.close(); - } - } finally { - if (checkErrorLogs) { - assertNoErrorLogs(); - } + if (checkErrorLogs) { + assertNoErrorLogs(); } } } @@ -271,6 +281,11 @@ private void launch() throws IOException { if (sessionToken != null) { command.add("-Ddd.trace.agent.test.session.token=" + sessionToken); } + if (checkTelemetry) { + // Emit telemetry promptly so app-started is captured before a (long-running server) app is + // killed at teardown — mirrors the Groovy base's telemetry tests. + command.add("-Ddd.telemetry.heartbeat.interval=1"); + } } for (String jvmArg : jvmArgs) { command.add(substitute(jvmArg)); @@ -377,15 +392,15 @@ public void assertNoErrorLogs() { } /** - * Asserts the agent emitted {@code app-started} telemetry for this app. Auto-invoked at teardown - * for agent-instrumented apps (unless {@link Builder#skipTelemetryCheck()}); telemetry flushes on - * app shutdown, so it runs after the process stops but while the backend is still up. + * Asserts the app's telemetry pipeline is active — at least one telemetry message reached the + * backend. Auto-invoked once at the first {@link #afterEach} (while the app + backend are still + * up) unless {@link Builder#skipTelemetryCheck()}. It intentionally asserts "telemetry is + * flowing" rather than a specific event: the once-only {@code app-started} is fragile under the + * per-method session clear, whereas heartbeats keep arriving; a test wanting a specific event can + * assert it with {@link #traces() backend}.{@code telemetry().waitForFlat(...)}. */ public void assertTelemetryReceived() { - backend - .telemetry() - .waitForFlat( - message -> "app-started".equals(message.get("request_type")), startupTimeoutSeconds); + backend.telemetry().waitForCount(1, startupTimeoutSeconds); } /** diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentBackend.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentBackend.java index dc9ca23d1ff..9349a4fe70f 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentBackend.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentBackend.java @@ -60,6 +60,7 @@ public final class TestAgentBackend implements TraceBackend { private final int externalPort; private final List enabledChecks; private final boolean shared; + private final boolean retainAcrossTests; private final String sessionToken; private final OkHttpClient client = new OkHttpClient(); @@ -72,6 +73,7 @@ private TestAgentBackend(Builder builder) { this.externalPort = builder.externalPort; this.enabledChecks = new ArrayList<>(builder.enabledChecks); this.shared = builder.shared; + this.retainAcrossTests = builder.retainAcrossTests; this.sessionToken = builder.sessionToken != null ? builder.sessionToken : "smoke-" + UUID.randomUUID(); } @@ -95,6 +97,11 @@ public boolean isShared() { return shared; } + @Override + public boolean clearsBetweenTests() { + return !retainAcrossTests; + } + @Override public void start() { if (baseUrl != null) { @@ -255,6 +262,7 @@ public static final class Builder { private int externalPort = AGENT_PORT; private final List enabledChecks = new ArrayList<>(DEFAULT_ENABLED_CHECKS); private boolean shared; + private boolean retainAcrossTests; private String sessionToken; private Builder() {} @@ -287,6 +295,16 @@ public Builder shared() { return this; } + /** + * Keeps received traces across test methods instead of clearing them before each (see {@link + * TraceBackend#clearsBetweenTests()}). Use when assertions cover app-startup traces, which a + * per-method clear would discard. + */ + public Builder retainAcrossTests() { + this.retainAcrossTests = true; + return this; + } + /** Overrides the auto-generated session token (mainly for deterministic tests). */ public Builder sessionToken(String token) { this.sessionToken = token; diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentTraceDecoder.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentTraceDecoder.java index 006a457d9e3..b004463fd3f 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentTraceDecoder.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TestAgentTraceDecoder.java @@ -65,6 +65,11 @@ private static final class RawTrace implements DecodedTrace { public List getSpans() { return spans; } + + @Override + public String toString() { + return "RawTrace[" + spans.toString(); + } } // Field names follow the standard v0.4 trace shape and are verified end-to-end against a live @@ -163,5 +168,41 @@ public Map getMetrics() { public String getType() { return type; } + + @Override + public String toString() { + return "RawSpan{" + + "service='" + + service + + '\'' + + ", name='" + + name + + '\'' + + ", resource='" + + resource + + '\'' + + ", type='" + + type + + '\'' + + ", traceId=" + + traceId + + ", spanId=" + + spanId + + ", parentId=" + + parentId + + ", start=" + + start + + ", duration=" + + duration + + ", error=" + + error + + ", meta=" + + meta + + ", metaStruct=" + + metaStruct + + ", metrics=" + + metrics + + '}'; + } } } diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java index 9a747e03e91..d20a4ad355d 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/TraceBackend.java @@ -59,6 +59,16 @@ default boolean isShared() { return false; } + /** + * Whether {@link #beforeEach} clears received traces so each test method sees only its own (the + * default). Return {@code false} to accumulate across methods — needed when the + * assertions cover traces emitted at app startup (before the first test method), which a + * per-method clear would discard. Overridden by the test agent's {@code retainAcrossTests()}. + */ + default boolean clearsBetweenTests() { + return true; + } + // JUnit lifecycle: a backend declared as its own `@RegisterExtension` field (shared across apps, // S6/Q8) drives its own start/clear/close. An inline backend passed to `SmokeApp.backend(...)` is // not registered as an extension, so `SmokeApp` drives it instead. start() is idempotent, so a @@ -71,7 +81,9 @@ default void beforeAll(ExtensionContext context) { @Override default void beforeEach(ExtensionContext context) { - clear(); + if (clearsBetweenTests()) { + clear(); + } } @Override diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Traces.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Traces.java index a047a8dd7c8..ec97ad5035e 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Traces.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Traces.java @@ -1,11 +1,13 @@ package datadog.smoketest.backend; import datadog.smoketest.trace.SmokeTraceAssertions; +import datadog.smoketest.trace.SpanMatcher; import datadog.smoketest.trace.TraceMatcher; import datadog.trace.test.agent.decoder.DecodedTrace; import datadog.trace.test.util.PollingConditions; import java.util.List; import java.util.function.Supplier; +import java.util.function.UnaryOperator; /** * Query facade over the traces a {@link TraceBackend} has received. It is backend-agnostic: it @@ -63,9 +65,65 @@ public Traces waitForTraceCount(int count, double timeoutSeconds) { * } */ public void assertTraces(TraceMatcher... matchers) { + int expectedTraceCount = matchers.length; + waitForTraceCount(expectedTraceCount); SmokeTraceAssertions.assertTraces(getTraces(), matchers); } + /** + * As {@link #assertTraces(TraceMatcher...)} with options (e.g. {@code + * SmokeTraceAssertions.IGNORE_ADDITIONAL_TRACES} / {@code SORT_BY_ROOT_SPAN_ID}). + */ + public void assertTraces( + UnaryOperator options, TraceMatcher... matchers) { + SmokeTraceAssertions.assertTraces(getTraces(), options, matchers); + } + + /** + * Polls (up to the default timeout) until some received trace contains a parent-child chain of + * spans matching {@code chain} (see {@link SmokeTraceAssertions#assertContainsChain}). Combines + * waiting for async arrival with the structural assertion, so it suits distributed traces whose + * full span set arrives piecemeal — assert the linkage you care about, ignore the rest: + * + * 
{@code
+   * app.traces().assertContainsChain(
+   *     span().service("web").operationName("servlet.request").root(),
+   *     span().service("web").operationName("amqp.command").resourceName(startsWith("basic.publish")));
+   * }
+ */ + public Traces assertContainsChain(SpanMatcher... chain) { + return assertContainsChain(DEFAULT_TIMEOUT_SECONDS, chain); + } + + /** As {@link #assertContainsChain(SpanMatcher...)}, but overriding the timeout for this call. */ + public Traces assertContainsChain(double timeoutSeconds, SpanMatcher... chain) { + new PollingConditions(timeoutSeconds) + .eventually(() -> SmokeTraceAssertions.assertContainsChain(getTraces(), chain)); + return this; + } + + /** + * Polls (up to {@code timeoutSeconds}) until at least {@code minMatches} received traces each + * contain a chain matching {@code chain} (see {@link SmokeTraceAssertions#countChainMatches}). + * Use to verify that N independent operations each produced the expected trace — e.g. one + * distributed trace per request — not merely that one did. + */ + public Traces assertContainsChain(int minMatches, double timeoutSeconds, SpanMatcher... chain) { + new PollingConditions(timeoutSeconds) + .eventually( + () -> { + long found = SmokeTraceAssertions.countChainMatches(getTraces(), chain); + if (found < minMatches) { + throw new AssertionError( + "Expected at least " + + minMatches + + " traces containing the chain but found " + + found); + } + }); + return this; + } + // Trace-invariant checks (ENABLED_CHECKS) are a test-agent-specific concern, validated by that // backend itself (TestAgentBackend#assertNoInvariantFailures, auto-run at container teardown per // Q5) rather than on this common facade, which stays portable across both backends (Q2). diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java index 2da717f4337..bdf8fae7000 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java @@ -1,7 +1,13 @@ package datadog.smoketest.trace; +import static java.util.function.UnaryOperator.identity; + +import datadog.trace.test.agent.decoder.DecodedSpan; import datadog.trace.test.agent.decoder.DecodedTrace; +import java.util.ArrayList; +import java.util.Comparator; import java.util.List; +import java.util.function.UnaryOperator; /** * Entry point for the thin smoke trace DSL. Assert a captured trace collection against one {@link @@ -15,21 +21,163 @@ * assertTraces(traces, trace(span().operationName("servlet.request").resourceName("GET /greeting"))); * } * - * The traces come from a {@code TraceBackend} (mock or test-agent), both decoded to {@link - * DecodedTrace}. + *

Traces are matched positionally, in the order received by default. For order-independence pass + * {@link #SORT_BY_START_TIME} / {@link #SORT_BY_ROOT_SPAN_ID}; to assert only a subset of the + * received traces pass {@link #IGNORE_ADDITIONAL_TRACES}. The traces come from a {@code + * TraceBackend} (mock or test-agent), both decoded to {@link DecodedTrace}. */ public final class SmokeTraceAssertions { + /** Sorts traces by the earliest span start time. */ + public static final Comparator> TRACE_START_TIME_COMPARATOR = + Comparator.comparingLong(SmokeTraceAssertions::earliestStart); + + /** Sorts traces by their root span id (the span with no parent). */ + public static final Comparator> TRACE_ROOT_SPAN_ID_COMPARATOR = + Comparator.comparingLong(SmokeTraceAssertions::rootSpanId); + + /** Do not fail when there are more traces than matchers (assert a subset). */ + public static final UnaryOperator IGNORE_ADDITIONAL_TRACES = + Options::ignoreAdditionalTraces; + + /** Sorts traces by earliest span start time before matching. */ + public static final UnaryOperator SORT_BY_START_TIME = + options -> options.sorter(TRACE_START_TIME_COMPARATOR); + + /** Sorts traces by root span id before matching. */ + public static final UnaryOperator SORT_BY_ROOT_SPAN_ID = + options -> options.sorter(TRACE_ROOT_SPAN_ID_COMPARATOR); + private SmokeTraceAssertions() {} + /** Asserts the traces against the matchers, one matcher per expected trace, in received order. */ public static void assertTraces(List traces, TraceMatcher... matchers) { - // TODO thin first cut: traces are matched positionally, in the order received. Add sorting - // (e.g. - // by root-span start time) when a smoke test needs order-independence. - if (traces.size() != matchers.length) { - throw new AssertionError("Expected " + matchers.length + " traces but got " + traces.size()); - } - for (int i = 0; i < matchers.length; i++) { - matchers[i].assertTrace(traces.get(i)); + assertTraces(traces, identity(), matchers); + } + + /** As {@link #assertTraces(List, TraceMatcher...)} with the given options. */ + public static void assertTraces( + List traces, UnaryOperator options, TraceMatcher... matchers) { + Options opts = options.apply(new Options()); + int expected = matchers.length; + int actual = traces.size(); + if (opts.ignoreAdditionalTraces) { + if (actual < expected) { + throw new AssertionError("Expected at least " + expected + " traces but got " + actual); + } + } else if (actual != expected) { + throw new AssertionError("Expected " + expected + " traces but got " + actual); + } + // Copy each trace's spans to a mutable list so the matcher can sort/inspect them. + List> spanLists = new ArrayList<>(actual); + for (DecodedTrace trace : traces) { + spanLists.add(new ArrayList<>(trace.getSpans())); + } + if (opts.sorter != null) { + spanLists.sort(opts.sorter); + } + for (int i = 0; i < expected; i++) { + matchers[i].assertTrace(spanLists.get(i), i); + } + } + + /** + * Asserts that some received trace contains a parent-child chain of spans matching + * {@code chain} in order: {@code chain[0]} matches a span (a trace root if it declares {@link + * SpanMatcher#root()}), and each {@code chain[i]} matches a direct child of {@code chain[i-1]}'s + * span. Unlike {@link #assertTraces}, this is a subset match — extra spans in the trace + * are ignored — which suits distributed traces whose full span set is large and timing-dependent + * (only span linkage, not exact shape, is asserted). Field constraints are declared per span; the + * chain declares the linkage, so the matchers should not also use {@code childOf*}. + */ + public static void assertContainsChain(List traces, SpanMatcher... chain) { + long found = countChainMatches(traces, chain); + if (found == 0) { + throw new AssertionError( + "No received trace contains a parent-child chain matching the " + + chain.length + + " given span matcher(s); traces: " + + traces); + } + } + + /** + * Counts how many received traces contain a parent-child chain matching {@code chain} (see {@link + * #assertContainsChain(List, SpanMatcher...)} for the chain semantics). Use to verify N + * independent operations each produced the expected trace — e.g. one distributed trace per + * request — rather than just that a single one did. + */ + public static long countChainMatches(List traces, SpanMatcher... chain) { + if (chain.length == 0) { + throw new IllegalArgumentException("countChainMatches requires at least one span matcher"); + } + long count = 0; + for (DecodedTrace trace : traces) { + if (containsChain(trace.getSpans(), chain)) { + count++; + } + } + return count; + } + + private static boolean containsChain(List spans, SpanMatcher[] chain) { + for (DecodedSpan first : spans) { + if (chain[0].rootRequired() && first.getParentId() != 0L) { + continue; + } + if (chain[0].matchesFields(first) && matchesChainFrom(spans, chain, 1, first)) { + return true; + } + } + return false; + } + + // Depth-first, backtracking: extend the chain by a direct child of `parent` matching + // chain[index]. + private static boolean matchesChainFrom( + List spans, SpanMatcher[] chain, int index, DecodedSpan parent) { + if (index == chain.length) { + return true; + } + for (DecodedSpan child : spans) { + if (child.getParentId() == parent.getSpanId() + && chain[index].matchesFields(child) + && matchesChainFrom(spans, chain, index + 1, child)) { + return true; + } + } + return false; + } + + private static long earliestStart(List spans) { + long earliest = Long.MAX_VALUE; + for (DecodedSpan span : spans) { + earliest = Math.min(earliest, span.getStart()); + } + return spans.isEmpty() ? 0L : earliest; + } + + private static long rootSpanId(List spans) { + for (DecodedSpan span : spans) { + if (span.getParentId() == 0L) { + return span.getSpanId(); + } + } + return spans.isEmpty() ? 0L : spans.get(0).getSpanId(); + } + + /** Trace-collection matching options. */ + public static final class Options { + boolean ignoreAdditionalTraces = false; + Comparator> sorter = null; // null => keep received order + + public Options ignoreAdditionalTraces() { + this.ignoreAdditionalTraces = true; + return this; + } + + public Options sorter(Comparator> sorter) { + this.sorter = sorter; + return this; } } } diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SpanMatcher.java b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SpanMatcher.java index cd703bb3263..4216baacfea 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SpanMatcher.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SpanMatcher.java @@ -1,14 +1,20 @@ package datadog.smoketest.trace; +import static datadog.trace.junit.utils.assertions.Matchers.any; import static datadog.trace.junit.utils.assertions.Matchers.assertValue; import static datadog.trace.junit.utils.assertions.Matchers.is; import static datadog.trace.junit.utils.assertions.Matchers.isFalse; import static datadog.trace.junit.utils.assertions.Matchers.isTrue; +import static datadog.trace.junit.utils.assertions.Matchers.matches; +import static datadog.trace.junit.utils.assertions.Matchers.validates; import datadog.trace.junit.utils.assertions.Matcher; import datadog.trace.test.agent.decoder.DecodedSpan; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.function.Predicate; +import java.util.regex.Pattern; /** * Thin span matcher for smoke tests, operating on the serialized {@link DecodedSpan} model @@ -20,16 +26,20 @@ * once serialized is matchable here: name/service/resource/type, error status, {@code meta} (string * tags), {@code metrics} (numeric tags), and parent linkage. * - *

Use the {@link #span()} factory as a fluent builder. Unset constraints are not checked (thin - * by design). + *

Parent linkage can be expressed by {@link #root()}, {@link #childOf(long) explicit id}, or — + * for structural assertions within a sorted trace — by position with {@link #childOfIndex(int)} / + * {@link #childOfPrevious()} (mirroring the instrumentation DSL). Use the {@link #span()} factory + * as a fluent builder; unset constraints are not checked. */ public final class SpanMatcher { private Matcher serviceMatcher; - private Matcher operationNameMatcher; - private Matcher resourceNameMatcher; + private Matcher operationNameMatcher; + private Matcher resourceNameMatcher; private Matcher typeMatcher; private Matcher errorMatcher; - private Long parentId; // null => parent not checked + private Long parentId; // explicit expected parent id (childOf/root); null => not checked by id + private int parentSpanIndex = -1; // >= 0 => parent is the span at this index in the sorted trace + private boolean childOfPrevious; // parent is the preceding span in the sorted trace private final Map> metaMatchers = new HashMap<>(); private final Map> metricMatchers = new HashMap<>(); @@ -50,17 +60,32 @@ public SpanMatcher operationName(String operationName) { return this; } + public SpanMatcher operationName(Pattern pattern) { + this.operationNameMatcher = matches(pattern); + return this; + } + public SpanMatcher resourceName(String resourceName) { this.resourceNameMatcher = is(resourceName); return this; } + public SpanMatcher resourceName(Pattern pattern) { + this.resourceNameMatcher = matches(pattern); + return this; + } + + public SpanMatcher resourceName(Predicate validator) { + this.resourceNameMatcher = validates(validator); + return this; + } + public SpanMatcher type(String type) { this.typeMatcher = is(type); return this; } - /** Matches a non-error span. */ + /** Matches the span error status. */ public SpanMatcher error(boolean errored) { this.errorMatcher = errored ? isTrue() : isFalse(); return this; @@ -69,12 +94,32 @@ public SpanMatcher error(boolean errored) { /** Matches a root span (no parent). */ public SpanMatcher root() { this.parentId = 0L; + this.parentSpanIndex = -1; + this.childOfPrevious = false; return this; } /** Matches a span whose parent is the given span id. */ public SpanMatcher childOf(long parentSpanId) { this.parentId = parentSpanId; + this.parentSpanIndex = -1; + this.childOfPrevious = false; + return this; + } + + /** Matches a span whose parent is the span at {@code parentSpanIndex} in the (sorted) trace. */ + public SpanMatcher childOfIndex(int parentSpanIndex) { + this.parentSpanIndex = parentSpanIndex; + this.parentId = null; + this.childOfPrevious = false; + return this; + } + + /** Matches a span whose parent is the immediately preceding span in the (sorted) trace. */ + public SpanMatcher childOfPrevious() { + this.childOfPrevious = true; + this.parentId = null; + this.parentSpanIndex = -1; return this; } @@ -96,21 +141,25 @@ public SpanMatcher metric(String name, Matcher matcher) { return this; } - // TODO thin first cut — add when a smoke test needs them: - // - operationName/resourceName by Pattern or Predicate, - // - childOf(SpanMatcher) resolving the parent within the trace, - // - exhaustive tag coverage (fail on unexpected meta/metrics), and span-links. + // TODO thin: exhaustive tag coverage (fail on unexpected meta/metrics) and span-links. + + /** Positional (count-exact) assertion: the span at {@code spanIndex} matches this matcher. */ + void assertSpan(List trace, int spanIndex) { + assertValue( + parentIdMatcher(trace, spanIndex), + trace.get(spanIndex).getParentId(), + "Unexpected parent id"); + assertFields(trace.get(spanIndex)); + } + /** Asserts a span's own fields (service/operation/resource/type/error/tags), ignoring linkage. */ @SuppressWarnings("unchecked") - void assertSpan(DecodedSpan span) { + private void assertFields(DecodedSpan span) { assertValue(serviceMatcher, span.getService(), "Unexpected service name"); assertValue(operationNameMatcher, span.getName(), "Unexpected operation name"); assertValue(resourceNameMatcher, span.getResource(), "Unexpected resource name"); assertValue(typeMatcher, span.getType(), "Unexpected span type"); assertValue(errorMatcher, span.getError() != 0, "Unexpected error status"); - if (parentId != null) { - assertValue(is(parentId), span.getParentId(), "Unexpected parent id"); - } Map meta = span.getMeta(); for (Map.Entry> entry : metaMatchers.entrySet()) { Object value = meta == null ? null : meta.get(entry.getKey()); @@ -126,4 +175,37 @@ void assertSpan(DecodedSpan span) { (Matcher) entry.getValue(), value, "Unexpected metric '" + entry.getKey() + "'"); } } + + /** + * Whether this matcher's own fields match {@code span}, ignoring parent linkage — the boolean + * counterpart of {@link #assertFields} used by subset/chain matching (see {@link + * SmokeTraceAssertions#assertContainsChain}), where linkage is verified positionally by the + * chain. + */ + boolean matchesFields(DecodedSpan span) { + try { + assertFields(span); + return true; + } catch (AssertionError ignored) { + return false; + } + } + + /** Whether this matcher requires its span to be a trace root (via {@link #root()}). */ + boolean rootRequired() { + return parentId != null && parentId == 0L; + } + + private Matcher parentIdMatcher(List trace, int spanIndex) { + if (this.parentSpanIndex >= 0) { + return is(trace.get(parentSpanIndex).getSpanId()); + } + if (this.childOfPrevious) { + if (spanIndex == 0) { + throw new IllegalStateException("childOfPrevious() cannot be used on the first span"); + } + return is(trace.get(spanIndex - 1).getSpanId()); + } + return this.parentId == null ? any() : is(this.parentId); + } } diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/TraceMatcher.java b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/TraceMatcher.java index 05880b3b032..bfbf2c8ce69 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/TraceMatcher.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/TraceMatcher.java @@ -1,40 +1,80 @@ package datadog.smoketest.trace; +import static java.util.Comparator.comparingLong; + import datadog.trace.test.agent.decoder.DecodedSpan; -import datadog.trace.test.agent.decoder.DecodedTrace; +import java.util.ArrayList; +import java.util.Comparator; import java.util.List; +import java.util.function.UnaryOperator; /** - * Thin trace matcher for smoke tests: one {@link SpanMatcher} per expected span in a {@link - * DecodedTrace}. Companion of {@link SpanMatcher}; see its class doc for the serialized-model - * rationale. + * Thin trace matcher for smoke tests: one {@link SpanMatcher} per expected span in a trace. + * Companion of {@link SpanMatcher}; see its class doc for the serialized-model rationale. + * + *

Spans are matched positionally. By default they are matched in the order received; pass {@link + * #SORT_BY_START_TIME} to {@link #trace(UnaryOperator, SpanMatcher...)} to sort them first, which + * makes {@link SpanMatcher#childOfIndex(int)} / {@link SpanMatcher#childOfPrevious()} + * deterministic. */ public final class TraceMatcher { + /** Sorts a trace's spans by start time. */ + public static final Comparator START_TIME_COMPARATOR = + comparingLong(DecodedSpan::getStart); + + /** Configures {@link #trace(UnaryOperator, SpanMatcher...)} to sort spans by start time. */ + public static final UnaryOperator SORT_BY_START_TIME = + options -> options.sorter(START_TIME_COMPARATOR); + + private final Options options; private final SpanMatcher[] spanMatchers; - private TraceMatcher(SpanMatcher[] spanMatchers) { + private TraceMatcher(Options options, SpanMatcher[] spanMatchers) { if (spanMatchers.length == 0) { throw new IllegalArgumentException("No span matchers provided"); } + this.options = options; this.spanMatchers = spanMatchers; } - /** Checks a trace structure, one matcher per expected span. */ + /** Checks a trace structure, one matcher per expected span, in the order received. */ public static TraceMatcher trace(SpanMatcher... matchers) { - return new TraceMatcher(matchers); + return new TraceMatcher(new Options(), matchers); + } + + /** Checks a trace structure with the given options (e.g. {@link #SORT_BY_START_TIME}). */ + public static TraceMatcher trace(UnaryOperator options, SpanMatcher... matchers) { + return new TraceMatcher(options.apply(new Options()), matchers); } - void assertTrace(DecodedTrace trace) { - List spans = trace.getSpans(); - // TODO thin first cut: spans are matched positionally, in the order received. Add sorting - // (e.g. by start time, mirroring the instrumentation DSL) and/or find-by-id matching when a - // smoke test needs order-independence. + void assertTrace(List spans, int traceIndex) { if (spans.size() != spanMatchers.length) { throw new AssertionError( - "Expected " + spanMatchers.length + " spans but got " + spans.size() + ": " + spans); + "Expected " + + spanMatchers.length + + " spans in trace " + + traceIndex + + " but got " + + spans.size() + + ": " + + spans); + } + List ordered = new ArrayList<>(spans); + if (options.sorter != null) { + ordered.sort(options.sorter); } for (int i = 0; i < spanMatchers.length; i++) { - spanMatchers[i].assertSpan(spans.get(i)); + spanMatchers[i].assertSpan(ordered, i); + } + } + + /** Per-trace matching options. */ + public static final class Options { + Comparator sorter = null; // null => keep received order + + public Options sorter(Comparator sorter) { + this.sorter = sorter; + return this; } } } diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/trace/SmokeMatcherTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/trace/SmokeMatcherTest.java new file mode 100644 index 00000000000..bd3ce7c157e --- /dev/null +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/trace/SmokeMatcherTest.java @@ -0,0 +1,161 @@ +package datadog.smoketest.trace; + +import static datadog.smoketest.trace.SmokeTraceAssertions.assertTraces; +import static datadog.smoketest.trace.SpanMatcher.span; +import static datadog.smoketest.trace.TraceMatcher.trace; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import datadog.smoketest.backend.TestAgentTraceDecoder; +import datadog.trace.test.agent.decoder.DecodedTrace; +import java.util.List; +import org.junit.jupiter.api.Test; + +/** + * Docker-free tests for the structural matcher extensions (span sorting + {@code childOfIndex} / + * {@code childOfPrevious}, and trace-collection sorting / {@code IGNORE_ADDITIONAL_TRACES}). Traces + * are built synthetically through the S1b JSON decoder so no backend is needed. + */ +class SmokeMatcherTest { + + // One trace, three spans forming root -> child -> grandchild, but delivered out of start order. + private static final String CHAIN_TRACE = + "[[" + + spanJson("grandchild", 300, 200, 30) + + "," + + spanJson("root", 100, 0, 10) + + "," + + spanJson("child", 200, 100, 20) + + "]]"; + + // Two single-span traces; root-b (id 400) has a smaller root span id than root-a (id 500). + private static final String TWO_TRACES = + "[[" + spanJson("root-a", 500, 0, 10) + "],[" + spanJson("root-b", 400, 0, 20) + "]]"; + + // Two traces with the same root -> child shape (distinct span ids). + private static final String TWO_CHAINS = + "[[" + + spanJson("root", 100, 0, 10) + + "," + + spanJson("child", 200, 100, 20) + + "],[" + + spanJson("root", 300, 0, 30) + + "," + + spanJson("child", 400, 300, 40) + + "]]"; + + @Test + void sortsSpansByStartTimeThenMatchesParentByPrevious() { + List traces = TestAgentTraceDecoder.decode(CHAIN_TRACE); + assertTraces( + traces, + trace( + TraceMatcher.SORT_BY_START_TIME, + span().operationName("root").root(), + span().operationName("child").childOfPrevious(), + span().operationName("grandchild").childOfPrevious())); + } + + @Test + void matchesParentByIndex() { + List traces = TestAgentTraceDecoder.decode(CHAIN_TRACE); + assertTraces( + traces, + trace( + TraceMatcher.SORT_BY_START_TIME, + span().operationName("root").root(), + span().operationName("child").childOfIndex(0), + span().operationName("grandchild").childOfIndex(1))); + } + + @Test + void wrongParentLinkFails() { + List traces = TestAgentTraceDecoder.decode(CHAIN_TRACE); + // grandchild's parent is child (index 1), not root (index 0). + assertThrows( + AssertionError.class, + () -> + assertTraces( + traces, + trace( + TraceMatcher.SORT_BY_START_TIME, + span().operationName("root").root(), + span().operationName("child").childOfIndex(0), + span().operationName("grandchild").childOfIndex(0)))); + } + + @Test + void ignoresAdditionalTraces() { + List traces = TestAgentTraceDecoder.decode(TWO_TRACES); + // Assert just the first received trace, ignoring the other. + assertTraces( + traces, + SmokeTraceAssertions.IGNORE_ADDITIONAL_TRACES, + trace(span().operationName("root-a").root())); + } + + @Test + void sortsTracesByRootSpanId() { + List traces = TestAgentTraceDecoder.decode(TWO_TRACES); + // Root-id order is [root-b (400), root-a (500)], the reverse of the received order. + assertTraces( + traces, + SmokeTraceAssertions.SORT_BY_ROOT_SPAN_ID, + trace(span().operationName("root-b").root()), + trace(span().operationName("root-a").root())); + } + + @Test + void assertContainsChainMatchesParentChildChain() { + List traces = TestAgentTraceDecoder.decode(CHAIN_TRACE); + // Full chain from the root. + SmokeTraceAssertions.assertContainsChain( + traces, + span().operationName("root").root(), + span().operationName("child"), + span().operationName("grandchild")); + // Subset chain that doesn't start at the root still matches (extra spans ignored). + SmokeTraceAssertions.assertContainsChain( + traces, span().operationName("child"), span().operationName("grandchild")); + } + + @Test + void assertContainsChainFailsOnBrokenLinkage() { + List traces = TestAgentTraceDecoder.decode(CHAIN_TRACE); + // grandchild is a child of child, not a direct child of root. + assertThrows( + AssertionError.class, + () -> + SmokeTraceAssertions.assertContainsChain( + traces, span().operationName("root").root(), span().operationName("grandchild"))); + } + + @Test + void countsTracesContainingChain() { + List traces = TestAgentTraceDecoder.decode(TWO_CHAINS); + // Both traces contain the root -> child chain. + assertEquals( + 2, + SmokeTraceAssertions.countChainMatches( + traces, span().operationName("root").root(), span().operationName("child"))); + // A chain present in neither trace counts zero. + assertEquals( + 0, + SmokeTraceAssertions.countChainMatches( + traces, span().operationName("root").root(), span().operationName("missing"))); + } + + private static String spanJson(String name, long id, long parent, long start) { + return "{\"service\":\"s\",\"name\":\"" + + name + + "\",\"resource\":\"" + + name + + "\",\"type\":\"web\",\"trace_id\":1,\"span_id\":" + + id + + ",\"parent_id\":" + + parent + + ",\"start\":" + + start + + ",\"duration\":1,\"error\":0,\"meta\":{},\"metrics\":{}}"; + } +} From 7118167e4f2414b25fa342ac0353afab51ba2a8c Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Tue, 14 Jul 2026 18:40:45 +0200 Subject: [PATCH 14/30] WIP --- .../spring-boot-rabbit/build.gradle | 1 + .../spring-boot-rabbit/gradle.lockfile | 3 +- .../smoketest/SpringBootRabbitSmokeTest.java | 27 +++++++++------ .../main/java/datadog/smoketest/SmokeApp.java | 34 ++++++++++++++++++- .../java/datadog/smoketest/SmokeAppTest.java | 13 ++++++- .../java/datadog/smoketest/TestServerApp.java | 7 +++- 6 files changed, 70 insertions(+), 15 deletions(-) diff --git a/dd-smoke-tests/spring-boot-rabbit/build.gradle b/dd-smoke-tests/spring-boot-rabbit/build.gradle index e7b91d57bfb..d7fd58b3e6a 100644 --- a/dd-smoke-tests/spring-boot-rabbit/build.gradle +++ b/dd-smoke-tests/spring-boot-rabbit/build.gradle @@ -26,6 +26,7 @@ dependencies { testImplementation project(':dd-smoke-tests') testImplementation group: 'org.testcontainers', name: 'rabbitmq', version: libs.versions.testcontainers.get() + testImplementation group: 'org.testcontainers', name: 'junit-jupiter', version: libs.versions.testcontainers.get() } tasks.withType(Test).configureEach { diff --git a/dd-smoke-tests/spring-boot-rabbit/gradle.lockfile b/dd-smoke-tests/spring-boot-rabbit/gradle.lockfile index f7fe086e170..a19d90f6502 100644 --- a/dd-smoke-tests/spring-boot-rabbit/gradle.lockfile +++ b/dd-smoke-tests/spring-boot-rabbit/gradle.lockfile @@ -80,7 +80,7 @@ org.apache.logging.log4j:log4j-to-slf4j:2.14.1=compileClasspath,runtimeClasspath org.apache.tomcat.embed:tomcat-embed-core:9.0.52=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.apache.tomcat.embed:tomcat-embed-el:9.0.52=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.apache.tomcat.embed:tomcat-embed-websocket:9.0.52=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath,testRuntimeClasspath org.codehaus.groovy:groovy-ant:3.0.23=codenarc org.codehaus.groovy:groovy-docgenerator:3.0.23=codenarc org.codehaus.groovy:groovy-groovydoc:3.0.23=codenarc @@ -158,6 +158,7 @@ org.springframework:spring-web:5.3.9=compileClasspath,runtimeClasspath,testCompi org.springframework:spring-webmvc:5.3.9=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.tabletest:tabletest-junit:1.2.1=testCompileClasspath,testRuntimeClasspath org.tabletest:tabletest-parser:1.2.0=testCompileClasspath,testRuntimeClasspath +org.testcontainers:junit-jupiter:1.21.4=testCompileClasspath,testRuntimeClasspath org.testcontainers:rabbitmq:1.21.4=testCompileClasspath,testRuntimeClasspath org.testcontainers:testcontainers:1.21.4=testCompileClasspath,testRuntimeClasspath org.xmlresolver:xmlresolver:5.3.3=spotbugs diff --git a/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java b/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java index 72b1321fb37..1d4ae5fbf0a 100644 --- a/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java +++ b/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java @@ -17,6 +17,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.testcontainers.containers.RabbitMQContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; import org.testcontainers.utility.DockerImageName; /** @@ -67,6 +69,7 @@ * */ @EnabledIfDockerAvailable +@Testcontainers class SpringBootRabbitSmokeTest { private static final int TIMEOUT_SECONDS = 60; private static final int RABBIT_AMQP_PORT = 5672; @@ -77,18 +80,17 @@ class SpringBootRabbitSmokeTest { "basic.qos", "basic.consume", "basic.ack", "queue.declare" }; - // Started in a static initializer so the app fields below can read its mapped port when built - // (before the JUnit lifecycle). Stopped by Testcontainers' JVM-shutdown hook. + // @Testcontainers starts/stops this static container in the class lifecycle (no static-block + // start()); as a class-level @ExtendWith it runs before the @RegisterExtension fields below, so + // RabbitMQ is up before the apps launch. Its mapped port is still read lazily at launch via + // placeholders, since the container is not yet started when these static fields initialize. + @Container private static final RabbitMQContainer RABBIT = new RabbitMQContainer(DockerImageName.parse("rabbitmq:3.9.20-alpine")); - static { - RABBIT.start(); - } - - // @Order pins the lifecycle: the shared agent starts before the apps (so its session is open when - // the consumers emit their startup traces) and is torn down after them. retainAcrossTests() keeps - // those startup traces for the single verifying method. + // @Order pins the @RegisterExtension order: the shared agent starts before the apps (so its + // session is open when the consumers emit startup traces) and is torn down after them. + // retainAcrossTests() keeps those startup traces for the one verifying method. @Order(1) @RegisterExtension static final TestAgentBackend agent = @@ -158,10 +160,13 @@ private static SmokeApp.Builder rabbitApp(int index) { .backend(agent) .jvmArgs( "-Ddd.service.name=spring-rabbit-" + index, "-Ddd.rabbit.legacy.tracing.enabled=false") + // Resolved at launch, after @Testcontainers has started RABBIT — not at build time. + .placeholder("rabbit.host", RABBIT::getHost) + .placeholder("rabbit.port", () -> String.valueOf(RABBIT.getMappedPort(RABBIT_AMQP_PORT))) .args( "--server.port=${app.httpPort}", - "--spring.rabbitmq.host=" + RABBIT.getHost(), - "--spring.rabbitmq.port=" + RABBIT.getMappedPort(RABBIT_AMQP_PORT)) + "--spring.rabbitmq.host=${rabbit.host}", + "--spring.rabbitmq.port=${rabbit.port}") // The broker connection is torn down noisily when the app is killed at teardown. .allowedErrorLogs( "Failed to check/redeclare auto-delete queue(s)", diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java b/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java index 470e2247d87..93a4c844797 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeApp.java @@ -14,6 +14,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -21,6 +22,7 @@ import java.util.concurrent.TimeoutException; import java.util.function.Function; import java.util.function.Predicate; +import java.util.function.Supplier; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; @@ -77,6 +79,7 @@ public final class SmokeApp private final String classpath; private final List jvmArgs; private final List programArgs; + private final Map> placeholders; private final Map extraEnv; private final File workingDirectory; private final TraceBackend backend; @@ -103,6 +106,7 @@ private SmokeApp(Builder builder) { builder.classpath != null ? builder.classpath : System.getProperty("java.class.path"); this.jvmArgs = new ArrayList<>(builder.jvmArgs); this.programArgs = new ArrayList<>(builder.programArgs); + this.placeholders = new LinkedHashMap<>(builder.placeholders); this.extraEnv = new HashMap<>(builder.extraEnv); this.workingDirectory = builder.workingDirectory; this.backend = builder.backend; @@ -338,7 +342,16 @@ private void stopProcess() { } private String substitute(String value) { - return value.replace(HTTP_PORT_PLACEHOLDER, Integer.toString(httpPort)); + String result = value.replace(HTTP_PORT_PLACEHOLDER, Integer.toString(httpPort)); + for (Map.Entry> placeholder : placeholders.entrySet()) { + String token = placeholder.getKey(); + if (result.contains(token)) { + // Resolved now (at launch), not when registered — the value's source (e.g. a container's + // mapped port) may not exist until test infrastructure has started. + result = result.replace(token, placeholder.getValue().get()); + } + } + return result; } private File resolveLogFile() { @@ -460,6 +473,7 @@ public static final class Builder { private String classpath; private final List jvmArgs = new ArrayList<>(); private final List programArgs = new ArrayList<>(); + private final Map> placeholders = new LinkedHashMap<>(); private final Map extraEnv = new HashMap<>(); private File workingDirectory; private TraceBackend backend; @@ -512,6 +526,24 @@ public Builder jvmArgs(String... jvmArgs) { return this; } + /** + * Registers a launch-time placeholder: occurrences of ${name} in {@link + * #jvmArgs(String...) jvmArgs} and {@link #args(String...) args} are replaced with {@code + * value.get()} when the app launches (in {@code beforeAll}), not when the builder + * runs. Use for values only known once test infrastructure has started — e.g. a Testcontainers + * mapped port, which is unavailable when the {@code static @RegisterExtension} fields + * initialize: + * + * 

{@code
+     * .placeholder("rabbit.port", () -> String.valueOf(RABBIT.container().getMappedPort(5672)))
+     * .args("--spring.rabbitmq.port=${rabbit.port}")
+     * }
+ */ + public Builder placeholder(String name, Supplier value) { + this.placeholders.put("${" + name + "}", value); + return this; + } + /** Sets an environment variable for the launched process (applied after the defaults). */ public Builder env(String key, String value) { this.extraEnv.put(key, value); diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/SmokeAppTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/SmokeAppTest.java index dc75df5151a..42443d6d6bc 100644 --- a/dd-smoke-tests/src/test/java/datadog/smoketest/SmokeAppTest.java +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/SmokeAppTest.java @@ -22,7 +22,8 @@ class SmokeAppTest { static final SmokeApp app = SmokeApp.named("test-server") .mainClass("datadog.smoketest.TestServerApp") - .args("--server.port=${app.httpPort}") + .placeholder("marker", () -> "resolved-at-launch") + .args("--server.port=${app.httpPort}", "--marker=${marker}") .backend(TraceBackend.mockAgent()) .noAgent() .build(); @@ -42,6 +43,16 @@ void capturesApplicationLogOutput() { "app stdout is captured during the test"); } + @Test + void substitutesCustomPlaceholderAtLaunch() { + app.get("/ping"); + // The app echoes its --marker launch arg; seeing the resolved value proves the custom ${marker} + // placeholder was substituted from its Supplier when the app launched. + assertTrue( + app.awaitLogLine(line -> line.contains("marker=resolved-at-launch")), + "custom placeholder was substituted into the launch args"); + } + @Test void ownsAndStartsItsBackend() { assertNotNull(app.backend().url(), "the owned backend was started before the app"); diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/TestServerApp.java b/dd-smoke-tests/src/test/java/datadog/smoketest/TestServerApp.java index 94647356e32..73b00c31123 100644 --- a/dd-smoke-tests/src/test/java/datadog/smoketest/TestServerApp.java +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/TestServerApp.java @@ -18,18 +18,23 @@ private TestServerApp() {} public static void main(String[] args) throws Exception { int port = 0; + String marker = ""; for (String arg : args) { if (arg.startsWith("--server.port=")) { port = Integer.parseInt(arg.substring("--server.port=".length())); + } else if (arg.startsWith("--marker=")) { + // Echoed on each request so tests can assert launch-arg (placeholder) substitution. + marker = arg.substring("--marker=".length()); } } + final String markerSuffix = marker.isEmpty() ? "" : " marker=" + marker; HttpServer server = HttpServer.create(new InetSocketAddress("localhost", port), 0); server.createContext( "/", exchange -> { String path = exchange.getRequestURI().getPath(); - System.out.println("REQUEST " + exchange.getRequestMethod() + " " + path); + System.out.println("REQUEST " + exchange.getRequestMethod() + " " + path + markerSuffix); if ("/error".equals(path)) { // Emit an error line so tests can exercise the no-error-logs check. System.out.println("ERROR simulated application error"); From 5f4bcaeddbe2a3b3b3d5a609e775a55b95a34771 Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Wed, 15 Jul 2026 14:06:24 +0200 Subject: [PATCH 15/30] WIP --- .../smoketest/SpringBootRabbitSmokeTest.java | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java b/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java index 1d4ae5fbf0a..02c2aa6631c 100644 --- a/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java +++ b/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java @@ -74,23 +74,15 @@ class SpringBootRabbitSmokeTest { private static final int TIMEOUT_SECONDS = 60; private static final int RABBIT_AMQP_PORT = 5672; private static final OkHttpClient CLIENT = new OkHttpClient(); - private static final String[] MESSAGES = {"foo", "bar", "baz"}; // AMQP connection-setup / ack commands each app emits as its own (single-span) trace. private static final String[] ADMIN_COMMANDS = { "basic.qos", "basic.consume", "basic.ack", "queue.declare" }; - // @Testcontainers starts/stops this static container in the class lifecycle (no static-block - // start()); as a class-level @ExtendWith it runs before the @RegisterExtension fields below, so - // RabbitMQ is up before the apps launch. Its mapped port is still read lazily at launch via - // placeholders, since the container is not yet started when these static fields initialize. @Container private static final RabbitMQContainer RABBIT = new RabbitMQContainer(DockerImageName.parse("rabbitmq:3.9.20-alpine")); - // @Order pins the @RegisterExtension order: the shared agent starts before the apps (so its - // session is open when the consumers emit startup traces) and is torn down after them. - // retainAcrossTests() keeps those startup traces for the one verifying method. @Order(1) @RegisterExtension static final TestAgentBackend agent = @@ -109,8 +101,9 @@ class SpringBootRabbitSmokeTest { @Test void roundtripsProduceFullAmqpTraceStructure() throws IOException { - // Drive 3 round-trips through the sender (matches the Groovy `where:` foo/bar/baz); each - // travels sender -> otherqueue -> receiver -> queue -> sender. + // Drive 3 round-trips through the sender; + // Each travels sender -> otherqueue -> receiver -> queue -> sender. + String[] MESSAGES = {"foo", "bar", "baz"}; for (String message : MESSAGES) { Request request = new Request.Builder().url(sender.url() + "/roundtrip/" + message).get().build(); From 808b90008541cde1b761b9ea856e52e1c2759ef7 Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Wed, 15 Jul 2026 14:11:17 +0200 Subject: [PATCH 16/30] WIP --- .../datadog/smoketest/trace/SpanMatcher.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SpanMatcher.java b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SpanMatcher.java index 4216baacfea..aaea4beb665 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SpanMatcher.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SpanMatcher.java @@ -1,14 +1,14 @@ package datadog.smoketest.trace; -import static datadog.trace.junit.utils.assertions.Matchers.any; -import static datadog.trace.junit.utils.assertions.Matchers.assertValue; -import static datadog.trace.junit.utils.assertions.Matchers.is; -import static datadog.trace.junit.utils.assertions.Matchers.isFalse; -import static datadog.trace.junit.utils.assertions.Matchers.isTrue; -import static datadog.trace.junit.utils.assertions.Matchers.matches; -import static datadog.trace.junit.utils.assertions.Matchers.validates; - -import datadog.trace.junit.utils.assertions.Matcher; +import static datadog.trace.test.junit.utils.assertions.Matchers.any; +import static datadog.trace.test.junit.utils.assertions.Matchers.assertValue; +import static datadog.trace.test.junit.utils.assertions.Matchers.is; +import static datadog.trace.test.junit.utils.assertions.Matchers.isFalse; +import static datadog.trace.test.junit.utils.assertions.Matchers.isTrue; +import static datadog.trace.test.junit.utils.assertions.Matchers.matches; +import static datadog.trace.test.junit.utils.assertions.Matchers.validates; + +import datadog.trace.test.junit.utils.assertions.Matcher; import datadog.trace.test.agent.decoder.DecodedSpan; import java.util.HashMap; import java.util.List; From 2ce5934d64e5a2f4c29cfcca2bf22fd60108a4bc Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Wed, 15 Jul 2026 18:48:33 +0200 Subject: [PATCH 17/30] WIP --- .../smoketest/SpringBootRabbitSmokeTest.java | 77 ++++++---- .../datadog/smoketest/backend/Traces.java | 67 ++------- .../smoketest/trace/SmokeTraceAssertions.java | 138 ++++++++---------- .../datadog/smoketest/trace/SpanMatcher.java | 22 +-- .../datadog/smoketest/trace/TraceMatcher.java | 109 +++++++++++++- .../smoketest/trace/SmokeMatcherTest.java | 79 +++++----- 6 files changed, 276 insertions(+), 216 deletions(-) diff --git a/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java b/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java index 02c2aa6631c..24e1dc2011b 100644 --- a/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java +++ b/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java @@ -1,15 +1,22 @@ package datadog.smoketest; +import static datadog.smoketest.trace.SmokeTraceAssertions.IGNORE_ADDITIONAL_TRACES; import static datadog.smoketest.trace.SpanMatcher.span; +import static datadog.smoketest.trace.TraceMatcher.SORT_BY_PARENT_CHAIN; +import static datadog.smoketest.trace.TraceMatcher.trace; import static org.junit.jupiter.api.Assertions.assertEquals; import datadog.smoketest.backend.EnabledIfDockerAvailable; import datadog.smoketest.backend.TestAgentBackend; import datadog.smoketest.backend.TraceBackend; import datadog.smoketest.backend.Traces; +import datadog.smoketest.trace.SmokeTraceAssertions; import datadog.smoketest.trace.SpanMatcher; +import datadog.smoketest.trace.TraceMatcher; import datadog.trace.test.agent.decoder.DecodedSpan; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; @@ -48,24 +55,27 @@ * spring-rabbit-0 spring.consume Receiver.receiveMessage (sender consumes reply) * * - *

Asserting that whole chain in one shot ({@link Traces#assertContainsChain}) is stronger than - * the Groovy set-membership check: it verifies every AMQP operation (publish/deliver/consume, both - * directions) and its cross-service linkage. The connection setup commands ({@code - * basic.qos}/{@code basic.consume}/{@code queue.declare}) and {@code basic.ack} are separate - * single-span traces, asserted per service. + *

The whole collection is asserted with the standard {@link Traces#assertTraces} DSL in {@link + * SmokeTraceAssertions#IGNORE_ADDITIONAL_TRACES order-independent subset} mode: each matcher + * matches a distinct received trace and extras are ignored. This is stronger than the Groovy + * set-membership check — each round-trip trace is matched count-exact (all 12 spans, in {@link + * TraceMatcher#SORT_BY_PARENT_CHAIN parent-chain order}), verifying every AMQP operation + * (publish/deliver/consume, both directions) and its cross-service linkage — while staying + * robust to the timing-dependent extras (the broker emits its connection-setup commands and per-ack + * traces as their own single-span traces, in non-deterministic count and order). * - *

Two things the Groovy base did implicitly that this port makes explicit: + *

Two design points, informed by inspecting the live trace collection: * *

    + *
  • Parent-chain order, not start time — the 12-span round-trip is a strict linear + * chain, but its spans start within the same tick and race, so {@code SORT_BY_START_TIME} is + * unstable across runs. {@code SORT_BY_PARENT_CHAIN} orders spans by parent links + * (timestamp-independent) and asserts the trace is exactly that chain. *
  • Accumulate, don't isolate — {@code .retainAcrossTests()} keeps traces from app * startup onward, because {@code basic.qos}/{@code basic.consume}/{@code queue.declare} are * emitted when the consumers start (before any test method), which a per-method session - * {@code clear()} would discard. The Groovy base verified once at {@code cleanupSpec} against - * everything written since launch; a single accumulating test method mirrors that. - *
  • Membership, not count-exact — {@code assertContainsChain} asserts the expected - * structure is present and ignores the rest (as the Groovy set-membership check did): the - * full distributed span set is large and timing-dependent, so a positional match over the - * whole collection is the wrong tool. + * {@code clear()} would discard. Mirrors the Groovy base verifying once at {@code + * cleanupSpec} against everything since launch. *
*/ @EnabledIfDockerAvailable @@ -113,15 +123,32 @@ void roundtripsProduceFullAmqpTraceStructure() throws IOException { } } - Traces traces = agent.traces(); + // One order-independent assertion over the whole collection (IGNORE_ADDITIONAL_TRACES): each + // matcher must match a distinct received trace; unrelated/duplicate traces (extra acks, etc.) + // are ignored. Assert one full round-trip trace per message plus each service's + // connection-setup/ack commands. + List expected = new ArrayList<>(); + for (int i = 0; i < MESSAGES.length; i++) { + expected.add(roundTrip()); // one full round-trip trace per message => all are verified + } + for (String service : new String[] {"spring-rabbit-0", "spring-rabbit-1"}) { + for (String command : ADMIN_COMMANDS) { + expected.add(admin(service, command)); + } + } + agent + .traces() + .assertTraces( + TIMEOUT_SECONDS, IGNORE_ADDITIONAL_TRACES, expected.toArray(new TraceMatcher[0])); + } - // Each round-trip is its own full distributed trace: a strict parent->child chain across both - // services and the broker, from the sender's HTTP entrypoint to the sender consuming the - // forwarded reply. Assert one such chain per message (MESSAGES.length), so all round-trips are - // verified, not just one. - traces.assertContainsChain( - MESSAGES.length, - TIMEOUT_SECONDS, + // The full distributed round-trip as one strict parent->child chain across both services and the + // broker. SORT_BY_PARENT_CHAIN orders the spans root->leaf and asserts the trace IS exactly this + // linear chain: HTTP entrypoint -> publish -> receiver consumes+forwards -> sender consumes + // reply. + private static TraceMatcher roundTrip() { + return trace( + SORT_BY_PARENT_CHAIN, sp("spring-rabbit-0", "servlet.request", "GET /roundtrip/{message}").root(), sp("spring-rabbit-0", "spring.handler", "WebController.roundtrip"), sp("spring-rabbit-0", "amqp.command", "basic.publish -> otherqueue"), @@ -134,13 +161,11 @@ void roundtripsProduceFullAmqpTraceStructure() throws IOException { sp("spring-rabbit-0", "amqp.command", "basic.deliver queue"), sp("spring-rabbit-0", "amqp.consume", "amqp.consume queue"), sp("spring-rabbit-0", "spring.consume", "Receiver.receiveMessage")); + } - // The connection-setup / ack commands each app emits as its own single-span trace. - for (String service : new String[] {"spring-rabbit-0", "spring-rabbit-1"}) { - for (String command : ADMIN_COMMANDS) { - traces.assertContainsChain(TIMEOUT_SECONDS, sp(service, "amqp.command", command).root()); - } - } + // A connection-setup / ack command emitted as its own single-span (root) trace. + private static TraceMatcher admin(String service, String command) { + return trace(sp(service, "amqp.command", command).root()); } private static SpanMatcher sp(String service, String operation, String resource) { diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Traces.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Traces.java index ec97ad5035e..e400f9e6370 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Traces.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Traces.java @@ -1,7 +1,8 @@ package datadog.smoketest.backend; +import static java.util.function.UnaryOperator.identity; + import datadog.smoketest.trace.SmokeTraceAssertions; -import datadog.smoketest.trace.SpanMatcher; import datadog.smoketest.trace.TraceMatcher; import datadog.trace.test.agent.decoder.DecodedTrace; import datadog.trace.test.util.PollingConditions; @@ -56,18 +57,18 @@ public Traces waitForTraceCount(int count, double timeoutSeconds) { } /** - * Asserts the received traces against the thin smoke matcher — one {@link TraceMatcher} per - * expected trace (matched positionally). Sugar over {@link SmokeTraceAssertions#assertTraces}: + * Polls (up to the default timeout) until the received traces satisfy the thin smoke matcher — + * one {@link TraceMatcher} per expected trace (matched positionally, count-exact). Polling + * absorbs the async arrival of traces from a separately-launched app. Sugar over {@link + * SmokeTraceAssertions#assertTraces}: * * 
{@code
-   * app.traces().waitForTraceCount(1)
-   *    .assertTraces(trace(span().operationName("servlet.request").resourceName("GET /greeting")));
+   * app.traces().assertTraces(
+   *     trace(span().operationName("servlet.request").resourceName("GET /greeting")));
    * }
*/ public void assertTraces(TraceMatcher... matchers) { - int expectedTraceCount = matchers.length; - waitForTraceCount(expectedTraceCount); - SmokeTraceAssertions.assertTraces(getTraces(), matchers); + assertTraces(identity(), matchers); } /** @@ -76,52 +77,16 @@ public void assertTraces(TraceMatcher... matchers) { */ public void assertTraces( UnaryOperator options, TraceMatcher... matchers) { - SmokeTraceAssertions.assertTraces(getTraces(), options, matchers); + assertTraces(DEFAULT_TIMEOUT_SECONDS, options, matchers); } - /** - * Polls (up to the default timeout) until some received trace contains a parent-child chain of - * spans matching {@code chain} (see {@link SmokeTraceAssertions#assertContainsChain}). Combines - * waiting for async arrival with the structural assertion, so it suits distributed traces whose - * full span set arrives piecemeal — assert the linkage you care about, ignore the rest: - * - * 
{@code
-   * app.traces().assertContainsChain(
-   *     span().service("web").operationName("servlet.request").root(),
-   *     span().service("web").operationName("amqp.command").resourceName(startsWith("basic.publish")));
-   * }
- */ - public Traces assertContainsChain(SpanMatcher... chain) { - return assertContainsChain(DEFAULT_TIMEOUT_SECONDS, chain); - } - - /** As {@link #assertContainsChain(SpanMatcher...)}, but overriding the timeout for this call. */ - public Traces assertContainsChain(double timeoutSeconds, SpanMatcher... chain) { - new PollingConditions(timeoutSeconds) - .eventually(() -> SmokeTraceAssertions.assertContainsChain(getTraces(), chain)); - return this; - } - - /** - * Polls (up to {@code timeoutSeconds}) until at least {@code minMatches} received traces each - * contain a chain matching {@code chain} (see {@link SmokeTraceAssertions#countChainMatches}). - * Use to verify that N independent operations each produced the expected trace — e.g. one - * distributed trace per request — not merely that one did. - */ - public Traces assertContainsChain(int minMatches, double timeoutSeconds, SpanMatcher... chain) { + /** As {@link #assertTraces(UnaryOperator, TraceMatcher...)}, overriding the timeout. */ + public void assertTraces( + double timeoutSeconds, + UnaryOperator options, + TraceMatcher... matchers) { new PollingConditions(timeoutSeconds) - .eventually( - () -> { - long found = SmokeTraceAssertions.countChainMatches(getTraces(), chain); - if (found < minMatches) { - throw new AssertionError( - "Expected at least " - + minMatches - + " traces containing the chain but found " - + found); - } - }); - return this; + .eventually(() -> SmokeTraceAssertions.assertTraces(getTraces(), options, matchers)); } // Trace-invariant checks (ENABLED_CHECKS) are a test-agent-specific concern, validated by that diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java index bdf8fae7000..b9938ad4fee 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java @@ -6,6 +6,7 @@ import datadog.trace.test.agent.decoder.DecodedTrace; import java.util.ArrayList; import java.util.Comparator; +import java.util.Iterator; import java.util.List; import java.util.function.UnaryOperator; @@ -21,10 +22,15 @@ * assertTraces(traces, trace(span().operationName("servlet.request").resourceName("GET /greeting"))); * } * - *

Traces are matched positionally, in the order received by default. For order-independence pass - * {@link #SORT_BY_START_TIME} / {@link #SORT_BY_ROOT_SPAN_ID}; to assert only a subset of the - * received traces pass {@link #IGNORE_ADDITIONAL_TRACES}. The traces come from a {@code - * TraceBackend} (mock or test-agent), both decoded to {@link DecodedTrace}. + *

By default traces are matched positionally (matcher i ↔ trace i), in + * the order received — pass {@link #SORT_BY_START_TIME} / {@link #SORT_BY_ROOT_SPAN_ID} for a + * stable order, and the count must match exactly. Pass {@link #IGNORE_ADDITIONAL_TRACES} to switch + * to order-independent subset matching instead: each matcher must match a + * distinct received trace, and any extra traces are ignored — the right mode when the + * collection contains unrelated or non-deterministically-ordered traces (e.g. a distributed test + * where connection-setup commands land as their own traces alongside the request traces you care + * about). The traces come from a {@code TraceBackend} (mock or test-agent), both decoded to {@link + * DecodedTrace}. */ public final class SmokeTraceAssertions { /** Sorts traces by the earliest span start time. */ @@ -35,7 +41,10 @@ public final class SmokeTraceAssertions { public static final Comparator> TRACE_ROOT_SPAN_ID_COMPARATOR = Comparator.comparingLong(SmokeTraceAssertions::rootSpanId); - /** Do not fail when there are more traces than matchers (assert a subset). */ + /** + * Switch to order-independent subset matching: each matcher must match a distinct received trace, + * and extra traces are ignored (instead of the default count-exact positional matching). + */ public static final UnaryOperator IGNORE_ADDITIONAL_TRACES = Options::ignoreAdditionalTraces; @@ -58,94 +67,69 @@ public static void assertTraces(List traces, TraceMatcher... match public static void assertTraces( List traces, UnaryOperator options, TraceMatcher... matchers) { Options opts = options.apply(new Options()); - int expected = matchers.length; - int actual = traces.size(); - if (opts.ignoreAdditionalTraces) { - if (actual < expected) { - throw new AssertionError("Expected at least " + expected + " traces but got " + actual); - } - } else if (actual != expected) { - throw new AssertionError("Expected " + expected + " traces but got " + actual); - } // Copy each trace's spans to a mutable list so the matcher can sort/inspect them. - List> spanLists = new ArrayList<>(actual); + List> spanLists = new ArrayList<>(traces.size()); for (DecodedTrace trace : traces) { spanLists.add(new ArrayList<>(trace.getSpans())); } - if (opts.sorter != null) { - spanLists.sort(opts.sorter); - } - for (int i = 0; i < expected; i++) { - matchers[i].assertTrace(spanLists.get(i), i); + if (opts.ignoreAdditionalTraces) { + assertContainsEach(spanLists, matchers); + } else { + assertPositionally(spanLists, opts, matchers); } } - /** - * Asserts that some received trace contains a parent-child chain of spans matching - * {@code chain} in order: {@code chain[0]} matches a span (a trace root if it declares {@link - * SpanMatcher#root()}), and each {@code chain[i]} matches a direct child of {@code chain[i-1]}'s - * span. Unlike {@link #assertTraces}, this is a subset match — extra spans in the trace - * are ignored — which suits distributed traces whose full span set is large and timing-dependent - * (only span linkage, not exact shape, is asserted). Field constraints are declared per span; the - * chain declares the linkage, so the matchers should not also use {@code childOf*}. - */ - public static void assertContainsChain(List traces, SpanMatcher... chain) { - long found = countChainMatches(traces, chain); - if (found == 0) { + // Order-independent subset: each matcher must match a distinct received trace; extras are + // ignored. + private static void assertContainsEach( + List> spanLists, TraceMatcher[] matchers) { + if (matchers.length > spanLists.size()) { throw new AssertionError( - "No received trace contains a parent-child chain matching the " - + chain.length - + " given span matcher(s); traces: " - + traces); - } - } - - /** - * Counts how many received traces contain a parent-child chain matching {@code chain} (see {@link - * #assertContainsChain(List, SpanMatcher...)} for the chain semantics). Use to verify N - * independent operations each produced the expected trace — e.g. one distributed trace per - * request — rather than just that a single one did. - */ - public static long countChainMatches(List traces, SpanMatcher... chain) { - if (chain.length == 0) { - throw new IllegalArgumentException("countChainMatches requires at least one span matcher"); + "Expected at least " + + matchers.length + + " traces but got " + + spanLists.size() + + ": " + + spanLists); } - long count = 0; - for (DecodedTrace trace : traces) { - if (containsChain(trace.getSpans(), chain)) { - count++; - } - } - return count; - } - - private static boolean containsChain(List spans, SpanMatcher[] chain) { - for (DecodedSpan first : spans) { - if (chain[0].rootRequired() && first.getParentId() != 0L) { - continue; + // Greedily assign each matcher to a distinct, not-yet-consumed trace. Adequate for smoke tests; + // overlapping matchers (one strictly more general than another) could mis-assign — write them + // specific enough to be unambiguous. + List> remaining = new ArrayList<>(spanLists); + for (int i = 0; i < matchers.length; i++) { + boolean matched = false; + for (Iterator> it = remaining.iterator(); it.hasNext(); ) { + if (matchers[i].matches(it.next(), i)) { + it.remove(); + matched = true; + break; + } } - if (chain[0].matchesFields(first) && matchesChainFrom(spans, chain, 1, first)) { - return true; + if (!matched) { + throw new AssertionError( + "No received trace matched trace matcher #" + + i + + " among " + + spanLists.size() + + " received trace(s): " + + spanLists); } } - return false; } - // Depth-first, backtracking: extend the chain by a direct child of `parent` matching - // chain[index]. - private static boolean matchesChainFrom( - List spans, SpanMatcher[] chain, int index, DecodedSpan parent) { - if (index == chain.length) { - return true; + // Count-exact positional: matcher i ↔ trace i, after the optional trace sort. + private static void assertPositionally( + List> spanLists, Options opts, TraceMatcher[] matchers) { + if (spanLists.size() != matchers.length) { + throw new AssertionError( + "Expected " + matchers.length + " traces but got " + spanLists.size()); } - for (DecodedSpan child : spans) { - if (child.getParentId() == parent.getSpanId() - && chain[index].matchesFields(child) - && matchesChainFrom(spans, chain, index + 1, child)) { - return true; - } + if (opts.sorter != null) { + spanLists.sort(opts.sorter); + } + for (int i = 0; i < matchers.length; i++) { + matchers[i].assertTrace(spanLists.get(i), i); } - return false; } private static long earliestStart(List spans) { diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SpanMatcher.java b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SpanMatcher.java index aaea4beb665..59d5e850f0c 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SpanMatcher.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SpanMatcher.java @@ -8,8 +8,8 @@ import static datadog.trace.test.junit.utils.assertions.Matchers.matches; import static datadog.trace.test.junit.utils.assertions.Matchers.validates; -import datadog.trace.test.junit.utils.assertions.Matcher; import datadog.trace.test.agent.decoder.DecodedSpan; +import datadog.trace.test.junit.utils.assertions.Matcher; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -176,26 +176,6 @@ private void assertFields(DecodedSpan span) { } } - /** - * Whether this matcher's own fields match {@code span}, ignoring parent linkage — the boolean - * counterpart of {@link #assertFields} used by subset/chain matching (see {@link - * SmokeTraceAssertions#assertContainsChain}), where linkage is verified positionally by the - * chain. - */ - boolean matchesFields(DecodedSpan span) { - try { - assertFields(span); - return true; - } catch (AssertionError ignored) { - return false; - } - } - - /** Whether this matcher requires its span to be a trace root (via {@link #root()}). */ - boolean rootRequired() { - return parentId != null && parentId == 0L; - } - private Matcher parentIdMatcher(List trace, int spanIndex) { if (this.parentSpanIndex >= 0) { return is(trace.get(parentSpanIndex).getSpanId()); diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/TraceMatcher.java b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/TraceMatcher.java index bfbf2c8ce69..b672f863231 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/TraceMatcher.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/TraceMatcher.java @@ -5,17 +5,29 @@ import datadog.trace.test.agent.decoder.DecodedSpan; import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.function.UnaryOperator; /** * Thin trace matcher for smoke tests: one {@link SpanMatcher} per expected span in a trace. * Companion of {@link SpanMatcher}; see its class doc for the serialized-model rationale. * - *

Spans are matched positionally. By default they are matched in the order received; pass {@link - * #SORT_BY_START_TIME} to {@link #trace(UnaryOperator, SpanMatcher...)} to sort them first, which - * makes {@link SpanMatcher#childOfIndex(int)} / {@link SpanMatcher#childOfPrevious()} - * deterministic. + *

Spans are matched positionally. By default they are matched in the order received; sort them + * first for a deterministic position: + * + *

    + *
  • {@link #SORT_BY_START_TIME} — by span start time. Beware: spans that start within the same + * clock tick (e.g. a publish and its broker-side deliver) can race, so this order is not + * stable for tightly-coupled concurrent spans. + *
  • {@link #SORT_BY_PARENT_CHAIN} — root→leaf following parent links. Stable regardless of + * timestamps, but only defined for a single linear chain (each span has at most one + * child); it throws otherwise. Ideal for a straight request→...→response distributed trace. + *
+ * + *

A sorted order also makes {@link SpanMatcher#childOfIndex(int)} / {@link + * SpanMatcher#childOfPrevious()} deterministic. */ public final class TraceMatcher { /** Sorts a trace's spans by start time. */ @@ -26,6 +38,14 @@ public final class TraceMatcher { public static final UnaryOperator SORT_BY_START_TIME = options -> options.sorter(START_TIME_COMPARATOR); + /** + * Configures {@link #trace(UnaryOperator, SpanMatcher...)} to order spans root→leaf by following + * parent links (see {@link Options#linearizeByParentChain()}). Timestamp-independent, so unlike + * {@link #SORT_BY_START_TIME} it is stable for concurrent spans — but only valid for a single + * linear chain. + */ + public static final UnaryOperator SORT_BY_PARENT_CHAIN = Options::linearizeByParentChain; + private final Options options; private final SpanMatcher[] spanMatchers; @@ -59,22 +79,97 @@ void assertTrace(List spans, int traceIndex) { + ": " + spans); } - List ordered = new ArrayList<>(spans); - if (options.sorter != null) { - ordered.sort(options.sorter); + List ordered; + if (options.linearizeByParentChain) { + ordered = chainOrder(spans); + } else { + ordered = new ArrayList<>(spans); + if (options.sorter != null) { + ordered.sort(options.sorter); + } } for (int i = 0; i < spanMatchers.length; i++) { spanMatchers[i].assertSpan(ordered, i); } } + /** + * Whether this matcher accepts {@code spans} as a whole trace, without throwing — the boolean + * counterpart of {@link #assertTrace} used by order-independent matching (see {@link + * SmokeTraceAssertions#assertTraces} with {@link SmokeTraceAssertions#IGNORE_ADDITIONAL_TRACES}), + * where each matcher must find some received trace it fits. A non-linear trace under {@link + * #SORT_BY_PARENT_CHAIN} counts as "does not match" here (rather than propagating) so probing can + * move on. + */ + boolean matches(List spans, int traceIndex) { + try { + assertTrace(spans, traceIndex); + return true; + } catch (AssertionError | IllegalStateException ignored) { + return false; + } + } + + /** + * Orders spans root→leaf by following parent links. Requires a single linear chain: exactly one + * root (parent id 0) and every span at most one child, with all spans reachable from the root. + * Throws {@link IllegalStateException} otherwise, so a non-chain trace fails loudly rather than + * being silently mis-ordered. + */ + private static List chainOrder(List spans) { + DecodedSpan root = null; + Map childByParent = new HashMap<>(); + for (DecodedSpan span : spans) { + if (span.getParentId() == 0L) { + if (root != null) { + throw new IllegalStateException( + "SORT_BY_PARENT_CHAIN requires a single root span (parent id 0); found more than one"); + } + root = span; + } else if (childByParent.put(span.getParentId(), span) != null) { + throw new IllegalStateException( + "SORT_BY_PARENT_CHAIN requires a linear chain, but span " + + span.getParentId() + + " has more than one child"); + } + } + if (root == null) { + throw new IllegalStateException("SORT_BY_PARENT_CHAIN requires a root span (parent id 0)"); + } + List ordered = new ArrayList<>(spans.size()); + for (DecodedSpan current = root; + current != null; + current = childByParent.get(current.getSpanId())) { + ordered.add(current); + } + if (ordered.size() != spans.size()) { + throw new IllegalStateException( + "SORT_BY_PARENT_CHAIN: trace is not a single chain (" + + ordered.size() + + " of " + + spans.size() + + " spans reachable from the root)"); + } + return ordered; + } + /** Per-trace matching options. */ public static final class Options { Comparator sorter = null; // null => keep received order + boolean linearizeByParentChain = false; public Options sorter(Comparator sorter) { this.sorter = sorter; return this; } + + /** + * Order spans root→leaf by parent links instead of by a comparator (see {@link + * #SORT_BY_PARENT_CHAIN}). + */ + public Options linearizeByParentChain() { + this.linearizeByParentChain = true; + return this; + } } } diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/trace/SmokeMatcherTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/trace/SmokeMatcherTest.java index bd3ce7c157e..364d151f1ec 100644 --- a/dd-smoke-tests/src/test/java/datadog/smoketest/trace/SmokeMatcherTest.java +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/trace/SmokeMatcherTest.java @@ -3,7 +3,6 @@ import static datadog.smoketest.trace.SmokeTraceAssertions.assertTraces; import static datadog.smoketest.trace.SpanMatcher.span; import static datadog.smoketest.trace.TraceMatcher.trace; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import datadog.smoketest.backend.TestAgentTraceDecoder; @@ -32,16 +31,14 @@ class SmokeMatcherTest { private static final String TWO_TRACES = "[[" + spanJson("root-a", 500, 0, 10) + "],[" + spanJson("root-b", 400, 0, 20) + "]]"; - // Two traces with the same root -> child shape (distinct span ids). - private static final String TWO_CHAINS = + // One trace whose root has two children (not a linear chain). + private static final String BRANCHING_TRACE = "[[" + spanJson("root", 100, 0, 10) + "," - + spanJson("child", 200, 100, 20) - + "],[" - + spanJson("root", 300, 0, 30) + + spanJson("a", 200, 100, 20) + "," - + spanJson("child", 400, 300, 40) + + spanJson("b", 300, 100, 30) + "]]"; @Test @@ -106,43 +103,57 @@ void sortsTracesByRootSpanId() { } @Test - void assertContainsChainMatchesParentChildChain() { + void sortsByParentChainRegardlessOfStartTime() { List traces = TestAgentTraceDecoder.decode(CHAIN_TRACE); - // Full chain from the root. - SmokeTraceAssertions.assertContainsChain( + // Spans arrive out of start order; parent-chain order recovers root -> child -> grandchild. + assertTraces( traces, - span().operationName("root").root(), - span().operationName("child"), - span().operationName("grandchild")); - // Subset chain that doesn't start at the root still matches (extra spans ignored). - SmokeTraceAssertions.assertContainsChain( - traces, span().operationName("child"), span().operationName("grandchild")); + trace( + TraceMatcher.SORT_BY_PARENT_CHAIN, + span().operationName("root").root(), + span().operationName("child").childOfPrevious(), + span().operationName("grandchild").childOfPrevious())); } @Test - void assertContainsChainFailsOnBrokenLinkage() { - List traces = TestAgentTraceDecoder.decode(CHAIN_TRACE); - // grandchild is a child of child, not a direct child of root. + void parentChainRejectsNonLinearTrace() { + List traces = TestAgentTraceDecoder.decode(BRANCHING_TRACE); + // root has two children, so a chain linearization is undefined -> loud failure, not mis-order. assertThrows( - AssertionError.class, + IllegalStateException.class, () -> - SmokeTraceAssertions.assertContainsChain( - traces, span().operationName("root").root(), span().operationName("grandchild"))); + assertTraces( + traces, + trace( + TraceMatcher.SORT_BY_PARENT_CHAIN, + span().operationName("root").root(), + span().operationName("a").childOfPrevious(), + span().operationName("b").childOfPrevious()))); } @Test - void countsTracesContainingChain() { - List traces = TestAgentTraceDecoder.decode(TWO_CHAINS); - // Both traces contain the root -> child chain. - assertEquals( - 2, - SmokeTraceAssertions.countChainMatches( - traces, span().operationName("root").root(), span().operationName("child"))); - // A chain present in neither trace counts zero. - assertEquals( - 0, - SmokeTraceAssertions.countChainMatches( - traces, span().operationName("root").root(), span().operationName("missing"))); + void ignoreAdditionalTracesMatchesAnyOrder() { + List traces = TestAgentTraceDecoder.decode(TWO_TRACES); + // Matchers in the opposite order of receipt still each find their trace (order-independent). + assertTraces( + traces, + SmokeTraceAssertions.IGNORE_ADDITIONAL_TRACES, + trace(span().operationName("root-b").root()), + trace(span().operationName("root-a").root())); + } + + @Test + void ignoreAdditionalTracesRequiresDistinctTraces() { + List traces = TestAgentTraceDecoder.decode(TWO_TRACES); + // Two matchers for the same trace can't both match: only one root-a trace exists. + assertThrows( + AssertionError.class, + () -> + assertTraces( + traces, + SmokeTraceAssertions.IGNORE_ADDITIONAL_TRACES, + trace(span().operationName("root-a").root()), + trace(span().operationName("root-a").root()))); } private static String spanJson(String name, long id, long parent, long start) { From 06e485078a4aef275ea8d1c80d58ae33f7d7d856 Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Thu, 16 Jul 2026 12:57:17 +0200 Subject: [PATCH 18/30] WIP --- .../smoketest/SpringBootRabbitSmokeTest.java | 30 ++-- .../smoketest/trace/SmokeTraceAssertions.java | 157 ++++++++++-------- .../smoketest/trace/SmokeMatcherTest.java | 33 +++- 3 files changed, 133 insertions(+), 87 deletions(-) diff --git a/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java b/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java index 24e1dc2011b..d75c58560b6 100644 --- a/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java +++ b/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java @@ -1,6 +1,5 @@ package datadog.smoketest; -import static datadog.smoketest.trace.SmokeTraceAssertions.IGNORE_ADDITIONAL_TRACES; import static datadog.smoketest.trace.SpanMatcher.span; import static datadog.smoketest.trace.TraceMatcher.SORT_BY_PARENT_CHAIN; import static datadog.smoketest.trace.TraceMatcher.trace; @@ -55,14 +54,15 @@ * spring-rabbit-0 spring.consume Receiver.receiveMessage (sender consumes reply) * * - *

The whole collection is asserted with the standard {@link Traces#assertTraces} DSL in {@link - * SmokeTraceAssertions#IGNORE_ADDITIONAL_TRACES order-independent subset} mode: each matcher - * matches a distinct received trace and extras are ignored. This is stronger than the Groovy - * set-membership check — each round-trip trace is matched count-exact (all 12 spans, in {@link - * TraceMatcher#SORT_BY_PARENT_CHAIN parent-chain order}), verifying every AMQP operation - * (publish/deliver/consume, both directions) and its cross-service linkage — while staying - * robust to the timing-dependent extras (the broker emits its connection-setup commands and per-ack - * traces as their own single-span traces, in non-deterministic count and order). + *

The whole collection is asserted with the standard {@link Traces#assertTraces} DSL in + * {@linkplain SmokeTraceAssertions order-independent subset} mode ({@code + * unordered().ignoreAdditionalTraces()}): each matcher matches a distinct received trace and extras + * are ignored. This is stronger than the Groovy set-membership check — each round-trip trace is + * matched count-exact (all 12 spans, in {@link TraceMatcher#SORT_BY_PARENT_CHAIN parent-chain + * order}), verifying every AMQP operation (publish/deliver/consume, both directions) and + * its cross-service linkage — while staying robust to the timing-dependent extras (the broker emits + * its connection-setup commands and per-ack traces as their own single-span traces, in + * non-deterministic count and order). * *

Two design points, informed by inspecting the live trace collection: * @@ -123,10 +123,10 @@ void roundtripsProduceFullAmqpTraceStructure() throws IOException { } } - // One order-independent assertion over the whole collection (IGNORE_ADDITIONAL_TRACES): each - // matcher must match a distinct received trace; unrelated/duplicate traces (extra acks, etc.) - // are ignored. Assert one full round-trip trace per message plus each service's - // connection-setup/ack commands. + // One order-independent subset assertion over the whole collection (unordered + + // ignoreAdditionalTraces): each matcher must match a distinct received trace, and + // unrelated/duplicate traces (extra acks, etc.) are ignored. Assert one full round-trip trace + // per message plus each service's connection-setup/ack commands. List expected = new ArrayList<>(); for (int i = 0; i < MESSAGES.length; i++) { expected.add(roundTrip()); // one full round-trip trace per message => all are verified @@ -139,7 +139,9 @@ void roundtripsProduceFullAmqpTraceStructure() throws IOException { agent .traces() .assertTraces( - TIMEOUT_SECONDS, IGNORE_ADDITIONAL_TRACES, expected.toArray(new TraceMatcher[0])); + TIMEOUT_SECONDS, + o -> o.unordered().ignoreAdditionalTraces(), + expected.toArray(new TraceMatcher[0])); } // The full distributed round-trip as one strict parent->child chain across both services and the diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java index b9938ad4fee..f37e685af46 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java @@ -6,7 +6,6 @@ import datadog.trace.test.agent.decoder.DecodedTrace; import java.util.ArrayList; import java.util.Comparator; -import java.util.Iterator; import java.util.List; import java.util.function.UnaryOperator; @@ -22,15 +21,26 @@ * assertTraces(traces, trace(span().operationName("servlet.request").resourceName("GET /greeting"))); * } * - *

By default traces are matched positionally (matcher i ↔ trace i), in - * the order received — pass {@link #SORT_BY_START_TIME} / {@link #SORT_BY_ROOT_SPAN_ID} for a - * stable order, and the count must match exactly. Pass {@link #IGNORE_ADDITIONAL_TRACES} to switch - * to order-independent subset matching instead: each matcher must match a - * distinct received trace, and any extra traces are ignored — the right mode when the - * collection contains unrelated or non-deterministically-ordered traces (e.g. a distributed test - * where connection-setup commands land as their own traces alongside the request traces you care - * about). The traces come from a {@code TraceBackend} (mock or test-agent), both decoded to {@link - * DecodedTrace}. + *

A single matching algorithm is tuned by three orthogonal, combinable {@link Options}: + * + *

    + *
  • {@link #SORT_BY_START_TIME} / {@link #SORT_BY_ROOT_SPAN_ID} (the {@code sorter}) — sort the + * received traces before matching. + *
  • {@link #UNORDERED} — a matcher may match any distinct (unconsumed) trace rather + * than the one at its position. Without a sorter this is fully order-independent; + * with a sorter it is forward-only (once a matcher takes the trace at sorted + * position p, the next matcher can only look at positions after p) — i.e. match + * as a subsequence in sorted order. + *
  • {@link #IGNORE_ADDITIONAL_TRACES} — don't require every received trace to be matched; extra + * traces are ignored (a subset assertion). Without it, the counts must be equal. + *
+ * + *

The default (no options) is thus count-exact positional matching (matcher i ↔ trace + * i, received order). Combine flags with the fluent {@link Options} — e.g. {@code o -> + * o.unordered().ignoreAdditionalTraces()} — to assert only the traces you care about in a + * collection that also holds unrelated or non-deterministically-ordered ones (e.g. a distributed + * test where connection-setup commands land as their own traces). The traces come from a {@code + * TraceBackend} (mock or test-agent), both decoded to {@link DecodedTrace}. */ public final class SmokeTraceAssertions { /** Sorts traces by the earliest span start time. */ @@ -41,13 +51,13 @@ public final class SmokeTraceAssertions { public static final Comparator> TRACE_ROOT_SPAN_ID_COMPARATOR = Comparator.comparingLong(SmokeTraceAssertions::rootSpanId); - /** - * Switch to order-independent subset matching: each matcher must match a distinct received trace, - * and extra traces are ignored (instead of the default count-exact positional matching). - */ + /** Don't require every received trace to be matched — ignore the extras (a subset assertion). */ public static final UnaryOperator IGNORE_ADDITIONAL_TRACES = Options::ignoreAdditionalTraces; + /** Let each matcher match any distinct trace rather than the one at its position. */ + public static final UnaryOperator UNORDERED = Options::unordered; + /** Sorts traces by earliest span start time before matching. */ public static final UnaryOperator SORT_BY_START_TIME = options -> options.sorter(TRACE_START_TIME_COMPARATOR); @@ -63,73 +73,76 @@ public static void assertTraces(List traces, TraceMatcher... match assertTraces(traces, identity(), matchers); } - /** As {@link #assertTraces(List, TraceMatcher...)} with the given options. */ + /** + * As {@link #assertTraces(List, TraceMatcher...)} with the given options. One matching pass, + * tuned by {@code sorter} / {@link Options#unordered() unordered} / {@link + * Options#ignoreAdditionalTraces() ignoreAdditionalTraces} — see the class doc for the flag + * semantics. + */ public static void assertTraces( List traces, UnaryOperator options, TraceMatcher... matchers) { Options opts = options.apply(new Options()); // Copy each trace's spans to a mutable list so the matcher can sort/inspect them. - List> spanLists = new ArrayList<>(traces.size()); + List> pool = new ArrayList<>(traces.size()); for (DecodedTrace trace : traces) { - spanLists.add(new ArrayList<>(trace.getSpans())); + pool.add(new ArrayList<>(trace.getSpans())); } - if (opts.ignoreAdditionalTraces) { - assertContainsEach(spanLists, matchers); - } else { - assertPositionally(spanLists, opts, matchers); - } - } - - // Order-independent subset: each matcher must match a distinct received trace; extras are - // ignored. - private static void assertContainsEach( - List> spanLists, TraceMatcher[] matchers) { - if (matchers.length > spanLists.size()) { - throw new AssertionError( - "Expected at least " - + matchers.length - + " traces but got " - + spanLists.size() - + ": " - + spanLists); + if (opts.sorter != null) { + pool.sort(opts.sorter); } - // Greedily assign each matcher to a distinct, not-yet-consumed trace. Adequate for smoke tests; - // overlapping matchers (one strictly more general than another) could mis-assign — write them - // specific enough to be unambiguous. - List> remaining = new ArrayList<>(spanLists); + // Walk the matchers, each consuming a distinct trace. `floor` is the lowest pool index a match + // may use: it advances past each match to enforce order, except in fully-unordered mode (no + // sorter) where it stays 0 so any remaining trace is a candidate. `consumed` keeps matches + // distinct when the floor doesn't advance. + boolean[] consumed = new boolean[pool.size()]; + int matched = 0; + int floor = 0; for (int i = 0; i < matchers.length; i++) { - boolean matched = false; - for (Iterator> it = remaining.iterator(); it.hasNext(); ) { - if (matchers[i].matches(it.next(), i)) { - it.remove(); - matched = true; - break; - } - } - if (!matched) { - throw new AssertionError( - "No received trace matched trace matcher #" - + i - + " among " - + spanLists.size() - + " received trace(s): " - + spanLists); + int idx = findMatch(pool, consumed, floor, opts, matchers[i], i); + consumed[idx] = true; + matched++; + if (!opts.unordered || opts.sorter != null) { + floor = idx + 1; } } - } - - // Count-exact positional: matcher i ↔ trace i, after the optional trace sort. - private static void assertPositionally( - List> spanLists, Options opts, TraceMatcher[] matchers) { - if (spanLists.size() != matchers.length) { + if (!opts.ignoreAdditionalTraces && matched != pool.size()) { throw new AssertionError( - "Expected " + matchers.length + " traces but got " + spanLists.size()); + "Expected " + matchers.length + " traces but got " + pool.size() + ": " + pool); } - if (opts.sorter != null) { - spanLists.sort(opts.sorter); + } + + // Finds the trace this matcher consumes, or throws. Scans from `floor`, skipping consumed traces + // and (unless matching strictly positionally) non-matching ones. A strictly-positional matcher + // that fails is re-run with the throwing assertion so the failure names the offending span. + private static int findMatch( + List> pool, + boolean[] consumed, + int floor, + Options opts, + TraceMatcher matcher, + int i) { + boolean strictPositional = !opts.unordered && !opts.ignoreAdditionalTraces; + for (int j = floor; j < pool.size(); j++) { + if (consumed[j]) { + continue; + } + if (matcher.matches(pool.get(j), i)) { + return j; + } + if (strictPositional) { + break; // the trace at this position must match; no skipping ahead + } } - for (int i = 0; i < matchers.length; i++) { - matchers[i].assertTrace(spanLists.get(i), i); + if (strictPositional && floor < pool.size() && !consumed[floor]) { + matcher.assertTrace(pool.get(floor), i); // throws with the per-span reason } + throw new AssertionError( + "No received trace matched trace matcher #" + + i + + " among " + + pool.size() + + " received trace(s): " + + pool); } private static long earliestStart(List spans) { @@ -149,16 +162,24 @@ private static long rootSpanId(List spans) { return spans.isEmpty() ? 0L : spans.get(0).getSpanId(); } - /** Trace-collection matching options. */ + /** Trace-collection matching options; see the class doc for how they combine. */ public static final class Options { boolean ignoreAdditionalTraces = false; + boolean unordered = false; Comparator> sorter = null; // null => keep received order + /** Ignore received traces beyond those the matchers consume (a subset assertion). */ public Options ignoreAdditionalTraces() { this.ignoreAdditionalTraces = true; return this; } + /** Let each matcher match any distinct trace (forward-only when a {@link #sorter} is set). */ + public Options unordered() { + this.unordered = true; + return this; + } + public Options sorter(Comparator> sorter) { this.sorter = sorter; return this; diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/trace/SmokeMatcherTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/trace/SmokeMatcherTest.java index 364d151f1ec..5bf45c0b49e 100644 --- a/dd-smoke-tests/src/test/java/datadog/smoketest/trace/SmokeMatcherTest.java +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/trace/SmokeMatcherTest.java @@ -132,18 +132,19 @@ void parentChainRejectsNonLinearTrace() { } @Test - void ignoreAdditionalTracesMatchesAnyOrder() { + void unorderedMatchesAnyOrder() { List traces = TestAgentTraceDecoder.decode(TWO_TRACES); - // Matchers in the opposite order of receipt still each find their trace (order-independent). + // Matchers in the opposite order of receipt still each find their trace (no sorter => any + // order). assertTraces( traces, - SmokeTraceAssertions.IGNORE_ADDITIONAL_TRACES, + options -> options.unordered().ignoreAdditionalTraces(), trace(span().operationName("root-b").root()), trace(span().operationName("root-a").root())); } @Test - void ignoreAdditionalTracesRequiresDistinctTraces() { + void unorderedRequiresDistinctTraces() { List traces = TestAgentTraceDecoder.decode(TWO_TRACES); // Two matchers for the same trace can't both match: only one root-a trace exists. assertThrows( @@ -151,11 +152,33 @@ void ignoreAdditionalTracesRequiresDistinctTraces() { () -> assertTraces( traces, - SmokeTraceAssertions.IGNORE_ADDITIONAL_TRACES, + options -> options.unordered().ignoreAdditionalTraces(), trace(span().operationName("root-a").root()), trace(span().operationName("root-a").root()))); } + @Test + void unorderedWithSorterIsForwardOnly() { + List traces = TestAgentTraceDecoder.decode(TWO_TRACES); + // Sorted by root span id => [root-b (400), root-a (500)]. With a sorter, an unordered match is + // forward-only: matchers in sorted order match... + assertTraces( + traces, + options -> options.unordered().sorter(SmokeTraceAssertions.TRACE_ROOT_SPAN_ID_COMPARATOR), + trace(span().operationName("root-b").root()), + trace(span().operationName("root-a").root())); + // ...but reversed they don't (root-a is matched at position 1, leaving nothing after it). + assertThrows( + AssertionError.class, + () -> + assertTraces( + traces, + options -> + options.unordered().sorter(SmokeTraceAssertions.TRACE_ROOT_SPAN_ID_COMPARATOR), + trace(span().operationName("root-a").root()), + trace(span().operationName("root-b").root()))); + } + private static String spanJson(String name, long id, long parent, long start) { return "{\"service\":\"s\",\"name\":\"" + name From 1b43c8b783825748dbe0e4ffcf00d200cc2c6770 Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Thu, 16 Jul 2026 14:22:30 +0200 Subject: [PATCH 19/30] WIP --- .../smoketest/trace/SmokeTraceAssertions.java | 150 ++++++++---------- .../smoketest/trace/SmokeMatcherTest.java | 53 +++---- 2 files changed, 90 insertions(+), 113 deletions(-) diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java index f37e685af46..519cc282586 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java @@ -1,6 +1,9 @@ package datadog.smoketest.trace; +import static java.lang.Long.MAX_VALUE; +import static java.lang.Math.min; import static java.util.function.UnaryOperator.identity; +import static org.junit.jupiter.api.AssertionFailureBuilder.assertionFailure; import datadog.trace.test.agent.decoder.DecodedSpan; import datadog.trace.test.agent.decoder.DecodedTrace; @@ -44,13 +47,9 @@ */ public final class SmokeTraceAssertions { /** Sorts traces by the earliest span start time. */ - public static final Comparator> TRACE_START_TIME_COMPARATOR = + public static final Comparator TRACE_START_TIME_COMPARATOR = Comparator.comparingLong(SmokeTraceAssertions::earliestStart); - /** Sorts traces by their root span id (the span with no parent). */ - public static final Comparator> TRACE_ROOT_SPAN_ID_COMPARATOR = - Comparator.comparingLong(SmokeTraceAssertions::rootSpanId); - /** Don't require every received trace to be matched — ignore the extras (a subset assertion). */ public static final UnaryOperator IGNORE_ADDITIONAL_TRACES = Options::ignoreAdditionalTraces; @@ -62,111 +61,100 @@ public final class SmokeTraceAssertions { public static final UnaryOperator SORT_BY_START_TIME = options -> options.sorter(TRACE_START_TIME_COMPARATOR); - /** Sorts traces by root span id before matching. */ - public static final UnaryOperator SORT_BY_ROOT_SPAN_ID = - options -> options.sorter(TRACE_ROOT_SPAN_ID_COMPARATOR); - private SmokeTraceAssertions() {} - /** Asserts the traces against the matchers, one matcher per expected trace, in received order. */ + /** + * Checks the structure of a trace collection. + * + * @param traces The trace collection to check. + * @param matchers The matchers to verify the trace collection, one matcher by expected trace. + */ public static void assertTraces(List traces, TraceMatcher... matchers) { assertTraces(traces, identity(), matchers); } /** - * As {@link #assertTraces(List, TraceMatcher...)} with the given options. One matching pass, - * tuned by {@code sorter} / {@link Options#unordered() unordered} / {@link - * Options#ignoreAdditionalTraces() ignoreAdditionalTraces} — see the class doc for the flag - * semantics. + * Checks the structure of a trace collection. + * + * @param traces The trace collection to check. + * @param options The {@link Options} to configure the checks. + * @param matchers The matchers to verify the trace collection, one matcher by expected trace. */ public static void assertTraces( List traces, UnaryOperator options, TraceMatcher... matchers) { Options opts = options.apply(new Options()); - // Copy each trace's spans to a mutable list so the matcher can sort/inspect them. - List> pool = new ArrayList<>(traces.size()); - for (DecodedTrace trace : traces) { - pool.add(new ArrayList<>(trace.getSpans())); + // Check trace count first + int traceCount = traces.size(); + if (opts.ignoreAdditionalTraces) { + if (traceCount < matchers.length) { + assertionFailure() + .message("Not enough of traces") + .expected(matchers.length) + .actual(traceCount) + .buildAndThrow(); + } + } else { + if (traceCount != matchers.length) { + assertionFailure() + .message("Invalid number of traces") + .expected(matchers.length) + .actual(traceCount) + .buildAndThrow(); + } } + // Apply sorter if (opts.sorter != null) { - pool.sort(opts.sorter); + traces = new ArrayList<>(traces); + traces.sort(opts.sorter); } - // Walk the matchers, each consuming a distinct trace. `floor` is the lowest pool index a match - // may use: it advances past each match to enforce order, except in fully-unordered mode (no - // sorter) where it stays 0 so any remaining trace is a candidate. `consumed` keeps matches - // distinct when the floor doesn't advance. - boolean[] consumed = new boolean[pool.size()]; - int matched = 0; + boolean strictPositional = !opts.unordered && !opts.ignoreAdditionalTraces; + // Record matched traces for unordered mode + boolean[] traceMatched = new boolean[traceCount]; + // Record last matching trace index for ignoreAdditionalTraces mode int floor = 0; - for (int i = 0; i < matchers.length; i++) { - int idx = findMatch(pool, consumed, floor, opts, matchers[i], i); - consumed[idx] = true; - matched++; - if (!opts.unordered || opts.sorter != null) { - floor = idx + 1; - } - } - if (!opts.ignoreAdditionalTraces && matched != pool.size()) { - throw new AssertionError( - "Expected " + matchers.length + " traces but got " + pool.size() + ": " + pool); - } - } - // Finds the trace this matcher consumes, or throws. Scans from `floor`, skipping consumed traces - // and (unless matching strictly positionally) non-matching ones. A strictly-positional matcher - // that fails is re-run with the throwing assertion so the failure names the offending span. - private static int findMatch( - List> pool, - boolean[] consumed, - int floor, - Options opts, - TraceMatcher matcher, - int i) { - boolean strictPositional = !opts.unordered && !opts.ignoreAdditionalTraces; - for (int j = floor; j < pool.size(); j++) { - if (consumed[j]) { + for (int matcherIndex = 0; matcherIndex < matchers.length; matcherIndex++) { + TraceMatcher matcher = matchers[matcherIndex]; + if (strictPositional) { + matcher.assertTrace(traces.get(matcherIndex).getSpans(), matcherIndex); continue; } - if (matcher.matches(pool.get(j), i)) { - return j; + + boolean matched = false; + for (int traceIndex = floor; traceIndex < traceCount; traceIndex++) { + // Skipped already matched traces - unordered mode only + if (traceMatched[traceIndex]) { + continue; + } + if (matcher.matches(traces.get(traceIndex).getSpans(), matcherIndex)) { + matched = true; + if (opts.unordered) { + traceMatched[traceIndex] = true; + } else { + floor = traceIndex + 1; + } + break; + } } - if (strictPositional) { - break; // the trace at this position must match; no skipping ahead + if (!matched) { + assertionFailure().message("No trace matches matcher # "+ matcherIndex).buildAndThrow(); } } - if (strictPositional && floor < pool.size() && !consumed[floor]) { - matcher.assertTrace(pool.get(floor), i); // throws with the per-span reason - } - throw new AssertionError( - "No received trace matched trace matcher #" - + i - + " among " - + pool.size() - + " received trace(s): " - + pool); } - private static long earliestStart(List spans) { - long earliest = Long.MAX_VALUE; - for (DecodedSpan span : spans) { - earliest = Math.min(earliest, span.getStart()); - } - return spans.isEmpty() ? 0L : earliest; - } - - private static long rootSpanId(List spans) { - for (DecodedSpan span : spans) { - if (span.getParentId() == 0L) { - return span.getSpanId(); - } + private static long earliestStart(DecodedTrace trace) { + long start = MAX_VALUE; + for (DecodedSpan span : trace.getSpans()) { + start = min(start, span.getStart()); } - return spans.isEmpty() ? 0L : spans.get(0).getSpanId(); + return start == MAX_VALUE ? 0L : start; } /** Trace-collection matching options; see the class doc for how they combine. */ public static final class Options { boolean ignoreAdditionalTraces = false; boolean unordered = false; - Comparator> sorter = null; // null => keep received order + Comparator sorter = null; // null => keep received order /** Ignore received traces beyond those the matchers consume (a subset assertion). */ public Options ignoreAdditionalTraces() { @@ -180,7 +168,7 @@ public Options unordered() { return this; } - public Options sorter(Comparator> sorter) { + public Options sorter(Comparator sorter) { this.sorter = sorter; return this; } diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/trace/SmokeMatcherTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/trace/SmokeMatcherTest.java index 5bf45c0b49e..b523c80bb20 100644 --- a/dd-smoke-tests/src/test/java/datadog/smoketest/trace/SmokeMatcherTest.java +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/trace/SmokeMatcherTest.java @@ -91,17 +91,6 @@ void ignoresAdditionalTraces() { trace(span().operationName("root-a").root())); } - @Test - void sortsTracesByRootSpanId() { - List traces = TestAgentTraceDecoder.decode(TWO_TRACES); - // Root-id order is [root-b (400), root-a (500)], the reverse of the received order. - assertTraces( - traces, - SmokeTraceAssertions.SORT_BY_ROOT_SPAN_ID, - trace(span().operationName("root-b").root()), - trace(span().operationName("root-a").root())); - } - @Test void sortsByParentChainRegardlessOfStartTime() { List traces = TestAgentTraceDecoder.decode(CHAIN_TRACE); @@ -157,27 +146,27 @@ void unorderedRequiresDistinctTraces() { trace(span().operationName("root-a").root()))); } - @Test - void unorderedWithSorterIsForwardOnly() { - List traces = TestAgentTraceDecoder.decode(TWO_TRACES); - // Sorted by root span id => [root-b (400), root-a (500)]. With a sorter, an unordered match is - // forward-only: matchers in sorted order match... - assertTraces( - traces, - options -> options.unordered().sorter(SmokeTraceAssertions.TRACE_ROOT_SPAN_ID_COMPARATOR), - trace(span().operationName("root-b").root()), - trace(span().operationName("root-a").root())); - // ...but reversed they don't (root-a is matched at position 1, leaving nothing after it). - assertThrows( - AssertionError.class, - () -> - assertTraces( - traces, - options -> - options.unordered().sorter(SmokeTraceAssertions.TRACE_ROOT_SPAN_ID_COMPARATOR), - trace(span().operationName("root-a").root()), - trace(span().operationName("root-b").root()))); - } + // @Test + // void unorderedWithSorterIsForwardOnly() { + // List traces = TestAgentTraceDecoder.decode(TWO_TRACES); + // // Sorted by root span id => [root-b (400), root-a (500)]. With a sorter, an unordered match is + // // forward-only: matchers in sorted order match... + // assertTraces( + // traces, + // options -> options.unordered().sorter(SmokeTraceAssertions.TRACE_ROOT_SPAN_ID_COMPARATOR), + // trace(span().operationName("root-b").root()), + // trace(span().operationName("root-a").root())); + // // ...but reversed they don't (root-a is matched at position 1, leaving nothing after it). + // assertThrows( + // AssertionError.class, + // () -> + // assertTraces( + // traces, + // options -> + // options.unordered().sorter(SmokeTraceAssertions.TRACE_ROOT_SPAN_ID_COMPARATOR), + // trace(span().operationName("root-a").root()), + // trace(span().operationName("root-b").root()))); + // } private static String spanJson(String name, long id, long parent, long start) { return "{\"service\":\"s\",\"name\":\"" From d291f052cdd3438ac0b9aa69889553cd4530f901 Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Thu, 16 Jul 2026 15:42:38 +0200 Subject: [PATCH 20/30] WIP --- .../smoketest/SpringBootRabbitSmokeTest.java | 2 +- .../datadog/smoketest/backend/Traces.java | 3 +- .../smoketest/trace/SmokeTraceAssertions.java | 66 ++++++++++--------- .../smoketest/trace/SmokeMatcherTest.java | 26 +------- 4 files changed, 39 insertions(+), 58 deletions(-) diff --git a/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java b/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java index d75c58560b6..abba5615b0f 100644 --- a/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java +++ b/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java @@ -140,7 +140,7 @@ void roundtripsProduceFullAmqpTraceStructure() throws IOException { .traces() .assertTraces( TIMEOUT_SECONDS, - o -> o.unordered().ignoreAdditionalTraces(), + o -> o.unorder().ignoreAdditionalTraces(), expected.toArray(new TraceMatcher[0])); } diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Traces.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Traces.java index e400f9e6370..484722d36c1 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Traces.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Traces.java @@ -73,7 +73,8 @@ public void assertTraces(TraceMatcher... matchers) { /** * As {@link #assertTraces(TraceMatcher...)} with options (e.g. {@code - * SmokeTraceAssertions.IGNORE_ADDITIONAL_TRACES} / {@code SORT_BY_ROOT_SPAN_ID}). + * SmokeTraceAssertions.UNORDERED} / {@code IGNORE_ADDITIONAL_TRACES} / {@code + * SORT_BY_START_TIME}). */ public void assertTraces( UnaryOperator options, TraceMatcher... matchers) { diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java index 519cc282586..c34a8a1b891 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/SmokeTraceAssertions.java @@ -8,6 +8,7 @@ import datadog.trace.test.agent.decoder.DecodedSpan; import datadog.trace.test.agent.decoder.DecodedTrace; import java.util.ArrayList; +import java.util.BitSet; import java.util.Comparator; import java.util.List; import java.util.function.UnaryOperator; @@ -24,26 +25,33 @@ * assertTraces(traces, trace(span().operationName("servlet.request").resourceName("GET /greeting"))); * } * - *

A single matching algorithm is tuned by three orthogonal, combinable {@link Options}: + *

A single matching algorithm is tuned by three combinable {@link Options}: * *

    - *
  • {@link #SORT_BY_START_TIME} / {@link #SORT_BY_ROOT_SPAN_ID} (the {@code sorter}) — sort the - * received traces before matching. + *
  • {@link #SORT_BY_START_TIME} (the {@code sorter}) — sort the received traces before matching, + * making positional matching deterministic. *
  • {@link #UNORDERED} — a matcher may match any distinct (unconsumed) trace rather - * than the one at its position. Without a sorter this is fully order-independent; - * with a sorter it is forward-only (once a matcher takes the trace at sorted - * position p, the next matcher can only look at positions after p) — i.e. match - * as a subsequence in sorted order. + * than the one at its position; the received order stops mattering (so a {@code sorter} has no + * effect in this mode). *
  • {@link #IGNORE_ADDITIONAL_TRACES} — don't require every received trace to be matched; extra * traces are ignored (a subset assertion). Without it, the counts must be equal. *
* - *

The default (no options) is thus count-exact positional matching (matcher i ↔ trace - * i, received order). Combine flags with the fluent {@link Options} — e.g. {@code o -> - * o.unordered().ignoreAdditionalTraces()} — to assert only the traces you care about in a - * collection that also holds unrelated or non-deterministically-ordered ones (e.g. a distributed - * test where connection-setup commands land as their own traces). The traces come from a {@code - * TraceBackend} (mock or test-agent), both decoded to {@link DecodedTrace}. + *

The useful combinations are: + * + *

    + *
  • default (optionally with a {@code sorter}) — count-exact positional: matcher + * i ↔ trace i. + *
  • {@code sorter} + {@code ignoreAdditionalTraces} — ordered subsequence: match in + * sorted order, skipping the traces no matcher claims. + *
  • {@code unordered} + {@code ignoreAdditionalTraces} — any-order subset: match the + * traces you care about and ignore the rest (e.g. a distributed test where connection-setup + * commands land as their own traces alongside the request traces). + *
+ * + *

Combine flags with the fluent {@link Options}, e.g. {@code o -> + * o.unordered().ignoreAdditionalTraces()}. The traces come from a {@code TraceBackend} (mock or + * test-agent), both decoded to {@link DecodedTrace}. */ public final class SmokeTraceAssertions { /** Sorts traces by the earliest span start time. */ @@ -55,11 +63,11 @@ public final class SmokeTraceAssertions { Options::ignoreAdditionalTraces; /** Let each matcher match any distinct trace rather than the one at its position. */ - public static final UnaryOperator UNORDERED = Options::unordered; + public static final UnaryOperator UNORDERED = Options::unorder; /** Sorts traces by earliest span start time before matching. */ public static final UnaryOperator SORT_BY_START_TIME = - options -> options.sorter(TRACE_START_TIME_COMPARATOR); + options -> options.sort(TRACE_START_TIME_COMPARATOR); private SmokeTraceAssertions() {} @@ -107,31 +115,26 @@ public static void assertTraces( traces = new ArrayList<>(traces); traces.sort(opts.sorter); } + // Assert traces boolean strictPositional = !opts.unordered && !opts.ignoreAdditionalTraces; - // Record matched traces for unordered mode - boolean[] traceMatched = new boolean[traceCount]; - // Record last matching trace index for ignoreAdditionalTraces mode - int floor = 0; - + BitSet skippedTraces = new BitSet(traceCount); for (int matcherIndex = 0; matcherIndex < matchers.length; matcherIndex++) { TraceMatcher matcher = matchers[matcherIndex]; if (strictPositional) { matcher.assertTrace(traces.get(matcherIndex).getSpans(), matcherIndex); continue; } - boolean matched = false; - for (int traceIndex = floor; traceIndex < traceCount; traceIndex++) { - // Skipped already matched traces - unordered mode only - if (traceMatched[traceIndex]) { + for (int traceIndex = skippedTraces.nextClearBit(0); traceIndex < traceCount; traceIndex++) { + if (skippedTraces.get(traceIndex)) { continue; } if (matcher.matches(traces.get(traceIndex).getSpans(), matcherIndex)) { matched = true; if (opts.unordered) { - traceMatched[traceIndex] = true; + skippedTraces.set(traceIndex); } else { - floor = traceIndex + 1; + skippedTraces.set(0, traceIndex); } break; } @@ -150,25 +153,24 @@ private static long earliestStart(DecodedTrace trace) { return start == MAX_VALUE ? 0L : start; } - /** Trace-collection matching options; see the class doc for how they combine. */ public static final class Options { boolean ignoreAdditionalTraces = false; boolean unordered = false; - Comparator sorter = null; // null => keep received order + Comparator sorter = null; - /** Ignore received traces beyond those the matchers consume (a subset assertion). */ public Options ignoreAdditionalTraces() { this.ignoreAdditionalTraces = true; return this; } - /** Let each matcher match any distinct trace (forward-only when a {@link #sorter} is set). */ - public Options unordered() { + public Options unorder() { this.unordered = true; + this.sorter = null; return this; } - public Options sorter(Comparator sorter) { + public Options sort(Comparator sorter) { + this.unordered = false; this.sorter = sorter; return this; } diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/trace/SmokeMatcherTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/trace/SmokeMatcherTest.java index b523c80bb20..9d48e79ec91 100644 --- a/dd-smoke-tests/src/test/java/datadog/smoketest/trace/SmokeMatcherTest.java +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/trace/SmokeMatcherTest.java @@ -127,7 +127,7 @@ void unorderedMatchesAnyOrder() { // order). assertTraces( traces, - options -> options.unordered().ignoreAdditionalTraces(), + options -> options.unorder().ignoreAdditionalTraces(), trace(span().operationName("root-b").root()), trace(span().operationName("root-a").root())); } @@ -141,33 +141,11 @@ void unorderedRequiresDistinctTraces() { () -> assertTraces( traces, - options -> options.unordered().ignoreAdditionalTraces(), + options -> options.unorder().ignoreAdditionalTraces(), trace(span().operationName("root-a").root()), trace(span().operationName("root-a").root()))); } - // @Test - // void unorderedWithSorterIsForwardOnly() { - // List traces = TestAgentTraceDecoder.decode(TWO_TRACES); - // // Sorted by root span id => [root-b (400), root-a (500)]. With a sorter, an unordered match is - // // forward-only: matchers in sorted order match... - // assertTraces( - // traces, - // options -> options.unordered().sorter(SmokeTraceAssertions.TRACE_ROOT_SPAN_ID_COMPARATOR), - // trace(span().operationName("root-b").root()), - // trace(span().operationName("root-a").root())); - // // ...but reversed they don't (root-a is matched at position 1, leaving nothing after it). - // assertThrows( - // AssertionError.class, - // () -> - // assertTraces( - // traces, - // options -> - // options.unordered().sorter(SmokeTraceAssertions.TRACE_ROOT_SPAN_ID_COMPARATOR), - // trace(span().operationName("root-a").root()), - // trace(span().operationName("root-b").root()))); - // } - private static String spanJson(String name, long id, long parent, long start) { return "{\"service\":\"s\",\"name\":\"" + name From 0de4f7d739e6ce9d20a2dac8a1773c4d1cf22a25 Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Thu, 16 Jul 2026 16:03:14 +0200 Subject: [PATCH 21/30] WIP --- .../smoketest/SpringBootRabbitSmokeTest.java | 2 +- .../java/datadog/smoketest/trace/TraceMatcher.java | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java b/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java index abba5615b0f..1f0ea008ba7 100644 --- a/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java +++ b/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java @@ -129,7 +129,7 @@ void roundtripsProduceFullAmqpTraceStructure() throws IOException { // per message plus each service's connection-setup/ack commands. List expected = new ArrayList<>(); for (int i = 0; i < MESSAGES.length; i++) { - expected.add(roundTrip()); // one full round-trip trace per message => all are verified + expected.add(roundTrip()); } for (String service : new String[] {"spring-rabbit-0", "spring-rabbit-1"}) { for (String command : ADMIN_COMMANDS) { diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/TraceMatcher.java b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/TraceMatcher.java index b672f863231..3f4c5ee7a0e 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/trace/TraceMatcher.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/trace/TraceMatcher.java @@ -36,7 +36,7 @@ public final class TraceMatcher { /** Configures {@link #trace(UnaryOperator, SpanMatcher...)} to sort spans by start time. */ public static final UnaryOperator SORT_BY_START_TIME = - options -> options.sorter(START_TIME_COMPARATOR); + options -> options.sort(START_TIME_COMPARATOR); /** * Configures {@link #trace(UnaryOperator, SpanMatcher...)} to order spans root→leaf by following @@ -84,8 +84,8 @@ void assertTrace(List spans, int traceIndex) { ordered = chainOrder(spans); } else { ordered = new ArrayList<>(spans); - if (options.sorter != null) { - ordered.sort(options.sorter); + if (options.comparator != null) { + ordered.sort(options.comparator); } } for (int i = 0; i < spanMatchers.length; i++) { @@ -153,13 +153,12 @@ private static List chainOrder(List spans) { return ordered; } - /** Per-trace matching options. */ public static final class Options { - Comparator sorter = null; // null => keep received order + Comparator comparator = null; // null => keep received order boolean linearizeByParentChain = false; - public Options sorter(Comparator sorter) { - this.sorter = sorter; + public Options sort(Comparator comparator) { + this.comparator = comparator; return this; } From 79915a556d051ede74d14a8465a63531ca6fd690 Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Thu, 16 Jul 2026 19:37:35 +0200 Subject: [PATCH 22/30] WIP --- .../smoketest/SpringBootRabbitSmokeTest.java | 26 +-- .../smoketest/trace/SmokeTraceAssertions.java | 29 ++- .../datadog/smoketest/trace/TraceMatcher.java | 185 ++++++++---------- .../smoketest/trace/SmokeMatcherTest.java | 28 ++- 4 files changed, 123 insertions(+), 145 deletions(-) diff --git a/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java b/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java index 1f0ea008ba7..b4402a9b2d7 100644 --- a/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java +++ b/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java @@ -1,7 +1,7 @@ package datadog.smoketest; import static datadog.smoketest.trace.SpanMatcher.span; -import static datadog.smoketest.trace.TraceMatcher.SORT_BY_PARENT_CHAIN; +import static datadog.smoketest.trace.TraceMatcher.SORT_BY_ANCESTRY; import static datadog.smoketest.trace.TraceMatcher.trace; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -58,19 +58,19 @@ * {@linkplain SmokeTraceAssertions order-independent subset} mode ({@code * unordered().ignoreAdditionalTraces()}): each matcher matches a distinct received trace and extras * are ignored. This is stronger than the Groovy set-membership check — each round-trip trace is - * matched count-exact (all 12 spans, in {@link TraceMatcher#SORT_BY_PARENT_CHAIN parent-chain - * order}), verifying every AMQP operation (publish/deliver/consume, both directions) and - * its cross-service linkage — while staying robust to the timing-dependent extras (the broker emits - * its connection-setup commands and per-ack traces as their own single-span traces, in + * matched count-exact (all 12 spans, in {@link TraceMatcher#SORT_BY_ANCESTRY ancestry order}), + * verifying every AMQP operation (publish/deliver/consume, both directions) and its + * cross-service linkage — while staying robust to the timing-dependent extras (the broker emits its + * connection-setup commands and per-ack traces as their own single-span traces, in * non-deterministic count and order). * *

Two design points, informed by inspecting the live trace collection: * *