diff --git a/solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/test.json b/solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/test.json
new file mode 100644
index 000000000000..8cd2d28437af
--- /dev/null
+++ b/solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/test.json
@@ -0,0 +1,111 @@
+{"phases":[
+ {
+ "description":"phase 0",
+ "spans":[{
+ "name":"post:/{collection}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1/update",
+ "http.params":"commit=true&wt=javabin",
+ "children":[
+ {
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard1_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard1_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=FROMLEADER&wt=javabin"},
+ {
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard1_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard1_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=FROMLEADER"},
+ {
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=TOLEADER&wt=javabin",
+ "children":[{
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=FROMLEADER&wt=javabin"}]},
+ {
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=TOLEADER",
+ "children":[{
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=FROMLEADER"}]}]}]},
+ {
+ "description":"phase 1",
+ "spans":[{
+ "name":"get:/{collection}/select",
+ "kind":"SERVER",
+ "db.instance":"collection1",
+ "db.type":"solr",
+ "http.request.method":"GET",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1/select",
+ "http.params":"q=*%3A*&wt=javabin",
+ "children":[
+ {
+ "name":"post:/{core}/select",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard1_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard1_replica_nN/select"},
+ {
+ "name":"post:/{core}/select",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard1_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard1_replica_nN/select"},
+ {
+ "name":"post:/{core}/select",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/select"},
+ {
+ "name":"post:/{core}/select",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/select"}]}]}]}
\ No newline at end of file
diff --git a/solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/testAdminApi.json b/solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/testAdminApi.json
new file mode 100644
index 000000000000..479f3cebb789
--- /dev/null
+++ b/solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/testAdminApi.json
@@ -0,0 +1,21 @@
+{"phases":[
+ {
+ "description":"phase 0",
+ "spans":[{
+ "name":"get:/admin/metrics",
+ "kind":"SERVER",
+ "db.type":"solr",
+ "http.request.method":"GET",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/admin/metrics",
+ "http.params":"wt=prometheus"}]},
+ {
+ "description":"phase 1",
+ "spans":[{
+ "name":"list:/admin/collections",
+ "kind":"SERVER",
+ "db.type":"solr",
+ "http.request.method":"GET",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/admin/collections",
+ "http.params":"action=LIST&wt=javabin"}]}]}
\ No newline at end of file
diff --git a/solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/testV2Api.json b/solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/testV2Api.json
new file mode 100644
index 000000000000..975aac135329
--- /dev/null
+++ b/solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/testV2Api.json
@@ -0,0 +1,143 @@
+{"phases":[
+ {
+ "description":"phase 0",
+ "spans":[
+ {
+ "name":"ReloadCollectionCmd",
+ "kind":"CLIENT",
+ "db.instance":"collection1",
+ "children":[
+ {
+ "name":"post:/admin/cores",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard1_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/admin/cores"},
+ {
+ "name":"post:/admin/cores",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard1_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/admin/cores"},
+ {
+ "name":"post:/admin/cores",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/admin/cores"},
+ {
+ "name":"post:/admin/cores",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/admin/cores"}]},
+ {
+ "name":"post:/collections/{collection}/reload",
+ "kind":"SERVER",
+ "db.instance":"collection1",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/____v2/collections/collection1/reload",
+ "http.params":"wt=javabin"}]},
+ {
+ "description":"phase 1",
+ "spans":[{
+ "name":"post:/c/{collection}/update/json",
+ "kind":"SERVER",
+ "db.instance":"collection1",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/____v2/c/collection1/update/json",
+ "http.params":"commit=true&wt=javabin",
+ "children":[
+ {
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard1_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard1_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=FROMLEADER"},
+ {
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=TOLEADER&wt=javabin",
+ "children":[{
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=FROMLEADER&wt=javabin"}]},
+ {
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=TOLEADER",
+ "children":[{
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=FROMLEADER"}]}]}]},
+ {
+ "description":"phase 2",
+ "spans":[{
+ "name":"get:/c/{collection}/select",
+ "kind":"SERVER",
+ "db.instance":"collection1",
+ "db.type":"solr",
+ "http.request.method":"GET",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/____v2/c/collection1/select",
+ "http.params":"q=id%3A9&wt=javabin",
+ "children":[
+ {
+ "name":"post:/{core}/select",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard1_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard1_replica_nN/select"},
+ {
+ "name":"post:/{core}/select",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/select"},
+ {
+ "name":"post:/{core}/select",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/select"}]}]}]}
\ No newline at end of file
diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/GoldFileTraceVerifier.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/GoldFileTraceVerifier.java
new file mode 100644
index 000000000000..adb32ac8bed1
--- /dev/null
+++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/GoldFileTraceVerifier.java
@@ -0,0 +1,476 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.opentelemetry;
+
+import io.opentelemetry.api.trace.SpanKind;
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.data.SpanData;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.util.RetryUtil;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.util.tracing.TraceUtils;
+import org.junit.Assert;
+
+/**
+ * Verifies collected OpenTelemetry spans against a committed gold file.
+ *
+ * Spans are normalized (IDs/timestamps stripped, URLs scrubbed) and arranged into a tree by
+ * parent-child relationships. The resulting JSON is compared against a gold file committed in
+ * {@code src/test-files/solr/tracing/}.
+ *
+ *
Usage
+ *
+ *
+ * var verifier = new GoldFileTraceVerifier("testV2Api");
+ * // ... perform operations ...
+ * verifier.verifyPhase(); // waits for spans, normalizes, compares
+ * // ... more operations ...
+ * verifier.verifyPhase();
+ * verifier.done();
+ *
+ *
+ * Regenerating Gold Files
+ *
+ * Requires {@code -Ptests.useSecurityManager=false} since it writes to the source tree.
+ *
+ *
+ * gradlew :solr:modules:opentelemetry:test --tests TestDistributedTracing.testV2Api \
+ * "-Ptests.jvmargs=-Dregenerate.golden.files=true" -Ptests.useSecurityManager=false
+ *
+ *
+ * Dumping Raw Traces (no normalization, all attributes)
+ *
+ * Also requires {@code -Ptests.useSecurityManager=false} (writes to source tree). Dump files are
+ * written alongside gold files as {@code {testMethod}-phase{N}-dump.json}. Output paths are logged
+ * to stdout (visible in {@code build/test-results/test/outputs/OUTPUT-*.txt}).
+ *
+ *
+ * gradlew :solr:modules:opentelemetry:test --tests TestDistributedTracing.testV2Api \
+ * "-Ptests.jvmargs=-Ddump.traces=true" -Ptests.useSecurityManager=false
+ *
+ */
+public class GoldFileTraceVerifier {
+
+ private static final boolean REGENERATE = Boolean.getBoolean("regenerate.golden.files");
+ private static final boolean DUMP_TRACES = Boolean.getBoolean("dump.traces");
+ private static final Pattern URL_HOST_PORT = Pattern.compile("https?://[^/]+/");
+ private static final Pattern REPLICA_SUFFIX = Pattern.compile("_replica_n\\d+");
+ private static final Pattern ENCODED_URL_IN_PARAMS = Pattern.compile("https?%3A%2F%2F[^&=]+");
+
+ private static final Comparator> SPAN_COMPARATOR =
+ Comparator.comparing((Map s) -> (String) s.get("name"))
+ .thenComparing(s -> s.getOrDefault(TraceUtils.TAG_DB.getKey(), "").toString());
+
+ private final String testName;
+ private final Path goldFilePath;
+ private final Map goldFile; // parsed gold file (phases list)
+ private final List>> recordedPhases = new ArrayList<>();
+ private int currentPhaseIndex = 0;
+
+ public GoldFileTraceVerifier(Class testClass, String testMethodName) {
+ this.testName = testMethodName;
+ this.goldFilePath = resolveGoldFilePath(testClass, testMethodName);
+ if (REGENERATE) {
+ this.goldFile = null;
+ } else {
+ this.goldFile = loadGoldFile(goldFilePath);
+ }
+ }
+
+ /**
+ * Waits for the expected number of spans (from the gold file), collects them, normalizes into a
+ * tree, and compares against the gold file's current phase.
+ */
+ @SuppressWarnings("unchecked")
+ public void verifyPhase() {
+ InMemorySpanExporter exporter = CustomTestOtelTracerConfigurator.getInMemorySpanExporter();
+
+ List spans;
+ if (REGENERATE) {
+ spans = waitForStableSpans(exporter);
+ } else {
+ List> phases = (List>) goldFile.get("phases");
+ Map phase = phases.get(currentPhaseIndex);
+ List> expectedSpans = (List>) phase.get("spans");
+ int expectedCount = countSpansRecursive(expectedSpans);
+ spans = waitForSpans(exporter, expectedCount);
+ }
+
+ exporter.reset();
+
+ if (DUMP_TRACES) {
+ dumpRawSpans(spans);
+ }
+
+ List> tree = buildTree(spans);
+ recordedPhases.add(tree);
+
+ if (!REGENERATE) {
+ comparePhase(currentPhaseIndex, tree);
+ }
+ currentPhaseIndex++;
+ }
+
+ /** Call after all phases are verified. In regenerate mode, writes the gold file. */
+ @SuppressWarnings("unchecked")
+ public void done() {
+ if (REGENERATE) {
+ writeGoldFile();
+ } else {
+ List> phases = (List>) goldFile.get("phases");
+ Assert.assertEquals(
+ "Unverified phases remain in gold file for " + testName,
+ phases.size(),
+ currentPhaseIndex);
+ }
+ }
+
+ // --- Span collection ---
+
+ private List waitForSpans(InMemorySpanExporter exporter, int expectedCount) {
+ try {
+ RetryUtil.retryUntil(
+ "Timed out waiting for " + expectedCount + " span(s) in phase " + currentPhaseIndex,
+ 500,
+ 20,
+ TimeUnit.MILLISECONDS,
+ () -> exporter.getFinishedSpanItems().size() >= expectedCount);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ return new ArrayList<>(exporter.getFinishedSpanItems());
+ }
+
+ private List waitForStableSpans(InMemorySpanExporter exporter) {
+ try {
+ RetryUtil.retryUntil(
+ "Timed out waiting for any spans in phase " + currentPhaseIndex,
+ 500,
+ 20,
+ TimeUnit.MILLISECONDS,
+ () -> !exporter.getFinishedSpanItems().isEmpty());
+ int lastCount = -1;
+ for (int i = 0; i < 10; i++) {
+ TimeUnit.MILLISECONDS.sleep(500);
+ int current = exporter.getFinishedSpanItems().size();
+ if (current == lastCount) {
+ break;
+ }
+ lastCount = current;
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ return new ArrayList<>(exporter.getFinishedSpanItems());
+ }
+
+ // --- Raw dump (for debugging, no normalization) ---
+
+ private void dumpRawSpans(List spans) {
+ // Sort by start timestamp for a chronological view
+ List sorted = new ArrayList<>(spans);
+ sorted.sort(Comparator.comparingLong(SpanData::getStartEpochNanos));
+
+ // Build tree from raw data (no normalization)
+ Map> nodesBySpanId = new HashMap<>();
+ List> roots = new ArrayList<>();
+
+ for (SpanData span : sorted) {
+ Map node = new LinkedHashMap<>();
+ node.put("name", span.getName());
+ node.put("kind", span.getKind().name());
+ node.put("traceId", span.getSpanContext().getTraceId());
+ node.put("spanId", span.getSpanContext().getSpanId());
+ if (span.getParentSpanContext().isValid()) {
+ node.put("parentSpanId", span.getParentSpanContext().getSpanId());
+ }
+ node.put("startEpochNanos", span.getStartEpochNanos());
+ node.put("endEpochNanos", span.getEndEpochNanos());
+ // All attributes
+ Map attrs = new LinkedHashMap<>();
+ span.getAttributes().forEach((key, value) -> attrs.put(key.getKey(), value));
+ if (!attrs.isEmpty()) {
+ node.put("attributes", attrs);
+ }
+ nodesBySpanId.put(span.getSpanContext().getSpanId(), node);
+ }
+
+ for (SpanData span : sorted) {
+ Map node = nodesBySpanId.get(span.getSpanContext().getSpanId());
+ String parentId = span.getParentSpanContext().getSpanId();
+ if (span.getParentSpanContext().isValid() && nodesBySpanId.containsKey(parentId)) {
+ @SuppressWarnings("unchecked")
+ List> children =
+ (List>)
+ nodesBySpanId.get(parentId).computeIfAbsent("children", k -> new ArrayList<>());
+ children.add(node);
+ } else {
+ roots.add(node);
+ }
+ }
+
+ String json = Utils.toJSONString(roots, 2);
+ Path dumpDir = goldFilePath.getParent();
+ Path dumpFile = dumpDir.resolve(testName + "-phase" + currentPhaseIndex + "-dump.json");
+ try {
+ Files.createDirectories(dumpDir);
+ Files.writeString(dumpFile, json);
+ System.out.println("Dumped raw traces: " + dumpFile.toAbsolutePath());
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to write trace dump", e);
+ }
+ }
+
+ // --- Tree building ---
+
+ private List> buildTree(List spans) {
+ Map> nodesBySpanId = new HashMap<>();
+ List> roots = new ArrayList<>();
+
+ for (SpanData span : spans) {
+ nodesBySpanId.put(span.getSpanContext().getSpanId(), normalize(span));
+ }
+
+ for (SpanData span : spans) {
+ Map node = nodesBySpanId.get(span.getSpanContext().getSpanId());
+ String parentId = span.getParentSpanContext().getSpanId();
+ if (span.getParentSpanContext().isValid() && nodesBySpanId.containsKey(parentId)) {
+ @SuppressWarnings("unchecked")
+ List> children =
+ (List>) nodesBySpanId.get(parentId).get("children");
+ if (children == null) {
+ children = new ArrayList<>();
+ nodesBySpanId.get(parentId).put("children", children);
+ }
+ children.add(node);
+ } else {
+ roots.add(node);
+ }
+ }
+
+ sortTree(roots);
+ return roots;
+ }
+
+ @SuppressWarnings("unchecked")
+ private void sortTree(List> nodes) {
+ nodes.sort(SPAN_COMPARATOR);
+ for (Map node : nodes) {
+ List> children = (List>) node.get("children");
+ if (children != null && !children.isEmpty()) {
+ sortTree(children);
+ }
+ }
+ }
+
+ private Map normalize(SpanData span) {
+ // Use LinkedHashMap for consistent key ordering in JSON output
+ Map node = new LinkedHashMap<>();
+ node.put("name", span.getName());
+ if (span.getKind() != SpanKind.INTERNAL) {
+ node.put("kind", span.getKind().name());
+ }
+
+ String dbInstance = normalizeDbInstance(span.getAttributes().get(TraceUtils.TAG_DB));
+ if (dbInstance != null) {
+ node.put(TraceUtils.TAG_DB.getKey(), dbInstance);
+ }
+
+ String dbType = span.getAttributes().get(TraceUtils.TAG_DB_TYPE);
+ if (dbType != null) {
+ node.put(TraceUtils.TAG_DB_TYPE.getKey(), dbType);
+ }
+
+ String httpMethod = span.getAttributes().get(TraceUtils.TAG_HTTP_METHOD);
+ if (httpMethod != null) {
+ node.put(TraceUtils.TAG_HTTP_METHOD.getKey(), httpMethod);
+ }
+
+ Long statusCode = span.getAttributes().get(TraceUtils.TAG_HTTP_STATUS);
+ if (statusCode != null) {
+ node.put(TraceUtils.TAG_HTTP_STATUS.getKey(), statusCode.intValue());
+ }
+
+ String httpUrl = span.getAttributes().get(TraceUtils.TAG_HTTP_URL);
+ if (httpUrl != null) {
+ node.put(TraceUtils.TAG_HTTP_URL.getKey(), normalizeUrl(httpUrl));
+ }
+
+ String httpParams = span.getAttributes().get(TraceUtils.TAG_HTTP_PARAMS);
+ if (httpParams != null) {
+ node.put(TraceUtils.TAG_HTTP_PARAMS.getKey(), normalizeParams(httpParams));
+ }
+
+ // children will be added later during tree building if needed
+ return node;
+ }
+
+ private String normalizeUrl(String url) {
+ url = URL_HOST_PORT.matcher(url).replaceFirst("http://NORMALIZED/");
+ url = REPLICA_SUFFIX.matcher(url).replaceAll("_replica_nN");
+ return url;
+ }
+
+ private String normalizeParams(String params) {
+ // Scrub embedded URLs (distrib.from=http%3A%2F%2F...) that contain ports
+ params = ENCODED_URL_IN_PARAMS.matcher(params).replaceAll("http%3A%2F%2FNORMALIZED");
+ String[] parts = params.split("&");
+ Arrays.sort(parts);
+ return String.join("&", parts);
+ }
+
+ private String normalizeDbInstance(String dbInstance) {
+ if (dbInstance == null) return null;
+ return REPLICA_SUFFIX.matcher(dbInstance).replaceAll("_replica_nN");
+ }
+
+ // --- Comparison ---
+
+ @SuppressWarnings("unchecked")
+ private void comparePhase(int phaseIndex, List> actual) {
+ List> phases = (List>) goldFile.get("phases");
+ Map phase = phases.get(phaseIndex);
+ List> expectedSpans = (List>) phase.get("spans");
+ String description = (String) phase.get("description");
+
+ String expectedJson = Utils.toJSONString(expectedSpans, 2);
+ String actualJson = Utils.toJSONString(actual, 2);
+
+ if (!expectedJson.equals(actualJson)) {
+ Path tempFile = writeTempJson(Utils.toJSONString(actual, 2), phaseIndex);
+ String message =
+ String.format(
+ """
+
+ Trace spans mismatch in phase %d%s
+
+ Expected (gold file):
+ %s
+
+ Actual output (written to temp file):
+ %s
+
+ To compare:
+ diff %s %s
+
+ If intentional, regenerate:
+ gradlew :solr:modules:opentelemetry:test --tests TestDistributedTracing.%s "-Ptests.jvmargs=-Dregenerate.golden.files=true" -Ptests.useSecurityManager=false
+ """,
+ phaseIndex,
+ description != null ? " \"" + description + "\"" : "",
+ goldFilePath.toAbsolutePath(),
+ tempFile.toAbsolutePath(),
+ goldFilePath.toAbsolutePath(),
+ tempFile.toAbsolutePath(),
+ testName);
+ Assert.assertEquals(message, expectedJson, actualJson);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private int countSpansRecursive(List> nodes) {
+ int count = 0;
+ for (Map node : nodes) {
+ count++;
+ List> children = (List>) node.get("children");
+ if (children != null && !children.isEmpty()) {
+ count += countSpansRecursive(children);
+ }
+ }
+ return count;
+ }
+
+ // --- Gold file I/O ---
+
+ private static Path resolveGoldFilePath(Class testClass, String testMethodName) {
+ // Gold files live at: {test-files}/solr/tracing/{TestClassName}/{testMethod}.json
+ // SolrTestCaseJ4.getFile() may resolve to the build output; walk up to find src/test-files.
+ Path solrDir = SolrTestCaseJ4.getFile("solr");
+ Path testFilesDir = solrDir.getParent();
+ // If we're in build output (e.g. build/resources/test/solr), find the source equivalent
+ String testFilesStr = testFilesDir.toString();
+ if (testFilesStr.contains("/build/")) {
+ testFilesDir =
+ Path.of(testFilesStr.substring(0, testFilesStr.indexOf("/build/")))
+ .resolve("src/test-files");
+ }
+ return testFilesDir
+ .resolve("solr/tracing")
+ .resolve(testClass.getSimpleName())
+ .resolve(testMethodName + ".json");
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Map loadGoldFile(Path path) {
+ if (!Files.exists(path)) {
+ Assert.fail(
+ "Gold file not found: "
+ + path.toAbsolutePath()
+ + "\nRegenerate with: gradlew ... \"-Ptests.jvmargs=-Dregenerate.golden.files=true\" -Ptests.useSecurityManager=false");
+ }
+ try {
+ byte[] bytes = Files.readAllBytes(path);
+ return (Map) Utils.fromJSON(bytes);
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to read gold file: " + path, e);
+ }
+ }
+
+ private void writeGoldFile() {
+ Map output = new LinkedHashMap<>();
+ List> phases = new ArrayList<>();
+ for (int i = 0; i < recordedPhases.size(); i++) {
+ Map phase = new LinkedHashMap<>();
+ phase.put("description", "phase " + i);
+ phase.put("spans", recordedPhases.get(i));
+ phases.add(phase);
+ }
+ output.put("phases", phases);
+ try {
+ Files.createDirectories(goldFilePath.getParent());
+ String json = Utils.toJSONString(output, 2);
+ Files.writeString(goldFilePath, json);
+ System.out.println("Regenerated gold file: " + goldFilePath.toAbsolutePath());
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to write gold file: " + goldFilePath, e);
+ }
+ }
+
+ private Path writeTempJson(String json, int phaseIndex) {
+ try {
+ Path tempFile =
+ Path.of(System.getProperty("java.io.tmpdir"), testName + "-phase" + phaseIndex + ".json");
+ Files.writeString(tempFile, json);
+ return tempFile;
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to write temp file", e);
+ }
+ }
+}
diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java
index 5433c11fd1d8..f73d47060032 100644
--- a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java
+++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java
@@ -17,11 +17,11 @@
package org.apache.solr.opentelemetry;
+import com.carrotsearch.randomizedtesting.annotations.Seed;
import io.opentelemetry.api.GlobalOpenTelemetry;
import io.opentelemetry.api.trace.TracerProvider;
import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
import io.opentelemetry.sdk.trace.data.SpanData;
-import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
@@ -30,11 +30,10 @@
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.MetricsRequest;
import org.apache.solr.client.solrj.request.SolrQuery;
+import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.request.V2Request;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.InputStreamResponseParser;
@@ -50,6 +49,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
+@Seed("0") // don't want randomization when testing observability
public class TestDistributedTracing extends SolrCloudTestCase {
private static final String COLLECTION = "collection1";
@@ -58,6 +58,8 @@ public class TestDistributedTracing extends SolrCloudTestCase {
public static void setupCluster() throws Exception {
// force early init
CustomTestOtelTracerConfigurator.prepareForTest();
+ // HTTP 2 clients can do things more asynchronously, leading to less determinism in what we test
+ System.setProperty("solr.http1", "true");
configureCluster(4)
.addConfig("config", TEST_PATH().resolve("collection1").resolve("conf"))
@@ -71,7 +73,28 @@ public static void setupCluster() throws Exception {
TracerProvider.noop(),
GlobalOpenTelemetry.get().getTracerProvider());
- CollectionAdminRequest.createCollection(COLLECTION, "config", 2, 2)
+ // Create collection with explicit replica placement for deterministic leader assignment.
+ // First replica on each shard becomes the leader.
+ String node0 = cluster.getJettySolrRunner(0).getNodeName();
+ String node1 = cluster.getJettySolrRunner(1).getNodeName();
+ String node2 = cluster.getJettySolrRunner(2).getNodeName();
+ String node3 = cluster.getJettySolrRunner(3).getNodeName();
+ CollectionAdminRequest.createCollection(COLLECTION, "config", 2, 1)
+ .setCreateNodeSet("EMPTY")
+ .process(cluster.getSolrClient());
+ // shard1: node0 = leader, node1 = follower
+ CollectionAdminRequest.addReplicaToShard(COLLECTION, "shard1")
+ .setNode(node0)
+ .process(cluster.getSolrClient());
+ CollectionAdminRequest.addReplicaToShard(COLLECTION, "shard1")
+ .setNode(node1)
+ .process(cluster.getSolrClient());
+ // shard2: node2 = leader, node3 = follower
+ CollectionAdminRequest.addReplicaToShard(COLLECTION, "shard2")
+ .setNode(node2)
+ .process(cluster.getSolrClient());
+ CollectionAdminRequest.addReplicaToShard(COLLECTION, "shard2")
+ .setNode(node3)
.process(cluster.getSolrClient());
cluster.waitForActiveCollection(COLLECTION, 2, 4);
}
@@ -87,121 +110,76 @@ private void resetSpanData() {
}
@Test
- public void test() throws IOException, SolrServerException {
- // TODO it would be clearer if we could compare the complete Span tree between reality
- // and what we assert it looks like in a structured visual way.
- CloudSolrClient cloudClient = cluster.getSolrClient();
+ public void test() throws Exception {
+ var verifier = new GoldFileTraceVerifier(getClass(), "test");
+ // TODO use a CloudSolrClient. However it's not yet deterministic due to use of random not
+ // aligned to the test seed.
+ var client = cluster.getJettySolrRunner(0).getSolrClient();
// Indexing
- cloudClient.add(COLLECTION, sdoc("id", "1"));
- var finishedSpans = getAndClearSpans(1);
- finishedSpans.removeIf(
- span ->
- span.getAttributes().get(TraceUtils.TAG_HTTP_URL) == null
- || !span.getAttributes().get(TraceUtils.TAG_HTTP_URL).endsWith("/update"));
- assertEquals(2, finishedSpans.size());
- assertOneSpanIsChildOfAnother(finishedSpans);
- // core because cloudClient routes to core
- assertEquals("post:/{core}/update", finishedSpans.get(0).getName());
- assertCoreName(finishedSpans.get(0), COLLECTION);
-
- cloudClient.add(COLLECTION, sdoc("id", "2"));
- cloudClient.add(COLLECTION, sdoc("id", "3"));
- cloudClient.add(COLLECTION, sdoc("id", "4"));
- cloudClient.commit(COLLECTION);
- getAndClearSpans();
+ new UpdateRequest()
+ .add(List.of(sdoc("id", "1"), sdoc("id", "2"), sdoc("id", "3"), sdoc("id", "4")))
+ .commit(client, COLLECTION);
+ verifier.verifyPhase();
// Searching
- cloudClient.query(COLLECTION, new SolrQuery("*:*"));
- finishedSpans = getAndClearSpans(1);
- finishedSpans.removeIf(
- span ->
- span.getAttributes().get(TraceUtils.TAG_HTTP_URL) == null
- || !span.getAttributes().get(TraceUtils.TAG_HTTP_URL).endsWith("/select"));
- // one from client to server, 2 for execute query, 2 for fetching documents
- assertEquals(5, finishedSpans.size());
- var parentTraceId = getRootTraceId(finishedSpans);
- for (var span : finishedSpans) {
- if (isRootSpan(span)) {
- continue;
- }
- assertEquals(span.getParentSpanContext().getTraceId(), parentTraceId);
- assertEquals(span.getTraceId(), parentTraceId);
- }
- assertEquals("get:/{core}/select", finishedSpans.get(0).getName());
- assertCoreName(finishedSpans.get(0), COLLECTION);
+ client.query(COLLECTION, new SolrQuery("*:*"));
+ verifier.verifyPhase();
+
+ verifier.done();
}
@Test
public void testAdminApi() throws Exception {
- CloudSolrClient cloudClient = cluster.getSolrClient();
+ var verifier = new GoldFileTraceVerifier(getClass(), "testAdminApi");
+ // TODO use a CloudSolrClient. However it's not yet deterministic due to use of random not
+ // aligned to the test seed.
+ var client = cluster.getJettySolrRunner(0).getSolrClient();
MetricsRequest request = new MetricsRequest();
request.setResponseParser(new InputStreamResponseParser(MetricUtils.PROMETHEUS_METRICS_WT));
- NamedList rsp = cloudClient.request(request);
+ NamedList rsp = client.request(request);
((InputStream) rsp.get("stream")).close();
- var finishedSpans = getAndClearSpans(1);
- assertEquals("get:/admin/metrics", finishedSpans.get(0).getName());
+ verifier.verifyPhase();
+
+ CollectionAdminRequest.listCollections(client);
+ verifier.verifyPhase();
- CollectionAdminRequest.listCollections(cloudClient);
- finishedSpans = getAndClearSpans(1);
- assertEquals("list:/admin/collections", finishedSpans.get(0).getName());
+ verifier.done();
}
@Test
public void testV2Api() throws Exception {
- CloudSolrClient cloudClient = cluster.getSolrClient();
+ var verifier = new GoldFileTraceVerifier(getClass(), "testV2Api");
+ // TODO use a CloudSolrClient. However it's not yet deterministic due to use of random not
+ // aligned to the test seed.
+ var client = cluster.getJettySolrRunner(0).getSolrClient();
new V2Request.Builder("/collections/" + COLLECTION + "/reload")
.withMethod(SolrRequest.METHOD.POST)
.withPayload("{}")
.build()
- .process(cloudClient);
- var finishedSpans = getAndClearSpans(1);
- assertEquals("post:/collections/{collection}/reload", finishedSpans.get(0).getName());
- assertCollectionName(finishedSpans.get(0), COLLECTION);
+ .process(client);
+ verifier.verifyPhase();
new V2Request.Builder("/c/" + COLLECTION + "/update/json")
.withMethod(SolrRequest.METHOD.POST)
- .withPayload("{\n" + " \"id\" : \"9\"\n" + "}")
+ .withPayload("{\"id\":\"9\"}")
.withParams(params("commit", "true"))
.build()
- .process(cloudClient);
- finishedSpans = getAndClearSpans(1);
- assertEquals("post:/c/{collection}/update/json", finishedSpans.get(0).getName());
- assertCollectionName(finishedSpans.get(0), COLLECTION);
+ .process(client);
+ verifier.verifyPhase();
final V2Response v2Response =
new V2Request.Builder("/c/" + COLLECTION + "/select")
.withMethod(SolrRequest.METHOD.GET)
.withParams(params("q", "id:9"))
.build()
- .process(cloudClient);
- finishedSpans = getAndClearSpans(1);
- assertEquals("get:/c/{collection}/select", finishedSpans.get(0).getName());
- assertCollectionName(finishedSpans.get(0), COLLECTION);
+ .process(client);
+ verifier.verifyPhase();
assertEquals(1, ((SolrDocumentList) v2Response.getResponse().get("response")).getNumFound());
- }
- /**
- * Best effort test of the apache http client tracing. the test assumes the request uses the http
- * client but there is no way to enforce it, so when the api will be rewritten this test will
- * become obsolete
- */
- @Test
- public void testApacheClient() throws Exception {
- CollectionAdminRequest.ColStatus a1 = CollectionAdminRequest.collectionStatus(COLLECTION);
- CollectionAdminResponse r1 = a1.process(cluster.getSolrClient());
- assertEquals(0, r1.getStatus());
- var finishedSpans = getAndClearSpans(1);
- var parentTraceId = getRootTraceId(finishedSpans);
- for (var span : finishedSpans) {
- if (isRootSpan(span)) {
- continue;
- }
- assertEquals(span.getParentSpanContext().getTraceId(), parentTraceId);
- assertEquals(span.getTraceId(), parentTraceId);
- }
+ verifier.done();
}
@Test
@@ -316,18 +294,6 @@ private static void assertCoreName(SpanData span, String collection) {
assertTrue(span.getAttributes().get(TraceUtils.TAG_DB).startsWith(collection + "_"));
}
- private void assertOneSpanIsChildOfAnother(List finishedSpans) {
- SpanData child = finishedSpans.get(0);
- SpanData parent = finishedSpans.get(1);
- if (isRootSpan(child)) {
- var temp = parent;
- parent = child;
- child = temp;
- }
- assertEquals(child.getParentSpanContext().getTraceId(), parent.getTraceId());
- assertEquals(child.getTraceId(), parent.getTraceId());
- }
-
static List getAndClearSpans() {
return getAndClearSpans(0);
}