feat: trace context propagation behavior

contrib/labstack/echo.v4/echotrace_test.go

@@ -720,6 +720,106 @@ func TestWithErrorCheck(t *testing.T) {
 	}
 }
 
+// TestPropagationBehaviorExtract is an integration test verifying DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT
+// through a real HTTP middleware stack. 
+// Echotrace was used because it was
+// already available; any HTTP middleware integration would work equivalently.
+func TestPropagationBehaviorExtract(t *testing.T) {
+	tests := []struct {
+		name            string
+		behavior        string
+		wantSameTraceID bool // server span continues the root trace
+		wantParentID    bool // server span has root as parent
+		wantSpanLinks   bool // server span has a span link to the root context
+		wantBaggage     bool // baggage from root is propagated to server span
+	}{
+		{
+			name:            "continue",
+			behavior:        "continue",
+			wantSameTraceID: true,
+			wantParentID:    true,
+			wantSpanLinks:   false,
+			wantBaggage:     true,
+		},
+		{
+			name:            "restart",
+			behavior:        "restart",
+			wantSameTraceID: false,
+			wantParentID:    false,
+			wantSpanLinks:   true,
+			wantBaggage:     true,
+		},
+		{
+			name:            "ignore",
+			behavior:        "ignore",
+			wantSameTraceID: false,
+			wantParentID:    false,
+			wantSpanLinks:   false,
+			wantBaggage:     false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Setenv("DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT", tc.behavior)
+
+			mt := mocktracer.Start()
+			defer mt.Stop()
+
+			router := echo.New()
+			router.Use(Middleware(WithService("test-service")))
+			router.GET("/test", func(c echo.Context) error {
+				return c.NoContent(200)
+			})
+
+			root := tracer.StartSpan("incoming-request")
+			root.SetBaggageItem("test-baggage", "baggage-value")
+
+			r := httptest.NewRequest("GET", "/test", nil)
+			err := tracer.Inject(root.Context(), tracer.HTTPHeadersCarrier(r.Header))
+			require.NoError(t, err)
+
+			router.ServeHTTP(httptest.NewRecorder(), r)
+
+			spans := mt.FinishedSpans()
+			require.Len(t, spans, 1)
+			span := spans[0]
+
+			if tc.wantSameTraceID {
+				assert.Equal(t, root.Context().TraceIDLower(), span.TraceID())
+			} else {
+				assert.NotEqual(t, root.Context().TraceIDLower(), span.TraceID())
+			}
+
+			if tc.wantParentID {
+				assert.Equal(t, root.Context().SpanID(), span.ParentID())
+			} else {
+				assert.Equal(t, uint64(0), span.ParentID())
+			}
+
+			links := span.Links()
+			if tc.wantSpanLinks {
+				require.Len(t, links, 1)
+				assert.Equal(t, root.Context().SpanID(), links[0].SpanID)
+				assert.Equal(t, map[string]string{"reason": "propagation_behavior_extract", "context_headers": "datadog"}, links[0].Attributes)
+			} else {
+				assert.Empty(t, links)
+			}
+
+			var baggageItems []string
+			span.Context().ForeachBaggageItem(func(k, v string) bool {
+				baggageItems = append(baggageItems, k+"="+v)
+				return true
+			})
+			if tc.wantBaggage {
+				assert.Contains(t, baggageItems, "test-baggage=baggage-value")
+			} else {
+				assert.Empty(t, baggageItems)
+			}
+		})
+	}
+}
+
 func BenchmarkEchoWithTracing(b *testing.B) {
 	tracer.Start(tracer.WithLogger(testutils.DiscardLogger()))
 	defer tracer.Stop()

ddtrace/tracer/spancontext.go

@@ -267,7 +267,11 @@
 			context.setBaggageItem(k, v)
 			return true
 		})
-	} else if traceID128BitEnabled.Load() {
+	}
+	// We generate a new upper trace ID when the trace is brand new (no parent)
+	// or when the parent is baggage only, since baggage only parents should
+	// not propagate their trace IDs
+	if (parent == nil || parent.baggageOnly) && traceID128BitEnabled.Load() { // +checklocksignore - Read-only after init.
 		// add 128 bit trace id, if enabled, formatted as big-endian:
 		// <32-bit unix seconds> <32 bits of zero> <64 random bits>
 		id128 := time.Duration(span.start) / time.Second
@@ -526,7 +530,7 @@
 	// reasonable as span is actually way bigger, and avoids re-allocating
 	// over and over. Could be fine-tuned at runtime.
 	traceStartSize = 10
 	traceMaxSize   = internalconfig.TraceMaxSize
 )
 
 // samplingPriorityCache holds pre-allocated pointers for the four standard

ddtrace/tracer/telemetry.go

@@ -81,6 +81,8 @@ func startTelemetry(c *config) telemetry.Client {
 			telemetry.Configuration{Name: "trace_propagation_style_inject", Value: chained.injectorNames})
 		telemetryConfigs = append(telemetryConfigs,
 			telemetry.Configuration{Name: "trace_propagation_style_extract", Value: chained.extractorsNames})
+		telemetryConfigs = append(telemetryConfigs,
+			telemetry.Configuration{Name: "trace_propagation_behavior_extract", Value: chained.propagationBehaviorExtract})
 	}
 	for k, v := range c.internalConfig.FeatureFlags() {
 		telemetryConfigs = append(telemetryConfigs, telemetry.Configuration{Name: k, Value: v})

ddtrace/tracer/textmap.go

@@ -69,6 +69,22 @@ func (c TextMapCarrier) ForeachKey(handler func(key, val string) error) error {
 }
 
 const (
+	// headerPropagationBehaviorExtract specifies how to handle incoming trace
+	// context. Allowed values:
+	// - "continue" (default): Continue the trace from incoming headers.
+	//   Baggage is propagated.
+	// - "restart": Start a new trace with a new trace ID and sampling
+	//   decision. The incoming context is referenced via a span link.
+	//   Baggage is propagated.
+	// - "ignore": Start a new trace with a new trace ID and sampling
+	//   decision. No span links are created. Baggage is dropped.
+	headerPropagationBehaviorExtract = "DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT"
+
+	propagationBehaviorExtractContinue = "continue"
+	propagationBehaviorExtractRestart  = "restart"
+	propagationBehaviorExtractIgnore   = "ignore"
+
+	headerPropagationExtractFirst = "DD_TRACE_PROPAGATION_EXTRACT_FIRST"
 	headerPropagationStyleInject  = "DD_TRACE_PROPAGATION_STYLE_INJECT"
 	headerPropagationStyleExtract = "DD_TRACE_PROPAGATION_STYLE_EXTRACT"
 	headerPropagationStyle        = "DD_TRACE_PROPAGATION_STYLE"
@@ -166,7 +182,17 @@ func NewPropagator(cfg *PropagatorConfig, propagators ...Propagator) Propagator
 		cfg.BaggageHeader = DefaultBaggageHeader
 	}
 	cp := new(chainedPropagator)
-	cp.onlyExtractFirst = internal.BoolEnv("DD_TRACE_PROPAGATION_EXTRACT_FIRST", false)
+	cp.onlyExtractFirst = internal.BoolEnv(headerPropagationExtractFirst, false)
+	cp.propagationBehaviorExtract = env.Get(headerPropagationBehaviorExtract)
+	switch cp.propagationBehaviorExtract {
+	case propagationBehaviorExtractContinue, propagationBehaviorExtractRestart, propagationBehaviorExtractIgnore:
+		// valid
+	default:
+		if cp.propagationBehaviorExtract != "" {
+			log.Warn("unrecognized propagation behavior: %s. Defaulting to continue", cp.propagationBehaviorExtract)
+		}
+		cp.propagationBehaviorExtract = propagationBehaviorExtractContinue
+	}
 	if len(propagators) > 0 {
 		cp.injectors = propagators
 		cp.extractors = propagators
@@ -183,11 +209,12 @@ func NewPropagator(cfg *PropagatorConfig, propagators ...Propagator) Propagator
 // When injecting, all injectors are called to propagate the span context.
 // When extracting, it tries each extractor, selecting the first successful one.
 type chainedPropagator struct {
-	injectors        []Propagator
-	extractors       []Propagator
-	injectorNames    string
-	extractorsNames  string
-	onlyExtractFirst bool // value of DD_TRACE_PROPAGATION_EXTRACT_FIRST
+	injectors                  []Propagator
+	extractors                 []Propagator
+	injectorNames              string
+	extractorsNames            string
+	onlyExtractFirst           bool   // value of DD_TRACE_PROPAGATION_EXTRACT_FIRST
+	propagationBehaviorExtract string // value of DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT
 }
 
 // getPropagators returns a list of propagators based on ps, which is a comma seperated
@@ -277,12 +304,86 @@ func (p *chainedPropagator) Inject(spanCtx *SpanContext, carrier any) error {
 // subsequent trace context has conflicting trace information, such information will
 // be relayed in the returned SpanContext with a SpanLink.
 func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) {
+	if p.propagationBehaviorExtract == propagationBehaviorExtractIgnore {
+		return nil, nil
+	}
+
+	incomingCtx, err := p.extractIncomingSpanContext(carrier)
+	if err != nil {
+		return nil, err
+	}
+
+	// "restart" propagation behavior starts a new trace with a new trace ID
+	// and sampling decision. The incoming context is referenced via a span
+	// link. Baggage is propagated.
+	if p.propagationBehaviorExtract == propagationBehaviorExtractRestart {
+		ctx := &SpanContext{
+			baggageOnly: true, // signals spanStart to generate new traceID/spanID
+		}
+
+		link := SpanLink{
+			TraceID:     incomingCtx.TraceIDLower(),
+			TraceIDHigh: incomingCtx.TraceIDUpper(),
+			SpanID:      incomingCtx.SpanID(),
+			Attributes: map[string]string{
+				"reason":          "propagation_behavior_extract",
+				"context_headers": getPropagatorName(p.extractors[0]),
+			},
+		}
+		if trace := incomingCtx.trace; trace != nil {
+			if prio := trace.priority.Load(); prio != nil && uint32(*prio) > 0 { // +checklocksignore - Initialization time, freshly extracted trace not yet shared.
+				link.Flags = 1
+			} else {
+				link.Flags = 0
+			}
+			link.Tracestate = trace.propagatingTag(tracestateHeader)
+		}
+		ctx.spanLinks = []SpanLink{link}
+
+		// When onlyExtractFirst is set, extractIncomingSpanContext returns after the
+		// first successful non-baggage extractor, so incomingCtx carries no baggage.
+		// Extract baggage explicitly so it is propagated regardless.
+		baggage := incomingCtx.baggage // +checklocksignore
+		if p.onlyExtractFirst {
+			baggage = p.extractBaggage(carrier)
+		}
+		if len(baggage) > 0 {
+			ctx.baggage = maps.Clone(baggage) // +checklocksignore
+			atomic.StoreUint32(&ctx.hasBaggage, 1)
+		}
+
+		return ctx, nil
+	}
+
+	// "continue" continues the trace from the incoming context. Baggage is
+	// propagated.
+	return incomingCtx, nil
+}
+
+// extractBaggage runs only the baggage propagator against the carrier and
+// returns the extracted items. Used when onlyExtractFirst has prevented the
+// baggage propagator from running inside extractIncomingSpanContext.
+func (p *chainedPropagator) extractBaggage(carrier any) map[string]string {
+	for _, v := range p.extractors {
+		if _, isBaggage := v.(*propagatorBaggage); !isBaggage {
+			continue
+		}
+		if baggageCtx, err := v.Extract(carrier); err == nil && baggageCtx != nil {
+			return baggageCtx.baggage // +checklocksignore - Initialization time, freshly extracted ctx not yet shared.
+		}
+		break // there is only one baggage propagator
+	}
+	return nil
+}
+
+func (p *chainedPropagator) extractIncomingSpanContext(carrier any) (*SpanContext, error) {
 	var ctx *SpanContext
 	var links []SpanLink
 	pendingBaggage := make(map[string]string) // used to store baggage items temporarily
 
 	for _, v := range p.extractors {
-		firstExtract := (ctx == nil) // ctx stores the most recently extracted ctx across iterations; if it's nil, no extractor has run yet
+		// If incomingCtx is nil, no extraction has run yet
+		firstExtraction := (ctx == nil)
 		extractedCtx, err := v.Extract(carrier)
 
 		// If this is the baggage propagator, just stash its items into pendingBaggage
@@ -293,7 +394,7 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) {
 			continue
 		}
 
-		if firstExtract {
+		if firstExtraction {
 			if err != nil {
 				if p.onlyExtractFirst { // Every error is relevant when we are relying on the first extractor
 					return nil, err
@@ -306,35 +407,34 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) {
 				return extractedCtx, nil
 			}
 			ctx = extractedCtx
-		} else { // A local trace context has already been extracted
-			extractedCtx2 := extractedCtx
-			ctx2 := ctx
-
-			// If we can't cast to spanContext, we can't propgate tracestate or create span links
-			if extractedCtx2.TraceID() == ctx2.TraceID() {
+		} else { // A trace context was already extracted by a previous propagator
+			// When trace IDs match, merge W3C tracestate and resolve parent ID conflicts.
+			// When trace IDs differ, create span links to preserve the terminated context.
+			if extractedCtx.TraceID() == ctx.TraceID() {
 				if pW3C, ok := v.(*propagatorW3c); ok {
-					pW3C.propagateTracestate(ctx2, extractedCtx2)
-					// If trace IDs match but span IDs do not, use spanID from `*propagatorW3c` extractedCtx for parenting
-					if extractedCtx2.SpanID() != ctx2.SpanID() {
+					pW3C.propagateTracestate(ctx, extractedCtx)
+					// W3C and Datadog headers may specify different parent span IDs.
+					// Prefer W3C's span ID for parenting, and record the Datadog span ID as reparentID.
+					if extractedCtx.SpanID() != ctx.SpanID() {
 						var ddCtx *SpanContext
-						// Grab the datadog-propagated spancontext again
 						if ddp := getDatadogPropagator(p); ddp != nil {
 							if ddSpanCtx, err := ddp.Extract(carrier); err == nil {
 								ddCtx = ddSpanCtx
 							}
 						}
-						overrideDatadogParentID(ctx2, extractedCtx2, ddCtx)
+						overrideDatadogParentID(ctx, extractedCtx, ddCtx)
 					}
 				}
-			} else if extractedCtx2 != nil { // Trace IDs do not match - create span links
-				link := SpanLink{TraceID: extractedCtx2.TraceIDLower(), SpanID: extractedCtx2.SpanID(), TraceIDHigh: extractedCtx2.TraceIDUpper(), Attributes: map[string]string{"reason": "terminated_context", "context_headers": getPropagatorName(v)}}
-				if trace := extractedCtx2.trace; trace != nil {
+			} else if extractedCtx != nil { // Trace IDs do not match - create span links
+				link := SpanLink{TraceID: extractedCtx.TraceIDLower(), SpanID: extractedCtx.SpanID(), TraceIDHigh: extractedCtx.TraceIDUpper(), Attributes: map[string]string{"reason": "terminated_context", "context_headers": getPropagatorName(v)}}
+				if trace := extractedCtx.trace; trace != nil {
 					if p := trace.priority.Load(); p != nil && uint32(*p) > 0 { // +checklocksignore - Initialization time, freshly extracted trace not yet shared.
+
 						link.Flags = 1
 					} else {
 						link.Flags = 0
 					}
-					link.Tracestate = extractedCtx2.trace.propagatingTag(tracestateHeader)
+					link.Tracestate = extractedCtx.trace.propagatingTag(tracestateHeader)
 				}
 				links = append(links, link)
 			}
@@ -587,9 +687,18 @@ func getDatadogPropagator(cp *chainedPropagator) *propagator {
 	return nil
 }
 
-// overrideDatadogParentID overrides the span ID of a context with the ID extracted from tracecontext headers.
-// If the reparenting ID is not set on the context, the span ID from datadog headers is used.
-// spanContexts are passed by reference to avoid copying lock value in spanContext type
+// overrideDatadogParentID overrides a context's:
+// 1. span ID with the span ID extracted from W3C tracecontext headers; and
+// 2. reparent ID with either:
+//   - the reparent ID from W3C tracecontext headers (if set), or
+//   - the span ID from Datadog headers (as fallback).
+//
+// reparent ID is the last known Datadog parent span ID, used by Datadog's
+// backend to fix broken parent-child relationships when non-Datadog tracers
+// in the path don't report spans to Datadog.
+//
+// SpanContexts are passed by reference to avoid copying lock information in
+// the SpanContext type.
 func overrideDatadogParentID(ctx, w3cCtx, ddCtx *SpanContext) {
 	if ctx == nil || w3cCtx == nil || ddCtx == nil {
 		return