From 1679462aa9b839d12db7e7338f6fe216c723c360 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 9 Apr 2026 11:37:43 +0200 Subject: [PATCH 01/40] refactor(tracer): improve variable names and comments in Extract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename firstExtract → firstExtraction - Remove extractedCtx2/ctx2 aliases, use extractedCtx/ctx directly - Improve comments explaining W3C/Datadog parent ID resolution - Improve overrideDatadogParentID docs explaining reparentID purpose --- ddtrace/tracer/textmap.go | 48 +++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index 8f965abe3b0..ab10727a1c1 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -282,7 +282,8 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { pendingBaggage := make(map[string]string) // used to store baggage items temporarily for _, v := range p.extractors { - firstExtract := (ctx == nil) // ctx stores the most recently extracted ctx across iterations; if it's nil, no extractor has run yet + // If ctx is nil, no extraction has run yet + firstExtraction := (ctx == nil) extractedCtx, err := v.Extract(carrier) // If this is the baggage propagator, just stash its items into pendingBaggage @@ -293,7 +294,7 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { continue } - if firstExtract { + if firstExtraction { if err != nil { if p.onlyExtractFirst { // Every error is relevant when we are relying on the first extractor return nil, err @@ -306,35 +307,33 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { return extractedCtx, nil } ctx = extractedCtx - } else { // A local trace context has already been extracted - extractedCtx2 := extractedCtx - ctx2 := ctx - - // If we can't cast to spanContext, we can't propgate tracestate or create span links - if extractedCtx2.TraceID() == ctx2.TraceID() { + } else { // A trace context was already extracted by a previous propagator + // When trace IDs match, merge W3C tracestate and resolve parent ID conflicts. + // When trace IDs differ, create span links to preserve the terminated context. + if extractedCtx.TraceID() == ctx.TraceID() { if pW3C, ok := v.(*propagatorW3c); ok { - pW3C.propagateTracestate(ctx2, extractedCtx2) - // If trace IDs match but span IDs do not, use spanID from `*propagatorW3c` extractedCtx for parenting - if extractedCtx2.SpanID() != ctx2.SpanID() { + pW3C.propagateTracestate(ctx, extractedCtx) + // W3C and Datadog headers may specify different parent span IDs. + // Prefer W3C's span ID for parenting, and record the Datadog span ID as reparentID. + if extractedCtx.SpanID() != ctx.SpanID() { var ddCtx *SpanContext - // Grab the datadog-propagated spancontext again if ddp := getDatadogPropagator(p); ddp != nil { if ddSpanCtx, err := ddp.Extract(carrier); err == nil { ddCtx = ddSpanCtx } } - overrideDatadogParentID(ctx2, extractedCtx2, ddCtx) + overrideDatadogParentID(ctx, extractedCtx, ddCtx) } } - } else if extractedCtx2 != nil { // Trace IDs do not match - create span links - link := SpanLink{TraceID: extractedCtx2.TraceIDLower(), SpanID: extractedCtx2.SpanID(), TraceIDHigh: extractedCtx2.TraceIDUpper(), Attributes: map[string]string{"reason": "terminated_context", "context_headers": getPropagatorName(v)}} - if trace := extractedCtx2.trace; trace != nil { + } else if extractedCtx != nil { // Trace IDs do not match - create span links + link := SpanLink{TraceID: extractedCtx.TraceIDLower(), SpanID: extractedCtx.SpanID(), TraceIDHigh: extractedCtx.TraceIDUpper(), Attributes: map[string]string{"reason": "terminated_context", "context_headers": getPropagatorName(v)}} + if trace := extractedCtx.trace; trace != nil { if p := trace.priority.Load(); p != nil && uint32(*p) > 0 { // +checklocksignore - Initialization time, freshly extracted trace not yet shared. link.Flags = 1 } else { link.Flags = 0 } - link.Tracestate = extractedCtx2.trace.propagatingTag(tracestateHeader) + link.Tracestate = extractedCtx.trace.propagatingTag(tracestateHeader) } links = append(links, link) } @@ -587,9 +586,18 @@ func getDatadogPropagator(cp *chainedPropagator) *propagator { return nil } -// overrideDatadogParentID overrides the span ID of a context with the ID extracted from tracecontext headers. -// If the reparenting ID is not set on the context, the span ID from datadog headers is used. -// spanContexts are passed by reference to avoid copying lock value in spanContext type +// overrideDatadogParentID overrides a context's: +// 1. span ID with the span ID extracted from W3C tracecontext headers; and +// 2. reparent ID with either: +// - the reparent ID from W3C tracecontext headers (if set), or +// - the span ID from Datadog headers (as fallback). +// +// reparent ID is the last known Datadog parent span ID, used by Datadog's +// backend to fix broken parent-child relationships when non-Datadog tracers +// in the path don't report spans to Datadog. +// +// SpanContexts are passed by reference to avoid copying lock information in +// the SpanContext type. func overrideDatadogParentID(ctx, w3cCtx, ddCtx *SpanContext) { if ctx == nil || w3cCtx == nil || ddCtx == nil { return From 721ee34f3b35321b18fed24dad9285f5a2c69db9 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 9 Apr 2026 11:40:47 +0200 Subject: [PATCH 02/40] feat: start naively adding support for DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT --- ddtrace/tracer/textmap.go | 134 +++++++++++++++++++++++++++++++------- 1 file changed, 110 insertions(+), 24 deletions(-) diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index ab10727a1c1..25066606d93 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -69,6 +69,18 @@ func (c TextMapCarrier) ForeachKey(handler func(key, val string) error) error { } const ( + // headerPropagationBehaviorExtract specifies how to handle incoming trace + // context. Allowed values: + // - "continue" (default): Continue the trace from incoming headers. + // Baggage is propagated. + // - "restart": Start a new trace with a new trace ID and sampling + // decision. The incoming context is referenced via a span link. + // Baggage is propagated. + // - "ignore": Start a new trace with a new trace ID and sampling + // decision. No span links are created. Baggage is dropped. + headerPropagationBehaviorExtract = "DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT" + + headerPropagationExtractFirst = "DD_TRACE_PROPAGATION_EXTRACT_FIRST" headerPropagationStyleInject = "DD_TRACE_PROPAGATION_STYLE_INJECT" headerPropagationStyleExtract = "DD_TRACE_PROPAGATION_STYLE_EXTRACT" headerPropagationStyle = "DD_TRACE_PROPAGATION_STYLE" @@ -166,7 +178,11 @@ func NewPropagator(cfg *PropagatorConfig, propagators ...Propagator) Propagator cfg.BaggageHeader = DefaultBaggageHeader } cp := new(chainedPropagator) - cp.onlyExtractFirst = internal.BoolEnv("DD_TRACE_PROPAGATION_EXTRACT_FIRST", false) + cp.onlyExtractFirst = internal.BoolEnv(headerPropagationExtractFirst, false) + cp.propagationBehaviorExtract = env.Get(headerPropagationBehaviorExtract) + if cp.propagationBehaviorExtract == "" { + cp.propagationBehaviorExtract = "continue" + } if len(propagators) > 0 { cp.injectors = propagators cp.extractors = propagators @@ -188,6 +204,7 @@ type chainedPropagator struct { injectorNames string extractorsNames string onlyExtractFirst bool // value of DD_TRACE_PROPAGATION_EXTRACT_FIRST + propagationBehaviorExtract string // value of DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT } // getPropagators returns a list of propagators based on ps, which is a comma seperated @@ -277,13 +294,80 @@ func (p *chainedPropagator) Inject(spanCtx *SpanContext, carrier any) error { // subsequent trace context has conflicting trace information, such information will // be relayed in the returned SpanContext with a SpanLink. func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { + // TODO: Return nil or an empty span context? + // If the propagation behavior is "ignore", return a new span context with no span links and no baggage. + if p.propagationBehaviorExtract == "ignore" { + return nil, nil + } + + incomingCtx, err := p.extractIncomingSpanContext(carrier) + if err != nil { + return nil, err + } + + // Continue the trace from incoming headers. Baggage is propagated. + if p.propagationBehaviorExtract == "continue" { + return incomingCtx, nil + } + + // Start a new trace with a new trace ID and sampling decision. + // The incoming context is referenced via a span link. + // Baggage is propagated. + if p.propagationBehaviorExtract == "restart" { + // TODO: Check if an empty span context will lead the tracer to generate a new trace ID and span ID when starting a new span. + ctx := &SpanContext{} + + link := SpanLink{ + TraceID: incomingCtx.TraceIDLower(), + TraceIDHigh: incomingCtx.TraceIDUpper(), + SpanID: incomingCtx.SpanID(), + Tracestate: func() string { + if incomingCtx.trace != nil { + return incomingCtx.trace.propagatingTag(tracestateHeader) + } + return "" + }(), + // TODO: What about the "new sampling decision"? If we prop the flag, isn't the sampling decision already set? + Flags: func() uint32 { + if incomingCtx.trace != nil { + if p, _ := incomingCtx.SamplingPriority(); p > 0 { + return 1 + } + } + return 0 + }(), + Attributes: map[string]string{ + "reason": "propagation_behavior_extract", + "context_headers": getPropagatorName(p.extractors[0]), + }, + } + + // NOTE: Span links from the incoming trace context do not need to be carried over. + ctx.spanLinks = []SpanLink{link} + + // NOTE: All baggage items from the incoming trace context. + // TODO: Copy or reference? Using copy to avoid shared state issues. + if incomingCtx.baggage != nil { + ctx.baggage = make(map[string]string, len(incomingCtx.baggage)) + maps.Copy(ctx.baggage, incomingCtx.baggage) + atomic.StoreUint32(&ctx.hasBaggage, 1) + } + + return ctx, nil + } + + return incomingCtx, nil +} + +func (p *chainedPropagator) extractIncomingSpanContext(carrier any) (*SpanContext, error) { var ctx *SpanContext var links []SpanLink pendingBaggage := make(map[string]string) // used to store baggage items temporarily + for _, v := range p.extractors { - // If ctx is nil, no extraction has run yet - firstExtraction := (ctx == nil) + // If incomingCtx is nil, no extraction has run yet + firstExtraction := (ctx == nil) extractedCtx, err := v.Extract(carrier) // If this is the baggage propagator, just stash its items into pendingBaggage @@ -328,7 +412,7 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { } else if extractedCtx != nil { // Trace IDs do not match - create span links link := SpanLink{TraceID: extractedCtx.TraceIDLower(), SpanID: extractedCtx.SpanID(), TraceIDHigh: extractedCtx.TraceIDUpper(), Attributes: map[string]string{"reason": "terminated_context", "context_headers": getPropagatorName(v)}} if trace := extractedCtx.trace; trace != nil { - if p := trace.priority.Load(); p != nil && uint32(*p) > 0 { // +checklocksignore - Initialization time, freshly extracted trace not yet shared. + if flags := uint32(*trace.priority); flags > 0 { // +checklocksignore - Initialization time, freshly extracted trace not yet shared. link.Flags = 1 } else { link.Flags = 0 @@ -368,23 +452,6 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { return ctx, nil } -func getPropagatorName(p Propagator) string { - switch p.(type) { - case *propagator: - return "datadog" - case *propagatorB3: - return "b3multi" - case *propagatorB3SingleHeader: - return "b3" - case *propagatorW3c: - return "tracecontext" - case *propagatorBaggage: - return "baggage" - default: - return "" - } -} - // propagateTracestate will add the tracestate propagating tag to the given // *spanContext. The W3C trace context will be extracted from the provided // carrier. The trace id of this W3C trace context must match the trace id @@ -414,6 +481,23 @@ func (p *propagatorW3c) propagateTracestate(ctx *SpanContext, w3cCtx *SpanContex ctx.isRemote = (w3cCtx.isRemote) } +func getPropagatorName(p Propagator) string { + switch p.(type) { + case *propagator: + return "datadog" + case *propagatorB3: + return "b3multi" + case *propagatorB3SingleHeader: + return "b3" + case *propagatorW3c: + return "tracecontext" + case *propagatorBaggage: + return "baggage" + default: + return "" + } +} + // propagator implements Propagator and injects/extracts span contexts // using datadog headers. Only TextMap carriers are supported. type propagator struct { @@ -586,17 +670,19 @@ func getDatadogPropagator(cp *chainedPropagator) *propagator { return nil } +// TODO: Verify if we should rename this function - e.g., "reconcile context span ID and reparent ID". maybe split this into two functions. one for reconciling the span ID and another for the reparent ID. + // overrideDatadogParentID overrides a context's: // 1. span ID with the span ID extracted from W3C tracecontext headers; and // 2. reparent ID with either: -// - the reparent ID from W3C tracecontext headers (if set), or -// - the span ID from Datadog headers (as fallback). +// - the reparent ID from W3C tracecontext headers (if set), or +// - the span ID from Datadog headers (as fallback). // // reparent ID is the last known Datadog parent span ID, used by Datadog's // backend to fix broken parent-child relationships when non-Datadog tracers // in the path don't report spans to Datadog. // -// SpanContexts are passed by reference to avoid copying lock information in +// SpanContexts are passed by reference to avoid copying lock information in // the SpanContext type. func overrideDatadogParentID(ctx, w3cCtx, ddCtx *SpanContext) { if ctx == nil || w3cCtx == nil || ddCtx == nil { From ec9d1fdd0d712252c7579b2d42c05142db362222 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 9 Apr 2026 11:47:23 +0200 Subject: [PATCH 03/40] feat: check allowed values --- ddtrace/tracer/textmap.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index 25066606d93..1668accba2a 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -180,9 +180,15 @@ func NewPropagator(cfg *PropagatorConfig, propagators ...Propagator) Propagator cp := new(chainedPropagator) cp.onlyExtractFirst = internal.BoolEnv(headerPropagationExtractFirst, false) cp.propagationBehaviorExtract = env.Get(headerPropagationBehaviorExtract) + + if cp.propagationBehaviorExtract != "continue" && cp.propagationBehaviorExtract != "restart" && cp.propagationBehaviorExtract != "ignore" { + log.Warn("unrecognized propagation behavior: %s\n. Defaulting to continue", cp.propagationBehaviorExtract) + cp.propagationBehaviorExtract = "continue" + } if cp.propagationBehaviorExtract == "" { cp.propagationBehaviorExtract = "continue" } + if len(propagators) > 0 { cp.injectors = propagators cp.extractors = propagators From 7166cb465abe308592d67674f0397a2cf2b3c34d Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 9 Apr 2026 11:49:09 +0200 Subject: [PATCH 04/40] feat: make it elegant --- ddtrace/tracer/textmap.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index 1668accba2a..cbc98bce280 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -180,15 +180,15 @@ func NewPropagator(cfg *PropagatorConfig, propagators ...Propagator) Propagator cp := new(chainedPropagator) cp.onlyExtractFirst = internal.BoolEnv(headerPropagationExtractFirst, false) cp.propagationBehaviorExtract = env.Get(headerPropagationBehaviorExtract) - - if cp.propagationBehaviorExtract != "continue" && cp.propagationBehaviorExtract != "restart" && cp.propagationBehaviorExtract != "ignore" { - log.Warn("unrecognized propagation behavior: %s\n. Defaulting to continue", cp.propagationBehaviorExtract) - cp.propagationBehaviorExtract = "continue" - } - if cp.propagationBehaviorExtract == "" { + switch cp.propagationBehaviorExtract { + case "continue", "restart", "ignore": + // valid + default: + if cp.propagationBehaviorExtract != "" { + log.Warn("unrecognized propagation behavior: %s. Defaulting to continue", cp.propagationBehaviorExtract) + } cp.propagationBehaviorExtract = "continue" } - if len(propagators) > 0 { cp.injectors = propagators cp.extractors = propagators From a31dfe120cffbf3f03c4451ea128ab9c98af111f Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 9 Apr 2026 11:51:37 +0200 Subject: [PATCH 05/40] refactor: start removing inline func() from Flags assignment --- ddtrace/tracer/textmap.go | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index cbc98bce280..1af4c15490a 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -327,26 +327,27 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { TraceID: incomingCtx.TraceIDLower(), TraceIDHigh: incomingCtx.TraceIDUpper(), SpanID: incomingCtx.SpanID(), - Tracestate: func() string { - if incomingCtx.trace != nil { - return incomingCtx.trace.propagatingTag(tracestateHeader) - } - return "" - }(), - // TODO: What about the "new sampling decision"? If we prop the flag, isn't the sampling decision already set? - Flags: func() uint32 { - if incomingCtx.trace != nil { - if p, _ := incomingCtx.SamplingPriority(); p > 0 { - return 1 - } - } - return 0 - }(), Attributes: map[string]string{ "reason": "propagation_behavior_extract", "context_headers": getPropagatorName(p.extractors[0]), }, } + // Tracestate: func() string { + // if incomingCtx.trace != nil { + // return incomingCtx.trace.propagatingTag(tracestateHeader) + // } + // return "" + // }(), + // // TODO: What about the "new sampling decision"? If we prop the flag, isn't the sampling decision already set? + // Flags: func() uint32 { + // if incomingCtx.trace != nil { + // if p, _ := incomingCtx.SamplingPriority(); p > 0 { + // return 1 + // } + // } + // return 0 + // }(), + } // NOTE: Span links from the incoming trace context do not need to be carried over. ctx.spanLinks = []SpanLink{link} From e5d5cf44442236c0e85c55544c9846bbd22ca8d5 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 9 Apr 2026 11:56:20 +0200 Subject: [PATCH 06/40] refactor: finish removing inline func() from Flags assignment --- ddtrace/tracer/textmap.go | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index 1af4c15490a..7f1b877677f 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -332,22 +332,15 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { "context_headers": getPropagatorName(p.extractors[0]), }, } - // Tracestate: func() string { - // if incomingCtx.trace != nil { - // return incomingCtx.trace.propagatingTag(tracestateHeader) - // } - // return "" - // }(), - // // TODO: What about the "new sampling decision"? If we prop the flag, isn't the sampling decision already set? - // Flags: func() uint32 { - // if incomingCtx.trace != nil { - // if p, _ := incomingCtx.SamplingPriority(); p > 0 { - // return 1 - // } - // } - // return 0 - // }(), - } + // TODO: +checklocksignore is needed here? + if trace := incomingCtx.trace; trace != nil { + if flags := uint32(*trace.priority); flags > 0 { // +checklocksignore - Initialization time, freshly extracted trace not yet shared. + link.Flags = 1 + } else { + link.Flags = 0 + } + link.Tracestate = trace.propagatingTag(tracestateHeader) + } // NOTE: Span links from the incoming trace context do not need to be carried over. ctx.spanLinks = []SpanLink{link} From d502c9790c2ea708d62bc8e4f920a57ff0fd1f14 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 9 Apr 2026 12:03:50 +0200 Subject: [PATCH 07/40] docs: improve explanations on Extract() about propagation modes, keep happy path with no indents --- ddtrace/tracer/textmap.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index 7f1b877677f..d2aafb53c09 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -301,7 +301,8 @@ func (p *chainedPropagator) Inject(spanCtx *SpanContext, carrier any) error { // be relayed in the returned SpanContext with a SpanLink. func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { // TODO: Return nil or an empty span context? - // If the propagation behavior is "ignore", return a new span context with no span links and no baggage. + // "ignore" propagation behavior returns a new span context with no span + // links and no baggage. if p.propagationBehaviorExtract == "ignore" { return nil, nil } @@ -311,14 +312,9 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { return nil, err } - // Continue the trace from incoming headers. Baggage is propagated. - if p.propagationBehaviorExtract == "continue" { - return incomingCtx, nil - } - - // Start a new trace with a new trace ID and sampling decision. - // The incoming context is referenced via a span link. - // Baggage is propagated. + // "restart" propagation behavior starts a new trace with a new trace ID + // and sampling decision. The incoming context is referenced via a span + // link. Baggage is propagated. if p.propagationBehaviorExtract == "restart" { // TODO: Check if an empty span context will lead the tracer to generate a new trace ID and span ID when starting a new span. ctx := &SpanContext{} @@ -356,6 +352,8 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { return ctx, nil } + // "continue" continues the trace from the incoming context. Baggage is + // propagated. return incomingCtx, nil } From e74e7650cde8214de59d6f09e30d178e950c91b5 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 9 Apr 2026 12:05:00 +0200 Subject: [PATCH 08/40] chore: remove notes we don't need anymore --- ddtrace/tracer/textmap.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index d2aafb53c09..be9541cc6a3 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -337,12 +337,8 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { } link.Tracestate = trace.propagatingTag(tracestateHeader) } - - // NOTE: Span links from the incoming trace context do not need to be carried over. ctx.spanLinks = []SpanLink{link} - // NOTE: All baggage items from the incoming trace context. - // TODO: Copy or reference? Using copy to avoid shared state issues. if incomingCtx.baggage != nil { ctx.baggage = make(map[string]string, len(incomingCtx.baggage)) maps.Copy(ctx.baggage, incomingCtx.baggage) From 6344be5b70c13f2192737382c449db8b4fb1f207 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 9 Apr 2026 14:54:10 +0200 Subject: [PATCH 09/40] fix: trace priority bug --- ddtrace/tracer/textmap.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index be9541cc6a3..4c247a97728 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -330,7 +330,7 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { } // TODO: +checklocksignore is needed here? if trace := incomingCtx.trace; trace != nil { - if flags := uint32(*trace.priority); flags > 0 { // +checklocksignore - Initialization time, freshly extracted trace not yet shared. + if p := trace.priority.Load(); p != nil && uint32(*p) > 0 { // +checklocksignore - Initialization time, freshly extracted trace not yet shared. link.Flags = 1 } else { link.Flags = 0 @@ -406,7 +406,8 @@ func (p *chainedPropagator) extractIncomingSpanContext(carrier any) (*SpanContex } else if extractedCtx != nil { // Trace IDs do not match - create span links link := SpanLink{TraceID: extractedCtx.TraceIDLower(), SpanID: extractedCtx.SpanID(), TraceIDHigh: extractedCtx.TraceIDUpper(), Attributes: map[string]string{"reason": "terminated_context", "context_headers": getPropagatorName(v)}} if trace := extractedCtx.trace; trace != nil { - if flags := uint32(*trace.priority); flags > 0 { // +checklocksignore - Initialization time, freshly extracted trace not yet shared. + if p := trace.priority.Load(); p != nil && uint32(*p) > 0 { // +checklocksignore - Initialization time, freshly extracted trace not yet shared. + link.Flags = 1 } else { link.Flags = 0 From dad691e4de10d8c13988a7bd6d267d4ecf9e8c11 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Wed, 15 Apr 2026 08:05:52 +0200 Subject: [PATCH 10/40] feat: set the restart SpanContext's baggageOnly to true to signal spanStart to generate a new traceID and span ID --- ddtrace/tracer/textmap.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index 4c247a97728..0a10d85258b 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -316,8 +316,9 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { // and sampling decision. The incoming context is referenced via a span // link. Baggage is propagated. if p.propagationBehaviorExtract == "restart" { - // TODO: Check if an empty span context will lead the tracer to generate a new trace ID and span ID when starting a new span. - ctx := &SpanContext{} + ctx := &SpanContext{ + baggageOnly: true, // signals spanStart to generate new traceID/spanID + } link := SpanLink{ TraceID: incomingCtx.TraceIDLower(), From 502e5fcab621141bde1880ae081f8aa75bd5e702 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Wed, 15 Apr 2026 08:06:12 +0200 Subject: [PATCH 11/40] chore: regen supported configurations --- internal/env/supported_configurations.gen.go | 5 +++ internal/env/supported_configurations.json | 37 +++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/internal/env/supported_configurations.gen.go b/internal/env/supported_configurations.gen.go index 4cd772232c5..88042a538ba 100644 --- a/internal/env/supported_configurations.gen.go +++ b/internal/env/supported_configurations.gen.go @@ -139,10 +139,14 @@ var SupportedConfigurations = map[string]struct{}{ "DD_TELEMETRY_METRICS_ENABLED": {}, "DD_TEST_AGENT_HOST": {}, "DD_TEST_AGENT_PORT": {}, + "DD_TEST_BOOL_ENV": {}, + "DD_TEST_FLOAT_ENV": {}, + "DD_TEST_INT_ENV": {}, "DD_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES": {}, "DD_TEST_MANAGEMENT_ENABLED": {}, "DD_TEST_OPTIMIZATION_ENV_DATA_FILE": {}, "DD_TEST_SESSION_NAME": {}, + "DD_TEST_STRING_ENV": {}, "DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED": {}, "DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED": {}, "DD_TRACE_ABANDONED_SPAN_TIMEOUT": {}, @@ -209,6 +213,7 @@ var SupportedConfigurations = map[string]struct{}{ "DD_TRACE_PARTIAL_FLUSH_MIN_SPANS": {}, "DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED": {}, "DD_TRACE_PEER_SERVICE_MAPPING": {}, + "DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT": {}, "DD_TRACE_PROPAGATION_EXTRACT_FIRST": {}, "DD_TRACE_PROPAGATION_STYLE": {}, "DD_TRACE_PROPAGATION_STYLE_EXTRACT": {}, diff --git a/internal/env/supported_configurations.json b/internal/env/supported_configurations.json index ae47ba556f5..6d60b2d0c5f 100644 --- a/internal/env/supported_configurations.json +++ b/internal/env/supported_configurations.json @@ -917,6 +917,27 @@ "default": null } ], + "DD_TEST_BOOL_ENV": [ + { + "implementation": "A", + "type": "FIX_ME", + "default": "FIX_ME" + } + ], + "DD_TEST_FLOAT_ENV": [ + { + "implementation": "A", + "type": "FIX_ME", + "default": "FIX_ME" + } + ], + "DD_TEST_INT_ENV": [ + { + "implementation": "A", + "type": "FIX_ME", + "default": "FIX_ME" + } + ], "DD_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES": [ { "implementation": "B", @@ -945,6 +966,13 @@ "default": null } ], + "DD_TEST_STRING_ENV": [ + { + "implementation": "A", + "type": "FIX_ME", + "default": "FIX_ME" + } + ], "DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED": [ { "implementation": "A", @@ -1407,6 +1435,13 @@ "default": "" } ], + "DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT": [ + { + "implementation": "A", + "type": "FIX_ME", + "default": "FIX_ME" + } + ], "DD_TRACE_PROPAGATION_EXTRACT_FIRST": [ { "implementation": "A", @@ -1842,4 +1877,4 @@ } ] } -} +} \ No newline at end of file From c1876e66c3c67ada6d84214d4dac9b5a0b93a84d Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Wed, 15 Apr 2026 08:07:33 +0200 Subject: [PATCH 12/40] test: add test drafts --- contrib/labstack/echo.v4/echotrace_test.go | 108 +++++++++++++++++++++ ddtrace/tracer/textmap_test.go | 1 + 2 files changed, 109 insertions(+) diff --git a/contrib/labstack/echo.v4/echotrace_test.go b/contrib/labstack/echo.v4/echotrace_test.go index 89f7be0e907..24870faf683 100644 --- a/contrib/labstack/echo.v4/echotrace_test.go +++ b/contrib/labstack/echo.v4/echotrace_test.go @@ -720,6 +720,114 @@ func TestWithErrorCheck(t *testing.T) { } } +func TestPropagationBehaviorExtract(t *testing.T) { + tests := []struct { + name string + propagationBehaviorExtract string + // TODO(human): Add expected behavior fields here + }{ + // { + // name: "continue-default", + // propagationBehaviorExtract: "continue", + // }, + { + name: "restart", + propagationBehaviorExtract: "restart", + }, + // { + // name: "ignore", + // propagationBehaviorExtract: "ignore", + // }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT", tc.propagationBehaviorExtract) + + mt := mocktracer.Start() + defer mt.Stop() + + router := echo.New() + router.Use(Middleware(WithService("test-service"))) + router.GET("/test", func(c echo.Context) error { + return c.NoContent(200) + }) + + // Create a "root" span simulating incoming trace context + root := tracer.StartSpan("incoming-request") + root.SetBaggageItem("test-baggage", "baggage-value") + + r := httptest.NewRequest("GET", "/test", nil) + err := tracer.Inject(root.Context(), tracer.HTTPHeadersCarrier(r.Header)) + require.NoError(t, err) + + fmt.Println("r") + fmt.Println(r.Header) + + w := httptest.NewRecorder() + router.ServeHTTP(w, r) + + spans := mt.FinishedSpans() + + fmt.Println("root") + fmt.Println(root) + + fmt.Println("spans") + fmt.Println(spans) + + + + // // verify traces look good + // assert.True(called) + // assert.True(traced) + + // spans := mt.FinishedSpans() + // assert.Len(spans, 1) + + // span := spans[0] + // assert.Equal("http.request", span.OperationName()) + // assert.Equal(ext.SpanTypeWeb, span.Tag(ext.SpanType)) + // assert.Equal("foobar", span.Tag(ext.ServiceName)) + // assert.Equal("echony", span.Tag("test.echo")) + // assert.Contains(span.Tag(ext.ResourceName), "/user/:id") + // assert.Equal("200", span.Tag(ext.HTTPCode)) + // assert.Equal("GET", span.Tag(ext.HTTPMethod)) + // assert.Equal(root.Context().SpanID(), span.ParentID()) + // assert.Equal("labstack/echo.v4", span.Tag(ext.Component)) + // assert.Equal(string(instrumentation.PackageLabstackEchoV4), span.Integration()) + // assert.Equal(ext.SpanKindServer, span.Tag(ext.SpanKind)) + + // assert.Equal("http://example.com/user/123", span.Tag(ext.HTTPURL)) + + + // TODO(human): Implement assertions for each propagation behavior. + // + // For "continue": + // - The server span should have the same trace ID as root + // - root.Context().SpanID() should equal span.ParentID() + // - No span links expected (unless conflicting trace contexts) + // + // For "restart": + // - The server span should have a NEW trace ID (different from root) + // - span.ParentID() should be 0 (no parent) + // - SpanLinks should contain one link to the incoming context + // - Baggage should still be propagated + // + // For "ignore": + // - The server span should have a NEW trace ID + // - No span links + // - Baggage should NOT be propagated + // + // Hints: + // - Use span.TraceID() and root.Context().TraceID() to compare trace IDs + // - Use mocktracer.MockSpan(span).SpanLinks() to check span links + // - Check baggage via the request context inside the handler + _ = spans + _ = root + }) + } +} + func BenchmarkEchoWithTracing(b *testing.B) { tracer.Start(tracer.WithLogger(testutils.DiscardLogger())) defer tracer.Stop() diff --git a/ddtrace/tracer/textmap_test.go b/ddtrace/tracer/textmap_test.go index 5aa49e453cf..407675076bb 100644 --- a/ddtrace/tracer/textmap_test.go +++ b/ddtrace/tracer/textmap_test.go @@ -105,6 +105,7 @@ func TestTextMapCarrierForeachKeyError(t *testing.T) { assert.Equal(t, got, want) } +// TODO: Split test into 3, one for continue, one for restart, one for ignore func TestTextMapExtractTracestatePropagation(t *testing.T) { tests := []struct { name, propagationStyle, traceparent string From 69414fcb892edc595cc73416e26a8725f4e1562f Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Wed, 15 Apr 2026 08:11:18 +0200 Subject: [PATCH 13/40] chore: (PLEASE REVERT) add _context --- ...ce-context-extraction-propagation-modes.md | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 _context/rfc-trace-context-extraction-propagation-modes.md diff --git a/_context/rfc-trace-context-extraction-propagation-modes.md b/_context/rfc-trace-context-extraction-propagation-modes.md new file mode 100644 index 00000000000..1c81fc01a38 --- /dev/null +++ b/_context/rfc-trace-context-extraction-propagation-modes.md @@ -0,0 +1,161 @@ +# RFC: Trace Context Propagation Extraction Modes + +Author: [Zach Montoya](mailto:zach.montoya@datadoghq.com) +Date: Dec 4, 2024 +Status: Approved +Previous Proposal(s): [Proposal: Fixing Trace Context Propagation Across Organizations](https://docs.google.com/document/d/1htJFBWR4RFpmK1i6wopxkZwlvYu9unkPJnMnTom4Xjo/edit?usp=sharing) +Problem Statement: [Escalation Review: Handling cases where the root service is an unrelated/unknown upstream service](https://docs.google.com/document/d/1zDQDQIRkW8OhTGZ1ua8IsGG_mf5MLdMoUYiAmn9G0pE/edit?tab=t.0) +Feature Parity Dashboard: [https://feature-parity.us1.prod.dog/\#/feature-health?viewType=tests\&features=353](https://feature-parity.us1.prod.dog/#/feature-health?viewType=tests&features=353) + +| Reviewer | Status | Notes | +| :---- | :---- | :---- | +| [Zach Groves](mailto:zach.groves@datadoghq.com) | Approved | | +| [Lucas Pimentel](mailto:lucas.pimentel@datadoghq.com) | Approved | | +| [Mikayla Toffler](mailto:mikayla.toffler@datadoghq.com) | Approved | | +| [Matthew Li](mailto:matthew.li@datadoghq.com) | Approved | | + +# Overview + +## Motivation + +As a result of increased adoption of W3C Trace Context (and possibly just increased adoption of Datadog APM), the APM Ecosystems engineering teams have observed an increasing number of customer escalations where context is propagated from the service of one organization into the service of another organization, leading to a degraded APM experience for the second organization. More context on these issues can be found here: [Escalation Review: Handling cases where the root service is an unrelated/unknown upstream service](https://docs.google.com/document/d/1zDQDQIRkW8OhTGZ1ua8IsGG_mf5MLdMoUYiAmn9G0pE/edit?tab=t.0#heading=h.fna027uqmyrt). + +We should provide our customers with tools to resolve this situation, specifically on the context extraction side as the changes can immediately remedy the situation for the downstream customer. We currently have one workaround which customers have successfully used, which is to configure DD\_TRACE\_PROPAGATION\_STYLE\_EXTRACT=none so that no incoming distributed tracing headers are checked for context extraction, but this solution is not extensible and conflates propagator formats with the context extraction logic. To provide an improved and more extensible user experience, we should add a new capability to the tracing library that separates the handling of incoming trace contexts from the selected propagator formats. + +Note: This work can also be extended to limit context injection from the upstream service, however its effects are not visible to the user so it seems prudent to first focus on the context extraction side. + +## Requirements + +1. By only updating the Datadog configuration of an instrumented service, a user can ignore any incoming trace context information + +# Out of scope + +* This RFC specifically doesn’t focus on an end-to-end UI workflow, and instead focuses on implementing the required fundamental behaviors in our tracing libraries that would enable such changes. +* Benchmarking is also out of scope, as the processing overhead is minimal in comparison to the rest of the context extraction operation. + +# Proposed Solution + +## Configuration + +Environment Variable: DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT +Accepted Values: + +* continue: The tracing library continues the trace from the incoming headers, if present. Also, incoming baggage is propagated. +* restart: The tracing library always starts a new trace with a new trace-id (and a new sampling decision). Context extraction will occur as otherwise configured, except that the local span will no longer have a parent-child span relationship with the incoming trace context, instead it will reference the incoming trace context via a span link. Also, incoming baggage is propagated. +* ignore: The tracing library always starts a new trace with a new trace-id (and a new sampling decision) *without creating any span links.* Also, incoming baggage is dropped. + +Default value: continue +Telemetry Key: DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT + +## Behavior + +This behavior is intricately linked to the [existing context extraction behaviors](https://docs.google.com/document/d/1xacBonCyuVk95D-L1STXGqdLtsOuf0_3jG-138uXHnQ/edit?tab=t.0#bookmark=id.e5pi9gdhj04t). This RFC proposes adding the following text to the existing logic: + +1. \[*Modified*\] Iterate through the propagators in precedence order and for each propagator apply the following logic to build the **incoming trace context** from the distributed tracing headers … +2. \[*New*\] Using the **incoming trace context** from the previous step, set the **local trace context** based on the configured value of DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT: + 1. If DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=continue, use the **incoming trace context** as the **local trace context.** + 2. If DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=restart, create a new **local trace context** containing the following information: + 1. The trace-id and span-id is set in such a way that the tracing library will generate a new trace-id and span-id when starting a new span from this trace context. + 2. A span link with the following properties: + 1. The TraceId, SpanId, TraceFlags, and TraceState fields that correspond to the **incoming trace context** + 2. Additionally, set the following Attributes on the span link: + 1. Key=reason, Value=propagation\_behavior\_extract + 2. Key=context\_headers, Value=\ + 1. Implementation detail: If multiple propagators read the same trace-id and were consolidated into one trace context, listing the first propagator is sufficient rather than listing all of the propagators. + 3. All baggage items from the **incoming trace context** + 1. Note: Span links from the incoming trace context do not need to be carried over. + 3. If DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=ignore, discard the entire **incoming trace context** and create a new **local trace context**. The result will be identical to no distributed tracing headers being found. + 1. Note: As a performance improvement, it is acceptable to skip the iteration of propagators when this configuration is detected since the contents of distributed tracing headers will not affect the results. + +# Testing + +Testing for this feature will be asserted through system-tests, and the new tests can be found [here](https://github.com/DataDog/system-tests/pull/3602). In order to pass the test cases, tracing libraries must implement the DD\_TRACE\_PROPAGATION\_EXTRACT\_FIRST configuration and the ability to add span links for **conflicting trace contexts** where the trace-ids do not match the **incoming trace context**. These behaviors are both outlined [here](https://docs.google.com/document/d/1xacBonCyuVk95D-L1STXGqdLtsOuf0_3jG-138uXHnQ/edit?tab=t.0#heading=h.4jm220mfuuo1). For the tests outlined below, we are assuming that the tracing library under test is configured with default extraction propagators: datadog,tracecontext. + +## Test Behavior + +* The weblog /make\_distant\_call endpoint is invoked with valid Datadog trace context headers, W3C Trace Context headers, and W3C Baggage headers. The Datadog tracing library should automatically perform context extraction and generate a span for this HTTP server request. +* The HTTP endpoint then creates an outbound HTTP request. The request and response headers for this request are sent in the response body for the original endpoint call. The Datadog tracing library should automatically perform context injection based on the local context. + +## Test Cases + +There are two test cases for each configuration: + +1. The distributed tracing headers share the same trace-id and span-id. +2. The distributed tracing headers have unique trace-ids and span-ids. + +Each test case will be run against the following tracing configurations. Due to the overhead of creating new test scenarios in the system-tests infrastructure, this omits testing the explicit configuration DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=continue. However, it is the default behavior, so it is still indirectly tested here. + +1. Default (equivalent to DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=continue) +2. DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=restart +3. DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=restart and DD\_TRACE\_PROPAGATION\_EXTRACT\_FIRST=true +4. DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=ignore + +## Test Results + +* Configuration: Default + 1. Test case: Same trace-id + 1. The HTTP server span has the same trace-id as the incoming Datadog trace context (i.e. the trace is continued). + 2. The HTTP server span has zero span links. + 3. Baggage is propagated. + 2. Test case: Unique trace-ids + 1. The HTTP server span has the same trace-id as the incoming Datadog trace context (i.e. the trace is continued). + 2. The HTTP server span has one span link, corresponding to the **conflicting trace context** in the W3C Trace Context headers. + 3. Baggage is propagated. +* Configuration: DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=restart + 1. Test case: Same trace-id + 1. The HTTP server span has a new trace-id (i.e. a new trace is started). + 2. The HTTP server span has one span link, corresponding to the **incoming trace context** extracted from the Datadog and W3C Trace Context. + 3. Baggage is propagated. + 2. Test case: Unique trace-ids + 1. The HTTP server span has a new trace-id (i.e. a new trace is started). + 2. The HTTP server span has one span link, corresponding to the **incoming trace context** extracted from the Datadog headers. + 3. Baggage is propagated. +* Configuration: DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=ignore + 1. Test case: Same trace-id + 1. The HTTP server span has a new trace-id (i.e. a new trace is started). + 2. The HTTP server span has no span links. + 3. Baggage is not propagated. + 2. Test case: Unique trace-ids + 1. The HTTP server span has a new trace-id (i.e. a new trace is started). + 2. The HTTP server span has no span links. + 3. Baggage is not propagated. +* Configuration: DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=restart and DD\_TRACE\_PROPAGATION\_EXTRACT\_FIRST=true + 1. Test case: Same trace-id + 1. The HTTP server span has a new trace-id (i.e. a new trace is started). + 2. The HTTP server span has one span link, corresponding to the **incoming trace context** extracted from the Datadog headers. + 3. Baggage is propagated. + 2. Test case: Unique trace-ids + 1. The HTTP server span has a new trace-id (i.e. a new trace is started). + 2. The HTTP server span has one span link, corresponding to the **incoming trace context** extracted from the Datadog headers. No **conflicting trace contexts** are identified because the extraction stops immediately, due to the configuration. + 3. Baggage is propagated. + +# Alternative Solutions + +All of the ideas explored in the [Longer Term Options](https://docs.google.com/document/d/1zDQDQIRkW8OhTGZ1ua8IsGG_mf5MLdMoUYiAmn9G0pE/edit?tab=t.0#heading=h.zcquizxtg7t9) section of [Escalation Review: Handling cases where the root service is an unrelated/unknown upstream service](https://docs.google.com/document/u/0/d/1zDQDQIRkW8OhTGZ1ua8IsGG_mf5MLdMoUYiAmn9G0pE/edit) fall into the following camps: + +* Sampling + * Option 1: Sampling Overrides + * Option 5: Implement non-”ParentBased” Samplers in line with OpenTelemetry +* Filter the context extraction + * Option 2: Designate a Head Service / Ignore Incoming Information (this one\!) + * Option 6: Implement configuration to filter extraction by host/headers (suggested in [Future Work](#future-work)) + +## Sampling-based solutions + +Sampling-based solutions don’t entirely solve the issue. With sampling, the customer can override the upstream sampling decision so they regain control of their ingestion, but it leaves their spans in an orphaned state, which causes a degraded UI experience. + +# Future Work {#future-work} + +There are several opportunities to expand on this feature: + +1. **Configuring the injection operation:** With this capability, customers could stop the propagation of their own trace context & baggage so that they don’t expose information to untrusted parties. This also prevents the issues being solved here by having upstream services remove unrecognized distributed tracing headers. However, this would also need to be applied to specific services so that users don’t break up their own traces. +2. **Datadog tracing library dynamically configures the propagation behavior per request:** Since the Datadog tracing library can see incoming/outgoing requests and is likely configured at known ingress/egress points, we may be able to configure this automatically to allow extraction (or injection, see above) to be dynamically configured so that we handle this issue automatically for customers, without them having to set configurations. The capability introduced in this RFC opens the door for those intelligent solutions, such as the alternative solution to filter extraction by host/headers). + +# Questions + +Ask away\! + +# References + +* [Escalation Review: Handling cases where the root service is an unrelated/unknown upstream service](https://docs.google.com/document/u/0/d/1zDQDQIRkW8OhTGZ1ua8IsGG_mf5MLdMoUYiAmn9G0pE/edit) +* [Proposal: Fixing Trace Context Propagation Across Organizations](https://docs.google.com/document/u/0/d/1htJFBWR4RFpmK1i6wopxkZwlvYu9unkPJnMnTom4Xjo/edit) From e86bc01d50408cb5113c35eee48ce387fc51f5df Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Wed, 15 Apr 2026 08:12:25 +0200 Subject: [PATCH 14/40] chore: add references to important files --- _context/references.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 _context/references.md diff --git a/_context/references.md b/_context/references.md new file mode 100644 index 00000000000..ab3af042fa1 --- /dev/null +++ b/_context/references.md @@ -0,0 +1 @@ +@/Users/augusto.deoliveira/Documents/obsidian-vaults/datadog-notes/tasks/26Q1/2026-03-18 (in-progress) Trace Context Propagation Extraction Modes.md From b3d13f8d94e31075acc6fcbeec50eebeaf9582a7 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Wed, 15 Apr 2026 10:39:23 +0200 Subject: [PATCH 15/40] feat: report propagation_behavior_extract in telemetry, fix supported_configurations --- _context/notes.md | 88 ++++++++++++++++++++++ ddtrace/tracer/telemetry.go | 2 + internal/env/supported_configurations.json | 34 +-------- 3 files changed, 93 insertions(+), 31 deletions(-) create mode 100644 _context/notes.md diff --git a/_context/notes.md b/_context/notes.md new file mode 100644 index 00000000000..ef9a2570e3a --- /dev/null +++ b/_context/notes.md @@ -0,0 +1,88 @@ +## 2026-04-15 + +### Status + +Core implementation of `DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT` is complete and verified. +Remaining work: unit tests in `ddtrace/tracer/textmap_test.go`. + +**Done:** +- `continue` (default): existing behavior, no changes needed +- `restart`: verified via echotrace — fresh trace-id, no parent, one span link with + `reason=propagation_behavior_extract` and `context_headers=datadog`, baggage propagated +- `ignore`: verified via echotrace — fresh trace-id, no parent, no span links, baggage + dropped. Returns `nil, nil` (not an error — same as receiving no headers at all) +- Telemetry: `trace_propagation_behavior_extract` now reported at startup alongside the + existing propagation style keys +- `supported_configurations.json`: fixed type/default, removed accidental `DD_TEST_*` + entries, regenerated `.gen.go` + +**Next:** ~~unit tests in `ddtrace/tracer/textmap_test.go`~~ done — see unit tests section below + +--- + + +### Why telemetry? + +Other config values in `chainedPropagator` (injection/extraction style names) are already +reported at startup via `startTelemetry()` in `telemetry.go`. The key +`"trace_propagation_behavior_extract"` follows the same pattern so the backend can observe +which mode customers are using. This is the RFC's "Telemetry Key: +DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT" requirement. + +See `ddtrace/tracer/telemetry.go`, the `if chained, ok := c.propagator.(*chainedPropagator)` +block where `trace_propagation_style_inject` and `trace_propagation_style_extract` are +already reported. + +### nil, nil on ignore — real-life behavior and rationale + +**How a user uses it:** Set `DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT=ignore` on a service. +Every incoming HTTP request — regardless of what trace headers the caller sends — produces +a fresh root span. No parent relationship, no span links, no baggage. The service starts +its own independent trace as if it had never received distributed tracing headers. This is +the option for a service that doesn't want to be adopted into an upstream org's trace. + +**Call path through echotrace/httptrace:** +1. `echotrace.Middleware` → `httptrace.StartRequestSpan` +2. `StartRequestSpan` calls `tracer.Extract(HTTPHeadersCarrier(r.Header))` → + returns `nil, nil` +3. Condition `extractErr == nil && parentCtx != nil` is false — `ChildOf` is never set, + span starts as a root ✓ +4. `parentCtx.ForeachBaggageItem(...)` is called unconditionally but `ForeachBaggageItem` + has a nil-receiver guard (`if c == nil { return }`) — safe ✓ + +**Why `nil, nil` and not `nil, ErrSpanContextNotFound`:** returning an error would trigger +misleading debug log "failed to extract span context" in `StartSpanFromPropagatedContext`, +implying something went wrong. `nil, nil` is the clean signal: no context, no problem. +Verified via `TestPropagationBehaviorExtract/ignore` in echotrace. + +### Unit tests (`ddtrace/tracer/textmap_test.go::TestPropagationBehaviorExtract`) + +5 sub-tests covering the RFC's 4 configurations, all passing: + +- `continue/same-trace-id`: trace continued from DD context, no span links, baggage propagated +- `continue/different-trace-ids`: trace continued from DD context, one `terminated_context` + span link for the conflicting W3C context, baggage propagated +- `restart`: zero trace-id (`baggageOnly=true`), one `propagation_behavior_extract` span link + pointing at the DD context. Tracestate is `dd=s:1;p:` (Datadog propagator enriches + it with the parent span ID sub-key). Flags=1 (priority > 0). Baggage propagated. +- `restart/extract-first`: same as restart but extraction stops after the Datadog propagator, + so tracestate is empty (Datadog headers carry no W3C tracestate). **Baggage is nil** — this + is a pre-existing limitation of `DD_TRACE_PROPAGATION_EXTRACT_FIRST`: it returns before the + baggage propagator runs, so baggage is lost regardless of the restart mode. Documented in + the test comment. +- `ignore`: returns `nil, nil` — asserted as `sctx == nil, err == nil` + +### extractFirst + restart interaction + +Covered explicitly in the RFC test matrix (configuration 3: +`DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT=restart` + `DD_TRACE_PROPAGATION_EXTRACT_FIRST=true`). +`extractIncomingSpanContext` returns after the first successful extraction, then `Extract()` +applies the restart behavior to that single context — so exactly one span link is created +and no conflicting-trace span links appear. Verified against .NET implementation. + +### supported_configurations.json + +`DD_TEST_BOOL_ENV`, `DD_TEST_FLOAT_ENV`, `DD_TEST_INT_ENV`, `DD_TEST_STRING_ENV` were +accidentally added to the JSON. Removed. They still appear in `.gen.go` because the +generator also scans Go source for `internal.*Env()` calls — that is correct behavior. +`DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT` fixed to `type: "String", default: "continue"`. diff --git a/ddtrace/tracer/telemetry.go b/ddtrace/tracer/telemetry.go index d1339447892..f5e1af2332a 100644 --- a/ddtrace/tracer/telemetry.go +++ b/ddtrace/tracer/telemetry.go @@ -82,6 +82,8 @@ func startTelemetry(c *config) telemetry.Client { telemetry.Configuration{Name: "trace_propagation_style_inject", Value: chained.injectorNames}) telemetryConfigs = append(telemetryConfigs, telemetry.Configuration{Name: "trace_propagation_style_extract", Value: chained.extractorsNames}) + telemetryConfigs = append(telemetryConfigs, + telemetry.Configuration{Name: "trace_propagation_behavior_extract", Value: chained.propagationBehaviorExtract}) } for k, v := range c.internalConfig.FeatureFlags() { telemetryConfigs = append(telemetryConfigs, telemetry.Configuration{Name: k, Value: v}) diff --git a/internal/env/supported_configurations.json b/internal/env/supported_configurations.json index 6d60b2d0c5f..1a64fc0b9e7 100644 --- a/internal/env/supported_configurations.json +++ b/internal/env/supported_configurations.json @@ -917,27 +917,6 @@ "default": null } ], - "DD_TEST_BOOL_ENV": [ - { - "implementation": "A", - "type": "FIX_ME", - "default": "FIX_ME" - } - ], - "DD_TEST_FLOAT_ENV": [ - { - "implementation": "A", - "type": "FIX_ME", - "default": "FIX_ME" - } - ], - "DD_TEST_INT_ENV": [ - { - "implementation": "A", - "type": "FIX_ME", - "default": "FIX_ME" - } - ], "DD_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES": [ { "implementation": "B", @@ -966,13 +945,6 @@ "default": null } ], - "DD_TEST_STRING_ENV": [ - { - "implementation": "A", - "type": "FIX_ME", - "default": "FIX_ME" - } - ], "DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED": [ { "implementation": "A", @@ -1438,8 +1410,8 @@ "DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT": [ { "implementation": "A", - "type": "FIX_ME", - "default": "FIX_ME" + "type": "String", + "default": "continue" } ], "DD_TRACE_PROPAGATION_EXTRACT_FIRST": [ @@ -1877,4 +1849,4 @@ } ] } -} \ No newline at end of file +} From b85623624f3ed9eea0a03feb4961b3a42d403211 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Wed, 15 Apr 2026 10:39:27 +0200 Subject: [PATCH 16/40] test: add unit tests for DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT --- contrib/labstack/echo.v4/echotrace_test.go | 8 +- ddtrace/tracer/textmap_test.go | 148 ++++++++++++++++++++- 2 files changed, 151 insertions(+), 5 deletions(-) diff --git a/contrib/labstack/echo.v4/echotrace_test.go b/contrib/labstack/echo.v4/echotrace_test.go index 24870faf683..b955e4bf342 100644 --- a/contrib/labstack/echo.v4/echotrace_test.go +++ b/contrib/labstack/echo.v4/echotrace_test.go @@ -734,10 +734,10 @@ func TestPropagationBehaviorExtract(t *testing.T) { name: "restart", propagationBehaviorExtract: "restart", }, - // { - // name: "ignore", - // propagationBehaviorExtract: "ignore", - // }, + { + name: "ignore", + propagationBehaviorExtract: "ignore", + }, } for _, tc := range tests { diff --git a/ddtrace/tracer/textmap_test.go b/ddtrace/tracer/textmap_test.go index 407675076bb..13c212a4d90 100644 --- a/ddtrace/tracer/textmap_test.go +++ b/ddtrace/tracer/textmap_test.go @@ -105,7 +105,6 @@ func TestTextMapCarrierForeachKeyError(t *testing.T) { assert.Equal(t, got, want) } -// TODO: Split test into 3, one for continue, one for restart, one for ignore func TestTextMapExtractTracestatePropagation(t *testing.T) { tests := []struct { name, propagationStyle, traceparent string @@ -2014,6 +2013,153 @@ func TestSpanLinks(t *testing.T) { }) } +// TestPropagationBehaviorExtract covers the four RFC-specified configurations for +// DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT. The default propagators are datadog,tracecontext,baggage. +// +// RFC test matrix: https://datadoghq.atlassian.net/wiki/x/RFC-trace-context-propagation-extraction-modes +func TestPropagationBehaviorExtract(t *testing.T) { + s, c := httpmem.ServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(404) + })) + defer s.Close() + + // Carrier with DD and W3C headers sharing the same trace ID and span ID. + sameIDCarrier := TextMapCarrier{ + DefaultTraceIDHeader: "1", + DefaultParentIDHeader: "1", + DefaultPriorityHeader: "1", + traceparentHeader: "00-00000000000000000000000000000001-0000000000000001-01", + tracestateHeader: "dd=s:1", + "baggage": "key=val", + } + + // Carrier with DD and W3C headers carrying different trace IDs. + diffIDCarrier := TextMapCarrier{ + DefaultTraceIDHeader: "1", + DefaultParentIDHeader: "1", + DefaultPriorityHeader: "1", + traceparentHeader: "00-00000000000000000000000000000002-0000000000000002-01", + tracestateHeader: "dd=s:1", + "baggage": "key=val", + } + + t.Run("continue/same-trace-id", func(t *testing.T) { + // Default behavior: trace is continued from the incoming Datadog context. + // Same trace ID across propagators means no conflicting span link is created. + tr, err := newTracer(WithHTTPClient(c)) + require.NoError(t, err) + defer tr.Stop() + + sctx, err := tr.Extract(sameIDCarrier) + require.NoError(t, err) + require.NotNil(t, sctx) + + assert.Equal(t, traceIDFrom64Bits(1).value, sctx.traceID.value) + assert.Empty(t, sctx.spanLinks) + assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) // +checklocksignore + }) + + t.Run("continue/different-trace-ids", func(t *testing.T) { + // Default behavior: trace is continued from the incoming Datadog context (first propagator). + // W3C context has a different trace ID, so a terminated_context span link is created for it. + tr, err := newTracer(WithHTTPClient(c)) + require.NoError(t, err) + defer tr.Stop() + + sctx, err := tr.Extract(diffIDCarrier) + require.NoError(t, err) + require.NotNil(t, sctx) + + assert.Equal(t, traceIDFrom64Bits(1).value, sctx.traceID.value) + require.Len(t, sctx.spanLinks, 1) + assert.Equal(t, SpanLink{ + TraceID: 2, + SpanID: 2, + Tracestate: "dd=s:1", + Flags: 1, + Attributes: map[string]string{"reason": "terminated_context", "context_headers": "tracecontext"}, + }, sctx.spanLinks[0]) + assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) // +checklocksignore + }) + + t.Run("restart", func(t *testing.T) { + // restart mode: a new local trace context is created regardless of the incoming + // trace ID. The incoming context is referenced via a span link with + // reason=propagation_behavior_extract. Baggage is propagated. + // + // Tracestate is enriched by the Datadog propagator with the p: sub-key (parent + // span ID). Flags=1 because sampling priority > 0. + t.Setenv(headerPropagationBehaviorExtract, "restart") + tr, err := newTracer(WithHTTPClient(c)) + require.NoError(t, err) + defer tr.Stop() + + sctx, err := tr.Extract(sameIDCarrier) + require.NoError(t, err) + require.NotNil(t, sctx) + + assert.True(t, sctx.baggageOnly) // signals spanStart to generate fresh IDs + assert.Equal(t, [16]byte{}, sctx.traceID.value) + require.Len(t, sctx.spanLinks, 1) + assert.Equal(t, SpanLink{ + TraceID: 1, + SpanID: 1, + Tracestate: "dd=s:1;p:0000000000000001", + Flags: 1, + Attributes: map[string]string{"reason": "propagation_behavior_extract", "context_headers": "datadog"}, + }, sctx.spanLinks[0]) + assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) // +checklocksignore + }) + + t.Run("restart/extract-first", func(t *testing.T) { + // restart + extract_first: extraction stops after the first propagator (Datadog). + // No conflicting span links appear because W3C headers are never examined. + // The single extracted context becomes the span link target. + // + // Note: baggage is not propagated in this configuration because extract_first + // returns before the baggage propagator runs. This is a pre-existing limitation + // of DD_TRACE_PROPAGATION_EXTRACT_FIRST, independent of restart mode. + // + // Tracestate is empty because the Datadog propagator does not carry W3C tracestate. + // Flags=1 because sampling priority > 0. + t.Setenv(headerPropagationBehaviorExtract, "restart") + t.Setenv(headerPropagationExtractFirst, "true") + tr, err := newTracer(WithHTTPClient(c)) + require.NoError(t, err) + defer tr.Stop() + + sctx, err := tr.Extract(diffIDCarrier) + require.NoError(t, err) + require.NotNil(t, sctx) + + assert.True(t, sctx.baggageOnly) + assert.Equal(t, [16]byte{}, sctx.traceID.value) + require.Len(t, sctx.spanLinks, 1) + assert.Equal(t, SpanLink{ + TraceID: 1, + SpanID: 1, + Tracestate: "", + Flags: 1, + Attributes: map[string]string{"reason": "propagation_behavior_extract", "context_headers": "datadog"}, + }, sctx.spanLinks[0]) + assert.Nil(t, sctx.baggage) // +checklocksignore + }) + + t.Run("ignore", func(t *testing.T) { + // ignore mode: the entire incoming trace context is discarded. Returns nil, nil — + // no error, no context — so callers produce a fresh root span with no parent, + // no span links, and no baggage. + t.Setenv(headerPropagationBehaviorExtract, "ignore") + tr, err := newTracer(WithHTTPClient(c)) + require.NoError(t, err) + defer tr.Stop() + + sctx, err := tr.Extract(sameIDCarrier) + assert.NoError(t, err) + assert.Nil(t, sctx) + }) +} + func TestW3CExtractsBaggage(t *testing.T) { tracer, err := newTracer() defer tracer.Stop() From abc8508b7df1b72e5136ee889a39e356a7ca5ecf Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Wed, 15 Apr 2026 11:44:04 +0200 Subject: [PATCH 17/40] fix: propagate baggage when extract-first and restart modes are combined --- _context/notes.md | 20 +++++++++++++----- ddtrace/tracer/textmap.go | 38 ++++++++++++++++++++++++++++------ ddtrace/tracer/textmap_test.go | 8 +++---- 3 files changed, 51 insertions(+), 15 deletions(-) diff --git a/_context/notes.md b/_context/notes.md index ef9a2570e3a..b0f086e245d 100644 --- a/_context/notes.md +++ b/_context/notes.md @@ -65,13 +65,23 @@ Verified via `TestPropagationBehaviorExtract/ignore` in echotrace. - `restart`: zero trace-id (`baggageOnly=true`), one `propagation_behavior_extract` span link pointing at the DD context. Tracestate is `dd=s:1;p:` (Datadog propagator enriches it with the parent span ID sub-key). Flags=1 (priority > 0). Baggage propagated. -- `restart/extract-first`: same as restart but extraction stops after the Datadog propagator, - so tracestate is empty (Datadog headers carry no W3C tracestate). **Baggage is nil** — this - is a pre-existing limitation of `DD_TRACE_PROPAGATION_EXTRACT_FIRST`: it returns before the - baggage propagator runs, so baggage is lost regardless of the restart mode. Documented in - the test comment. +- `restart/extract-first`: extraction stops after the Datadog propagator (no conflicting + W3C span link), tracestate is empty (Datadog headers carry no W3C tracestate), Flags=1. + Baggage is propagated — see fix below. - `ignore`: returns `nil, nil` — asserted as `sctx == nil, err == nil` +### Bug fix: baggage lost with extract-first + +`extractIncomingSpanContext` returns immediately when `onlyExtractFirst=true` (after the +first non-baggage propagator succeeds), so the baggage propagator never runs inside the +loop. This caused baggage to be nil in `restart+extract_first` mode — violating the RFC. + +**Fix** (`ddtrace/tracer/textmap.go`, `Extract()`): after `extractIncomingSpanContext` +returns and `onlyExtractFirst` is set, iterate `p.extractors` looking for the baggage +propagator and run it explicitly, merging results into `incomingCtx`. This is isolated to +`Extract()` so `extractIncomingSpanContext` is unchanged. All existing extract-first tests +still pass. + ### extractFirst + restart interaction Covered explicitly in the RFC test matrix (configuration 3: diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index 0a10d85258b..3c76fe701d0 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -312,8 +312,35 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { return nil, err } - // "restart" propagation behavior starts a new trace with a new trace ID - // and sampling decision. The incoming context is referenced via a span + // When onlyExtractFirst is set, extractIncomingSpanContext returns after the + // first successful non-baggage extractor, so the baggage propagator never + // runs inside the loop. Run it explicitly here so baggage is always available + // to callers regardless of the extract-first setting. + if p.onlyExtractFirst { + for _, v := range p.extractors { + if _, isBaggage := v.(*propagatorBaggage); !isBaggage { + continue + } + if baggageCtx, err := v.Extract(carrier); err == nil && baggageCtx != nil { + if incomingCtx == nil { + incomingCtx = baggageCtx + } else { + baggageCtx.ForeachBaggageItem(func(k, val string) bool { // +checklocksignore - Initialization time, freshly extracted ctx not yet shared. + if incomingCtx.baggage == nil { // +checklocksignore + incomingCtx.baggage = make(map[string]string) // +checklocksignore + } + incomingCtx.baggage[k] = val // +checklocksignore + atomic.StoreUint32(&incomingCtx.hasBaggage, 1) + return true + }) + } + } + break // there is only one baggage propagator + } + } + + // "restart" propagation behavior starts a new trace with a new trace ID + // and sampling decision. The incoming context is referenced via a span // link. Baggage is propagated. if p.propagationBehaviorExtract == "restart" { ctx := &SpanContext{ @@ -329,7 +356,6 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { "context_headers": getPropagatorName(p.extractors[0]), }, } - // TODO: +checklocksignore is needed here? if trace := incomingCtx.trace; trace != nil { if p := trace.priority.Load(); p != nil && uint32(*p) > 0 { // +checklocksignore - Initialization time, freshly extracted trace not yet shared. link.Flags = 1 @@ -340,9 +366,9 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { } ctx.spanLinks = []SpanLink{link} - if incomingCtx.baggage != nil { - ctx.baggage = make(map[string]string, len(incomingCtx.baggage)) - maps.Copy(ctx.baggage, incomingCtx.baggage) + if incomingCtx.baggage != nil { // +checklocksignore + ctx.baggage = make(map[string]string, len(incomingCtx.baggage)) // +checklocksignore + maps.Copy(ctx.baggage, incomingCtx.baggage) // +checklocksignore atomic.StoreUint32(&ctx.hasBaggage, 1) } diff --git a/ddtrace/tracer/textmap_test.go b/ddtrace/tracer/textmap_test.go index 13c212a4d90..6d3528a4fe3 100644 --- a/ddtrace/tracer/textmap_test.go +++ b/ddtrace/tracer/textmap_test.go @@ -2116,9 +2116,9 @@ func TestPropagationBehaviorExtract(t *testing.T) { // No conflicting span links appear because W3C headers are never examined. // The single extracted context becomes the span link target. // - // Note: baggage is not propagated in this configuration because extract_first - // returns before the baggage propagator runs. This is a pre-existing limitation - // of DD_TRACE_PROPAGATION_EXTRACT_FIRST, independent of restart mode. + // Baggage is still propagated: Extract() runs the baggage propagator explicitly + // after extractIncomingSpanContext returns, so the extract-first short-circuit + // does not suppress baggage. // // Tracestate is empty because the Datadog propagator does not carry W3C tracestate. // Flags=1 because sampling priority > 0. @@ -2142,7 +2142,7 @@ func TestPropagationBehaviorExtract(t *testing.T) { Flags: 1, Attributes: map[string]string{"reason": "propagation_behavior_extract", "context_headers": "datadog"}, }, sctx.spanLinks[0]) - assert.Nil(t, sctx.baggage) // +checklocksignore + assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) // +checklocksignore }) t.Run("ignore", func(t *testing.T) { From 3b072e0d2d4956ae1664a7bb191e398ca7aca683 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 09:14:34 +0200 Subject: [PATCH 18/40] chore: document remaining work and system-tests requirements --- _context/notes.md | 60 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/_context/notes.md b/_context/notes.md index b0f086e245d..584c230f34e 100644 --- a/_context/notes.md +++ b/_context/notes.md @@ -1,3 +1,63 @@ +## 2026-04-16 + +### Remaining work to pass system-tests + +Source for all items below: `~/go/src/github.com/DataDog/system-tests/` + +--- + +**1. Remove `missing_feature` markers from `manifests/golang.yml`** + +File: `manifests/golang.yml` lines 1334–1337: +``` +tests/test_library_conf.py::Test_ExtractBehavior_Default: missing_feature (baggage should be implemented and conflicting trace contexts should generate span link in v1.71.0) +tests/test_library_conf.py::Test_ExtractBehavior_Ignore: missing_feature (extract behavior not implemented) +tests/test_library_conf.py::Test_ExtractBehavior_Restart: missing_feature (extract behavior not implemented) +tests/test_library_conf.py::Test_ExtractBehavior_Restart_With_Extract_First: missing_feature (extract behavior not implemented) +``` +These markers skip the tests for Go. They need to be removed in a PR to the system-tests +repo once the implementation is confirmed working. + +--- + +**2. Outbound injection through `/make_distant_call` needs smoke-testing** + +All four test classes call `/make_distant_call?url=http://weblog:7777/` and assert on +the *outbound* headers that the weblog sends to the downstream service +(`data["request_headers"][...]`). Source: `tests/test_library_conf.py`: + +- `restart` (line 610–612): outbound `x-datadog-trace-id` must differ from `"1"`; + `_dd.p.tid=1111111111111111` must NOT be in outbound tags; `key1=value1` must be + in outbound `baggage` +- `ignore` (line 767–769): outbound `x-datadog-trace-id` != incoming; outbound + `baggage` header must be absent entirely +- `restart+extract_first` (line 860–862): same as restart + +This tests injection, not just extraction. The Go weblog `/make_distant_call` endpoint +exists and was confirmed by the MCP, but these assertions have not been run against the +actual system-test harness yet. Need CI to confirm. + +--- + +**3. `restart` — same trace ID, different span IDs (`test_multiple_tracecontexts_with_overrides`)** + +Source: `tests/test_library_conf.py` lines 667–718, `Test_ExtractBehavior_Restart`. + +This test sends DD and W3C headers sharing the same trace ID (`1111111111111111...0001`) +but with *different* span IDs (DD=`1`, W3C=`0x1234567890123456`). The assertion at +line 708: +```python +assert int(link["spanID"]) == 1311768467284833366 # 0x1234567890123456 +``` +The span link's `spanID` is expected to be the **W3C span ID**, not the DD span ID. +This is the `overrideDatadogParentID` code path. The current `restart` implementation +builds the span link from `incomingCtx` which is whatever `extractIncomingSpanContext` +returns — need to verify that `overrideDatadogParentID` runs before `Extract()` applies +the restart behavior, and that the resulting `incomingCtx.SpanID()` is the W3C span ID. +Not yet verified with a unit test. + +--- + ## 2026-04-15 ### Status From 13b2517626358f800920aea6263573ed3f506f62 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 09:23:15 +0200 Subject: [PATCH 19/40] chore: answer open questions in notes --- _context/notes.md | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/_context/notes.md b/_context/notes.md index 584c230f34e..511694e755a 100644 --- a/_context/notes.md +++ b/_context/notes.md @@ -9,12 +9,14 @@ Source for all items below: `~/go/src/github.com/DataDog/system-tests/` **1. Remove `missing_feature` markers from `manifests/golang.yml`** File: `manifests/golang.yml` lines 1334–1337: + ``` tests/test_library_conf.py::Test_ExtractBehavior_Default: missing_feature (baggage should be implemented and conflicting trace contexts should generate span link in v1.71.0) tests/test_library_conf.py::Test_ExtractBehavior_Ignore: missing_feature (extract behavior not implemented) tests/test_library_conf.py::Test_ExtractBehavior_Restart: missing_feature (extract behavior not implemented) tests/test_library_conf.py::Test_ExtractBehavior_Restart_With_Extract_First: missing_feature (extract behavior not implemented) ``` + These markers skip the tests for Go. They need to be removed in a PR to the system-tests repo once the implementation is confirmed working. @@ -46,9 +48,11 @@ Source: `tests/test_library_conf.py` lines 667–718, `Test_ExtractBehavior_Rest This test sends DD and W3C headers sharing the same trace ID (`1111111111111111...0001`) but with *different* span IDs (DD=`1`, W3C=`0x1234567890123456`). The assertion at line 708: + ```python assert int(link["spanID"]) == 1311768467284833366 # 0x1234567890123456 ``` + The span link's `spanID` is expected to be the **W3C span ID**, not the DD span ID. This is the `overrideDatadogParentID` code path. The current `restart` implementation builds the span link from `incomingCtx` which is whatever `extractIncomingSpanContext` @@ -66,6 +70,7 @@ Core implementation of `DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT` is complete and v Remaining work: unit tests in `ddtrace/tracer/textmap_test.go`. **Done:** + - `continue` (default): existing behavior, no changes needed - `restart`: verified via echotrace — fresh trace-id, no parent, one span link with `reason=propagation_behavior_extract` and `context_headers=datadog`, baggage propagated @@ -80,7 +85,6 @@ Remaining work: unit tests in `ddtrace/tracer/textmap_test.go`. --- - ### Why telemetry? Other config values in `chainedPropagator` (injection/extraction style names) are already @@ -102,6 +106,7 @@ its own independent trace as if it had never received distributed tracing header the option for a service that doesn't want to be adopted into an upstream org's trace. **Call path through echotrace/httptrace:** + 1. `echotrace.Middleware` → `httptrace.StartRequestSpan` 2. `StartRequestSpan` calls `tracer.Extract(HTTPHeadersCarrier(r.Header))` → returns `nil, nil` @@ -115,6 +120,8 @@ misleading debug log "failed to extract span context" in `StartSpanFromPropagate implying something went wrong. `nil, nil` is the clean signal: no context, no problem. Verified via `TestPropagationBehaviorExtract/ignore` in echotrace. +See + ### Unit tests (`ddtrace/tracer/textmap_test.go::TestPropagationBehaviorExtract`) 5 sub-tests covering the RFC's 4 configurations, all passing: @@ -122,9 +129,24 @@ Verified via `TestPropagationBehaviorExtract/ignore` in echotrace. - `continue/same-trace-id`: trace continued from DD context, no span links, baggage propagated - `continue/different-trace-ids`: trace continued from DD context, one `terminated_context` span link for the conflicting W3C context, baggage propagated + + > **Q: Was `terminated_context` added by us?** + > + > No. It pre-existed on `main` before this branch. Confirmed by `git show 7d2d81401:ddtrace/tracer/textmap.go | grep terminated_context` — it was already at line 330. We did not add it; we only added `propagation_behavior_extract` (the restart mode span link reason). + - `restart`: zero trace-id (`baggageOnly=true`), one `propagation_behavior_extract` span link pointing at the DD context. Tracestate is `dd=s:1;p:` (Datadog propagator enriches it with the parent span ID sub-key). Flags=1 (priority > 0). Baggage propagated. + + > **Q: Is it OK to have a zero trace-id in the restart context?** + > + > Yes. The zero trace-id is intentional and never reaches a span. `baggageOnly=true` causes + > `spanStart` to skip the `span.traceID = context.traceID.Lower()` assignment entirely + > (`tracer.go` line 846: `if context != nil && !context.baggageOnly`). The span instead + > gets a freshly generated ID from `generateSpanID(startTime)` (`tracer.go` line 830), + > which is a 63-bit random value. The zero trace-id in the restart context is only an + > intermediate value that never escapes. + - `restart/extract-first`: extraction stops after the Datadog propagator (no conflicting W3C span link), tracestate is empty (Datadog headers carry no W3C tracestate), Flags=1. Baggage is propagated — see fix below. @@ -142,6 +164,15 @@ propagator and run it explicitly, merging results into `incomingCtx`. This is is `Extract()` so `extractIncomingSpanContext` is unchanged. All existing extract-first tests still pass. +> **Q: Doesn't this duplicate the work of the OTel extractor?** +> +> No. There is no separate OTel baggage extractor in this path. `getDDorOtelConfig("propagationStyle")` +> (`otel_dd_mappings.go`) only determines *which* propagator types to instantiate (datadog, +> tracecontext, etc.) — it does not add an extra extraction pass. The fix runs +> `propagatorBaggage.Extract()` exactly once, on the same `p.extractors` slice, same carrier. +> The baggage propagator was already in `p.extractors`; we are just calling it at the right +> moment instead of relying on the loop that `onlyExtractFirst` short-circuits. + ### extractFirst + restart interaction Covered explicitly in the RFC test matrix (configuration 3: From f240849c35a22ec00114dae13cd0f0866661b0df Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 09:27:16 +0200 Subject: [PATCH 20/40] chore: format --- ddtrace/tracer/textmap.go | 25 +++++------ ddtrace/tracer/textmap_test.go | 81 ++++++++++++++++++++++++++++++---- 2 files changed, 85 insertions(+), 21 deletions(-) diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index 3c76fe701d0..652b42a3ed3 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -205,11 +205,11 @@ func NewPropagator(cfg *PropagatorConfig, propagators ...Propagator) Propagator // When injecting, all injectors are called to propagate the span context. // When extracting, it tries each extractor, selecting the first successful one. type chainedPropagator struct { - injectors []Propagator - extractors []Propagator - injectorNames string - extractorsNames string - onlyExtractFirst bool // value of DD_TRACE_PROPAGATION_EXTRACT_FIRST + injectors []Propagator + extractors []Propagator + injectorNames string + extractorsNames string + onlyExtractFirst bool // value of DD_TRACE_PROPAGATION_EXTRACT_FIRST propagationBehaviorExtract string // value of DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT } @@ -301,7 +301,7 @@ func (p *chainedPropagator) Inject(spanCtx *SpanContext, carrier any) error { // be relayed in the returned SpanContext with a SpanLink. func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { // TODO: Return nil or an empty span context? - // "ignore" propagation behavior returns a new span context with no span + // "ignore" propagation behavior returns a new span context with no span // links and no baggage. if p.propagationBehaviorExtract == "ignore" { return nil, nil @@ -326,8 +326,8 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { incomingCtx = baggageCtx } else { baggageCtx.ForeachBaggageItem(func(k, val string) bool { // +checklocksignore - Initialization time, freshly extracted ctx not yet shared. - if incomingCtx.baggage == nil { // +checklocksignore - incomingCtx.baggage = make(map[string]string) // +checklocksignore + if incomingCtx.baggage == nil { // +checklocksignore + incomingCtx.baggage = make(map[string]string) // +checklocksignore } incomingCtx.baggage[k] = val // +checklocksignore atomic.StoreUint32(&incomingCtx.hasBaggage, 1) @@ -385,10 +385,9 @@ func (p *chainedPropagator) extractIncomingSpanContext(carrier any) (*SpanContex var links []SpanLink pendingBaggage := make(map[string]string) // used to store baggage items temporarily - for _, v := range p.extractors { // If incomingCtx is nil, no extraction has run yet - firstExtraction := (ctx == nil) + firstExtraction := (ctx == nil) extractedCtx, err := v.Extract(carrier) // If this is the baggage propagator, just stash its items into pendingBaggage @@ -697,14 +696,14 @@ func getDatadogPropagator(cp *chainedPropagator) *propagator { // overrideDatadogParentID overrides a context's: // 1. span ID with the span ID extracted from W3C tracecontext headers; and // 2. reparent ID with either: -// - the reparent ID from W3C tracecontext headers (if set), or -// - the span ID from Datadog headers (as fallback). +// - the reparent ID from W3C tracecontext headers (if set), or +// - the span ID from Datadog headers (as fallback). // // reparent ID is the last known Datadog parent span ID, used by Datadog's // backend to fix broken parent-child relationships when non-Datadog tracers // in the path don't report spans to Datadog. // -// SpanContexts are passed by reference to avoid copying lock information in +// SpanContexts are passed by reference to avoid copying lock information in // the SpanContext type. func overrideDatadogParentID(ctx, w3cCtx, ddCtx *SpanContext) { if ctx == nil || w3cCtx == nil || ddCtx == nil { diff --git a/ddtrace/tracer/textmap_test.go b/ddtrace/tracer/textmap_test.go index 6d3528a4fe3..7729b6154bd 100644 --- a/ddtrace/tracer/textmap_test.go +++ b/ddtrace/tracer/textmap_test.go @@ -2111,14 +2111,66 @@ func TestPropagationBehaviorExtract(t *testing.T) { assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) // +checklocksignore }) - t.Run("restart/extract-first", func(t *testing.T) { - // restart + extract_first: extraction stops after the first propagator (Datadog). - // No conflicting span links appear because W3C headers are never examined. - // The single extracted context becomes the span link target. + t.Run("restart/different-trace-ids", func(t *testing.T) { + // restart mode with unique trace IDs: the outcome is the same as same-trace-id. + // A new trace is started, and the span link points to the Datadog context (the + // first propagator). The W3C conflicting context is not included in the span link + // because restart discards the incoming context entirely and replaces it. + t.Setenv(headerPropagationBehaviorExtract, "restart") + tr, err := newTracer(WithHTTPClient(c)) + require.NoError(t, err) + defer tr.Stop() + + sctx, err := tr.Extract(diffIDCarrier) + require.NoError(t, err) + require.NotNil(t, sctx) + + assert.True(t, sctx.baggageOnly) + assert.Equal(t, [16]byte{}, sctx.traceID.value) + require.Len(t, sctx.spanLinks, 1) + assert.Equal(t, SpanLink{ + TraceID: 1, + SpanID: 1, + Tracestate: "dd=s:1;p:0000000000000001", + Flags: 1, + Attributes: map[string]string{"reason": "propagation_behavior_extract", "context_headers": "datadog"}, + }, sctx.spanLinks[0]) + assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) // +checklocksignore + }) + + t.Run("restart/extract-first/same-trace-id", func(t *testing.T) { + // restart + extract_first with same trace ID: extraction stops after Datadog. + // One span link to the Datadog context. Baggage propagated. // - // Baggage is still propagated: Extract() runs the baggage propagator explicitly - // after extractIncomingSpanContext returns, so the extract-first short-circuit - // does not suppress baggage. + // Tracestate is enriched by the Datadog propagator with the p: sub-key. + // Flags=1 because sampling priority > 0. + t.Setenv(headerPropagationBehaviorExtract, "restart") + t.Setenv(headerPropagationExtractFirst, "true") + tr, err := newTracer(WithHTTPClient(c)) + require.NoError(t, err) + defer tr.Stop() + + sctx, err := tr.Extract(sameIDCarrier) + require.NoError(t, err) + require.NotNil(t, sctx) + + assert.True(t, sctx.baggageOnly) + assert.Equal(t, [16]byte{}, sctx.traceID.value) + require.Len(t, sctx.spanLinks, 1) + assert.Equal(t, SpanLink{ + TraceID: 1, + SpanID: 1, + Tracestate: "dd=s:1;p:0000000000000001", + Flags: 1, + Attributes: map[string]string{"reason": "propagation_behavior_extract", "context_headers": "datadog"}, + }, sctx.spanLinks[0]) + assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) // +checklocksignore + }) + + t.Run("restart/extract-first/different-trace-ids", func(t *testing.T) { + // restart + extract_first with unique trace IDs: extraction stops after Datadog, + // so the W3C conflicting context is never seen. One span link to the Datadog context. + // Baggage is still propagated via the explicit baggage pass in Extract(). // // Tracestate is empty because the Datadog propagator does not carry W3C tracestate. // Flags=1 because sampling priority > 0. @@ -2145,7 +2197,7 @@ func TestPropagationBehaviorExtract(t *testing.T) { assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) // +checklocksignore }) - t.Run("ignore", func(t *testing.T) { + t.Run("ignore/same-trace-id", func(t *testing.T) { // ignore mode: the entire incoming trace context is discarded. Returns nil, nil — // no error, no context — so callers produce a fresh root span with no parent, // no span links, and no baggage. @@ -2158,6 +2210,19 @@ func TestPropagationBehaviorExtract(t *testing.T) { assert.NoError(t, err) assert.Nil(t, sctx) }) + + t.Run("ignore/different-trace-ids", func(t *testing.T) { + // ignore mode with unique trace IDs: same result as same-trace-id. + // All incoming context is discarded regardless of what headers are present. + t.Setenv(headerPropagationBehaviorExtract, "ignore") + tr, err := newTracer(WithHTTPClient(c)) + require.NoError(t, err) + defer tr.Stop() + + sctx, err := tr.Extract(diffIDCarrier) + assert.NoError(t, err) + assert.Nil(t, sctx) + }) } func TestW3CExtractsBaggage(t *testing.T) { From 9857535116cd01b16faefc6215eeb5e536494e73 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 09:29:05 +0200 Subject: [PATCH 21/40] test: update tests --- contrib/labstack/echo.v4/echotrace_test.go | 3 --- ddtrace/tracer/textmap_test.go | 21 ++++++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/contrib/labstack/echo.v4/echotrace_test.go b/contrib/labstack/echo.v4/echotrace_test.go index b955e4bf342..10110c1a261 100644 --- a/contrib/labstack/echo.v4/echotrace_test.go +++ b/contrib/labstack/echo.v4/echotrace_test.go @@ -775,8 +775,6 @@ func TestPropagationBehaviorExtract(t *testing.T) { fmt.Println("spans") fmt.Println(spans) - - // // verify traces look good // assert.True(called) // assert.True(traced) @@ -799,7 +797,6 @@ func TestPropagationBehaviorExtract(t *testing.T) { // assert.Equal("http://example.com/user/123", span.Tag(ext.HTTPURL)) - // TODO(human): Implement assertions for each propagation behavior. // // For "continue": diff --git a/ddtrace/tracer/textmap_test.go b/ddtrace/tracer/textmap_test.go index 7729b6154bd..528b4e1fa4c 100644 --- a/ddtrace/tracer/textmap_test.go +++ b/ddtrace/tracer/textmap_test.go @@ -2112,10 +2112,12 @@ func TestPropagationBehaviorExtract(t *testing.T) { }) t.Run("restart/different-trace-ids", func(t *testing.T) { - // restart mode with unique trace IDs: the outcome is the same as same-trace-id. - // A new trace is started, and the span link points to the Datadog context (the - // first propagator). The W3C conflicting context is not included in the span link - // because restart discards the incoming context entirely and replaces it. + // restart mode with unique trace IDs: a new trace is started, span link points to + // the Datadog context (first propagator). The W3C conflicting context is not + // included because restart applies before conflicting-trace span links are generated. + // + // Tracestate is empty: the W3C traceparent has a different trace ID, so + // propagateTracestate is never called and no p: sub-key is added to propagating tags. t.Setenv(headerPropagationBehaviorExtract, "restart") tr, err := newTracer(WithHTTPClient(c)) require.NoError(t, err) @@ -2131,7 +2133,7 @@ func TestPropagationBehaviorExtract(t *testing.T) { assert.Equal(t, SpanLink{ TraceID: 1, SpanID: 1, - Tracestate: "dd=s:1;p:0000000000000001", + Tracestate: "", Flags: 1, Attributes: map[string]string{"reason": "propagation_behavior_extract", "context_headers": "datadog"}, }, sctx.spanLinks[0]) @@ -2139,10 +2141,11 @@ func TestPropagationBehaviorExtract(t *testing.T) { }) t.Run("restart/extract-first/same-trace-id", func(t *testing.T) { - // restart + extract_first with same trace ID: extraction stops after Datadog. - // One span link to the Datadog context. Baggage propagated. + // restart + extract_first with same trace ID: extraction stops after Datadog, + // so the W3C propagator never runs. One span link to the Datadog context. Baggage propagated. // - // Tracestate is enriched by the Datadog propagator with the p: sub-key. + // Tracestate is empty: the W3C propagator (which would add the p: sub-key via + // propagateTracestate) never runs because onlyExtractFirst stops the loop early. // Flags=1 because sampling priority > 0. t.Setenv(headerPropagationBehaviorExtract, "restart") t.Setenv(headerPropagationExtractFirst, "true") @@ -2160,7 +2163,7 @@ func TestPropagationBehaviorExtract(t *testing.T) { assert.Equal(t, SpanLink{ TraceID: 1, SpanID: 1, - Tracestate: "dd=s:1;p:0000000000000001", + Tracestate: "", Flags: 1, Attributes: map[string]string{"reason": "propagation_behavior_extract", "context_headers": "datadog"}, }, sctx.spanLinks[0]) From 0e1e163d7579bb4d58aaab4a6ed71d94b870829d Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 09:48:37 +0200 Subject: [PATCH 22/40] =?UTF-8?q?refactor:=20clean=20up=20Extract()=20?= =?UTF-8?q?=E2=80=94=20remove=20TODO,=20extract=20extractBaggage=20helper,?= =?UTF-8?q?=20move=20baggage=20fix=20inside=20restart=20branch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- _context/notes.md | 23 ++++++++++------ ddtrace/tracer/textmap.go | 58 +++++++++++++++++---------------------- 2 files changed, 39 insertions(+), 42 deletions(-) diff --git a/_context/notes.md b/_context/notes.md index 511694e755a..b22cbfb219f 100644 --- a/_context/notes.md +++ b/_context/notes.md @@ -152,26 +152,31 @@ See **Q: Doesn't this duplicate the work of the OTel extractor?** > > No. There is no separate OTel baggage extractor in this path. `getDDorOtelConfig("propagationStyle")` > (`otel_dd_mappings.go`) only determines *which* propagator types to instantiate (datadog, -> tracecontext, etc.) — it does not add an extra extraction pass. The fix runs +> tracecontext, etc.) — it does not add an extra extraction pass. `extractBaggage()` runs > `propagatorBaggage.Extract()` exactly once, on the same `p.extractors` slice, same carrier. -> The baggage propagator was already in `p.extractors`; we are just calling it at the right -> moment instead of relying on the loop that `onlyExtractFirst` short-circuits. ### extractFirst + restart interaction diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index 652b42a3ed3..285f9291d7b 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -300,9 +300,6 @@ func (p *chainedPropagator) Inject(spanCtx *SpanContext, carrier any) error { // subsequent trace context has conflicting trace information, such information will // be relayed in the returned SpanContext with a SpanLink. func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { - // TODO: Return nil or an empty span context? - // "ignore" propagation behavior returns a new span context with no span - // links and no baggage. if p.propagationBehaviorExtract == "ignore" { return nil, nil } @@ -312,33 +309,6 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { return nil, err } - // When onlyExtractFirst is set, extractIncomingSpanContext returns after the - // first successful non-baggage extractor, so the baggage propagator never - // runs inside the loop. Run it explicitly here so baggage is always available - // to callers regardless of the extract-first setting. - if p.onlyExtractFirst { - for _, v := range p.extractors { - if _, isBaggage := v.(*propagatorBaggage); !isBaggage { - continue - } - if baggageCtx, err := v.Extract(carrier); err == nil && baggageCtx != nil { - if incomingCtx == nil { - incomingCtx = baggageCtx - } else { - baggageCtx.ForeachBaggageItem(func(k, val string) bool { // +checklocksignore - Initialization time, freshly extracted ctx not yet shared. - if incomingCtx.baggage == nil { // +checklocksignore - incomingCtx.baggage = make(map[string]string) // +checklocksignore - } - incomingCtx.baggage[k] = val // +checklocksignore - atomic.StoreUint32(&incomingCtx.hasBaggage, 1) - return true - }) - } - } - break // there is only one baggage propagator - } - } - // "restart" propagation behavior starts a new trace with a new trace ID // and sampling decision. The incoming context is referenced via a span // link. Baggage is propagated. @@ -366,9 +336,15 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { } ctx.spanLinks = []SpanLink{link} - if incomingCtx.baggage != nil { // +checklocksignore - ctx.baggage = make(map[string]string, len(incomingCtx.baggage)) // +checklocksignore - maps.Copy(ctx.baggage, incomingCtx.baggage) // +checklocksignore + // When onlyExtractFirst is set, extractIncomingSpanContext returns after the + // first successful non-baggage extractor, so incomingCtx carries no baggage. + // Extract baggage explicitly so it is propagated regardless. + baggage := incomingCtx.baggage // +checklocksignore + if p.onlyExtractFirst { + baggage = p.extractBaggage(carrier) + } + if len(baggage) > 0 { + ctx.baggage = maps.Clone(baggage) // +checklocksignore atomic.StoreUint32(&ctx.hasBaggage, 1) } @@ -380,6 +356,22 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { return incomingCtx, nil } +// extractBaggage runs only the baggage propagator against the carrier and +// returns the extracted items. Used when onlyExtractFirst has prevented the +// baggage propagator from running inside extractIncomingSpanContext. +func (p *chainedPropagator) extractBaggage(carrier any) map[string]string { + for _, v := range p.extractors { + if _, isBaggage := v.(*propagatorBaggage); !isBaggage { + continue + } + if baggageCtx, err := v.Extract(carrier); err == nil && baggageCtx != nil { + return baggageCtx.baggage // +checklocksignore - Initialization time, freshly extracted ctx not yet shared. + } + break // there is only one baggage propagator + } + return nil +} + func (p *chainedPropagator) extractIncomingSpanContext(carrier any) (*SpanContext, error) { var ctx *SpanContext var links []SpanLink From 7aef04eb98370146fbc2bb28fc8ca8ccc666a2c9 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 09:53:01 +0200 Subject: [PATCH 23/40] chore: remove TODO above overrideDatadogParentID --- ddtrace/tracer/textmap.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index 285f9291d7b..d12051ac64e 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -683,8 +683,6 @@ func getDatadogPropagator(cp *chainedPropagator) *propagator { return nil } -// TODO: Verify if we should rename this function - e.g., "reconcile context span ID and reparent ID". maybe split this into two functions. one for reconciling the span ID and another for the reparent ID. - // overrideDatadogParentID overrides a context's: // 1. span ID with the span ID extracted from W3C tracecontext headers; and // 2. reparent ID with either: From 564bc64ed7055c1b347747bbca66f4f985f61ffe Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 09:53:27 +0200 Subject: [PATCH 24/40] chore: remove redundant test comment --- ddtrace/tracer/textmap_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/ddtrace/tracer/textmap_test.go b/ddtrace/tracer/textmap_test.go index 528b4e1fa4c..2563c555030 100644 --- a/ddtrace/tracer/textmap_test.go +++ b/ddtrace/tracer/textmap_test.go @@ -2013,9 +2013,6 @@ func TestSpanLinks(t *testing.T) { }) } -// TestPropagationBehaviorExtract covers the four RFC-specified configurations for -// DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT. The default propagators are datadog,tracecontext,baggage. -// // RFC test matrix: https://datadoghq.atlassian.net/wiki/x/RFC-trace-context-propagation-extraction-modes func TestPropagationBehaviorExtract(t *testing.T) { s, c := httpmem.ServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { From acbd99acc4fa5c9b4afcd05a5d5b70995d03ed2f Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 09:53:56 +0200 Subject: [PATCH 25/40] chore: untrack _context, keep locally --- .gitignore | 1 + _context/notes.md | 194 ------------------ _context/references.md | 1 - ...ce-context-extraction-propagation-modes.md | 161 --------------- 4 files changed, 1 insertion(+), 356 deletions(-) delete mode 100644 _context/notes.md delete mode 100644 _context/references.md delete mode 100644 _context/rfc-trace-context-extraction-propagation-modes.md diff --git a/.gitignore b/.gitignore index 47ece489df7..6b48471552f 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ coverage-*.txt /.claude/* !/.claude/commands/ !/.claude/settings.json +_context/ diff --git a/_context/notes.md b/_context/notes.md deleted file mode 100644 index b22cbfb219f..00000000000 --- a/_context/notes.md +++ /dev/null @@ -1,194 +0,0 @@ -## 2026-04-16 - -### Remaining work to pass system-tests - -Source for all items below: `~/go/src/github.com/DataDog/system-tests/` - ---- - -**1. Remove `missing_feature` markers from `manifests/golang.yml`** - -File: `manifests/golang.yml` lines 1334–1337: - -``` -tests/test_library_conf.py::Test_ExtractBehavior_Default: missing_feature (baggage should be implemented and conflicting trace contexts should generate span link in v1.71.0) -tests/test_library_conf.py::Test_ExtractBehavior_Ignore: missing_feature (extract behavior not implemented) -tests/test_library_conf.py::Test_ExtractBehavior_Restart: missing_feature (extract behavior not implemented) -tests/test_library_conf.py::Test_ExtractBehavior_Restart_With_Extract_First: missing_feature (extract behavior not implemented) -``` - -These markers skip the tests for Go. They need to be removed in a PR to the system-tests -repo once the implementation is confirmed working. - ---- - -**2. Outbound injection through `/make_distant_call` needs smoke-testing** - -All four test classes call `/make_distant_call?url=http://weblog:7777/` and assert on -the *outbound* headers that the weblog sends to the downstream service -(`data["request_headers"][...]`). Source: `tests/test_library_conf.py`: - -- `restart` (line 610–612): outbound `x-datadog-trace-id` must differ from `"1"`; - `_dd.p.tid=1111111111111111` must NOT be in outbound tags; `key1=value1` must be - in outbound `baggage` -- `ignore` (line 767–769): outbound `x-datadog-trace-id` != incoming; outbound - `baggage` header must be absent entirely -- `restart+extract_first` (line 860–862): same as restart - -This tests injection, not just extraction. The Go weblog `/make_distant_call` endpoint -exists and was confirmed by the MCP, but these assertions have not been run against the -actual system-test harness yet. Need CI to confirm. - ---- - -**3. `restart` — same trace ID, different span IDs (`test_multiple_tracecontexts_with_overrides`)** - -Source: `tests/test_library_conf.py` lines 667–718, `Test_ExtractBehavior_Restart`. - -This test sends DD and W3C headers sharing the same trace ID (`1111111111111111...0001`) -but with *different* span IDs (DD=`1`, W3C=`0x1234567890123456`). The assertion at -line 708: - -```python -assert int(link["spanID"]) == 1311768467284833366 # 0x1234567890123456 -``` - -The span link's `spanID` is expected to be the **W3C span ID**, not the DD span ID. -This is the `overrideDatadogParentID` code path. The current `restart` implementation -builds the span link from `incomingCtx` which is whatever `extractIncomingSpanContext` -returns — need to verify that `overrideDatadogParentID` runs before `Extract()` applies -the restart behavior, and that the resulting `incomingCtx.SpanID()` is the W3C span ID. -Not yet verified with a unit test. - ---- - -## 2026-04-15 - -### Status - -Core implementation of `DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT` is complete and verified. -Remaining work: unit tests in `ddtrace/tracer/textmap_test.go`. - -**Done:** - -- `continue` (default): existing behavior, no changes needed -- `restart`: verified via echotrace — fresh trace-id, no parent, one span link with - `reason=propagation_behavior_extract` and `context_headers=datadog`, baggage propagated -- `ignore`: verified via echotrace — fresh trace-id, no parent, no span links, baggage - dropped. Returns `nil, nil` (not an error — same as receiving no headers at all) -- Telemetry: `trace_propagation_behavior_extract` now reported at startup alongside the - existing propagation style keys -- `supported_configurations.json`: fixed type/default, removed accidental `DD_TEST_*` - entries, regenerated `.gen.go` - -**Next:** ~~unit tests in `ddtrace/tracer/textmap_test.go`~~ done — see unit tests section below - ---- - -### Why telemetry? - -Other config values in `chainedPropagator` (injection/extraction style names) are already -reported at startup via `startTelemetry()` in `telemetry.go`. The key -`"trace_propagation_behavior_extract"` follows the same pattern so the backend can observe -which mode customers are using. This is the RFC's "Telemetry Key: -DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT" requirement. - -See `ddtrace/tracer/telemetry.go`, the `if chained, ok := c.propagator.(*chainedPropagator)` -block where `trace_propagation_style_inject` and `trace_propagation_style_extract` are -already reported. - -### nil, nil on ignore — real-life behavior and rationale - -**How a user uses it:** Set `DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT=ignore` on a service. -Every incoming HTTP request — regardless of what trace headers the caller sends — produces -a fresh root span. No parent relationship, no span links, no baggage. The service starts -its own independent trace as if it had never received distributed tracing headers. This is -the option for a service that doesn't want to be adopted into an upstream org's trace. - -**Call path through echotrace/httptrace:** - -1. `echotrace.Middleware` → `httptrace.StartRequestSpan` -2. `StartRequestSpan` calls `tracer.Extract(HTTPHeadersCarrier(r.Header))` → - returns `nil, nil` -3. Condition `extractErr == nil && parentCtx != nil` is false — `ChildOf` is never set, - span starts as a root ✓ -4. `parentCtx.ForeachBaggageItem(...)` is called unconditionally but `ForeachBaggageItem` - has a nil-receiver guard (`if c == nil { return }`) — safe ✓ - -**Why `nil, nil` and not `nil, ErrSpanContextNotFound`:** returning an error would trigger -misleading debug log "failed to extract span context" in `StartSpanFromPropagatedContext`, -implying something went wrong. `nil, nil` is the clean signal: no context, no problem. -Verified via `TestPropagationBehaviorExtract/ignore` in echotrace. - -See - -### Unit tests (`ddtrace/tracer/textmap_test.go::TestPropagationBehaviorExtract`) - -5 sub-tests covering the RFC's 4 configurations, all passing: - -- `continue/same-trace-id`: trace continued from DD context, no span links, baggage propagated -- `continue/different-trace-ids`: trace continued from DD context, one `terminated_context` - span link for the conflicting W3C context, baggage propagated - - > **Q: Was `terminated_context` added by us?** - > - > No. It pre-existed on `main` before this branch. Confirmed by `git show 7d2d81401:ddtrace/tracer/textmap.go | grep terminated_context` — it was already at line 330. We did not add it; we only added `propagation_behavior_extract` (the restart mode span link reason). - -- `restart`: zero trace-id (`baggageOnly=true`), one `propagation_behavior_extract` span link - pointing at the DD context. Tracestate is `dd=s:1;p:` (Datadog propagator enriches - it with the parent span ID sub-key). Flags=1 (priority > 0). Baggage propagated. - - > **Q: Is it OK to have a zero trace-id in the restart context?** - > - > Yes. The zero trace-id is intentional and never reaches a span. `baggageOnly=true` causes - > `spanStart` to skip the `span.traceID = context.traceID.Lower()` assignment entirely - > (`tracer.go` line 846: `if context != nil && !context.baggageOnly`). The span instead - > gets a freshly generated ID from `generateSpanID(startTime)` (`tracer.go` line 830), - > which is a 63-bit random value. The zero trace-id in the restart context is only an - > intermediate value that never escapes. - -- `restart/extract-first`: extraction stops after the Datadog propagator (no conflicting - W3C span link), tracestate is empty (Datadog headers carry no W3C tracestate), Flags=1. - Baggage is propagated — see fix below. -- `ignore`: returns `nil, nil` — asserted as `sctx == nil, err == nil` - -### Bug fix: baggage lost with restart+extract-first - -`extractIncomingSpanContext` returns immediately when `onlyExtractFirst=true` (after the -first non-baggage propagator succeeds), so the baggage propagator never runs inside the -loop. This caused baggage to be nil in `restart+extract_first` mode — violating the RFC. - -Note: `continue+extract_first` dropping baggage is intentional — you asked for the first -propagator only, baggage wasn't it, done. No RFC governs that case. - -**Fix** (`ddtrace/tracer/textmap.go`): added `extractBaggage(carrier)` helper that runs -only the baggage propagator and returns a `map[string]string`, mirroring the -`pendingBaggage` pattern in `extractIncomingSpanContext`. Inside the `restart` branch, -`incomingCtx.baggage` is used directly in the normal path; `extractBaggage()` is called -only when `onlyExtractFirst=true`. No duplication between the two paths. - -The `onlyExtractFirst` baggage block was initially placed at the top of `Extract()` (before -the `restart` branch), which fixed both `continue` and `restart`. Moved inside `restart` -only after review — `continue+extract_first` dropping baggage is not a bug. - -> **Q: Doesn't this duplicate the work of the OTel extractor?** -> -> No. There is no separate OTel baggage extractor in this path. `getDDorOtelConfig("propagationStyle")` -> (`otel_dd_mappings.go`) only determines *which* propagator types to instantiate (datadog, -> tracecontext, etc.) — it does not add an extra extraction pass. `extractBaggage()` runs -> `propagatorBaggage.Extract()` exactly once, on the same `p.extractors` slice, same carrier. - -### extractFirst + restart interaction - -Covered explicitly in the RFC test matrix (configuration 3: -`DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT=restart` + `DD_TRACE_PROPAGATION_EXTRACT_FIRST=true`). -`extractIncomingSpanContext` returns after the first successful extraction, then `Extract()` -applies the restart behavior to that single context — so exactly one span link is created -and no conflicting-trace span links appear. Verified against .NET implementation. - -### supported_configurations.json - -`DD_TEST_BOOL_ENV`, `DD_TEST_FLOAT_ENV`, `DD_TEST_INT_ENV`, `DD_TEST_STRING_ENV` were -accidentally added to the JSON. Removed. They still appear in `.gen.go` because the -generator also scans Go source for `internal.*Env()` calls — that is correct behavior. -`DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT` fixed to `type: "String", default: "continue"`. diff --git a/_context/references.md b/_context/references.md deleted file mode 100644 index ab3af042fa1..00000000000 --- a/_context/references.md +++ /dev/null @@ -1 +0,0 @@ -@/Users/augusto.deoliveira/Documents/obsidian-vaults/datadog-notes/tasks/26Q1/2026-03-18 (in-progress) Trace Context Propagation Extraction Modes.md diff --git a/_context/rfc-trace-context-extraction-propagation-modes.md b/_context/rfc-trace-context-extraction-propagation-modes.md deleted file mode 100644 index 1c81fc01a38..00000000000 --- a/_context/rfc-trace-context-extraction-propagation-modes.md +++ /dev/null @@ -1,161 +0,0 @@ -# RFC: Trace Context Propagation Extraction Modes - -Author: [Zach Montoya](mailto:zach.montoya@datadoghq.com) -Date: Dec 4, 2024 -Status: Approved -Previous Proposal(s): [Proposal: Fixing Trace Context Propagation Across Organizations](https://docs.google.com/document/d/1htJFBWR4RFpmK1i6wopxkZwlvYu9unkPJnMnTom4Xjo/edit?usp=sharing) -Problem Statement: [Escalation Review: Handling cases where the root service is an unrelated/unknown upstream service](https://docs.google.com/document/d/1zDQDQIRkW8OhTGZ1ua8IsGG_mf5MLdMoUYiAmn9G0pE/edit?tab=t.0) -Feature Parity Dashboard: [https://feature-parity.us1.prod.dog/\#/feature-health?viewType=tests\&features=353](https://feature-parity.us1.prod.dog/#/feature-health?viewType=tests&features=353) - -| Reviewer | Status | Notes | -| :---- | :---- | :---- | -| [Zach Groves](mailto:zach.groves@datadoghq.com) | Approved | | -| [Lucas Pimentel](mailto:lucas.pimentel@datadoghq.com) | Approved | | -| [Mikayla Toffler](mailto:mikayla.toffler@datadoghq.com) | Approved | | -| [Matthew Li](mailto:matthew.li@datadoghq.com) | Approved | | - -# Overview - -## Motivation - -As a result of increased adoption of W3C Trace Context (and possibly just increased adoption of Datadog APM), the APM Ecosystems engineering teams have observed an increasing number of customer escalations where context is propagated from the service of one organization into the service of another organization, leading to a degraded APM experience for the second organization. More context on these issues can be found here: [Escalation Review: Handling cases where the root service is an unrelated/unknown upstream service](https://docs.google.com/document/d/1zDQDQIRkW8OhTGZ1ua8IsGG_mf5MLdMoUYiAmn9G0pE/edit?tab=t.0#heading=h.fna027uqmyrt). - -We should provide our customers with tools to resolve this situation, specifically on the context extraction side as the changes can immediately remedy the situation for the downstream customer. We currently have one workaround which customers have successfully used, which is to configure DD\_TRACE\_PROPAGATION\_STYLE\_EXTRACT=none so that no incoming distributed tracing headers are checked for context extraction, but this solution is not extensible and conflates propagator formats with the context extraction logic. To provide an improved and more extensible user experience, we should add a new capability to the tracing library that separates the handling of incoming trace contexts from the selected propagator formats. - -Note: This work can also be extended to limit context injection from the upstream service, however its effects are not visible to the user so it seems prudent to first focus on the context extraction side. - -## Requirements - -1. By only updating the Datadog configuration of an instrumented service, a user can ignore any incoming trace context information - -# Out of scope - -* This RFC specifically doesn’t focus on an end-to-end UI workflow, and instead focuses on implementing the required fundamental behaviors in our tracing libraries that would enable such changes. -* Benchmarking is also out of scope, as the processing overhead is minimal in comparison to the rest of the context extraction operation. - -# Proposed Solution - -## Configuration - -Environment Variable: DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT -Accepted Values: - -* continue: The tracing library continues the trace from the incoming headers, if present. Also, incoming baggage is propagated. -* restart: The tracing library always starts a new trace with a new trace-id (and a new sampling decision). Context extraction will occur as otherwise configured, except that the local span will no longer have a parent-child span relationship with the incoming trace context, instead it will reference the incoming trace context via a span link. Also, incoming baggage is propagated. -* ignore: The tracing library always starts a new trace with a new trace-id (and a new sampling decision) *without creating any span links.* Also, incoming baggage is dropped. - -Default value: continue -Telemetry Key: DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT - -## Behavior - -This behavior is intricately linked to the [existing context extraction behaviors](https://docs.google.com/document/d/1xacBonCyuVk95D-L1STXGqdLtsOuf0_3jG-138uXHnQ/edit?tab=t.0#bookmark=id.e5pi9gdhj04t). This RFC proposes adding the following text to the existing logic: - -1. \[*Modified*\] Iterate through the propagators in precedence order and for each propagator apply the following logic to build the **incoming trace context** from the distributed tracing headers … -2. \[*New*\] Using the **incoming trace context** from the previous step, set the **local trace context** based on the configured value of DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT: - 1. If DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=continue, use the **incoming trace context** as the **local trace context.** - 2. If DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=restart, create a new **local trace context** containing the following information: - 1. The trace-id and span-id is set in such a way that the tracing library will generate a new trace-id and span-id when starting a new span from this trace context. - 2. A span link with the following properties: - 1. The TraceId, SpanId, TraceFlags, and TraceState fields that correspond to the **incoming trace context** - 2. Additionally, set the following Attributes on the span link: - 1. Key=reason, Value=propagation\_behavior\_extract - 2. Key=context\_headers, Value=\ - 1. Implementation detail: If multiple propagators read the same trace-id and were consolidated into one trace context, listing the first propagator is sufficient rather than listing all of the propagators. - 3. All baggage items from the **incoming trace context** - 1. Note: Span links from the incoming trace context do not need to be carried over. - 3. If DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=ignore, discard the entire **incoming trace context** and create a new **local trace context**. The result will be identical to no distributed tracing headers being found. - 1. Note: As a performance improvement, it is acceptable to skip the iteration of propagators when this configuration is detected since the contents of distributed tracing headers will not affect the results. - -# Testing - -Testing for this feature will be asserted through system-tests, and the new tests can be found [here](https://github.com/DataDog/system-tests/pull/3602). In order to pass the test cases, tracing libraries must implement the DD\_TRACE\_PROPAGATION\_EXTRACT\_FIRST configuration and the ability to add span links for **conflicting trace contexts** where the trace-ids do not match the **incoming trace context**. These behaviors are both outlined [here](https://docs.google.com/document/d/1xacBonCyuVk95D-L1STXGqdLtsOuf0_3jG-138uXHnQ/edit?tab=t.0#heading=h.4jm220mfuuo1). For the tests outlined below, we are assuming that the tracing library under test is configured with default extraction propagators: datadog,tracecontext. - -## Test Behavior - -* The weblog /make\_distant\_call endpoint is invoked with valid Datadog trace context headers, W3C Trace Context headers, and W3C Baggage headers. The Datadog tracing library should automatically perform context extraction and generate a span for this HTTP server request. -* The HTTP endpoint then creates an outbound HTTP request. The request and response headers for this request are sent in the response body for the original endpoint call. The Datadog tracing library should automatically perform context injection based on the local context. - -## Test Cases - -There are two test cases for each configuration: - -1. The distributed tracing headers share the same trace-id and span-id. -2. The distributed tracing headers have unique trace-ids and span-ids. - -Each test case will be run against the following tracing configurations. Due to the overhead of creating new test scenarios in the system-tests infrastructure, this omits testing the explicit configuration DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=continue. However, it is the default behavior, so it is still indirectly tested here. - -1. Default (equivalent to DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=continue) -2. DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=restart -3. DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=restart and DD\_TRACE\_PROPAGATION\_EXTRACT\_FIRST=true -4. DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=ignore - -## Test Results - -* Configuration: Default - 1. Test case: Same trace-id - 1. The HTTP server span has the same trace-id as the incoming Datadog trace context (i.e. the trace is continued). - 2. The HTTP server span has zero span links. - 3. Baggage is propagated. - 2. Test case: Unique trace-ids - 1. The HTTP server span has the same trace-id as the incoming Datadog trace context (i.e. the trace is continued). - 2. The HTTP server span has one span link, corresponding to the **conflicting trace context** in the W3C Trace Context headers. - 3. Baggage is propagated. -* Configuration: DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=restart - 1. Test case: Same trace-id - 1. The HTTP server span has a new trace-id (i.e. a new trace is started). - 2. The HTTP server span has one span link, corresponding to the **incoming trace context** extracted from the Datadog and W3C Trace Context. - 3. Baggage is propagated. - 2. Test case: Unique trace-ids - 1. The HTTP server span has a new trace-id (i.e. a new trace is started). - 2. The HTTP server span has one span link, corresponding to the **incoming trace context** extracted from the Datadog headers. - 3. Baggage is propagated. -* Configuration: DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=ignore - 1. Test case: Same trace-id - 1. The HTTP server span has a new trace-id (i.e. a new trace is started). - 2. The HTTP server span has no span links. - 3. Baggage is not propagated. - 2. Test case: Unique trace-ids - 1. The HTTP server span has a new trace-id (i.e. a new trace is started). - 2. The HTTP server span has no span links. - 3. Baggage is not propagated. -* Configuration: DD\_TRACE\_PROPAGATION\_BEHAVIOR\_EXTRACT=restart and DD\_TRACE\_PROPAGATION\_EXTRACT\_FIRST=true - 1. Test case: Same trace-id - 1. The HTTP server span has a new trace-id (i.e. a new trace is started). - 2. The HTTP server span has one span link, corresponding to the **incoming trace context** extracted from the Datadog headers. - 3. Baggage is propagated. - 2. Test case: Unique trace-ids - 1. The HTTP server span has a new trace-id (i.e. a new trace is started). - 2. The HTTP server span has one span link, corresponding to the **incoming trace context** extracted from the Datadog headers. No **conflicting trace contexts** are identified because the extraction stops immediately, due to the configuration. - 3. Baggage is propagated. - -# Alternative Solutions - -All of the ideas explored in the [Longer Term Options](https://docs.google.com/document/d/1zDQDQIRkW8OhTGZ1ua8IsGG_mf5MLdMoUYiAmn9G0pE/edit?tab=t.0#heading=h.zcquizxtg7t9) section of [Escalation Review: Handling cases where the root service is an unrelated/unknown upstream service](https://docs.google.com/document/u/0/d/1zDQDQIRkW8OhTGZ1ua8IsGG_mf5MLdMoUYiAmn9G0pE/edit) fall into the following camps: - -* Sampling - * Option 1: Sampling Overrides - * Option 5: Implement non-”ParentBased” Samplers in line with OpenTelemetry -* Filter the context extraction - * Option 2: Designate a Head Service / Ignore Incoming Information (this one\!) - * Option 6: Implement configuration to filter extraction by host/headers (suggested in [Future Work](#future-work)) - -## Sampling-based solutions - -Sampling-based solutions don’t entirely solve the issue. With sampling, the customer can override the upstream sampling decision so they regain control of their ingestion, but it leaves their spans in an orphaned state, which causes a degraded UI experience. - -# Future Work {#future-work} - -There are several opportunities to expand on this feature: - -1. **Configuring the injection operation:** With this capability, customers could stop the propagation of their own trace context & baggage so that they don’t expose information to untrusted parties. This also prevents the issues being solved here by having upstream services remove unrecognized distributed tracing headers. However, this would also need to be applied to specific services so that users don’t break up their own traces. -2. **Datadog tracing library dynamically configures the propagation behavior per request:** Since the Datadog tracing library can see incoming/outgoing requests and is likely configured at known ingress/egress points, we may be able to configure this automatically to allow extraction (or injection, see above) to be dynamically configured so that we handle this issue automatically for customers, without them having to set configurations. The capability introduced in this RFC opens the door for those intelligent solutions, such as the alternative solution to filter extraction by host/headers). - -# Questions - -Ask away\! - -# References - -* [Escalation Review: Handling cases where the root service is an unrelated/unknown upstream service](https://docs.google.com/document/u/0/d/1zDQDQIRkW8OhTGZ1ua8IsGG_mf5MLdMoUYiAmn9G0pE/edit) -* [Proposal: Fixing Trace Context Propagation Across Organizations](https://docs.google.com/document/u/0/d/1htJFBWR4RFpmK1i6wopxkZwlvYu9unkPJnMnTom4Xjo/edit) From 7a49acc9d309161b2844cbc943c0ce05a57d0f74 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 10:02:56 +0200 Subject: [PATCH 26/40] refactor: restore getPropagatorName position, implement echotrace assertions, remove stale comments --- contrib/labstack/echo.v4/echotrace_test.go | 127 +++++++++------------ ddtrace/tracer/textmap.go | 34 +++--- ddtrace/tracer/textmap_test.go | 1 - 3 files changed, 74 insertions(+), 88 deletions(-) diff --git a/contrib/labstack/echo.v4/echotrace_test.go b/contrib/labstack/echo.v4/echotrace_test.go index 10110c1a261..a76d657012a 100644 --- a/contrib/labstack/echo.v4/echotrace_test.go +++ b/contrib/labstack/echo.v4/echotrace_test.go @@ -720,29 +720,40 @@ func TestWithErrorCheck(t *testing.T) { } } +// TestPropagationBehaviorExtract is an integration test verifying DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT +// through a real HTTP middleware stack. It checks that the server span's trace ID, parent ID, span +// links, and baggage match the expected behavior for each mode. Echotrace was used because it was +// already available; any HTTP middleware integration would work equivalently. func TestPropagationBehaviorExtract(t *testing.T) { tests := []struct { - name string - propagationBehaviorExtract string - // TODO(human): Add expected behavior fields here + name string + behavior string + wantSameTraceID bool // server span continues the root trace + wantParentID bool // server span has root as parent + wantSpanLinks bool // server span has a span link to the root context + wantBaggage bool // baggage from root is propagated to server span }{ - // { - // name: "continue-default", - // propagationBehaviorExtract: "continue", - // }, { - name: "restart", - propagationBehaviorExtract: "restart", + name: "restart", + behavior: "restart", + wantSameTraceID: false, + wantParentID: false, + wantSpanLinks: true, + wantBaggage: true, }, { - name: "ignore", - propagationBehaviorExtract: "ignore", + name: "ignore", + behavior: "ignore", + wantSameTraceID: false, + wantParentID: false, + wantSpanLinks: false, + wantBaggage: false, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - t.Setenv("DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT", tc.propagationBehaviorExtract) + t.Setenv("DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT", tc.behavior) mt := mocktracer.Start() defer mt.Stop() @@ -753,7 +764,6 @@ func TestPropagationBehaviorExtract(t *testing.T) { return c.NoContent(200) }) - // Create a "root" span simulating incoming trace context root := tracer.StartSpan("incoming-request") root.SetBaggageItem("test-baggage", "baggage-value") @@ -761,66 +771,43 @@ func TestPropagationBehaviorExtract(t *testing.T) { err := tracer.Inject(root.Context(), tracer.HTTPHeadersCarrier(r.Header)) require.NoError(t, err) - fmt.Println("r") - fmt.Println(r.Header) - - w := httptest.NewRecorder() - router.ServeHTTP(w, r) + router.ServeHTTP(httptest.NewRecorder(), r) spans := mt.FinishedSpans() + require.Len(t, spans, 1) + span := spans[0] + + if tc.wantSameTraceID { + assert.Equal(t, root.Context().TraceID(), span.TraceID()) + } else { + assert.NotEqual(t, root.Context().TraceID(), span.TraceID()) + } + + if tc.wantParentID { + assert.Equal(t, root.Context().SpanID(), span.ParentID()) + } else { + assert.Equal(t, uint64(0), span.ParentID()) + } + + links := span.Links() + if tc.wantSpanLinks { + require.Len(t, links, 1) + assert.Equal(t, root.Context().SpanID(), links[0].SpanID) + assert.Equal(t, map[string]string{"reason": "propagation_behavior_extract", "context_headers": "datadog"}, links[0].Attributes) + } else { + assert.Empty(t, links) + } - fmt.Println("root") - fmt.Println(root) - - fmt.Println("spans") - fmt.Println(spans) - - // // verify traces look good - // assert.True(called) - // assert.True(traced) - - // spans := mt.FinishedSpans() - // assert.Len(spans, 1) - - // span := spans[0] - // assert.Equal("http.request", span.OperationName()) - // assert.Equal(ext.SpanTypeWeb, span.Tag(ext.SpanType)) - // assert.Equal("foobar", span.Tag(ext.ServiceName)) - // assert.Equal("echony", span.Tag("test.echo")) - // assert.Contains(span.Tag(ext.ResourceName), "/user/:id") - // assert.Equal("200", span.Tag(ext.HTTPCode)) - // assert.Equal("GET", span.Tag(ext.HTTPMethod)) - // assert.Equal(root.Context().SpanID(), span.ParentID()) - // assert.Equal("labstack/echo.v4", span.Tag(ext.Component)) - // assert.Equal(string(instrumentation.PackageLabstackEchoV4), span.Integration()) - // assert.Equal(ext.SpanKindServer, span.Tag(ext.SpanKind)) - - // assert.Equal("http://example.com/user/123", span.Tag(ext.HTTPURL)) - - // TODO(human): Implement assertions for each propagation behavior. - // - // For "continue": - // - The server span should have the same trace ID as root - // - root.Context().SpanID() should equal span.ParentID() - // - No span links expected (unless conflicting trace contexts) - // - // For "restart": - // - The server span should have a NEW trace ID (different from root) - // - span.ParentID() should be 0 (no parent) - // - SpanLinks should contain one link to the incoming context - // - Baggage should still be propagated - // - // For "ignore": - // - The server span should have a NEW trace ID - // - No span links - // - Baggage should NOT be propagated - // - // Hints: - // - Use span.TraceID() and root.Context().TraceID() to compare trace IDs - // - Use mocktracer.MockSpan(span).SpanLinks() to check span links - // - Check baggage via the request context inside the handler - _ = spans - _ = root + var baggageItems []string + span.Context().ForeachBaggageItem(func(k, v string) bool { + baggageItems = append(baggageItems, k+"="+v) + return true + }) + if tc.wantBaggage { + assert.Contains(t, baggageItems, "test-baggage=baggage-value") + } else { + assert.Empty(t, baggageItems) + } }) } } diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index d12051ac64e..dd952bbad75 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -465,6 +465,23 @@ func (p *chainedPropagator) extractIncomingSpanContext(carrier any) (*SpanContex return ctx, nil } +func getPropagatorName(p Propagator) string { + switch p.(type) { + case *propagator: + return "datadog" + case *propagatorB3: + return "b3multi" + case *propagatorB3SingleHeader: + return "b3" + case *propagatorW3c: + return "tracecontext" + case *propagatorBaggage: + return "baggage" + default: + return "" + } +} + // propagateTracestate will add the tracestate propagating tag to the given // *spanContext. The W3C trace context will be extracted from the provided // carrier. The trace id of this W3C trace context must match the trace id @@ -494,23 +511,6 @@ func (p *propagatorW3c) propagateTracestate(ctx *SpanContext, w3cCtx *SpanContex ctx.isRemote = (w3cCtx.isRemote) } -func getPropagatorName(p Propagator) string { - switch p.(type) { - case *propagator: - return "datadog" - case *propagatorB3: - return "b3multi" - case *propagatorB3SingleHeader: - return "b3" - case *propagatorW3c: - return "tracecontext" - case *propagatorBaggage: - return "baggage" - default: - return "" - } -} - // propagator implements Propagator and injects/extracts span contexts // using datadog headers. Only TextMap carriers are supported. type propagator struct { diff --git a/ddtrace/tracer/textmap_test.go b/ddtrace/tracer/textmap_test.go index 2563c555030..6c66abb0325 100644 --- a/ddtrace/tracer/textmap_test.go +++ b/ddtrace/tracer/textmap_test.go @@ -2013,7 +2013,6 @@ func TestSpanLinks(t *testing.T) { }) } -// RFC test matrix: https://datadoghq.atlassian.net/wiki/x/RFC-trace-context-propagation-extraction-modes func TestPropagationBehaviorExtract(t *testing.T) { s, c := httpmem.ServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(404) From c0d292eb4d86f5ecdcf2ffc7fbd939da3d1e8a72 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 10:03:22 +0200 Subject: [PATCH 27/40] chore: remove checklocksignore from test file --- ddtrace/tracer/textmap_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ddtrace/tracer/textmap_test.go b/ddtrace/tracer/textmap_test.go index 6c66abb0325..9e6c8dae644 100644 --- a/ddtrace/tracer/textmap_test.go +++ b/ddtrace/tracer/textmap_test.go @@ -2052,7 +2052,7 @@ func TestPropagationBehaviorExtract(t *testing.T) { assert.Equal(t, traceIDFrom64Bits(1).value, sctx.traceID.value) assert.Empty(t, sctx.spanLinks) - assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) // +checklocksignore + assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) }) t.Run("continue/different-trace-ids", func(t *testing.T) { @@ -2075,7 +2075,7 @@ func TestPropagationBehaviorExtract(t *testing.T) { Flags: 1, Attributes: map[string]string{"reason": "terminated_context", "context_headers": "tracecontext"}, }, sctx.spanLinks[0]) - assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) // +checklocksignore + assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) }) t.Run("restart", func(t *testing.T) { @@ -2104,7 +2104,7 @@ func TestPropagationBehaviorExtract(t *testing.T) { Flags: 1, Attributes: map[string]string{"reason": "propagation_behavior_extract", "context_headers": "datadog"}, }, sctx.spanLinks[0]) - assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) // +checklocksignore + assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) }) t.Run("restart/different-trace-ids", func(t *testing.T) { @@ -2133,7 +2133,7 @@ func TestPropagationBehaviorExtract(t *testing.T) { Flags: 1, Attributes: map[string]string{"reason": "propagation_behavior_extract", "context_headers": "datadog"}, }, sctx.spanLinks[0]) - assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) // +checklocksignore + assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) }) t.Run("restart/extract-first/same-trace-id", func(t *testing.T) { @@ -2163,7 +2163,7 @@ func TestPropagationBehaviorExtract(t *testing.T) { Flags: 1, Attributes: map[string]string{"reason": "propagation_behavior_extract", "context_headers": "datadog"}, }, sctx.spanLinks[0]) - assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) // +checklocksignore + assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) }) t.Run("restart/extract-first/different-trace-ids", func(t *testing.T) { @@ -2193,7 +2193,7 @@ func TestPropagationBehaviorExtract(t *testing.T) { Flags: 1, Attributes: map[string]string{"reason": "propagation_behavior_extract", "context_headers": "datadog"}, }, sctx.spanLinks[0]) - assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) // +checklocksignore + assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) }) t.Run("ignore/same-trace-id", func(t *testing.T) { From adf76911f23677b036ef75bda36aff9e5f9c3441 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 10:05:13 +0200 Subject: [PATCH 28/40] chore: align test names with RFC wording (unique/same trace-id) --- ddtrace/tracer/textmap_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ddtrace/tracer/textmap_test.go b/ddtrace/tracer/textmap_test.go index 9e6c8dae644..7faf97b3b82 100644 --- a/ddtrace/tracer/textmap_test.go +++ b/ddtrace/tracer/textmap_test.go @@ -2055,7 +2055,7 @@ func TestPropagationBehaviorExtract(t *testing.T) { assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) }) - t.Run("continue/different-trace-ids", func(t *testing.T) { + t.Run("continue/unique-trace-ids", func(t *testing.T) { // Default behavior: trace is continued from the incoming Datadog context (first propagator). // W3C context has a different trace ID, so a terminated_context span link is created for it. tr, err := newTracer(WithHTTPClient(c)) @@ -2078,7 +2078,7 @@ func TestPropagationBehaviorExtract(t *testing.T) { assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) }) - t.Run("restart", func(t *testing.T) { + t.Run("restart/same-trace-id", func(t *testing.T) { // restart mode: a new local trace context is created regardless of the incoming // trace ID. The incoming context is referenced via a span link with // reason=propagation_behavior_extract. Baggage is propagated. @@ -2107,7 +2107,7 @@ func TestPropagationBehaviorExtract(t *testing.T) { assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) }) - t.Run("restart/different-trace-ids", func(t *testing.T) { + t.Run("restart/unique-trace-ids", func(t *testing.T) { // restart mode with unique trace IDs: a new trace is started, span link points to // the Datadog context (first propagator). The W3C conflicting context is not // included because restart applies before conflicting-trace span links are generated. @@ -2166,7 +2166,7 @@ func TestPropagationBehaviorExtract(t *testing.T) { assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) }) - t.Run("restart/extract-first/different-trace-ids", func(t *testing.T) { + t.Run("restart/extract-first/unique-trace-ids", func(t *testing.T) { // restart + extract_first with unique trace IDs: extraction stops after Datadog, // so the W3C conflicting context is never seen. One span link to the Datadog context. // Baggage is still propagated via the explicit baggage pass in Extract(). @@ -2210,7 +2210,7 @@ func TestPropagationBehaviorExtract(t *testing.T) { assert.Nil(t, sctx) }) - t.Run("ignore/different-trace-ids", func(t *testing.T) { + t.Run("ignore/unique-trace-ids", func(t *testing.T) { // ignore mode with unique trace IDs: same result as same-trace-id. // All incoming context is discarded regardless of what headers are present. t.Setenv(headerPropagationBehaviorExtract, "ignore") From cb9a60b3a588dafcd6d17dd1ed449c735080a616 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 10:10:36 +0200 Subject: [PATCH 29/40] test: start spans after Extract to verify produced trace ID and span links --- ddtrace/tracer/textmap_test.go | 76 ++++++++++++++++++++++++---------- 1 file changed, 54 insertions(+), 22 deletions(-) diff --git a/ddtrace/tracer/textmap_test.go b/ddtrace/tracer/textmap_test.go index 7faf97b3b82..3b3e8fcfca5 100644 --- a/ddtrace/tracer/textmap_test.go +++ b/ddtrace/tracer/textmap_test.go @@ -2050,7 +2050,10 @@ func TestPropagationBehaviorExtract(t *testing.T) { require.NoError(t, err) require.NotNil(t, sctx) - assert.Equal(t, traceIDFrom64Bits(1).value, sctx.traceID.value) + span := tr.StartSpan("test", ChildOf(sctx)) + defer span.Finish() + + assert.Equal(t, uint64(1), span.traceID) assert.Empty(t, sctx.spanLinks) assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) }) @@ -2066,7 +2069,10 @@ func TestPropagationBehaviorExtract(t *testing.T) { require.NoError(t, err) require.NotNil(t, sctx) - assert.Equal(t, traceIDFrom64Bits(1).value, sctx.traceID.value) + span := tr.StartSpan("test", ChildOf(sctx)) + defer span.Finish() + + assert.Equal(t, uint64(1), span.traceID) require.Len(t, sctx.spanLinks, 1) assert.Equal(t, SpanLink{ TraceID: 2, @@ -2094,16 +2100,19 @@ func TestPropagationBehaviorExtract(t *testing.T) { require.NoError(t, err) require.NotNil(t, sctx) - assert.True(t, sctx.baggageOnly) // signals spanStart to generate fresh IDs - assert.Equal(t, [16]byte{}, sctx.traceID.value) - require.Len(t, sctx.spanLinks, 1) + span := tr.StartSpan("test", ChildOf(sctx), WithSpanLinks(sctx.SpanLinks())) + defer span.Finish() + + assert.NotEqual(t, uint64(1), span.traceID) + assert.Equal(t, uint64(0), span.parentID) + require.Len(t, span.spanLinks, 1) assert.Equal(t, SpanLink{ TraceID: 1, SpanID: 1, Tracestate: "dd=s:1;p:0000000000000001", Flags: 1, Attributes: map[string]string{"reason": "propagation_behavior_extract", "context_headers": "datadog"}, - }, sctx.spanLinks[0]) + }, span.spanLinks[0]) assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) }) @@ -2123,16 +2132,19 @@ func TestPropagationBehaviorExtract(t *testing.T) { require.NoError(t, err) require.NotNil(t, sctx) - assert.True(t, sctx.baggageOnly) - assert.Equal(t, [16]byte{}, sctx.traceID.value) - require.Len(t, sctx.spanLinks, 1) + span := tr.StartSpan("test", ChildOf(sctx), WithSpanLinks(sctx.SpanLinks())) + defer span.Finish() + + assert.NotEqual(t, uint64(1), span.traceID) + assert.Equal(t, uint64(0), span.parentID) + require.Len(t, span.spanLinks, 1) assert.Equal(t, SpanLink{ TraceID: 1, SpanID: 1, Tracestate: "", Flags: 1, Attributes: map[string]string{"reason": "propagation_behavior_extract", "context_headers": "datadog"}, - }, sctx.spanLinks[0]) + }, span.spanLinks[0]) assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) }) @@ -2153,16 +2165,19 @@ func TestPropagationBehaviorExtract(t *testing.T) { require.NoError(t, err) require.NotNil(t, sctx) - assert.True(t, sctx.baggageOnly) - assert.Equal(t, [16]byte{}, sctx.traceID.value) - require.Len(t, sctx.spanLinks, 1) + span := tr.StartSpan("test", ChildOf(sctx), WithSpanLinks(sctx.SpanLinks())) + defer span.Finish() + + assert.NotEqual(t, uint64(1), span.traceID) + assert.Equal(t, uint64(0), span.parentID) + require.Len(t, span.spanLinks, 1) assert.Equal(t, SpanLink{ TraceID: 1, SpanID: 1, Tracestate: "", Flags: 1, Attributes: map[string]string{"reason": "propagation_behavior_extract", "context_headers": "datadog"}, - }, sctx.spanLinks[0]) + }, span.spanLinks[0]) assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) }) @@ -2183,16 +2198,19 @@ func TestPropagationBehaviorExtract(t *testing.T) { require.NoError(t, err) require.NotNil(t, sctx) - assert.True(t, sctx.baggageOnly) - assert.Equal(t, [16]byte{}, sctx.traceID.value) - require.Len(t, sctx.spanLinks, 1) + span := tr.StartSpan("test", ChildOf(sctx), WithSpanLinks(sctx.SpanLinks())) + defer span.Finish() + + assert.NotEqual(t, uint64(1), span.traceID) + assert.Equal(t, uint64(0), span.parentID) + require.Len(t, span.spanLinks, 1) assert.Equal(t, SpanLink{ TraceID: 1, SpanID: 1, Tracestate: "", Flags: 1, Attributes: map[string]string{"reason": "propagation_behavior_extract", "context_headers": "datadog"}, - }, sctx.spanLinks[0]) + }, span.spanLinks[0]) assert.Equal(t, map[string]string{"key": "val"}, sctx.baggage) }) @@ -2206,8 +2224,15 @@ func TestPropagationBehaviorExtract(t *testing.T) { defer tr.Stop() sctx, err := tr.Extract(sameIDCarrier) - assert.NoError(t, err) - assert.Nil(t, sctx) + require.NoError(t, err) + require.Nil(t, sctx) + + span := tr.StartSpan("test") + defer span.Finish() + + assert.NotEqual(t, uint64(1), span.traceID) + assert.Equal(t, uint64(0), span.parentID) + assert.Empty(t, span.spanLinks) }) t.Run("ignore/unique-trace-ids", func(t *testing.T) { @@ -2219,8 +2244,15 @@ func TestPropagationBehaviorExtract(t *testing.T) { defer tr.Stop() sctx, err := tr.Extract(diffIDCarrier) - assert.NoError(t, err) - assert.Nil(t, sctx) + require.NoError(t, err) + require.Nil(t, sctx) + + span := tr.StartSpan("test") + defer span.Finish() + + assert.NotEqual(t, uint64(1), span.traceID) + assert.Equal(t, uint64(0), span.parentID) + assert.Empty(t, span.spanLinks) }) } From 72b092f89dbb5545e3a2411271bb10beb704b24a Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 10:12:45 +0200 Subject: [PATCH 30/40] chore: explain why WithSpanLinks is needed alongside ChildOf in restart tests --- ddtrace/tracer/textmap_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ddtrace/tracer/textmap_test.go b/ddtrace/tracer/textmap_test.go index 3b3e8fcfca5..c017af3d153 100644 --- a/ddtrace/tracer/textmap_test.go +++ b/ddtrace/tracer/textmap_test.go @@ -2100,6 +2100,8 @@ func TestPropagationBehaviorExtract(t *testing.T) { require.NoError(t, err) require.NotNil(t, sctx) + // WithSpanLinks is required here: ChildOf does not transfer span links from the context. + // StartSpanFromPropagatedContext handles this automatically; plain StartSpan does not. span := tr.StartSpan("test", ChildOf(sctx), WithSpanLinks(sctx.SpanLinks())) defer span.Finish() @@ -2132,6 +2134,8 @@ func TestPropagationBehaviorExtract(t *testing.T) { require.NoError(t, err) require.NotNil(t, sctx) + // WithSpanLinks is required here: ChildOf does not transfer span links from the context. + // StartSpanFromPropagatedContext handles this automatically; plain StartSpan does not. span := tr.StartSpan("test", ChildOf(sctx), WithSpanLinks(sctx.SpanLinks())) defer span.Finish() @@ -2165,6 +2169,8 @@ func TestPropagationBehaviorExtract(t *testing.T) { require.NoError(t, err) require.NotNil(t, sctx) + // WithSpanLinks is required here: ChildOf does not transfer span links from the context. + // StartSpanFromPropagatedContext handles this automatically; plain StartSpan does not. span := tr.StartSpan("test", ChildOf(sctx), WithSpanLinks(sctx.SpanLinks())) defer span.Finish() @@ -2198,6 +2204,8 @@ func TestPropagationBehaviorExtract(t *testing.T) { require.NoError(t, err) require.NotNil(t, sctx) + // WithSpanLinks is required here: ChildOf does not transfer span links from the context. + // StartSpanFromPropagatedContext handles this automatically; plain StartSpan does not. span := tr.StartSpan("test", ChildOf(sctx), WithSpanLinks(sctx.SpanLinks())) defer span.Finish() From 5c35603bd5c13a783ba5a0263aa41bca002cb895 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 10:19:23 +0200 Subject: [PATCH 31/40] chore: format, remove _context from .gitignore --- .gitignore | 1 - contrib/labstack/echo.v4/echotrace_test.go | 12 ++++++------ ddtrace/tracer/textmap.go | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 6b48471552f..47ece489df7 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,3 @@ coverage-*.txt /.claude/* !/.claude/commands/ !/.claude/settings.json -_context/ diff --git a/contrib/labstack/echo.v4/echotrace_test.go b/contrib/labstack/echo.v4/echotrace_test.go index a76d657012a..309b6591278 100644 --- a/contrib/labstack/echo.v4/echotrace_test.go +++ b/contrib/labstack/echo.v4/echotrace_test.go @@ -726,12 +726,12 @@ func TestWithErrorCheck(t *testing.T) { // already available; any HTTP middleware integration would work equivalently. func TestPropagationBehaviorExtract(t *testing.T) { tests := []struct { - name string - behavior string - wantSameTraceID bool // server span continues the root trace - wantParentID bool // server span has root as parent - wantSpanLinks bool // server span has a span link to the root context - wantBaggage bool // baggage from root is propagated to server span + name string + behavior string + wantSameTraceID bool // server span continues the root trace + wantParentID bool // server span has root as parent + wantSpanLinks bool // server span has a span link to the root context + wantBaggage bool // baggage from root is propagated to server span }{ { name: "restart", diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index dd952bbad75..caf9d7d28f3 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -344,7 +344,7 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { baggage = p.extractBaggage(carrier) } if len(baggage) > 0 { - ctx.baggage = maps.Clone(baggage) // +checklocksignore + ctx.baggage = maps.Clone(baggage) // +checklocksignore atomic.StoreUint32(&ctx.hasBaggage, 1) } From 463dda083ec552803257f3d3835b7dff7b72a3fd Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 10:25:19 +0200 Subject: [PATCH 32/40] fix: rename p shadow variable, fix JSON type casing --- ddtrace/tracer/textmap.go | 2 +- internal/env/supported_configurations.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index caf9d7d28f3..a452264ea32 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -327,7 +327,7 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { }, } if trace := incomingCtx.trace; trace != nil { - if p := trace.priority.Load(); p != nil && uint32(*p) > 0 { // +checklocksignore - Initialization time, freshly extracted trace not yet shared. + if prio := trace.priority.Load(); prio != nil && uint32(*prio) > 0 { // +checklocksignore - Initialization time, freshly extracted trace not yet shared. link.Flags = 1 } else { link.Flags = 0 diff --git a/internal/env/supported_configurations.json b/internal/env/supported_configurations.json index 1a64fc0b9e7..3b51cfcd7ab 100644 --- a/internal/env/supported_configurations.json +++ b/internal/env/supported_configurations.json @@ -1410,7 +1410,7 @@ "DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT": [ { "implementation": "A", - "type": "String", + "type": "string", "default": "continue" } ], From cb0dc18bf8a9b2b532bfa71f20d29fe886f5baea Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 10:34:03 +0200 Subject: [PATCH 33/40] test: add benchmarks for restart and ignore extraction modes --- ddtrace/tracer/textmap_test.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/ddtrace/tracer/textmap_test.go b/ddtrace/tracer/textmap_test.go index c017af3d153..b87579db8e1 100644 --- a/ddtrace/tracer/textmap_test.go +++ b/ddtrace/tracer/textmap_test.go @@ -2580,6 +2580,34 @@ func BenchmarkExtractW3C(b *testing.B) { } } +func BenchmarkExtractRestart(b *testing.B) { + b.Setenv(headerPropagationBehaviorExtract, "restart") + propagator := NewPropagator(nil) + carrier := TextMapCarrier(map[string]string{ + DefaultTraceIDHeader: "1123123132131312313123123", + DefaultParentIDHeader: "1212321131231312312312312", + DefaultPriorityHeader: "-1", + }) + b.ResetTimer() + for b.Loop() { + propagator.Extract(carrier) + } +} + +func BenchmarkExtractIgnore(b *testing.B) { + b.Setenv(headerPropagationBehaviorExtract, "ignore") + propagator := NewPropagator(nil) + carrier := TextMapCarrier(map[string]string{ + DefaultTraceIDHeader: "1123123132131312313123123", + DefaultParentIDHeader: "1212321131231312312312312", + DefaultPriorityHeader: "-1", + }) + b.ResetTimer() + for b.Loop() { + propagator.Extract(carrier) + } +} + func FuzzMarshalPropagatingTags(f *testing.F) { f.Add("testA", "testB", "testC", "testD", "testG", "testF") f.Fuzz(func(t *testing.T, key1 string, val1 string, From 5a09f0a0aae3ae556fffba883445293045dd6986 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 11:09:46 +0200 Subject: [PATCH 34/40] test: remove unhelpful restart/ignore benchmarks --- ddtrace/tracer/textmap_test.go | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/ddtrace/tracer/textmap_test.go b/ddtrace/tracer/textmap_test.go index b87579db8e1..c017af3d153 100644 --- a/ddtrace/tracer/textmap_test.go +++ b/ddtrace/tracer/textmap_test.go @@ -2580,34 +2580,6 @@ func BenchmarkExtractW3C(b *testing.B) { } } -func BenchmarkExtractRestart(b *testing.B) { - b.Setenv(headerPropagationBehaviorExtract, "restart") - propagator := NewPropagator(nil) - carrier := TextMapCarrier(map[string]string{ - DefaultTraceIDHeader: "1123123132131312313123123", - DefaultParentIDHeader: "1212321131231312312312312", - DefaultPriorityHeader: "-1", - }) - b.ResetTimer() - for b.Loop() { - propagator.Extract(carrier) - } -} - -func BenchmarkExtractIgnore(b *testing.B) { - b.Setenv(headerPropagationBehaviorExtract, "ignore") - propagator := NewPropagator(nil) - carrier := TextMapCarrier(map[string]string{ - DefaultTraceIDHeader: "1123123132131312313123123", - DefaultParentIDHeader: "1212321131231312312312312", - DefaultPriorityHeader: "-1", - }) - b.ResetTimer() - for b.Loop() { - propagator.Extract(carrier) - } -} - func FuzzMarshalPropagatingTags(f *testing.F) { f.Add("testA", "testB", "testC", "testD", "testG", "testF") f.Fuzz(func(t *testing.T, key1 string, val1 string, From 9ce00bed399be298d5161a94e5e9b0fc7a95cf07 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 14:26:39 +0200 Subject: [PATCH 35/40] test: add continue case to echotrace propagation behavior test --- contrib/labstack/echo.v4/echotrace_test.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/contrib/labstack/echo.v4/echotrace_test.go b/contrib/labstack/echo.v4/echotrace_test.go index 309b6591278..27864aa6221 100644 --- a/contrib/labstack/echo.v4/echotrace_test.go +++ b/contrib/labstack/echo.v4/echotrace_test.go @@ -733,6 +733,14 @@ func TestPropagationBehaviorExtract(t *testing.T) { wantSpanLinks bool // server span has a span link to the root context wantBaggage bool // baggage from root is propagated to server span }{ + { + name: "continue", + behavior: "continue", + wantSameTraceID: true, + wantParentID: true, + wantSpanLinks: false, + wantBaggage: true, + }, { name: "restart", behavior: "restart", @@ -778,9 +786,9 @@ func TestPropagationBehaviorExtract(t *testing.T) { span := spans[0] if tc.wantSameTraceID { - assert.Equal(t, root.Context().TraceID(), span.TraceID()) + assert.Equal(t, root.Context().TraceIDLower(), span.TraceID()) } else { - assert.NotEqual(t, root.Context().TraceID(), span.TraceID()) + assert.NotEqual(t, root.Context().TraceIDLower(), span.TraceID()) } if tc.wantParentID { From 920565f95f22bc7955db4019dd99ba9fd5c73374 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 14:48:10 +0200 Subject: [PATCH 36/40] refactor: collapse duplicate generateUpperTraceID calls in newSpanContext --- ddtrace/tracer/spancontext.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/ddtrace/tracer/spancontext.go b/ddtrace/tracer/spancontext.go index d33eb922ac8..0933e6fcc00 100644 --- a/ddtrace/tracer/spancontext.go +++ b/ddtrace/tracer/spancontext.go @@ -267,15 +267,9 @@ func newSpanContext(span *Span, parent *SpanContext) *SpanContext { context.setBaggageItem(k, v) return true }) - } else if traceID128BitEnabled.Load() { - // add 128 bit trace id, if enabled, formatted as big-endian: - // <32-bit unix seconds> <32 bits of zero> <64 random bits> - id128 := time.Duration(span.start) / time.Second - // casting from int64 -> uint32 should be safe since the start time won't be - // negative, and the seconds should fit within 32-bits for the foreseeable future. - // (We only want 32 bits of time, then the rest is zero) - tUp := uint64(uint32(id128)) << 32 // We need the time at the upper 32 bits of the uint - context.traceID.SetUpper(tUp) + } + if (parent == nil || parent.baggageOnly) && traceID128BitEnabled.Load() { // +checklocksignore - Read-only after init. + context.traceID.SetUpper(generateUpperTraceID(span, context)) } if context.trace == nil { context.trace = newTrace() @@ -293,6 +287,17 @@ func newSpanContext(span *Span, parent *SpanContext) *SpanContext { return context } +// generateUpperTraceID generates the upper 32 bits of the trace ID. +func generateUpperTraceID(span *Span, context *SpanContext) uint64 { + // add 128 bit trace id, if enabled, formatted as big-endian: + // <32-bit unix seconds> <32 bits of zero> <64 random bits> + id128 := time.Duration(span.start) / time.Second + // casting from int64 -> uint32 should be safe since the start time won't be + // negative, and the seconds should fit within 32-bits for the foreseeable future. + // (We only want 32 bits of time, then the rest is zero) + return uint64(uint32(id128)) << 32 // We need the time at the upper 32 bits of the uint +} + // SpanID implements ddtrace.SpanContext. func (c *SpanContext) SpanID() uint64 { if c == nil { From ddd05adef9f33df967ad29cb0edaf209765a3f88 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 14:50:59 +0200 Subject: [PATCH 37/40] refactor --- ddtrace/tracer/spancontext.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/ddtrace/tracer/spancontext.go b/ddtrace/tracer/spancontext.go index 0933e6fcc00..9f0e55c4a51 100644 --- a/ddtrace/tracer/spancontext.go +++ b/ddtrace/tracer/spancontext.go @@ -268,8 +268,17 @@ func newSpanContext(span *Span, parent *SpanContext) *SpanContext { return true }) } + // We generate a new upper trace ID when the trace is brand new (no parent) + // or when the parent is baggage only, since baggage only parents should + // not propagate their trace IDs if (parent == nil || parent.baggageOnly) && traceID128BitEnabled.Load() { // +checklocksignore - Read-only after init. - context.traceID.SetUpper(generateUpperTraceID(span, context)) + // add 128 bit trace id, if enabled, formatted as big-endian: + // <32-bit unix seconds> <32 bits of zero> <64 random bits> + id128 := time.Duration(span.start) / time.Second + // casting from int64 -> uint32 should be safe since the start time won't be + // negative, and the seconds should fit within 32-bits for the foreseeable future. + // (We only want 32 bits of time, then the rest is zero) + context.traceID.SetUpper(uint64(uint32(id128)) << 32) // We need the time at the upper 32 bits of the uint } if context.trace == nil { context.trace = newTrace() @@ -287,17 +296,6 @@ func newSpanContext(span *Span, parent *SpanContext) *SpanContext { return context } -// generateUpperTraceID generates the upper 32 bits of the trace ID. -func generateUpperTraceID(span *Span, context *SpanContext) uint64 { - // add 128 bit trace id, if enabled, formatted as big-endian: - // <32-bit unix seconds> <32 bits of zero> <64 random bits> - id128 := time.Duration(span.start) / time.Second - // casting from int64 -> uint32 should be safe since the start time won't be - // negative, and the seconds should fit within 32-bits for the foreseeable future. - // (We only want 32 bits of time, then the rest is zero) - return uint64(uint32(id128)) << 32 // We need the time at the upper 32 bits of the uint -} - // SpanID implements ddtrace.SpanContext. func (c *SpanContext) SpanID() uint64 { if c == nil { From b16c4d8e383658890a2b91d7b1afd796ceab7130 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 14:52:29 +0200 Subject: [PATCH 38/40] refactor: use named constants for propagation behavior values --- ddtrace/tracer/textmap.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index a452264ea32..45f294f0f40 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -80,6 +80,10 @@ const ( // decision. No span links are created. Baggage is dropped. headerPropagationBehaviorExtract = "DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT" + propagationBehaviorExtractContinue = "continue" + propagationBehaviorExtractRestart = "restart" + propagationBehaviorExtractIgnore = "ignore" + headerPropagationExtractFirst = "DD_TRACE_PROPAGATION_EXTRACT_FIRST" headerPropagationStyleInject = "DD_TRACE_PROPAGATION_STYLE_INJECT" headerPropagationStyleExtract = "DD_TRACE_PROPAGATION_STYLE_EXTRACT" @@ -181,13 +185,13 @@ func NewPropagator(cfg *PropagatorConfig, propagators ...Propagator) Propagator cp.onlyExtractFirst = internal.BoolEnv(headerPropagationExtractFirst, false) cp.propagationBehaviorExtract = env.Get(headerPropagationBehaviorExtract) switch cp.propagationBehaviorExtract { - case "continue", "restart", "ignore": + case propagationBehaviorExtractContinue, propagationBehaviorExtractRestart, propagationBehaviorExtractIgnore: // valid default: if cp.propagationBehaviorExtract != "" { log.Warn("unrecognized propagation behavior: %s. Defaulting to continue", cp.propagationBehaviorExtract) } - cp.propagationBehaviorExtract = "continue" + cp.propagationBehaviorExtract = propagationBehaviorExtractContinue } if len(propagators) > 0 { cp.injectors = propagators @@ -300,7 +304,7 @@ func (p *chainedPropagator) Inject(spanCtx *SpanContext, carrier any) error { // subsequent trace context has conflicting trace information, such information will // be relayed in the returned SpanContext with a SpanLink. func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { - if p.propagationBehaviorExtract == "ignore" { + if p.propagationBehaviorExtract == propagationBehaviorExtractIgnore { return nil, nil } @@ -312,7 +316,7 @@ func (p *chainedPropagator) Extract(carrier any) (*SpanContext, error) { // "restart" propagation behavior starts a new trace with a new trace ID // and sampling decision. The incoming context is referenced via a span // link. Baggage is propagated. - if p.propagationBehaviorExtract == "restart" { + if p.propagationBehaviorExtract == propagationBehaviorExtractRestart { ctx := &SpanContext{ baggageOnly: true, // signals spanStart to generate new traceID/spanID } From 5bfb26ba2da86af5dc073e6d534dfccc7b279bde Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 14:55:22 +0200 Subject: [PATCH 39/40] revert --- ddtrace/tracer/spancontext.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ddtrace/tracer/spancontext.go b/ddtrace/tracer/spancontext.go index 9f0e55c4a51..96211f2b130 100644 --- a/ddtrace/tracer/spancontext.go +++ b/ddtrace/tracer/spancontext.go @@ -278,7 +278,8 @@ func newSpanContext(span *Span, parent *SpanContext) *SpanContext { // casting from int64 -> uint32 should be safe since the start time won't be // negative, and the seconds should fit within 32-bits for the foreseeable future. // (We only want 32 bits of time, then the rest is zero) - context.traceID.SetUpper(uint64(uint32(id128)) << 32) // We need the time at the upper 32 bits of the uint + tUp := uint64(uint32(id128)) << 32 // We need the time at the upper 32 bits of the uint + context.traceID.SetUpper(tUp) } if context.trace == nil { context.trace = newTrace() From 50b4212ff6a7da39a861df7ca5db38cd4e027257 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 16 Apr 2026 15:00:15 +0200 Subject: [PATCH 40/40] docs: improve echotrace_test comment --- contrib/labstack/echo.v4/echotrace_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/labstack/echo.v4/echotrace_test.go b/contrib/labstack/echo.v4/echotrace_test.go index 27864aa6221..b012118af46 100644 --- a/contrib/labstack/echo.v4/echotrace_test.go +++ b/contrib/labstack/echo.v4/echotrace_test.go @@ -721,8 +721,8 @@ func TestWithErrorCheck(t *testing.T) { } // TestPropagationBehaviorExtract is an integration test verifying DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT -// through a real HTTP middleware stack. It checks that the server span's trace ID, parent ID, span -// links, and baggage match the expected behavior for each mode. Echotrace was used because it was +// through a real HTTP middleware stack. +// Echotrace was used because it was // already available; any HTTP middleware integration would work equivalently. func TestPropagationBehaviorExtract(t *testing.T) { tests := []struct {