diff --git a/rust/observability/src/otel.rs b/rust/observability/src/otel.rs index 1d79437..fc15880 100644 --- a/rust/observability/src/otel.rs +++ b/rust/observability/src/otel.rs @@ -131,6 +131,17 @@ fn build_resource(options: &SetupOtelOptions) -> Resource { } fn build_and_install(options: SetupOtelOptions) -> OtelSdkHandle { + // Install the global W3C Trace Context propagator so traceparent/tracestate + // headers are injected on outbound calls and extracted on inbound ones (see + // the `tower` / `reqwest-middleware` integrations). This is always-on and + // cheap — it just registers the text-map propagator the integrations consult + // via `opentelemetry::global::get_text_map_propagator`. Without it that + // global defaults to a no-op propagator and traces never link across + // services. (SMOODEV-2024) + opentelemetry::global::set_text_map_propagator( + opentelemetry_sdk::propagation::TraceContextPropagator::new(), + ); + let resource = build_resource(&options); let tracer_provider = match build_span_exporter(&options) { diff --git a/rust/observability/src/reqwest_mw.rs b/rust/observability/src/reqwest_mw.rs index c6bae36..5767d99 100644 --- a/rust/observability/src/reqwest_mw.rs +++ b/rust/observability/src/reqwest_mw.rs @@ -63,7 +63,7 @@ impl OtelReqwestMiddleware { impl Middleware for OtelReqwestMiddleware { async fn handle( &self, - req: Request, + mut req: Request, extensions: &mut http::Extensions, next: Next<'_>, ) -> reqwest_middleware::Result { @@ -93,6 +93,21 @@ impl Middleware for OtelReqwestMiddleware { // `with_context` (vs `attach`) keeps the future `Send` — `ContextGuard` // is `!Send` and can't be held across this `.await`. let cx = Context::current_with_span(span); + + // Inject W3C trace context (traceparent/tracestate) into the outbound + // request headers so the downstream service can EXTRACT it and continue + // this trace instead of starting a disconnected root. We inject the + // freshly-created client span's context, so downstream server spans + // parent off this CLIENT span — yielding a single linked trace across + // the service hop. If no propagator is installed (it always is once + // `setup_otel_sdk` ran) this is a no-op. (SMOODEV-2024) + opentelemetry::global::get_text_map_propagator(|propagator| { + propagator.inject_context( + &cx, + &mut opentelemetry_http::HeaderInjector(req.headers_mut()), + ); + }); + let result = next.run(req, extensions).with_context(cx.clone()).await; let span = cx.span(); @@ -233,4 +248,65 @@ mod tests { fn middleware_is_constructible() { let _m = OtelReqwestMiddleware::new(); } + + // --- W3C trace context propagation (SMOODEV-2024) ---------------------- + // + // These exercise the inject side of the middleware's behavior directly via + // the global propagator, independent of a live reqwest call. The + // inbound/extract counterpart lives in `tower.rs`; a full inject->extract + // round-trip is covered there too. + + use opentelemetry::propagation::TextMapPropagator; + use opentelemetry::trace::{SpanContext, TraceContextExt, TraceState}; + use opentelemetry::{Context, SpanId, TraceFlags, TraceId}; + use opentelemetry_sdk::propagation::TraceContextPropagator; + + /// A context carrying a known sampled remote span context. + fn known_context() -> (Context, TraceId, SpanId) { + let trace_id = TraceId::from_hex("0af7651916cd43dd8448eb211c80319c").unwrap(); + let span_id = SpanId::from_hex("b7ad6b7169203331").unwrap(); + let sc = SpanContext::new( + trace_id, + span_id, + TraceFlags::SAMPLED, + true, + TraceState::default(), + ); + let cx = Context::new().with_remote_span_context(sc); + (cx, trace_id, span_id) + } + + #[test] + fn inject_writes_traceparent_header() { + let propagator = TraceContextPropagator::new(); + let (cx, trace_id, span_id) = known_context(); + + let mut headers = http::HeaderMap::new(); + propagator.inject_context(&cx, &mut opentelemetry_http::HeaderInjector(&mut headers)); + + let traceparent = headers + .get("traceparent") + .expect("traceparent header injected") + .to_str() + .unwrap(); + // W3C format: 00--- + assert_eq!( + traceparent, + format!("00-{trace_id}-{span_id}-01"), + "traceparent encodes the active span context" + ); + } + + #[test] + fn inject_with_no_active_context_is_noop() { + let propagator = TraceContextPropagator::new(); + // An empty context has no valid span context, so nothing is injected. + let cx = Context::new(); + let mut headers = http::HeaderMap::new(); + propagator.inject_context(&cx, &mut opentelemetry_http::HeaderInjector(&mut headers)); + assert!( + headers.get("traceparent").is_none(), + "no traceparent header when there is no active span context" + ); + } } diff --git a/rust/observability/src/tower.rs b/rust/observability/src/tower.rs index 3bde41d..a307843 100644 --- a/rust/observability/src/tower.rs +++ b/rust/observability/src/tower.rs @@ -149,8 +149,20 @@ where .unwrap_or_else(|| req.uri().path().to_owned()); let protocol = http_version_str(req.version()); + // Extract any upstream W3C trace context (traceparent/tracestate) the + // caller injected into the request headers. When present, the server + // span we open below CONTINUES that trace (parents off the remote span) + // instead of starting a disconnected root — this is what links traces + // across service hops. With no traceparent header (or no propagator + // installed) this yields an empty context and the span becomes a fresh + // root, exactly as before. (SMOODEV-2024) + let parent_cx = opentelemetry::global::get_text_map_propagator(|propagator| { + propagator.extract(&opentelemetry_http::HeaderExtractor(req.headers())) + }); + // Open a server span on the GLOBAL tracer (the one setup_otel_sdk // installed). If no provider is installed it's a cheap no-op span. + // `with_parent_context` ties it to the extracted upstream context. let tracer = opentelemetry::global::tracer(TRACER_NAME); let span: BoxedSpan = tracer .span_builder(format!("{method} {route}")) @@ -160,7 +172,7 @@ where KeyValue::new("url.path", req.uri().path().to_owned()), KeyValue::new("network.protocol.version", protocol), ]) - .start(&tracer); + .start_with_context(&tracer, &parent_cx); // Build the request context that holds the server span. The inner // service is called WITHOUT the context attached here — `ResponseFuture` @@ -327,4 +339,64 @@ mod tests { assert_eq!(http_version_str(http::Version::HTTP_11), "1.1"); assert_eq!(http_version_str(http::Version::HTTP_2), "2"); } + + // --- W3C trace context propagation (SMOODEV-2024) ---------------------- + + use opentelemetry::propagation::TextMapPropagator; + use opentelemetry::trace::{SpanContext, TraceState}; + use opentelemetry::{SpanId, TraceFlags, TraceId}; + use opentelemetry_sdk::propagation::TraceContextPropagator; + + #[test] + fn inject_extract_round_trip_preserves_trace_id() { + let propagator = TraceContextPropagator::new(); + + let trace_id = TraceId::from_hex("0af7651916cd43dd8448eb211c80319c").unwrap(); + let span_id = SpanId::from_hex("b7ad6b7169203331").unwrap(); + let sc = SpanContext::new( + trace_id, + span_id, + TraceFlags::SAMPLED, + true, + TraceState::default(), + ); + let cx = Context::new().with_remote_span_context(sc); + + // Inject into a fresh HeaderMap (the outbound side). + let mut headers = http::HeaderMap::new(); + propagator.inject_context(&cx, &mut opentelemetry_http::HeaderInjector(&mut headers)); + + // Extract it back out (the inbound side, as the tower layer does). + let extracted = propagator.extract(&opentelemetry_http::HeaderExtractor(&headers)); + let extracted_sc = extracted.span().span_context().clone(); + + assert!(extracted_sc.is_valid(), "extracted context is valid"); + assert_eq!( + extracted_sc.trace_id(), + trace_id, + "trace_id survives the round-trip" + ); + assert_eq!( + extracted_sc.span_id(), + span_id, + "parent span_id survives the round-trip" + ); + assert!( + extracted_sc.is_remote(), + "extracted context is marked remote" + ); + } + + #[test] + fn extract_with_no_headers_yields_invalid_context() { + let propagator = TraceContextPropagator::new(); + let headers = http::HeaderMap::new(); + let extracted = propagator.extract(&opentelemetry_http::HeaderExtractor(&headers)); + // With no traceparent header the extracted span context is invalid, so + // the server span the layer opens becomes a fresh root. + assert!( + !extracted.span().span_context().is_valid(), + "no headers -> invalid (empty) span context" + ); + } }