From 6e322598f64946b45ceac851508b7dc494231fbf Mon Sep 17 00:00:00 2001 From: STHITAPRAJNAS Date: Tue, 24 Mar 2026 02:43:13 +0000 Subject: [PATCH 1/2] fix(tracing): instrument FastAPI ASGI layer for inbound traceparent propagation Fixes #4767. When get_fast_api_app() is used in production behind an OTel-instrumented caller (e.g. a Next.js service using @opentelemetry/sdk-node), every inbound request carried a W3C traceparent header that ADK silently discarded. The TracerProvider and W3C propagator were already wired up correctly by _setup_telemetry(), but without an ASGI-level hook to extract the header, each request spawned a new trace root instead of continuing the caller's trace. The fix calls FastAPIInstrumentor.instrument_app(app) immediately after the FastAPI instance is created, but only when an OTel export pipeline is actually active (OTLP env vars or otel_to_cloud=True). The instrumentation is applied before the CORS and origin-check middleware are registered so that Starlette's reverse-registration order leaves the security wrappers outermost in the stack. The call is best-effort: if opentelemetry-instrumentation-fastapi is absent a debug-level message is emitted and the server starts normally. The package is added to the otel-gcp extras so users who install google-adk[otel-gcp] get end-to-end distributed tracing out of the box without any manual post-instrumentation step. opentelemetry-instrumentation-fastapi is also added to the test extras so the new unit tests can import it directly. https://github.com/google/adk-python/issues/4767 --- pyproject.toml | 8 +- src/google/adk/cli/adk_web_server.py | 21 +++ tests/unittests/cli/test_fast_api.py | 186 +++++++++++++++++++++++++++ 3 files changed, 214 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2789bcf82a..9e989385ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,6 +129,7 @@ test = [ "litellm>=1.75.5, <=1.82.6", # For LiteLLM tests. Upper bound pinned: versions 1.82.7+ compromised in supply chain attack. "llama-index-readers-file>=0.4.0", # For retrieval tests "openai>=1.100.2", # For LiteLLM + "opentelemetry-instrumentation-fastapi>=0.48b0, <1.0.0", "opentelemetry-instrumentation-google-genai>=0.3b0, <1.0.0", "pypika>=0.50.0", # For crewai->chromadb dependency "pytest-asyncio>=0.25.0", @@ -168,7 +169,12 @@ extensions = [ "toolbox-adk>=1.0.0, <2.0.0", # For tools.toolbox_toolset.ToolboxToolset ] -otel-gcp = ["opentelemetry-instrumentation-google-genai>=0.6b0, <1.0.0"] +otel-gcp = [ + # go/keep-sorted start + "opentelemetry-instrumentation-fastapi>=0.48b0, <1.0.0", + "opentelemetry-instrumentation-google-genai>=0.6b0, <1.0.0", + # go/keep-sorted end +] toolbox = ["toolbox-adk>=1.0.0, <2.0.0"] diff --git a/src/google/adk/cli/adk_web_server.py b/src/google/adk/cli/adk_web_server.py index 927cd7ad03..5b9e53ec0a 100644 --- a/src/google/adk/cli/adk_web_server.py +++ b/src/google/adk/cli/adk_web_server.py @@ -910,6 +910,27 @@ async def internal_lifespan(app: FastAPI): # Run the FastAPI server. app = FastAPI(lifespan=internal_lifespan) + # When an OTel pipeline is active, instrument the ASGI layer so that + # inbound W3C traceparent headers are extracted and agent spans are + # correctly parented to the caller's trace. Without this, every request + # starts a new trace root even though the TracerProvider and W3C propagator + # are already configured. Applied before other middleware so that + # security/CORS wrappers are outermost in the stack (Starlette applies + # middleware in reverse-registration order). + if otel_to_cloud or _otel_env_vars_enabled(): + try: + from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor # pylint: disable=g-import-not-at-top + + FastAPIInstrumentor.instrument_app(app) + logger.debug("FastAPI OpenTelemetry instrumentation enabled.") + except ImportError: + logger.debug( + "opentelemetry-instrumentation-fastapi is not installed; inbound" + " traceparent headers will not be propagated into agent spans." + " Install it alongside the OTel extras: pip install" + " opentelemetry-instrumentation-fastapi" + ) + has_configured_allowed_origins = bool(allow_origins) if allow_origins: literal_origins, combined_regex = _parse_cors_origins(allow_origins) diff --git a/tests/unittests/cli/test_fast_api.py b/tests/unittests/cli/test_fast_api.py index 15bc908ddb..2209235b5f 100755 --- a/tests/unittests/cli/test_fast_api.py +++ b/tests/unittests/cli/test_fast_api.py @@ -1950,5 +1950,191 @@ async def run_async_session_not_found(self, **kwargs): assert "Session not found" in response.json()["detail"] +# --------------------------------------------------------------------------- +# OpenTelemetry FastAPI instrumentation tests +# --------------------------------------------------------------------------- + +_OTEL_ENV_ENABLED = "google.adk.cli.adk_web_server._otel_env_vars_enabled" +_SETUP_TELEMETRY = "google.adk.cli.adk_web_server._setup_telemetry" +_FASTAPI_INSTRUMENTOR = "opentelemetry.instrumentation.fastapi.FastAPIInstrumentor" + + +def _make_otel_test_client( + mock_session_service, + mock_artifact_service, + mock_memory_service, + mock_agent_loader, + mock_eval_sets_manager, + mock_eval_set_results_manager, + **app_kwargs, +): + """Like _create_test_client but suppresses real OTel provider setup.""" + defaults = dict( + agents_dir=".", + web=False, + session_service_uri="", + artifact_service_uri="", + memory_service_uri="", + a2a=False, + host="127.0.0.1", + port=8000, + ) + defaults.update(app_kwargs) + with ( + patch.object(signal, "signal", autospec=True, return_value=None), + patch.object( + fast_api_module, + "create_session_service_from_options", + autospec=True, + return_value=mock_session_service, + ), + patch.object( + fast_api_module, + "create_artifact_service_from_options", + autospec=True, + return_value=mock_artifact_service, + ), + patch.object( + fast_api_module, + "create_memory_service_from_options", + autospec=True, + return_value=mock_memory_service, + ), + patch.object( + fast_api_module, + "AgentLoader", + autospec=True, + return_value=mock_agent_loader, + ), + patch.object( + fast_api_module, + "LocalEvalSetsManager", + autospec=True, + return_value=mock_eval_sets_manager, + ), + patch.object( + fast_api_module, + "LocalEvalSetResultsManager", + autospec=True, + return_value=mock_eval_set_results_manager, + ), + # Suppress real OTel provider / exporter setup so tests stay isolated. + patch(_SETUP_TELEMETRY), + ): + app = get_fast_api_app(**defaults) + return TestClient(app) + + +def test_fastapi_instrumented_when_otlp_env_var_set( + mock_session_service, + mock_artifact_service, + mock_memory_service, + mock_agent_loader, + mock_eval_sets_manager, + mock_eval_set_results_manager, +): + """FastAPIInstrumentor.instrument_app is called when an OTLP env var is set.""" + with ( + patch(_OTEL_ENV_ENABLED, return_value=True), + patch(_FASTAPI_INSTRUMENTOR) as mock_instrumentor_cls, + ): + _make_otel_test_client( + mock_session_service, + mock_artifact_service, + mock_memory_service, + mock_agent_loader, + mock_eval_sets_manager, + mock_eval_set_results_manager, + ) + + mock_instrumentor_cls.instrument_app.assert_called_once() + + +def test_fastapi_instrumented_when_otel_to_cloud_enabled( + mock_session_service, + mock_artifact_service, + mock_memory_service, + mock_agent_loader, + mock_eval_sets_manager, + mock_eval_set_results_manager, +): + """FastAPIInstrumentor.instrument_app is called when otel_to_cloud=True.""" + with ( + # otel_to_cloud=True triggers the instrumentation regardless of env vars. + patch(_OTEL_ENV_ENABLED, return_value=False), + patch(_FASTAPI_INSTRUMENTOR) as mock_instrumentor_cls, + ): + _make_otel_test_client( + mock_session_service, + mock_artifact_service, + mock_memory_service, + mock_agent_loader, + mock_eval_sets_manager, + mock_eval_set_results_manager, + otel_to_cloud=True, + ) + + mock_instrumentor_cls.instrument_app.assert_called_once() + + +def test_fastapi_not_instrumented_without_otel_config( + mock_session_service, + mock_artifact_service, + mock_memory_service, + mock_agent_loader, + mock_eval_sets_manager, + mock_eval_set_results_manager, +): + """FastAPIInstrumentor.instrument_app is NOT called when OTel is not configured.""" + with ( + patch(_OTEL_ENV_ENABLED, return_value=False), + patch(_FASTAPI_INSTRUMENTOR) as mock_instrumentor_cls, + ): + _make_otel_test_client( + mock_session_service, + mock_artifact_service, + mock_memory_service, + mock_agent_loader, + mock_eval_sets_manager, + mock_eval_set_results_manager, + otel_to_cloud=False, + ) + + mock_instrumentor_cls.instrument_app.assert_not_called() + + +def test_missing_fastapi_instrumentor_does_not_prevent_startup( + mock_session_service, + mock_artifact_service, + mock_memory_service, + mock_agent_loader, + mock_eval_sets_manager, + mock_eval_set_results_manager, +): + """App starts normally when opentelemetry-instrumentation-fastapi is absent.""" + import sys + + # Simulate the package not being installed by removing it from sys.modules + # and making the import raise ImportError. + with ( + patch(_OTEL_ENV_ENABLED, return_value=True), + patch.dict( + sys.modules, + {"opentelemetry.instrumentation.fastapi": None}, + ), + ): + client = _make_otel_test_client( + mock_session_service, + mock_artifact_service, + mock_memory_service, + mock_agent_loader, + mock_eval_sets_manager, + mock_eval_set_results_manager, + ) + + response = client.get("/health") + assert response.status_code == 200 + + if __name__ == "__main__": pytest.main(["-xvs", __file__]) From 8df3dba818fec6d229be13fc3b0830de50ee1ff1 Mon Sep 17 00:00:00 2001 From: STHITAPRAJNAS Date: Wed, 25 Mar 2026 01:19:54 +0000 Subject: [PATCH 2/2] style: apply pyink formatting to test_fast_api.py --- tests/unittests/cli/test_fast_api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unittests/cli/test_fast_api.py b/tests/unittests/cli/test_fast_api.py index 2209235b5f..fde5fd6e9e 100755 --- a/tests/unittests/cli/test_fast_api.py +++ b/tests/unittests/cli/test_fast_api.py @@ -1956,7 +1956,9 @@ async def run_async_session_not_found(self, **kwargs): _OTEL_ENV_ENABLED = "google.adk.cli.adk_web_server._otel_env_vars_enabled" _SETUP_TELEMETRY = "google.adk.cli.adk_web_server._setup_telemetry" -_FASTAPI_INSTRUMENTOR = "opentelemetry.instrumentation.fastapi.FastAPIInstrumentor" +_FASTAPI_INSTRUMENTOR = ( + "opentelemetry.instrumentation.fastapi.FastAPIInstrumentor" +) def _make_otel_test_client(