From 31b6da5ef44efc80981c29e8449a680c30bd4c4f Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Thu, 23 Jul 2026 21:03:30 -0400 Subject: [PATCH] fix(opentelemetry): stop OTLP logs export from re-exporting the tracer's own logs When DD_LOGS_OTEL_ENABLED=true, the OpenTelemetry SDK attaches a LoggingHandler to the Python root logger that captures every propagated log record. Because ddtrace's internal loggers (ddtrace.*) and the OpenTelemetry exporter's loggers (opentelemetry.*) propagate to root, the exporter's own log records -- the per-batch export debug line and the exporter's export-failure warnings/errors -- were captured, exported, and captured again, producing a self-amplifying export loop (triggered by DD_TRACE_DEBUG or by export failures at the default log level). Attach a logging.Filter to the OTLP LoggingHandler that rejects records whose logger name is in the ddtrace or opentelemetry namespaces, so the tracer never feeds its own telemetry-pipeline logs back into its own log export. Application logs are still captured and exported. The filter is attached only to the handler ddtrace itself installs (identified by diffing the root logger's handlers), so application-configured handlers are left untouched. Because it binds to the handler instance, it also persists across logging.basicConfig reconfiguration, which the OpenTelemetry SDK patches to re-add that same handler instance (dictConfig/fileConfig are not patched by the SDK). Co-Authored-By: Claude Opus 4.8 (1M context) --- ddtrace/internal/opentelemetry/logs.py | 42 +++++++++++++++ ...elemetry-export-loop-d62e76e236121d32.yaml | 8 +++ tests/opentelemetry/test_logs.py | 52 +++++++++++++++++++ 3 files changed, 102 insertions(+) create mode 100644 releasenotes/notes/fix-otlp-logs-self-telemetry-export-loop-d62e76e236121d32.yaml diff --git a/ddtrace/internal/opentelemetry/logs.py b/ddtrace/internal/opentelemetry/logs.py index 2d7ab585931..e7c0d5dad24 100644 --- a/ddtrace/internal/opentelemetry/logs.py +++ b/ddtrace/internal/opentelemetry/logs.py @@ -1,3 +1,4 @@ +import logging from typing import Any from typing import Optional @@ -168,6 +169,43 @@ def _import_exporter(protocol): return None +class _SelfTelemetryLogFilter(logging.Filter): + """Reject log records emitted by ddtrace's own loggers and by the OpenTelemetry SDK/exporter loggers. + + ``_init_logging`` attaches a ``LoggingHandler`` to the root logger that captures every log record + that propagates to root. Without this filter, the tracer's own telemetry-pipeline logs (``ddtrace.*``, + e.g. the per-batch export debug line) and the OpenTelemetry exporter's logs (``opentelemetry.*``, + e.g. export-failure warnings/errors) would be captured, exported, and -- because exporting emits + further log records -- captured again, producing a self-amplifying export loop. A tracer must never + feed its own telemetry-pipeline logs back into its own telemetry export. + """ + + _EXCLUDED_NAMESPACES = ("ddtrace", "opentelemetry") + + def filter(self, record: logging.LogRecord) -> bool: + name = record.name + return not any(name == namespace or name.startswith(namespace + ".") for namespace in self._EXCLUDED_NAMESPACES) + + +def _exclude_self_telemetry_from_otlp_handler(preexisting_handler_ids: set[int]) -> None: + """Attach the self-telemetry filter to the OTLP logs handler that ``_init_logging`` just added. + + Only the newly added handler is filtered (found by diffing against ``preexisting_handler_ids``), + leaving application-configured handlers untouched. The filter binds to the handler instance, so it + persists when the OpenTelemetry SDK re-adds that same instance after ``logging.basicConfig`` (the + one reconfiguration API the SDK patches; ``dictConfig``/``fileConfig`` are not). + """ + try: + from opentelemetry.sdk._logs import LoggingHandler + except ImportError: + return + + telemetry_filter = _SelfTelemetryLogFilter() + for handler in logging.getLogger().handlers: + if isinstance(handler, LoggingHandler) and id(handler) not in preexisting_handler_ids: + handler.addFilter(telemetry_filter) + + def _initialize_logging(exporter_class, protocol, resource): """Configures and sets up the OpenTelemetry Logs exporter.""" try: @@ -176,7 +214,11 @@ def _initialize_logging(exporter_class, protocol, resource): # Ensure logs exporter is configured to send payloads to a Datadog Agent. if "OTEL_EXPORTER_OTLP_ENDPOINT" not in env and "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT" not in env: env["OTEL_EXPORTER_OTLP_LOGS_ENDPOINT"] = otel_config.exporter.LOGS_ENDPOINT + preexisting_handler_ids = {id(handler) for handler in logging.getLogger().handlers} _init_logging({protocol: exporter_class}, resource=resource) + # Stop the OTLP logs handler ddtrace just installed from capturing and re-exporting ddtrace's + # own log records (and the OpenTelemetry exporter's), which would create a self-amplifying loop. + _exclude_self_telemetry_from_otlp_handler(preexisting_handler_ids) return True except ImportError as e: log.warning( diff --git a/releasenotes/notes/fix-otlp-logs-self-telemetry-export-loop-d62e76e236121d32.yaml b/releasenotes/notes/fix-otlp-logs-self-telemetry-export-loop-d62e76e236121d32.yaml new file mode 100644 index 00000000000..aa58b5837b6 --- /dev/null +++ b/releasenotes/notes/fix-otlp-logs-self-telemetry-export-loop-d62e76e236121d32.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + OpenTelemetry logs: Fixes an issue where enabling OpenTelemetry logs collection + (``DD_LOGS_OTEL_ENABLED=true``) could cause the tracer to export its own internal log + records, producing a growing volume of exported logs. The tracer's own log records and + the OpenTelemetry exporter's log records are now excluded from OpenTelemetry logs export, + while application logs continue to be collected and exported as before. diff --git a/tests/opentelemetry/test_logs.py b/tests/opentelemetry/test_logs.py index 2c3ae69b009..c37ce3a7e80 100644 --- a/tests/opentelemetry/test_logs.py +++ b/tests/opentelemetry/test_logs.py @@ -506,6 +506,58 @@ def test_otel_trace_log_correlation(): ) +@pytest.mark.skipif( + EXPORTER_VERSION < MINIMUM_SUPPORTED_VERSION, + reason=f"OpenTelemetry exporter version {MINIMUM_SUPPORTED_VERSION} is required to export logs", +) +@pytest.mark.subprocess(ddtrace_run=True, env={"DD_LOGS_OTEL_ENABLED": "true"}, err=None) +def test_otel_logs_exporter_excludes_self_telemetry(): + """Test that the OTLP logs handler does not capture the tracer's own telemetry. + + Records emitted by ddtrace's own loggers (``ddtrace.*``) and by the OpenTelemetry + SDK/exporter loggers (``opentelemetry.*``) must not be captured and exported by the + OTLP logs handler. Otherwise the exporter's own debug/warning/error log records feed + back into the export pipeline and create a self-amplifying export loop. Application + log records must still be captured and exported. + """ + from logging import getLogger + + from opentelemetry._logs import get_logger_provider + + from tests.opentelemetry.test_logs import create_mock_grpc_server + + mock_service, server = create_mock_grpc_server() + try: + server.start() + # Emulates the tracer's own export-pipeline debug line and the OpenTelemetry + # exporter's export-failure warning/error logs -- both must be excluded. + getLogger("ddtrace.internal.opentelemetry.logs").warning("ddtrace_self_telemetry_log_record") + getLogger("opentelemetry.exporter.otlp").warning("opentelemetry_self_telemetry_log_record") + # A normal application log record must still be captured and exported. + getLogger("my_application_logger").warning("application_log_record") + get_logger_provider().force_flush() + finally: + server.stop(2) + + exported_messages = [ + record.body.string_value + for request in mock_service.received_requests + for resource_logs in request.resource_logs + for scope_logs in resource_logs.scope_logs + for record in scope_logs.log_records + ] + + assert any("application_log_record" in message for message in exported_messages), ( + f"Application log record should have been exported but was not found in: {exported_messages}" + ) + assert not any("ddtrace_self_telemetry_log_record" in message for message in exported_messages), ( + f"ddtrace self-telemetry log record must not be exported but was found in: {exported_messages}" + ) + assert not any("opentelemetry_self_telemetry_log_record" in message for message in exported_messages), ( + f"OpenTelemetry self-telemetry log record must not be exported but was found in: {exported_messages}" + ) + + @pytest.mark.skipif( EXPORTER_VERSION < MINIMUM_SUPPORTED_VERSION, reason=f"OpenTelemetry exporter version {MINIMUM_SUPPORTED_VERSION} is required to export logs",