diff --git a/ddtrace/internal/opentelemetry/logs.py b/ddtrace/internal/opentelemetry/logs.py index 2d7ab585931..e7c0d5dad24 100644 --- a/ddtrace/internal/opentelemetry/logs.py +++ b/ddtrace/internal/opentelemetry/logs.py @@ -1,3 +1,4 @@ +import logging from typing import Any from typing import Optional @@ -168,6 +169,43 @@ def _import_exporter(protocol): return None +class _SelfTelemetryLogFilter(logging.Filter): + """Reject log records emitted by ddtrace's own loggers and by the OpenTelemetry SDK/exporter loggers. + + ``_init_logging`` attaches a ``LoggingHandler`` to the root logger that captures every log record + that propagates to root. Without this filter, the tracer's own telemetry-pipeline logs (``ddtrace.*``, + e.g. the per-batch export debug line) and the OpenTelemetry exporter's logs (``opentelemetry.*``, + e.g. export-failure warnings/errors) would be captured, exported, and -- because exporting emits + further log records -- captured again, producing a self-amplifying export loop. A tracer must never + feed its own telemetry-pipeline logs back into its own telemetry export. + """ + + _EXCLUDED_NAMESPACES = ("ddtrace", "opentelemetry") + + def filter(self, record: logging.LogRecord) -> bool: + name = record.name + return not any(name == namespace or name.startswith(namespace + ".") for namespace in self._EXCLUDED_NAMESPACES) + + +def _exclude_self_telemetry_from_otlp_handler(preexisting_handler_ids: set[int]) -> None: + """Attach the self-telemetry filter to the OTLP logs handler that ``_init_logging`` just added. + + Only the newly added handler is filtered (found by diffing against ``preexisting_handler_ids``), + leaving application-configured handlers untouched. The filter binds to the handler instance, so it + persists when the OpenTelemetry SDK re-adds that same instance after ``logging.basicConfig`` (the + one reconfiguration API the SDK patches; ``dictConfig``/``fileConfig`` are not). + """ + try: + from opentelemetry.sdk._logs import LoggingHandler + except ImportError: + return + + telemetry_filter = _SelfTelemetryLogFilter() + for handler in logging.getLogger().handlers: + if isinstance(handler, LoggingHandler) and id(handler) not in preexisting_handler_ids: + handler.addFilter(telemetry_filter) + + def _initialize_logging(exporter_class, protocol, resource): """Configures and sets up the OpenTelemetry Logs exporter.""" try: @@ -176,7 +214,11 @@ def _initialize_logging(exporter_class, protocol, resource): # Ensure logs exporter is configured to send payloads to a Datadog Agent. if "OTEL_EXPORTER_OTLP_ENDPOINT" not in env and "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT" not in env: env["OTEL_EXPORTER_OTLP_LOGS_ENDPOINT"] = otel_config.exporter.LOGS_ENDPOINT + preexisting_handler_ids = {id(handler) for handler in logging.getLogger().handlers} _init_logging({protocol: exporter_class}, resource=resource) + # Stop the OTLP logs handler ddtrace just installed from capturing and re-exporting ddtrace's + # own log records (and the OpenTelemetry exporter's), which would create a self-amplifying loop. + _exclude_self_telemetry_from_otlp_handler(preexisting_handler_ids) return True except ImportError as e: log.warning( diff --git a/releasenotes/notes/fix-otlp-logs-self-telemetry-export-loop-d62e76e236121d32.yaml b/releasenotes/notes/fix-otlp-logs-self-telemetry-export-loop-d62e76e236121d32.yaml new file mode 100644 index 00000000000..aa58b5837b6 --- /dev/null +++ b/releasenotes/notes/fix-otlp-logs-self-telemetry-export-loop-d62e76e236121d32.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + OpenTelemetry logs: Fixes an issue where enabling OpenTelemetry logs collection + (``DD_LOGS_OTEL_ENABLED=true``) could cause the tracer to export its own internal log + records, producing a growing volume of exported logs. The tracer's own log records and + the OpenTelemetry exporter's log records are now excluded from OpenTelemetry logs export, + while application logs continue to be collected and exported as before. diff --git a/tests/opentelemetry/test_logs.py b/tests/opentelemetry/test_logs.py index 2c3ae69b009..c37ce3a7e80 100644 --- a/tests/opentelemetry/test_logs.py +++ b/tests/opentelemetry/test_logs.py @@ -506,6 +506,58 @@ def test_otel_trace_log_correlation(): ) +@pytest.mark.skipif( + EXPORTER_VERSION < MINIMUM_SUPPORTED_VERSION, + reason=f"OpenTelemetry exporter version {MINIMUM_SUPPORTED_VERSION} is required to export logs", +) +@pytest.mark.subprocess(ddtrace_run=True, env={"DD_LOGS_OTEL_ENABLED": "true"}, err=None) +def test_otel_logs_exporter_excludes_self_telemetry(): + """Test that the OTLP logs handler does not capture the tracer's own telemetry. + + Records emitted by ddtrace's own loggers (``ddtrace.*``) and by the OpenTelemetry + SDK/exporter loggers (``opentelemetry.*``) must not be captured and exported by the + OTLP logs handler. Otherwise the exporter's own debug/warning/error log records feed + back into the export pipeline and create a self-amplifying export loop. Application + log records must still be captured and exported. + """ + from logging import getLogger + + from opentelemetry._logs import get_logger_provider + + from tests.opentelemetry.test_logs import create_mock_grpc_server + + mock_service, server = create_mock_grpc_server() + try: + server.start() + # Emulates the tracer's own export-pipeline debug line and the OpenTelemetry + # exporter's export-failure warning/error logs -- both must be excluded. + getLogger("ddtrace.internal.opentelemetry.logs").warning("ddtrace_self_telemetry_log_record") + getLogger("opentelemetry.exporter.otlp").warning("opentelemetry_self_telemetry_log_record") + # A normal application log record must still be captured and exported. + getLogger("my_application_logger").warning("application_log_record") + get_logger_provider().force_flush() + finally: + server.stop(2) + + exported_messages = [ + record.body.string_value + for request in mock_service.received_requests + for resource_logs in request.resource_logs + for scope_logs in resource_logs.scope_logs + for record in scope_logs.log_records + ] + + assert any("application_log_record" in message for message in exported_messages), ( + f"Application log record should have been exported but was not found in: {exported_messages}" + ) + assert not any("ddtrace_self_telemetry_log_record" in message for message in exported_messages), ( + f"ddtrace self-telemetry log record must not be exported but was found in: {exported_messages}" + ) + assert not any("opentelemetry_self_telemetry_log_record" in message for message in exported_messages), ( + f"OpenTelemetry self-telemetry log record must not be exported but was found in: {exported_messages}" + ) + + @pytest.mark.skipif( EXPORTER_VERSION < MINIMUM_SUPPORTED_VERSION, reason=f"OpenTelemetry exporter version {MINIMUM_SUPPORTED_VERSION} is required to export logs",