fix(opentelemetry): prevent OTLP logs self-export loop

ddtrace/internal/opentelemetry/logs.py

@@ -1,3 +1,4 @@
+import logging
 from typing import Any
 from typing import Optional
 
@@ -168,6 +169,43 @@ def _import_exporter(protocol):
         return None
 
 
+class _SelfTelemetryLogFilter(logging.Filter):
+    """Reject log records emitted by ddtrace's own loggers and by the OpenTelemetry SDK/exporter loggers.
+
+    ``_init_logging`` attaches a ``LoggingHandler`` to the root logger that captures every log record
+    that propagates to root. Without this filter, the tracer's own telemetry-pipeline logs (``ddtrace.*``,
+    e.g. the per-batch export debug line) and the OpenTelemetry exporter's logs (``opentelemetry.*``,
+    e.g. export-failure warnings/errors) would be captured, exported, and -- because exporting emits
+    further log records -- captured again, producing a self-amplifying export loop. A tracer must never
+    feed its own telemetry-pipeline logs back into its own telemetry export.
+    """
+
+    _EXCLUDED_NAMESPACES = ("ddtrace", "opentelemetry")
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        name = record.name
+        return not any(name == namespace or name.startswith(namespace + ".") for namespace in self._EXCLUDED_NAMESPACES)
+
+
+def _exclude_self_telemetry_from_otlp_handler(preexisting_handler_ids: set[int]) -> None:
+    """Attach the self-telemetry filter to the OTLP logs handler that ``_init_logging`` just added.
+
+    Only the newly added handler is filtered (found by diffing against ``preexisting_handler_ids``),
+    leaving application-configured handlers untouched. The filter binds to the handler instance, so it
+    persists when the OpenTelemetry SDK re-adds that same instance after ``logging.basicConfig`` (the
+    one reconfiguration API the SDK patches; ``dictConfig``/``fileConfig`` are not).
+    """
+    try:
+        from opentelemetry.sdk._logs import LoggingHandler
+    except ImportError:
+        return
+
+    telemetry_filter = _SelfTelemetryLogFilter()
+    for handler in logging.getLogger().handlers:
+        if isinstance(handler, LoggingHandler) and id(handler) not in preexisting_handler_ids:
+            handler.addFilter(telemetry_filter)
+
+
 def _initialize_logging(exporter_class, protocol, resource):
     """Configures and sets up the OpenTelemetry Logs exporter."""
     try:
@@ -176,7 +214,11 @@ def _initialize_logging(exporter_class, protocol, resource):
         # Ensure logs exporter is configured to send payloads to a Datadog Agent.
         if "OTEL_EXPORTER_OTLP_ENDPOINT" not in env and "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT" not in env:
             env["OTEL_EXPORTER_OTLP_LOGS_ENDPOINT"] = otel_config.exporter.LOGS_ENDPOINT
+        preexisting_handler_ids = {id(handler) for handler in logging.getLogger().handlers}
         _init_logging({protocol: exporter_class}, resource=resource)
+        # Stop the OTLP logs handler ddtrace just installed from capturing and re-exporting ddtrace's
+        # own log records (and the OpenTelemetry exporter's), which would create a self-amplifying loop.
+        _exclude_self_telemetry_from_otlp_handler(preexisting_handler_ids)
         return True
     except ImportError as e:
         log.warning(

releasenotes/notes/fix-otlp-logs-self-telemetry-export-loop-d62e76e236121d32.yaml

@@ -0,0 +1,8 @@
+---
+fixes:
+  - |
+    OpenTelemetry logs: Fixes an issue where enabling OpenTelemetry logs collection
+    (``DD_LOGS_OTEL_ENABLED=true``) could cause the tracer to export its own internal log
+    records, producing a growing volume of exported logs. The tracer's own log records and
+    the OpenTelemetry exporter's log records are now excluded from OpenTelemetry logs export,
+    while application logs continue to be collected and exported as before.

tests/opentelemetry/test_logs.py

@@ -506,6 +506,58 @@ def test_otel_trace_log_correlation():
     )
 
 
+@pytest.mark.skipif(
+    EXPORTER_VERSION < MINIMUM_SUPPORTED_VERSION,
+    reason=f"OpenTelemetry exporter version {MINIMUM_SUPPORTED_VERSION} is required to export logs",
+)
+@pytest.mark.subprocess(ddtrace_run=True, env={"DD_LOGS_OTEL_ENABLED": "true"}, err=None)
+def test_otel_logs_exporter_excludes_self_telemetry():
+    """Test that the OTLP logs handler does not capture the tracer's own telemetry.
+
+    Records emitted by ddtrace's own loggers (``ddtrace.*``) and by the OpenTelemetry
+    SDK/exporter loggers (``opentelemetry.*``) must not be captured and exported by the
+    OTLP logs handler. Otherwise the exporter's own debug/warning/error log records feed
+    back into the export pipeline and create a self-amplifying export loop. Application
+    log records must still be captured and exported.
+    """
+    from logging import getLogger
+
+    from opentelemetry._logs import get_logger_provider
+
+    from tests.opentelemetry.test_logs import create_mock_grpc_server
+
+    mock_service, server = create_mock_grpc_server()
+    try:
+        server.start()
+        # Emulates the tracer's own export-pipeline debug line and the OpenTelemetry
+        # exporter's export-failure warning/error logs -- both must be excluded.
+        getLogger("ddtrace.internal.opentelemetry.logs").warning("ddtrace_self_telemetry_log_record")
+        getLogger("opentelemetry.exporter.otlp").warning("opentelemetry_self_telemetry_log_record")
+        # A normal application log record must still be captured and exported.
+        getLogger("my_application_logger").warning("application_log_record")
+        get_logger_provider().force_flush()
+    finally:
+        server.stop(2)
+
+    exported_messages = [
+        record.body.string_value
+        for request in mock_service.received_requests
+        for resource_logs in request.resource_logs
+        for scope_logs in resource_logs.scope_logs
+        for record in scope_logs.log_records
+    ]
+
+    assert any("application_log_record" in message for message in exported_messages), (
+        f"Application log record should have been exported but was not found in: {exported_messages}"
+    )
+    assert not any("ddtrace_self_telemetry_log_record" in message for message in exported_messages), (
+        f"ddtrace self-telemetry log record must not be exported but was found in: {exported_messages}"
+    )
+    assert not any("opentelemetry_self_telemetry_log_record" in message for message in exported_messages), (
+        f"OpenTelemetry self-telemetry log record must not be exported but was found in: {exported_messages}"
+    )
+
+
 @pytest.mark.skipif(
     EXPORTER_VERSION < MINIMUM_SUPPORTED_VERSION,
     reason=f"OpenTelemetry exporter version {MINIMUM_SUPPORTED_VERSION} is required to export logs",