[WIP] Add log handler configuration to autoinstrumentation

CHANGELOG.md

@@ -195,6 +195,11 @@ can cause a deadlock to occur over `logging._lock` in some cases ([#4636](https:
 
 - typecheck: add sdk/resources and drop mypy
   ([#4578](https://github.com/open-telemetry/opentelemetry-python/pull/4578))
+- Refactor `BatchLogRecordProcessor` to simplify code and make the control flow more
+  clear ([#4562](https://github.com/open-telemetry/opentelemetry-python/pull/4562/)
+  and [#4535](https://github.com/open-telemetry/opentelemetry-python/pull/4535)).
+- Enable configuration of logging format and level in auto-instrumentation
+  ([#4203](https://github.com/open-telemetry/opentelemetry-python/pull/4203))
 - Use PEP702 for marking deprecations
   ([#4522](https://github.com/open-telemetry/opentelemetry-python/pull/4522))
 - Refactor `BatchLogRecordProcessor` and `BatchSpanProcessor` to simplify code

opentelemetry-sdk/src/opentelemetry/sdk/_configuration/__init__.py

@@ -52,6 +52,8 @@
     OTEL_EXPORTER_OTLP_METRICS_PROTOCOL,
     OTEL_EXPORTER_OTLP_PROTOCOL,
     OTEL_EXPORTER_OTLP_TRACES_PROTOCOL,
+    OTEL_PYTHON_LOG_FORMAT,
+    OTEL_PYTHON_LOG_HANDLER_LEVEL,
     OTEL_PYTHON_TRACER_CONFIGURATOR,
     OTEL_TRACES_SAMPLER,
     OTEL_TRACES_SAMPLER_ARG,
@@ -101,6 +103,15 @@
 
 _OTEL_SAMPLER_ENTRY_POINT_GROUP = "opentelemetry_traces_sampler"
 
+_OTEL_PYTHON_LOG_HANDLER_LEVEL_BY_NAME = {
+    "notset": logging.NOTSET,
+    "debug": logging.DEBUG,
+    "info": logging.INFO,
+    "warn": logging.WARNING,
+    "warning": logging.WARNING,
+    "error": logging.ERROR,
+}
+
 _logger = logging.getLogger(__name__)
 
 ExporterArgsMap = Mapping[
@@ -167,6 +178,13 @@ def _get_id_generator() -> str:
     return environ.get(OTEL_PYTHON_ID_GENERATOR, _DEFAULT_ID_GENERATOR)
 
 
+def _get_log_level() -> int:
+    return _OTEL_PYTHON_LOG_HANDLER_LEVEL_BY_NAME.get(
+        environ.get(OTEL_PYTHON_LOG_HANDLER_LEVEL, "notset").lower().strip(),
+        logging.NOTSET,
+    )
+
+
 def _get_tracer_configurator() -> str | None:
     return environ.get(OTEL_PYTHON_TRACER_CONFIGURATOR, None)
 
@@ -330,6 +348,15 @@ def _init_logging(
         handler = LoggingHandler(
             level=logging.NOTSET, logger_provider=provider
         )
+        # Log level
+        if OTEL_PYTHON_LOG_HANDLER_LEVEL in environ:
+            handler.setLevel(_get_log_level())
+        # Log format
+        if OTEL_PYTHON_LOG_FORMAT in environ:
+            log_format = environ.get(
+                OTEL_PYTHON_LOG_FORMAT, logging.BASIC_FORMAT
+            )
+            handler.setFormatter(logging.Formatter(log_format))
         logging.getLogger().addHandler(handler)
         _overwrite_logging_config_fns(handler)
 

opentelemetry-sdk/src/opentelemetry/sdk/environment_variables/__init__.py

@@ -54,6 +54,22 @@
 Default: "info"
 """
 
+OTEL_PYTHON_LOG_FORMAT = "OTEL_PYTHON_LOG_FORMAT"
+"""
+.. envvar:: OTEL_PYTHON_LOG_FORMAT
+
+The :envvar:`OTEL_PYTHON_LOG_FORMAT` environment variable sets the log format for the OpenTelemetry LoggingHandler's Formatter
+Default: "logging.BASIC_FORMAT"
+"""
+
+OTEL_PYTHON_LOG_HANDLER_LEVEL = "OTEL_PYTHON_LOG_HANDLER_LEVEL"
+"""
+.. envvar:: OTEL_PYTHON_LOG_HANDLER_LEVEL
+
+The :envvar:`OTEL_PYTHON_LOG_HANDLER_LEVEL` environment variable sets the log level for the OpenTelemetry LoggingHandler
+Default: "logging.NOTSET"
+"""
+
 OTEL_TRACES_SAMPLER = "OTEL_TRACES_SAMPLER"
 """
 .. envvar:: OTEL_TRACES_SAMPLER

opentelemetry-sdk/tests/test_configurator.py

@@ -18,7 +18,14 @@
 
 import logging
 import logging.config
-from logging import WARNING, getLogger
+from logging import (
+    DEBUG,
+    ERROR,
+    INFO,
+    NOTSET,
+    WARNING,
+    getLogger,
+)
 from os import environ
 from typing import Iterable, Optional, Sequence
 from unittest import TestCase, mock
@@ -35,6 +42,7 @@
     _EXPORTER_OTLP_PROTO_HTTP,
     _get_exporter_names,
     _get_id_generator,
+    _get_log_level,
     _get_sampler,
     _get_tracer_configurator,
     _import_config_components,
@@ -87,6 +95,8 @@
 from opentelemetry.trace.span import TraceState
 from opentelemetry.util.types import Attributes
 
+CUSTOM_LOG_FORMAT = "CUSTOM FORMAT %(levelname)s:%(name)s:%(message)s"
+
 
 class Provider:
     def __init__(
@@ -718,6 +728,8 @@ def setUp(self):
             self.set_event_logger_provider_patch.start()
         )
 
+        getLogger().handlers.clear()
+
     def tearDown(self):
         self.processor_patch.stop()
         self.set_provider_patch.stop()
@@ -818,7 +830,123 @@ def test_logging_init_custom_log_record_processors(self):
 
     @patch.dict(
         environ,
-        {"OTEL_RESOURCE_ATTRIBUTES": "service.name=otlp-service"},
+        {
+            "OTEL_RESOURCE_ATTRIBUTES": "service.name=otlp-service",
+            "OTEL_PYTHON_LOG_HANDLER_LEVEL": "CUSTOM_LOG_LEVEL",
+        },
+    )
+    @patch("opentelemetry.sdk._configuration._get_log_level", return_value=39)
+    def test_logging_init_exporter_level_under(self, log_level_mock):
+        resource = Resource.create({})
+        _init_logging(
+            {"otlp": DummyOTLPLogExporter},
+            resource=resource,
+        )
+        self.assertEqual(self.set_provider_mock.call_count, 1)
+        provider = self.set_provider_mock.call_args[0][0]
+        self.assertIsInstance(provider, DummyLoggerProvider)
+        self.assertIsInstance(provider.resource, Resource)
+        self.assertEqual(
+            provider.resource.attributes.get("service.name"),
+            "otlp-service",
+        )
+        self.assertEqual(len(provider.processors), 1)
+        self.assertIsInstance(provider.processors[0], DummyLogRecordProcessor)
+        self.assertIsInstance(
+            provider.processors[0].exporter, DummyOTLPLogExporter
+        )
+        getLogger(__name__).error("hello")
+        self.assertTrue(provider.processors[0].exporter.export_called)
+        root_logger = getLogger()
+        self.assertEqual(root_logger.level, WARNING)
+        handler_present = False
+        for handler in root_logger.handlers:
+            if isinstance(handler, LoggingHandler):
+                handler_present = True
+                self.assertEqual(handler.level, 39)
+        self.assertTrue(handler_present)
+
+    @patch.dict(
+        environ,
+        {
+            "OTEL_RESOURCE_ATTRIBUTES": "service.name=otlp-service",
+            "OTEL_PYTHON_LOG_HANDLER_LEVEL": "CUSTOM_LOG_LEVEL",
+        },
+        clear=True,
+    )
+    @patch("opentelemetry.sdk._configuration._get_log_level", return_value=41)
+    def test_logging_init_exporter_level_over(self, log_level_mock):
+        resource = Resource.create({})
+        _init_logging(
+            {"otlp": DummyOTLPLogExporter},
+            resource=resource,
+        )
+        self.assertEqual(self.set_provider_mock.call_count, 1)
+        provider = self.set_provider_mock.call_args[0][0]
+        self.assertIsInstance(provider, DummyLoggerProvider)
+        self.assertIsInstance(provider.resource, Resource)
+        self.assertEqual(
+            provider.resource.attributes.get("service.name"),
+            "otlp-service",
+        )
+        self.assertEqual(len(provider.processors), 1)
+        self.assertIsInstance(provider.processors[0], DummyLogRecordProcessor)
+        self.assertIsInstance(
+            provider.processors[0].exporter, DummyOTLPLogExporter
+        )
+        getLogger(__name__).error("hello")
+        self.assertFalse(provider.processors[0].exporter.export_called)
+        root_logger = getLogger()
+        handler_present = False
+        for handler in root_logger.handlers:
+            if isinstance(handler, LoggingHandler):
+                handler_present = True
+                self.assertEqual(handler.level, 41)
+        self.assertTrue(handler_present)
+
+    @patch.dict(
+        environ,
+        {
+            "OTEL_RESOURCE_ATTRIBUTES": "service.name=otlp-service",
+            "OTEL_PYTHON_LOG_FORMAT": CUSTOM_LOG_FORMAT,
+        },
+    )
+    def test_logging_init_exporter_format(self):
+        resource = Resource.create({})
+        _init_logging(
+            {"otlp": DummyOTLPLogExporter},
+            resource=resource,
+        )
+        self.assertEqual(self.set_provider_mock.call_count, 1)
+        provider = self.set_provider_mock.call_args[0][0]
+        self.assertIsInstance(provider, DummyLoggerProvider)
+        self.assertIsInstance(provider.resource, Resource)
+        self.assertEqual(
+            provider.resource.attributes.get("service.name"),
+            "otlp-service",
+        )
+        self.assertEqual(len(provider.processors), 1)
+        self.assertIsInstance(provider.processors[0], DummyLogRecordProcessor)
+        self.assertIsInstance(
+            provider.processors[0].exporter, DummyOTLPLogExporter
+        )
+        getLogger(__name__).error("hello")
+        self.assertTrue(provider.processors[0].exporter.export_called)
+        root_logger = getLogger()
+        self.assertEqual(root_logger.level, WARNING)
+        handler_present = False
+        for handler in root_logger.handlers:
+            if isinstance(handler, LoggingHandler):
+                self.assertEqual(handler.formatter._fmt, CUSTOM_LOG_FORMAT)
+                handler_present = True
+        self.assertTrue(handler_present)
+
+    @patch.dict(
+        environ,
+        {
+            "OTEL_RESOURCE_ATTRIBUTES": "service.name=otlp-service",
+            "OTEL_PYTHON_LOG_FORMAT": CUSTOM_LOG_FORMAT,
+        },
     )
     def test_logging_init_exporter_without_handler_setup(self):
         resource = Resource.create({})
@@ -842,6 +970,60 @@ def test_logging_init_exporter_without_handler_setup(self):
         )
         getLogger(__name__).error("hello")
         self.assertFalse(provider.processors[0].exporter.export_called)
+        root_logger = getLogger()
+        self.assertEqual(root_logger.level, WARNING)
+        handler_present = False
+        for handler in root_logger.handlers:
+            if isinstance(handler, LoggingHandler):
+                self.assertEqual(handler.formatter._fmt, CUSTOM_LOG_FORMAT)
+                handler_present = True
+        self.assertTrue(handler_present)
+
+    @patch.dict(environ, {}, clear=True)
+    def test_otel_log_level_by_name_default(self):
+        self.assertEqual(_get_log_level(), NOTSET)
+
+    @patch.dict(
+        environ, {"OTEL_PYTHON_LOG_HANDLER_LEVEL": "NOTSET "}, clear=True
+    )
+    def test_otel_log_level_by_name_notset(self):
+        self.assertEqual(_get_log_level(), NOTSET)
+
+    @patch.dict(
+        environ, {"OTEL_PYTHON_LOG_HANDLER_LEVEL": " DeBug "}, clear=True
+    )
+    def test_otel_log_level_by_name_debug(self):
+        self.assertEqual(_get_log_level(), DEBUG)
+
+    @patch.dict(
+        environ, {"OTEL_PYTHON_LOG_HANDLER_LEVEL": " info "}, clear=True
+    )
+    def test_otel_log_level_by_name_info(self):
+        self.assertEqual(_get_log_level(), INFO)
+
+    @patch.dict(
+        environ, {"OTEL_PYTHON_LOG_HANDLER_LEVEL": " warn"}, clear=True
+    )
+    def test_otel_log_level_by_name_warn(self):
+        self.assertEqual(_get_log_level(), WARNING)
+
+    @patch.dict(
+        environ, {"OTEL_PYTHON_LOG_HANDLER_LEVEL": " warnING "}, clear=True
+    )
+    def test_otel_log_level_by_name_warning(self):
+        self.assertEqual(_get_log_level(), WARNING)
+
+    @patch.dict(
+        environ, {"OTEL_PYTHON_LOG_HANDLER_LEVEL": " eRroR"}, clear=True
+    )
+    def test_otel_log_level_by_name_error(self):
+        self.assertEqual(_get_log_level(), ERROR)
+
+    @patch.dict(
+        environ, {"OTEL_PYTHON_LOG_HANDLER_LEVEL": "foobar"}, clear=True
+    )
+    def test_otel_log_level_by_name_invalid(self):
+        self.assertEqual(_get_log_level(), NOTSET)
 
     @patch.dict(
         environ,
@@ -1074,6 +1256,60 @@ def test_basicConfig_preserves_otel_handler(self):
                 "Should still have exactly one OpenTelemetry LoggingHandler",
             )
 
+    @patch.dict(
+        environ,
+        {
+            "OTEL_TRACES_EXPORTER": _EXPORTER_OTLP,
+            "OTEL_METRICS_EXPORTER": _EXPORTER_OTLP_PROTO_GRPC,
+            "OTEL_LOGS_EXPORTER": _EXPORTER_OTLP_PROTO_HTTP,
+        },
+    )
+    @patch.dict(
+        environ,
+        {
+            "OTEL_RESOURCE_ATTRIBUTES": "service.name=otlp-service, custom.key.1=env-value",
+            "OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED": "False",
+        },
+    )
+    @patch("opentelemetry.sdk._configuration.Resource")
+    @patch("opentelemetry.sdk._configuration._import_exporters")
+    @patch("opentelemetry.sdk._configuration._get_exporter_names")
+    @patch("opentelemetry.sdk._configuration._init_tracing")
+    @patch("opentelemetry.sdk._configuration._init_logging")
+    @patch("opentelemetry.sdk._configuration._init_metrics")
+    def test_initialize_components_kwargs_disable_logging_handler(
+        self,
+        metrics_mock,
+        logging_mock,
+        tracing_mock,
+        exporter_names_mock,
+        import_exporters_mock,
+        resource_mock,
+    ):
+        exporter_names_mock.return_value = [
+            "env_var_exporter_1",
+            "env_var_exporter_2",
+        ]
+        import_exporters_mock.return_value = (
+            "TEST_SPAN_EXPORTERS_DICT",
+            "TEST_METRICS_EXPORTERS_DICT",
+            "TEST_LOG_EXPORTERS_DICT",
+        )
+        resource_mock.create.return_value = "TEST_RESOURCE"
+        kwargs = {
+            "auto_instrumentation_version": "auto-version",
+            "trace_exporter_names": ["custom_span_exporter"],
+            "metric_exporter_names": ["custom_metric_exporter"],
+            "log_exporter_names": ["custom_log_exporter"],
+            "sampler": "TEST_SAMPLER",
+            "resource_attributes": {
+                "custom.key.1": "pass-in-value-1",
+                "custom.key.2": "pass-in-value-2",
+            },
+            "id_generator": "TEST_GENERATOR",
+        }
+        _initialize_components(**kwargs)
+
     def test_dictConfig_preserves_otel_handler(self):
         with ResetGlobalLoggingState():
             _init_logging(
@@ -1394,7 +1630,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         logging.config.fileConfig = self.original_file_config
 
 
-class TestClearLoggingHandlers(TestCase):
+class TestResetGlobalLoggingState(TestCase):
     def test_preserves_handlers(self):
         root_logger = getLogger()
         initial_handlers = root_logger.handlers[:]