From a66a58503b58181d06a4c1867cbbb7bfc76b0dae Mon Sep 17 00:00:00 2001 From: nubtron Date: Thu, 16 Jul 2026 12:24:45 +0000 Subject: [PATCH 1/3] Add OpenMetrics max_returned_metrics telemetry --- datadog_checks_base/changelog.d/0.added | 1 + .../datadog_checks/base/checks/base.py | 32 ++- .../base/checks/openmetrics/base_check.py | 1 + .../base/checks/openmetrics/v2/base.py | 1 + .../datadog_checks/base/utils/limiter.py | 6 +- .../tests/base/checks/test_agent_check.py | 247 ++++++++++++++++++ .../tests/base/utils/test_utils.py | 55 ++++ 7 files changed, 339 insertions(+), 4 deletions(-) create mode 100644 datadog_checks_base/changelog.d/0.added diff --git a/datadog_checks_base/changelog.d/0.added b/datadog_checks_base/changelog.d/0.added new file mode 100644 index 0000000000000..b126424a7da09 --- /dev/null +++ b/datadog_checks_base/changelog.d/0.added @@ -0,0 +1 @@ +Add best-effort OpenMetrics metric-limit Agent telemetry when ``max_returned_metrics`` is reached. diff --git a/datadog_checks_base/datadog_checks/base/checks/base.py b/datadog_checks_base/datadog_checks/base/checks/base.py index 22511a6d6fe83..98768f5c83eb3 100644 --- a/datadog_checks_base/datadog_checks/base/checks/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/base.py @@ -171,6 +171,7 @@ class except the `check` method but sometimes it might be useful for a Check to # be sent to the aggregator, the rest are dropped. The state is reset after each run. # See https://github.com/DataDog/integrations-core/pull/2093 for more information. DEFAULT_METRIC_LIMIT = 0 + OPENMETRICS_MAX_RETURNED_METRICS_TELEMETRY = False # Allow tracing for classic integrations def __init_subclass__(cls, *args, **kwargs): @@ -391,10 +392,30 @@ def _get_metric_limiter(self, name, instance=None): # See Performance Optimizations in this package's README.md. from datadog_checks.base.utils.limiter import Limiter - return Limiter(name, 'metrics', limit, self.warning) + on_limit_reached = None + if self.OPENMETRICS_MAX_RETURNED_METRICS_TELEMETRY: + on_limit_reached = self._emit_openmetrics_max_returned_metrics_reached + + return Limiter(name, 'metrics', limit, self.warning, on_limit_reached_func=on_limit_reached) return None + def _emit_openmetrics_max_returned_metrics_reached(self): + emit_agent_telemetry_with_labels = getattr(datadog_agent, 'emit_agent_telemetry_with_labels', None) + if not callable(emit_agent_telemetry_with_labels): + return + + try: + emit_agent_telemetry_with_labels( + "openmetrics", + "max_returned_metrics_reached", + 1, + "counter", + {"check_name": self.name}, + ) + except Exception: + self.log.debug("Unable to emit OpenMetrics max_returned_metrics Agent telemetry", exc_info=True) + def _get_metric_limit(self, instance=None): # type: (InstanceType) -> int if instance is None: @@ -1617,8 +1638,13 @@ def run(self): self.metric_limiter.reset() tags = self.get_debug_metric_tags() - for metric_name, value in debug_metrics: - self.gauge(metric_name, value, tags=tags, raw=True) + on_limit_reached = self.metric_limiter.on_limit_reached + self.metric_limiter.on_limit_reached = None + try: + for metric_name, value in debug_metrics: + self.gauge(metric_name, value, tags=tags, raw=True) + finally: + self.metric_limiter.on_limit_reached = on_limit_reached self.metric_limiter.reset() diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/base_check.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/base_check.py index 3a65a8dc83efb..2bd229cff4a1e 100644 --- a/datadog_checks_base/datadog_checks/base/checks/openmetrics/base_check.py +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/base_check.py @@ -56,6 +56,7 @@ class OpenMetricsBaseCheck(OpenMetricsScraperMixin, AgentCheck): """ DEFAULT_METRIC_LIMIT = 2000 + OPENMETRICS_MAX_RETURNED_METRICS_TELEMETRY = True HTTP_CONFIG_REMAPPER = { 'ssl_verify': {'name': 'tls_verify'}, diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/base.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/base.py index 2fb5062e146bf..636f4193fc3ad 100644 --- a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/base.py @@ -41,6 +41,7 @@ class OpenMetricsBaseCheckV2(AgentCheck): """ DEFAULT_METRIC_LIMIT = 2000 + OPENMETRICS_MAX_RETURNED_METRICS_TELEMETRY = True METRICS_MAP: tuple[MetricsMapping, ...] = () """YAML files with metric name mappings to load automatically. diff --git a/datadog_checks_base/datadog_checks/base/utils/limiter.py b/datadog_checks_base/datadog_checks/base/utils/limiter.py index 323f069d35e89..66993ed77fea3 100644 --- a/datadog_checks_base/datadog_checks/base/utils/limiter.py +++ b/datadog_checks_base/datadog_checks/base/utils/limiter.py @@ -11,14 +11,16 @@ class Limiter(object): that can be set by an instance. """ - def __init__(self, check_name, object_name, object_limit, warning_func=None): + def __init__(self, check_name, object_name, object_limit, warning_func=None, on_limit_reached_func=None): """ :param check_name: name of the check using this limiter :param object_name: (plural) name of counted objects for warning wording :param object_limit: maximum number of objects to accept before limiting :param warning_func: callback function, called with a string when limit is exceeded + :param on_limit_reached_func: callback function, called when limit is first exceeded """ self.warning = warning_func + self.on_limit_reached = on_limit_reached_func self.name = object_name self.limit = object_limit self.check_name = check_name @@ -70,6 +72,8 @@ def is_reached(self, uid=None): "Check %s exceeded limit of %s %s, ignoring next ones", self.check_name, self.limit, self.name ) self.reached_limit = True + if self.on_limit_reached: + self.on_limit_reached() return True return False diff --git a/datadog_checks_base/tests/base/checks/test_agent_check.py b/datadog_checks_base/tests/base/checks/test_agent_check.py index 8cb0e6a8c1c0b..3b7fe924a26da 100644 --- a/datadog_checks_base/tests/base/checks/test_agent_check.py +++ b/datadog_checks_base/tests/base/checks/test_agent_check.py @@ -1162,6 +1162,10 @@ def check(self, _): self.gauge('foo', i) +class OpenMetricsTelemetryCheck(LimitedCheck): + OPENMETRICS_MAX_RETURNED_METRICS_TELEMETRY = True + + class TestLimits: def test_context_uid(self, aggregator): check = LimitedCheck() @@ -1208,6 +1212,141 @@ def test_metric_limit_count(self, aggregator): assert len(check.get_warnings()) == 1 assert len(aggregator.metrics("metric")) == 29 + def test_openmetrics_metric_limit_telemetry(self, aggregator, monkeypatch): + emit_agent_telemetry = mock.MagicMock() + monkeypatch.setattr( + 'datadog_checks.base.checks.base.datadog_agent.emit_agent_telemetry_with_labels', + emit_agent_telemetry, + raising=False, + ) + check = OpenMetricsTelemetryCheck('canonical_openmetrics', {}, [{'max_returned_metrics': 2}]) + + check.gauge('metric', 0) + check.gauge('metric', 0) + emit_agent_telemetry.assert_not_called() + assert len(aggregator.metrics('metric')) == 2 + + check.gauge('metric', 0) + check.gauge('metric', 0) + + emit_agent_telemetry.assert_called_once_with( + 'openmetrics', + 'max_returned_metrics_reached', + 1, + 'counter', + {'check_name': 'canonical_openmetrics'}, + ) + assert len(check.get_warnings()) == 1 + assert len(aggregator.metrics('metric')) == 2 + + def test_openmetrics_metric_limit_telemetry_aggregates_same_check_name(self, monkeypatch): + emit_agent_telemetry = mock.MagicMock() + monkeypatch.setattr( + 'datadog_checks.base.checks.base.datadog_agent.emit_agent_telemetry_with_labels', + emit_agent_telemetry, + raising=False, + ) + first_check = OpenMetricsTelemetryCheck('canonical_openmetrics', {}, [{'max_returned_metrics': 1}]) + second_check = OpenMetricsTelemetryCheck('canonical_openmetrics', {}, [{'max_returned_metrics': 1}]) + + first_check.gauge('metric', 0) + first_check.gauge('metric', 0) + second_check.gauge('metric', 0) + second_check.gauge('metric', 0) + + assert emit_agent_telemetry.call_args_list == [ + mock.call( + 'openmetrics', 'max_returned_metrics_reached', 1, 'counter', {'check_name': 'canonical_openmetrics'} + ), + mock.call( + 'openmetrics', 'max_returned_metrics_reached', 1, 'counter', {'check_name': 'canonical_openmetrics'} + ), + ] + + def test_openmetrics_metric_limit_telemetry_uid_dedup(self, aggregator, monkeypatch): + emit_agent_telemetry = mock.MagicMock() + monkeypatch.setattr( + 'datadog_checks.base.checks.base.datadog_agent.emit_agent_telemetry_with_labels', + emit_agent_telemetry, + raising=False, + ) + check = OpenMetricsTelemetryCheck('canonical_openmetrics', {}, [{'max_returned_metrics': 2}]) + + for _ in range(5): + check.count('metric', 0, hostname='same-host') + assert len(aggregator.metrics('metric')) == 5 + emit_agent_telemetry.assert_not_called() + + check.count('metric', 0, hostname='second-host') + check.count('metric', 0, hostname='third-host') + check.count('metric', 0, hostname='fourth-host') + + emit_agent_telemetry.assert_called_once_with( + 'openmetrics', + 'max_returned_metrics_reached', + 1, + 'counter', + {'check_name': 'canonical_openmetrics'}, + ) + assert len(check.get_warnings()) == 1 + assert len(aggregator.metrics('metric')) == 6 + + def test_metric_limit_telemetry_skips_non_openmetrics(self, aggregator, monkeypatch): + emit_agent_telemetry = mock.MagicMock() + monkeypatch.setattr( + 'datadog_checks.base.checks.base.datadog_agent.emit_agent_telemetry_with_labels', + emit_agent_telemetry, + raising=False, + ) + check = LimitedCheck('generic_check', {}, [{'max_returned_metrics': 1}]) + + check.gauge('metric', 0) + check.gauge('metric', 0) + + emit_agent_telemetry.assert_not_called() + assert len(check.get_warnings()) == 1 + assert len(aggregator.metrics('metric')) == 1 + + def test_openmetrics_metric_limit_telemetry_skips_without_labeled_bridge(self, aggregator, monkeypatch): + emit_agent_telemetry = mock.MagicMock() + monkeypatch.delattr( + 'datadog_checks.base.checks.base.datadog_agent.emit_agent_telemetry_with_labels', raising=False + ) + monkeypatch.setattr('datadog_checks.base.checks.base.datadog_agent.emit_agent_telemetry', emit_agent_telemetry) + check = OpenMetricsTelemetryCheck('canonical_openmetrics', {}, [{'max_returned_metrics': 1}]) + + check.gauge('metric', 0) + check.gauge('metric', 0) + + emit_agent_telemetry.assert_not_called() + assert len(check.get_warnings()) == 1 + assert len(aggregator.metrics('metric')) == 1 + + def test_openmetrics_metric_limit_telemetry_bridge_failure(self, aggregator, caplog, monkeypatch): + emit_agent_telemetry = mock.MagicMock(side_effect=Exception('bridge failed')) + monkeypatch.setattr( + 'datadog_checks.base.checks.base.datadog_agent.emit_agent_telemetry_with_labels', + emit_agent_telemetry, + raising=False, + ) + caplog.set_level(logging.DEBUG) + check = OpenMetricsTelemetryCheck('canonical_openmetrics', {}, [{'max_returned_metrics': 1}]) + + check.gauge('metric', 0) + check.gauge('metric', 0) + check.gauge('metric', 0) + + emit_agent_telemetry.assert_called_once_with( + 'openmetrics', + 'max_returned_metrics_reached', + 1, + 'counter', + {'check_name': 'canonical_openmetrics'}, + ) + assert len(check.get_warnings()) == 1 + assert len(aggregator.metrics('metric')) == 1 + assert 'Unable to emit OpenMetrics max_returned_metrics Agent telemetry' in caplog.text + def test_metric_limit_instance_config(self, aggregator): instances = [{"max_returned_metrics": 42}] check = AgentCheck("test", {}, instances) @@ -1294,6 +1433,114 @@ def test_debug_metrics_over_limit(self, aggregator, dd_run_check): aggregator.assert_metric('datadog.agent.metrics.contexts.limit', 3) aggregator.assert_metric('datadog.agent.metrics.contexts.total', 5) + @pytest.mark.parametrize('debug_metrics', [False, True]) + def test_openmetrics_metric_limit_telemetry_once_per_run( + self, aggregator, dd_run_check, debug_metrics, monkeypatch + ): + emit_agent_telemetry = mock.MagicMock() + monkeypatch.setattr( + 'datadog_checks.base.checks.base.datadog_agent.emit_agent_telemetry_with_labels', + emit_agent_telemetry, + raising=False, + ) + instance = {'max_returned_metrics': 3} + if debug_metrics: + instance['debug_metrics'] = {'metric_contexts': True} + check = OpenMetricsTelemetryCheck('canonical_openmetrics', {}, [instance]) + + dd_run_check(check) + dd_run_check(check) + + assert emit_agent_telemetry.call_args_list == [ + mock.call( + 'openmetrics', 'max_returned_metrics_reached', 1, 'counter', {'check_name': 'canonical_openmetrics'} + ), + mock.call( + 'openmetrics', 'max_returned_metrics_reached', 1, 'counter', {'check_name': 'canonical_openmetrics'} + ), + ] + assert len(aggregator.metrics('foo')) == 6 + if debug_metrics: + aggregator.assert_metric('datadog.agent.metrics.contexts.limit', 3) + aggregator.assert_metric('datadog.agent.metrics.contexts.total', 5) + + def test_openmetrics_metric_limit_telemetry_not_emitted_for_debug_metrics( + self, aggregator, dd_run_check, monkeypatch + ): + emit_agent_telemetry = mock.MagicMock() + monkeypatch.setattr( + 'datadog_checks.base.checks.base.datadog_agent.emit_agent_telemetry_with_labels', + emit_agent_telemetry, + raising=False, + ) + instance = {'debug_metrics': {'metric_contexts': True}, 'max_returned_metrics': 1} + check = OpenMetricsTelemetryCheck('canonical_openmetrics', {}, [instance]) + + dd_run_check(check) + + emit_agent_telemetry.assert_called_once_with( + 'openmetrics', + 'max_returned_metrics_reached', + 1, + 'counter', + {'check_name': 'canonical_openmetrics'}, + ) + assert len(aggregator.metrics('foo')) == 1 + aggregator.assert_metric('datadog.agent.metrics.contexts.limit', 1) + + @pytest.mark.parametrize( + 'openmetrics_version, instance', + [ + pytest.param( + 'v1', + { + 'prometheus_url': 'http://example.com/metrics', + 'namespace': 'test', + 'metrics': [], + 'max_returned_metrics': 1, + }, + id='v1', + ), + pytest.param( + 'v2', + { + 'openmetrics_endpoint': 'http://example.com/metrics', + 'namespace': 'test', + 'metrics': [], + 'max_returned_metrics': 1, + }, + id='v2', + ), + ], + ) + def test_openmetrics_base_classes_opt_in_to_metric_limit_telemetry( + self, aggregator, openmetrics_version, instance, monkeypatch + ): + from datadog_checks.base.checks.openmetrics.base_check import OpenMetricsBaseCheck + from datadog_checks.base.checks.openmetrics.v2.base import OpenMetricsBaseCheckV2 + + check_class = OpenMetricsBaseCheck if openmetrics_version == 'v1' else OpenMetricsBaseCheckV2 + emit_agent_telemetry = mock.MagicMock() + monkeypatch.setattr( + 'datadog_checks.base.checks.base.datadog_agent.emit_agent_telemetry_with_labels', + emit_agent_telemetry, + raising=False, + ) + check = check_class('canonical_openmetrics', {}, [instance]) + + check.gauge('metric', 0) + check.gauge('metric', 0) + + emit_agent_telemetry.assert_called_once_with( + 'openmetrics', + 'max_returned_metrics_reached', + 1, + 'counter', + {'check_name': 'canonical_openmetrics'}, + ) + assert len(check.get_warnings()) == 1 + assert len(aggregator.metrics('metric')) == 1 + class TestCheckInitializations: def test_success_only_once(self): diff --git a/datadog_checks_base/tests/base/utils/test_utils.py b/datadog_checks_base/tests/base/utils/test_utils.py index db42e45720916..ae75bbf00bdde 100644 --- a/datadog_checks_base/tests/base/utils/test_utils.py +++ b/datadog_checks_base/tests/base/utils/test_utils.py @@ -116,6 +116,61 @@ def test_reset(self): assert limiter.is_reached("dummy1") is False assert limiter.get_status() == (1, 10, False) + def test_on_limit_reached_callback(self): + callback = mock.MagicMock() + limiter = Limiter("my_check", "names", 2, on_limit_reached_func=callback) + + assert limiter.is_reached() is False + assert limiter.is_reached() is False + callback.assert_not_called() + + assert limiter.is_reached() is True + callback.assert_called_once_with() + + assert limiter.is_reached() is True + callback.assert_called_once_with() + + def test_on_limit_reached_callback_with_uid(self): + callback = mock.MagicMock() + limiter = Limiter("my_check", "names", 2, on_limit_reached_func=callback) + + for _ in range(3): + assert limiter.is_reached("dummy1") is False + assert limiter.get_status() == (1, 2, False) + callback.assert_not_called() + + assert limiter.is_reached("dummy2") is False + assert limiter.is_reached("dummy3") is True + assert limiter.get_status() == (3, 2, True) + callback.assert_called_once_with() + + assert limiter.is_reached("dummy4") is True + assert limiter.get_status() == (4, 2, True) + callback.assert_called_once_with() + + def test_on_limit_reached_callback_order(self): + calls = [] + limiter = None + + def warning(*args): + calls.append(("warning", limiter.reached_limit, args)) + + def callback(): + calls.append(("callback", limiter.reached_limit)) + + limiter = Limiter("my_check", "names", 1, warning_func=warning, on_limit_reached_func=callback) + assert limiter.is_reached() is False + assert limiter.is_reached() is True + + assert calls == [ + ( + "warning", + False, + ("Check %s exceeded limit of %s %s, ignoring next ones", "my_check", 1, "names"), + ), + ("callback", True), + ] + class TestRounding: def test_round_half_up(self): From aa31d2d64dcf6aa878bfce6cd0bb13c150cd305c Mon Sep 17 00:00:00 2001 From: nubtron Date: Thu, 16 Jul 2026 12:50:24 +0000 Subject: [PATCH 2/3] Remove placeholder changelog fragment --- datadog_checks_base/changelog.d/0.added | 1 - 1 file changed, 1 deletion(-) delete mode 100644 datadog_checks_base/changelog.d/0.added diff --git a/datadog_checks_base/changelog.d/0.added b/datadog_checks_base/changelog.d/0.added deleted file mode 100644 index b126424a7da09..0000000000000 --- a/datadog_checks_base/changelog.d/0.added +++ /dev/null @@ -1 +0,0 @@ -Add best-effort OpenMetrics metric-limit Agent telemetry when ``max_returned_metrics`` is reached. From 4e49c1cc8bf862d5aa43d09f3be82aa9dc7942b7 Mon Sep 17 00:00:00 2001 From: nubtron Date: Thu, 16 Jul 2026 15:47:32 +0000 Subject: [PATCH 3/3] Add datadog_checks_base changelog entry --- datadog_checks_base/changelog.d/24577.added | 1 + 1 file changed, 1 insertion(+) create mode 100644 datadog_checks_base/changelog.d/24577.added diff --git a/datadog_checks_base/changelog.d/24577.added b/datadog_checks_base/changelog.d/24577.added new file mode 100644 index 0000000000000..b126424a7da09 --- /dev/null +++ b/datadog_checks_base/changelog.d/24577.added @@ -0,0 +1 @@ +Add best-effort OpenMetrics metric-limit Agent telemetry when ``max_returned_metrics`` is reached.