From 040fd0ae35e1dee36b4708f2dc21891248db02b0 Mon Sep 17 00:00:00 2001 From: Rajdeep Singh Date: Fri, 3 Apr 2026 16:51:22 +0530 Subject: [PATCH 1/5] Fixes #26555: Fix db_connections_total Prometheus metric always zero --- .../monitoring/OpenMetadataMetrics.java | 23 ++++---- .../monitoring/OpenMetadataMetricsTest.java | 54 +++++++++++++++++++ 2 files changed, 67 insertions(+), 10 deletions(-) create mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/monitoring/OpenMetadataMetricsTest.java diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/monitoring/OpenMetadataMetrics.java b/openmetadata-service/src/main/java/org/openmetadata/service/monitoring/OpenMetadataMetrics.java index a8612e80b9bd..d51b92672cf4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/monitoring/OpenMetadataMetrics.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/monitoring/OpenMetadataMetrics.java @@ -4,6 +4,7 @@ import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.DistributionSummary; +import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Timer; import java.time.Duration; @@ -19,7 +20,6 @@ public class OpenMetadataMetrics { private final DistributionSummary httpResponseSize; private final Timer jdbiQueryTimer; - private final Counter jdbiConnectionCounter; private final Counter jdbiErrorCounter; private final Counter entityCreatedCounter; @@ -50,10 +50,9 @@ public OpenMetadataMetrics(MeterRegistry meterRegistry) { .sla(LATENCY_SLA_BUCKETS) .register(meterRegistry); - this.jdbiConnectionCounter = - Counter.builder("db.connections.total") - .description("Total database connections created") - .register(meterRegistry); + Gauge.builder("db.connections", () -> poolTotalConnections()) + .description("Total connections in the database connection pool") + .register(meterRegistry); this.jdbiErrorCounter = Counter.builder("db.errors.total") @@ -135,10 +134,6 @@ public void recordDatabaseQuery(String queryType, long durationMs) { .record(Duration.ofMillis(durationMs)); } - public void incrementDatabaseConnections() { - jdbiConnectionCounter.increment(); - } - public void incrementDatabaseErrors(String errorType) { meterRegistry.counter("db.errors.total", "type", errorType).increment(); } @@ -200,7 +195,7 @@ public void recordAuthenticationFailure(String authType, String reason) { // Gauge registration methods public void registerGauge( String name, java.util.function.Supplier supplier, String description) { - io.micrometer.core.instrument.Gauge.builder(name, () -> supplier.get().doubleValue()) + Gauge.builder(name, () -> supplier.get().doubleValue()) .description(description) .register(meterRegistry); } @@ -208,4 +203,12 @@ public void registerGauge( public MeterRegistry getMeterRegistry() { return meterRegistry; } + + private double poolTotalConnections() { + Gauge active = meterRegistry.find("hikaricp.connections.active").gauge(); + Gauge idle = meterRegistry.find("hikaricp.connections.idle").gauge(); + double activeVal = active != null ? active.value() : 0.0; + double idleVal = idle != null ? idle.value() : 0.0; + return activeVal + idleVal; + } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/OpenMetadataMetricsTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/OpenMetadataMetricsTest.java new file mode 100644 index 000000000000..b699518a19c1 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/OpenMetadataMetricsTest.java @@ -0,0 +1,54 @@ +package org.openmetadata.service.monitoring; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class OpenMetadataMetricsTest { + + private SimpleMeterRegistry registry; + + @BeforeEach + void setUp() { + registry = new SimpleMeterRegistry(); + } + + @Test + void dbConnectionsTotalReflectsHikariCPPoolSize() { + AtomicInteger activeConnections = new AtomicInteger(5); + AtomicInteger idleConnections = new AtomicInteger(15); + + Gauge.builder("hikaricp.connections.active", activeConnections, AtomicInteger::doubleValue) + .register(registry); + Gauge.builder("hikaricp.connections.idle", idleConnections, AtomicInteger::doubleValue) + .register(registry); + + new OpenMetadataMetrics(registry); + + Gauge total = registry.find("db.connections").gauge(); + assertNotNull(total, "db.connections gauge should be registered"); + assertEquals(20.0, total.value(), 0.01, "Should equal active + idle"); + + activeConnections.set(10); + idleConnections.set(10); + assertEquals(20.0, total.value(), 0.01, "Should reflect updated pool state"); + + activeConnections.set(0); + idleConnections.set(0); + assertEquals(0.0, total.value(), 0.01, "Should be zero when pool is empty"); + } + + @Test + void dbConnectionsTotalReturnsZeroWithoutHikariCP() { + new OpenMetadataMetrics(registry); + + Gauge total = registry.find("db.connections").gauge(); + assertNotNull(total, "db.connections gauge should be registered"); + assertEquals(0.0, total.value(), 0.01, "Should be zero when HikariCP metrics are absent"); + } +} From 5c5accdd3772a6335aedef7b8203445dabfbb688 Mon Sep 17 00:00:00 2001 From: Rajdeep Singh Date: Sat, 4 Apr 2026 22:10:07 +0530 Subject: [PATCH 2/5] =?UTF-8?q?fix:=20Address=20review=20comments=20?= =?UTF-8?q?=E2=80=94=20restore=20metric=20name=20and=20add=20Prometheus=20?= =?UTF-8?q?scrape=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Restore gauge name to 'db.connections.total' so Micrometer's Prometheus naming convention exposes it as 'db_connections_total', preserving backward compatibility with existing dashboards. - Add prometheusScrapExposesDbConnectionsTotal() test that uses PrometheusMeterRegistry to verify the Prometheus scrape output contains 'db_connections_total' as a gauge type. --- .../monitoring/OpenMetadataMetrics.java | 2 +- .../monitoring/OpenMetadataMetricsTest.java | 34 ++++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/monitoring/OpenMetadataMetrics.java b/openmetadata-service/src/main/java/org/openmetadata/service/monitoring/OpenMetadataMetrics.java index d51b92672cf4..3077fdc03cff 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/monitoring/OpenMetadataMetrics.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/monitoring/OpenMetadataMetrics.java @@ -50,7 +50,7 @@ public OpenMetadataMetrics(MeterRegistry meterRegistry) { .sla(LATENCY_SLA_BUCKETS) .register(meterRegistry); - Gauge.builder("db.connections", () -> poolTotalConnections()) + Gauge.builder("db.connections.total", () -> poolTotalConnections()) .description("Total connections in the database connection pool") .register(meterRegistry); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/OpenMetadataMetricsTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/OpenMetadataMetricsTest.java index b699518a19c1..1767cc02ad4b 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/OpenMetadataMetricsTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/OpenMetadataMetricsTest.java @@ -2,9 +2,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import io.micrometer.prometheusmetrics.PrometheusConfig; +import io.micrometer.prometheusmetrics.PrometheusMeterRegistry; import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -30,8 +33,8 @@ void dbConnectionsTotalReflectsHikariCPPoolSize() { new OpenMetadataMetrics(registry); - Gauge total = registry.find("db.connections").gauge(); - assertNotNull(total, "db.connections gauge should be registered"); + Gauge total = registry.find("db.connections.total").gauge(); + assertNotNull(total, "db.connections.total gauge should be registered"); assertEquals(20.0, total.value(), 0.01, "Should equal active + idle"); activeConnections.set(10); @@ -47,8 +50,31 @@ void dbConnectionsTotalReflectsHikariCPPoolSize() { void dbConnectionsTotalReturnsZeroWithoutHikariCP() { new OpenMetadataMetrics(registry); - Gauge total = registry.find("db.connections").gauge(); - assertNotNull(total, "db.connections gauge should be registered"); + Gauge total = registry.find("db.connections.total").gauge(); + assertNotNull(total, "db.connections.total gauge should be registered"); assertEquals(0.0, total.value(), 0.01, "Should be zero when HikariCP metrics are absent"); } + + @Test + void prometheusScrapExposesDbConnectionsTotal() { + PrometheusMeterRegistry promRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); + + AtomicInteger activeConnections = new AtomicInteger(3); + AtomicInteger idleConnections = new AtomicInteger(7); + + Gauge.builder("hikaricp.connections.active", activeConnections, AtomicInteger::doubleValue) + .register(promRegistry); + Gauge.builder("hikaricp.connections.idle", idleConnections, AtomicInteger::doubleValue) + .register(promRegistry); + + new OpenMetadataMetrics(promRegistry); + + String scrape = promRegistry.scrape(); + assertTrue( + scrape.contains("db_connections_total"), + "Prometheus scrape should contain db_connections_total metric"); + assertTrue( + scrape.contains("# TYPE db_connections_total gauge"), + "db_connections_total should be exposed as a gauge type"); + } } From 7ef92aa57361b1ec9eed23245c0eae538ef2e521 Mon Sep 17 00:00:00 2001 From: Rajdeep Singh Date: Sat, 4 Apr 2026 22:27:40 +0530 Subject: [PATCH 3/5] Fix typo: prometheusScrap -> prometheusScrape --- .../service/monitoring/OpenMetadataMetricsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/OpenMetadataMetricsTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/OpenMetadataMetricsTest.java index 1767cc02ad4b..afd973e43f23 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/OpenMetadataMetricsTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/OpenMetadataMetricsTest.java @@ -56,7 +56,7 @@ void dbConnectionsTotalReturnsZeroWithoutHikariCP() { } @Test - void prometheusScrapExposesDbConnectionsTotal() { + void prometheusScrapeExposesDbConnectionsTotal() { PrometheusMeterRegistry promRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); AtomicInteger activeConnections = new AtomicInteger(3); From 90bd5725b18baa7abd4f26eeb259537cb8ec321d Mon Sep 17 00:00:00 2001 From: Rajdeep Singh Date: Sun, 5 Apr 2026 00:24:26 +0530 Subject: [PATCH 4/5] fix: Rename gauge to db.pool.connections to avoid Prometheus _total suffix The new Prometheus Java client (used by PrometheusMeterRegistry in micrometer 1.14.5) strips the _total suffix from metric names because it is reserved for counters. The gauge db.connections.total was being sanitized to db_connections, causing the test to fail when looking for db_connections_total in the scrape output. Renamed to db.pool.connections which exports as db_pool_connections in Prometheus format, avoiding suffix conflicts. --- .../service/monitoring/OpenMetadataMetrics.java | 2 +- .../monitoring/OpenMetadataMetricsTest.java | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/monitoring/OpenMetadataMetrics.java b/openmetadata-service/src/main/java/org/openmetadata/service/monitoring/OpenMetadataMetrics.java index 3077fdc03cff..b84d21fb4233 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/monitoring/OpenMetadataMetrics.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/monitoring/OpenMetadataMetrics.java @@ -50,7 +50,7 @@ public OpenMetadataMetrics(MeterRegistry meterRegistry) { .sla(LATENCY_SLA_BUCKETS) .register(meterRegistry); - Gauge.builder("db.connections.total", () -> poolTotalConnections()) + Gauge.builder("db.pool.connections", () -> poolTotalConnections()) .description("Total connections in the database connection pool") .register(meterRegistry); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/OpenMetadataMetricsTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/OpenMetadataMetricsTest.java index afd973e43f23..85ed457e6b06 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/OpenMetadataMetricsTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/OpenMetadataMetricsTest.java @@ -33,8 +33,8 @@ void dbConnectionsTotalReflectsHikariCPPoolSize() { new OpenMetadataMetrics(registry); - Gauge total = registry.find("db.connections.total").gauge(); - assertNotNull(total, "db.connections.total gauge should be registered"); + Gauge total = registry.find("db.pool.connections").gauge(); + assertNotNull(total, "db.pool.connections gauge should be registered"); assertEquals(20.0, total.value(), 0.01, "Should equal active + idle"); activeConnections.set(10); @@ -50,8 +50,8 @@ void dbConnectionsTotalReflectsHikariCPPoolSize() { void dbConnectionsTotalReturnsZeroWithoutHikariCP() { new OpenMetadataMetrics(registry); - Gauge total = registry.find("db.connections.total").gauge(); - assertNotNull(total, "db.connections.total gauge should be registered"); + Gauge total = registry.find("db.pool.connections").gauge(); + assertNotNull(total, "db.pool.connections gauge should be registered"); assertEquals(0.0, total.value(), 0.01, "Should be zero when HikariCP metrics are absent"); } @@ -71,10 +71,10 @@ void prometheusScrapeExposesDbConnectionsTotal() { String scrape = promRegistry.scrape(); assertTrue( - scrape.contains("db_connections_total"), - "Prometheus scrape should contain db_connections_total metric"); + scrape.contains("db_pool_connections"), + "Prometheus scrape should contain db_pool_connections metric"); assertTrue( - scrape.contains("# TYPE db_connections_total gauge"), - "db_connections_total should be exposed as a gauge type"); + scrape.contains("# TYPE db_pool_connections gauge"), + "db_pool_connections should be exposed as a gauge type"); } } From 105c956574a9a425ff039564bab44cda58fdab3e Mon Sep 17 00:00:00 2001 From: Rajdeep Singh Date: Fri, 17 Apr 2026 07:26:30 +0530 Subject: [PATCH 5/5] test: rename db-connection tests to match db_pool_connections assertions --- .../service/monitoring/OpenMetadataMetricsTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/OpenMetadataMetricsTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/OpenMetadataMetricsTest.java index 85ed457e6b06..4b821e2ccd46 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/OpenMetadataMetricsTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/OpenMetadataMetricsTest.java @@ -22,7 +22,7 @@ void setUp() { } @Test - void dbConnectionsTotalReflectsHikariCPPoolSize() { + void dbPoolConnectionsGaugeReflectsHikariCPPoolSize() { AtomicInteger activeConnections = new AtomicInteger(5); AtomicInteger idleConnections = new AtomicInteger(15); @@ -47,7 +47,7 @@ void dbConnectionsTotalReflectsHikariCPPoolSize() { } @Test - void dbConnectionsTotalReturnsZeroWithoutHikariCP() { + void dbPoolConnectionsGaugeReturnsZeroWithoutHikariCP() { new OpenMetadataMetrics(registry); Gauge total = registry.find("db.pool.connections").gauge(); @@ -56,7 +56,7 @@ void dbConnectionsTotalReturnsZeroWithoutHikariCP() { } @Test - void prometheusScrapeExposesDbConnectionsTotal() { + void prometheusScrapeExposesDbPoolConnectionsGauge() { PrometheusMeterRegistry promRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); AtomicInteger activeConnections = new AtomicInteger(3);