From 89f6937d1d54d9e313a83e94e66ea91aeb661407 Mon Sep 17 00:00:00 2001 From: sean wibisono Date: Thu, 14 May 2026 14:07:27 +1000 Subject: [PATCH] UID2-7063: Emit salts_rotated_last_cycle Prometheus gauge Adds a Micrometer gauge `uid2_salts_rotated_last_cycle` that records the count of salts rotated in the most recent successful salt rotation cycle. The gauge reports NaN until the first rotation completes so the volume alert does not fire on cold start. Lets the salt-rotation volume alert key off a Prometheus metric instead of Loki log scraping, which has produced false positives when log ingestion glitches (Alloy re-streams, Loki push failures). --- src/main/java/com/uid2/admin/Main.java | 2 + .../admin/monitoring/SaltRotationMetrics.java | 29 ++++++++++ .../com/uid2/admin/salt/SaltRotation.java | 2 + .../com/uid2/admin/salt/SaltRotationTest.java | 57 +++++++++++++++++++ 4 files changed, 90 insertions(+) create mode 100644 src/main/java/com/uid2/admin/monitoring/SaltRotationMetrics.java diff --git a/src/main/java/com/uid2/admin/Main.java b/src/main/java/com/uid2/admin/Main.java index 91dd9e902..725abdb5f 100644 --- a/src/main/java/com/uid2/admin/Main.java +++ b/src/main/java/com/uid2/admin/Main.java @@ -14,6 +14,7 @@ import com.uid2.admin.legacy.RotatingLegacyClientKeyProvider; import com.uid2.admin.managers.KeysetManager; import com.uid2.admin.monitoring.DataStoreMetrics; +import com.uid2.admin.monitoring.SaltRotationMetrics; import com.uid2.admin.salt.SaltRotation; import com.uid2.admin.secret.*; import com.uid2.admin.store.*; @@ -329,6 +330,7 @@ public void run() { DataStoreMetrics.addDataStoreMetrics("service_link", serviceLinkProvider); DataStoreMetrics.addDataStoreServiceLinkEntryCount("snowflake", serviceLinkProvider, serviceProvider); + SaltRotationMetrics.register(Metrics.globalRegistry); ReplaceSharingTypesWithSitesJob replaceSharingTypesWithSitesJob = new ReplaceSharingTypesWithSitesJob(config, writeLock, adminKeysetProvider, keysetProvider, keysetStoreWriter, siteProvider); jobDispatcher.enqueue(replaceSharingTypesWithSitesJob); diff --git a/src/main/java/com/uid2/admin/monitoring/SaltRotationMetrics.java b/src/main/java/com/uid2/admin/monitoring/SaltRotationMetrics.java new file mode 100644 index 000000000..b395eee78 --- /dev/null +++ b/src/main/java/com/uid2/admin/monitoring/SaltRotationMetrics.java @@ -0,0 +1,29 @@ +package com.uid2.admin.monitoring; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; + +import java.util.concurrent.atomic.AtomicLong; + +public final class SaltRotationMetrics { + private static final AtomicLong lastRotatedSaltCount = new AtomicLong(-1); + + public static void register(MeterRegistry registry) { + // Reports NaN until the first rotation completes, so the alert does not fire on cold start. + Gauge.builder("uid2_salts_rotated_last_cycle", lastRotatedSaltCount, SaltRotationMetrics::asGaugeValue) + .description("Number of salts rotated in the most recent successful salt rotation cycle") + .strongReference(true) + .register(registry); + } + + public static void recordRotated(int count) { + lastRotatedSaltCount.set(count); + } + + private static double asGaugeValue(AtomicLong ref) { + long value = ref.get(); + return value < 0 ? Double.NaN : (double) value; + } + + private SaltRotationMetrics() {} +} diff --git a/src/main/java/com/uid2/admin/salt/SaltRotation.java b/src/main/java/com/uid2/admin/salt/SaltRotation.java index 3cb4cfa0d..9ef3b0c90 100644 --- a/src/main/java/com/uid2/admin/salt/SaltRotation.java +++ b/src/main/java/com/uid2/admin/salt/SaltRotation.java @@ -1,6 +1,7 @@ package com.uid2.admin.salt; import com.uid2.admin.AdminConst; +import com.uid2.admin.monitoring.SaltRotationMetrics; import com.uid2.shared.model.SaltEntry; import com.uid2.shared.secret.IKeyGenerator; @@ -63,6 +64,7 @@ public Result rotateSalts( logSaltAges("rotated-salts", targetDate, saltsToRotate); logSaltAges("total-salts", targetDate, Arrays.asList(postRotationSalts)); logBucketFormatCount(targetDate, postRotationSalts); + SaltRotationMetrics.recordRotated(saltsToRotate.size()); var nextSnapshot = new SaltSnapshot( nextEffective, diff --git a/src/test/java/com/uid2/admin/salt/SaltRotationTest.java b/src/test/java/com/uid2/admin/salt/SaltRotationTest.java index 56344bab4..6fd5cd306 100644 --- a/src/test/java/com/uid2/admin/salt/SaltRotationTest.java +++ b/src/test/java/com/uid2/admin/salt/SaltRotationTest.java @@ -3,10 +3,12 @@ import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.read.ListAppender; import com.uid2.admin.AdminConst; +import com.uid2.admin.monitoring.SaltRotationMetrics; import com.uid2.admin.salt.helper.SaltBuilder; import com.uid2.admin.salt.helper.SaltSnapshotBuilder; import com.uid2.shared.model.SaltEntry; import com.uid2.shared.secret.IKeyGenerator; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import io.vertx.core.json.JsonObject; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -440,6 +442,61 @@ private int countEntriesWithLastUpdated(SaltEntry[] entries, Instant lastUpdated return (int) Arrays.stream(entries).filter(e -> e.lastUpdated() == lastUpdated.toEpochMilli()).count(); } + @Test + void testRotateSaltsRecordsLastCycleMetric() throws Exception { + var registry = new SimpleMeterRegistry(); + SaltRotationMetrics.register(registry); + SaltRotationMetrics.recordRotated(-1); + + final Duration[] minAges = { + Duration.ofDays(1), + Duration.ofDays(2), + }; + var lastSnapshot = SaltSnapshotBuilder.start() + .entries(10, daysEarlier(10), targetDate()) + .build(); + + var result = saltRotation.rotateSalts(lastSnapshot, minAges, 0.2, targetDate()); + assertTrue(result.hasSnapshot()); + + var rotatedCount = countEntriesWithLastUpdated(result.getSnapshot().getAllRotatingSalts(), result.getSnapshot().getEffective()); + var gauge = registry.find("uid2_salts_rotated_last_cycle").gauge(); + assertThat(gauge).isNotNull(); + assertThat(gauge.value()).isEqualTo((double) rotatedCount); + } + + @Test + void testRotateSaltsNoSnapshotLeavesMetricUnchanged() throws Exception { + var registry = new SimpleMeterRegistry(); + SaltRotationMetrics.register(registry); + SaltRotationMetrics.recordRotated(42); + + final Duration[] minAges = { + Duration.ofDays(1), + Duration.ofDays(2), + }; + var lastSnapshot = SaltSnapshotBuilder.start() + .entries(10, targetDate(), targetDate()) + .build(); + + var result = saltRotation.rotateSalts(lastSnapshot, minAges, 0.2, targetDate()); + assertFalse(result.hasSnapshot()); + + var gauge = registry.find("uid2_salts_rotated_last_cycle").gauge(); + assertThat(gauge.value()).isEqualTo(42.0); + } + + @Test + void testSaltRotationMetricReportsNaNBeforeFirstRotation() { + var registry = new SimpleMeterRegistry(); + SaltRotationMetrics.register(registry); + SaltRotationMetrics.recordRotated(-1); + + var gauge = registry.find("uid2_salts_rotated_last_cycle").gauge(); + assertThat(gauge).isNotNull(); + assertThat(Double.isNaN(gauge.value())).isTrue(); + } + @Test void testRotateSaltsZeroDoesntRotateSaltsButUpdatesRefreshFrom() throws Exception { var lastSnapshot = SaltSnapshotBuilder.start()