apollo_dashboard: alert on frozen exchange-rate oracle (eth_to_strk, strk_to_usd) (#14706)

crates/apollo_dashboard/resources/dev_grafana_alerts.json

@@ -898,6 +898,32 @@
       "for": "1m",
       "severity": "p5"
     },
+    {
+      "name": "eth_to_strk_rate_frozen",
+      "title": "Eth to Strk rate frozen",
+      "ruleGroup": "evaluation_rate_default",
+      "expr": "sum(changes(eth_to_strk_rate{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1h]))",
+      "conditions": [
+        {
+          "evaluator": {
+            "params": [
+              1.0
+            ],
+            "type": "lt"
+          },
+          "operator": {
+            "type": "and"
+          },
+          "reducer": {
+            "params": [],
+            "type": "avg"
+          },
+          "type": "query"
+        }
+      ],
+      "for": "30s",
+      "severity": "$$$eth_to_strk_rate_frozen-severity$$$"
+    },
     {
       "name": "eth_to_strk_success_count",
       "title": "Eth to Strk success count",
@@ -2172,6 +2198,32 @@
       "for": "1m",
       "severity": "p5"
     },
+    {
+      "name": "strk_to_usd_rate_frozen",
+      "title": "Strk to Usd rate frozen",
+      "ruleGroup": "evaluation_rate_default",
+      "expr": "sum(changes(snip35_strk_usd_rate{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1h]))",
+      "conditions": [
+        {
+          "evaluator": {
+            "params": [
+              1.0
+            ],
+            "type": "lt"
+          },
+          "operator": {
+            "type": "and"
+          },
+          "reducer": {
+            "params": [],
+            "type": "avg"
+          },
+          "type": "query"
+        }
+      ],
+      "for": "30s",
+      "severity": "$$$strk_to_usd_rate_frozen-severity$$$"
+    },
     {
       "name": "strk_to_usd_success_count",
       "title": "Strk to Usd success count",

crates/apollo_dashboard/src/alert_definitions.rs

@@ -71,10 +71,12 @@ use crate::alert_scenarios::infra_alerts::{
 use crate::alert_scenarios::l1_endpoints::get_primary_l1_endpoint_down_too_long_alerts;
 use crate::alert_scenarios::l1_gas_prices::{
     get_eth_to_strk_error_count_alert,
+    get_eth_to_strk_rate_frozen_alert,
     get_eth_to_strk_success_count_alert,
     get_l1_gas_price_provider_insufficient_history_alert,
     get_l1_gas_price_scraper_success_count_alert,
     get_strk_to_usd_error_count_alert,
+    get_strk_to_usd_rate_frozen_alert,
     get_strk_to_usd_success_count_alert,
 };
 use crate::alert_scenarios::l1_handlers::{
@@ -640,6 +642,8 @@ pub fn get_apollo_alerts() -> Alerts {
     alerts.push(get_consensus_round_high());
     alerts.push(get_eth_to_strk_success_count_alert());
     alerts.push(get_strk_to_usd_success_count_alert());
+    alerts.push(get_eth_to_strk_rate_frozen_alert());
+    alerts.push(get_strk_to_usd_rate_frozen_alert());
     alerts.append(&mut get_general_pod_memory_utilization_vec());
     alerts.append(&mut get_general_pod_disk_utilization_vec());
     alerts.push(get_http_server_avg_add_tx_latency_alert());

crates/apollo_dashboard/src/alert_scenarios/l1_gas_prices.rs

@@ -1,9 +1,11 @@
 use apollo_l1_gas_price::metrics::{
     ETH_TO_STRK_ERROR_COUNT,
+    ETH_TO_STRK_RATE,
     ETH_TO_STRK_SUCCESS_COUNT,
     L1_GAS_PRICE_PROVIDER_INSUFFICIENT_HISTORY,
     L1_GAS_PRICE_SCRAPER_SUCCESS_COUNT,
     SNIP35_STRK_USD_ERROR_COUNT,
+    SNIP35_STRK_USD_RATE,
     SNIP35_STRK_USD_SUCCESS_COUNT,
 };
 use apollo_metrics::metrics::MetricQueryName;
@@ -59,6 +61,26 @@ pub(crate) fn get_strk_to_usd_error_count_alert() -> Alert {
     )
 }
 
+pub(crate) fn get_eth_to_strk_rate_frozen_alert() -> Alert {
+    const ALERT_NAME: &str = "eth_to_strk_rate_frozen";
+    oracle_rate_frozen_alert(
+        ALERT_NAME,
+        "Eth to Strk rate frozen",
+        &ETH_TO_STRK_RATE,
+        SeverityValueOrPlaceholder::Placeholder(ALERT_NAME.to_string()),
+    )
+}
+
+pub(crate) fn get_strk_to_usd_rate_frozen_alert() -> Alert {
+    const ALERT_NAME: &str = "strk_to_usd_rate_frozen";
+    oracle_rate_frozen_alert(
+        ALERT_NAME,
+        "Strk to Usd rate frozen",
+        &SNIP35_STRK_USD_RATE,
+        SeverityValueOrPlaceholder::Placeholder(ALERT_NAME.to_string()),
+    )
+}
+
 /// Alert if had no successful l1 gas price scrape in the last hour.
 ///
 /// Uses `sum_increase` for the same spot-eviction reason as `get_eth_to_strk_success_count_alert`.
@@ -138,3 +160,32 @@ fn oracle_error_count_alert(
         ObserverApplicability::NotApplicable,
     )
 }
+
+/// Alert if an exchange-rate oracle's rate gauge has not changed at all in the last hour.
+///
+/// Detects a *frozen feed*: the oracle keeps resolving successfully (so the success count and
+/// last-success timestamp stay healthy) while serving a stale, unchanging price. `changes` over 1h
+/// is 0 only when the value never moved across the ~4 update buckets in that window — effectively
+/// impossible for a live 18-decimal rate. Unlike the error-count alert there is deliberately no
+/// `or vector(0)`: an absent gauge must stay no-data (so an oracle that never resolves doesn't look
+/// "frozen"); only a present-but-flat gauge trips this.
+///
+/// Applies to observers too: a frozen upstream feed is env-wide and observer nodes run the same
+/// oracle client, so the alert should fire regardless of node role.
+fn oracle_rate_frozen_alert(
+    name: &str,
+    title: &str,
+    rate_metric: &dyn MetricQueryName,
+    severity: impl Into<SeverityValueOrPlaceholder>,
+) -> Alert {
+    Alert::new(
+        name,
+        title,
+        EvaluationRate::Default,
+        format!("sum(changes({}[1h]))", rate_metric.get_name_with_filter()),
+        vec![AlertCondition::new(AlertComparisonOp::LessThan, 1.0, AlertLogicalOp::And)],
+        PENDING_DURATION_DEFAULT,
+        severity,
+        ObserverApplicability::Applicable,
+    )
+}