From a620b9af39f13d457769e4236f2fb64dcf09b7bb Mon Sep 17 00:00:00 2001 From: Matan Lior Date: Tue, 7 Jul 2026 12:57:35 +0300 Subject: [PATCH] apollo_dashboard: alert on frozen exchange-rate oracle (eth_to_strk, strk_to_usd) (#14706) Co-authored-by: Claude Opus 4.8 (1M context) (cherry picked from commit e871c40c4d26ca6c4867661b335644ec45e68f48) --- .../resources/dev_grafana_alerts.json | 52 +++++++++++++++++++ .../apollo_dashboard/src/alert_definitions.rs | 4 ++ .../src/alert_scenarios/l1_gas_prices.rs | 51 ++++++++++++++++++ 3 files changed, 107 insertions(+) diff --git a/crates/apollo_dashboard/resources/dev_grafana_alerts.json b/crates/apollo_dashboard/resources/dev_grafana_alerts.json index c6d930e222f..e2e03e43e83 100644 --- a/crates/apollo_dashboard/resources/dev_grafana_alerts.json +++ b/crates/apollo_dashboard/resources/dev_grafana_alerts.json @@ -898,6 +898,32 @@ "for": "1m", "severity": "p5" }, + { + "name": "eth_to_strk_rate_frozen", + "title": "Eth to Strk rate frozen", + "ruleGroup": "evaluation_rate_default", + "expr": "sum(changes(eth_to_strk_rate{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1h]))", + "conditions": [ + { + "evaluator": { + "params": [ + 1.0 + ], + "type": "lt" + }, + "operator": { + "type": "and" + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "for": "30s", + "severity": "$$$eth_to_strk_rate_frozen-severity$$$" + }, { "name": "eth_to_strk_success_count", "title": "Eth to Strk success count", @@ -2172,6 +2198,32 @@ "for": "1m", "severity": "p5" }, + { + "name": "strk_to_usd_rate_frozen", + "title": "Strk to Usd rate frozen", + "ruleGroup": "evaluation_rate_default", + "expr": "sum(changes(snip35_strk_usd_rate{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1h]))", + "conditions": [ + { + "evaluator": { + "params": [ + 1.0 + ], + "type": "lt" + }, + "operator": { + "type": "and" + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "for": "30s", + "severity": "$$$strk_to_usd_rate_frozen-severity$$$" + }, { "name": "strk_to_usd_success_count", "title": "Strk to Usd success count", diff --git a/crates/apollo_dashboard/src/alert_definitions.rs b/crates/apollo_dashboard/src/alert_definitions.rs index 55cd11781b7..0e0473c9e77 100644 --- a/crates/apollo_dashboard/src/alert_definitions.rs +++ b/crates/apollo_dashboard/src/alert_definitions.rs @@ -71,10 +71,12 @@ use crate::alert_scenarios::infra_alerts::{ use crate::alert_scenarios::l1_endpoints::get_primary_l1_endpoint_down_too_long_alerts; use crate::alert_scenarios::l1_gas_prices::{ get_eth_to_strk_error_count_alert, + get_eth_to_strk_rate_frozen_alert, get_eth_to_strk_success_count_alert, get_l1_gas_price_provider_insufficient_history_alert, get_l1_gas_price_scraper_success_count_alert, get_strk_to_usd_error_count_alert, + get_strk_to_usd_rate_frozen_alert, get_strk_to_usd_success_count_alert, }; use crate::alert_scenarios::l1_handlers::{ @@ -640,6 +642,8 @@ pub fn get_apollo_alerts() -> Alerts { alerts.push(get_consensus_round_high()); alerts.push(get_eth_to_strk_success_count_alert()); alerts.push(get_strk_to_usd_success_count_alert()); + alerts.push(get_eth_to_strk_rate_frozen_alert()); + alerts.push(get_strk_to_usd_rate_frozen_alert()); alerts.append(&mut get_general_pod_memory_utilization_vec()); alerts.append(&mut get_general_pod_disk_utilization_vec()); alerts.push(get_http_server_avg_add_tx_latency_alert()); diff --git a/crates/apollo_dashboard/src/alert_scenarios/l1_gas_prices.rs b/crates/apollo_dashboard/src/alert_scenarios/l1_gas_prices.rs index 9304cce737a..01cc7c19d43 100644 --- a/crates/apollo_dashboard/src/alert_scenarios/l1_gas_prices.rs +++ b/crates/apollo_dashboard/src/alert_scenarios/l1_gas_prices.rs @@ -1,9 +1,11 @@ use apollo_l1_gas_price::metrics::{ ETH_TO_STRK_ERROR_COUNT, + ETH_TO_STRK_RATE, ETH_TO_STRK_SUCCESS_COUNT, L1_GAS_PRICE_PROVIDER_INSUFFICIENT_HISTORY, L1_GAS_PRICE_SCRAPER_SUCCESS_COUNT, SNIP35_STRK_USD_ERROR_COUNT, + SNIP35_STRK_USD_RATE, SNIP35_STRK_USD_SUCCESS_COUNT, }; use apollo_metrics::metrics::MetricQueryName; @@ -59,6 +61,26 @@ pub(crate) fn get_strk_to_usd_error_count_alert() -> Alert { ) } +pub(crate) fn get_eth_to_strk_rate_frozen_alert() -> Alert { + const ALERT_NAME: &str = "eth_to_strk_rate_frozen"; + oracle_rate_frozen_alert( + ALERT_NAME, + "Eth to Strk rate frozen", + Ð_TO_STRK_RATE, + SeverityValueOrPlaceholder::Placeholder(ALERT_NAME.to_string()), + ) +} + +pub(crate) fn get_strk_to_usd_rate_frozen_alert() -> Alert { + const ALERT_NAME: &str = "strk_to_usd_rate_frozen"; + oracle_rate_frozen_alert( + ALERT_NAME, + "Strk to Usd rate frozen", + &SNIP35_STRK_USD_RATE, + SeverityValueOrPlaceholder::Placeholder(ALERT_NAME.to_string()), + ) +} + /// Alert if had no successful l1 gas price scrape in the last hour. /// /// Uses `sum_increase` for the same spot-eviction reason as `get_eth_to_strk_success_count_alert`. @@ -138,3 +160,32 @@ fn oracle_error_count_alert( ObserverApplicability::NotApplicable, ) } + +/// Alert if an exchange-rate oracle's rate gauge has not changed at all in the last hour. +/// +/// Detects a *frozen feed*: the oracle keeps resolving successfully (so the success count and +/// last-success timestamp stay healthy) while serving a stale, unchanging price. `changes` over 1h +/// is 0 only when the value never moved across the ~4 update buckets in that window — effectively +/// impossible for a live 18-decimal rate. Unlike the error-count alert there is deliberately no +/// `or vector(0)`: an absent gauge must stay no-data (so an oracle that never resolves doesn't look +/// "frozen"); only a present-but-flat gauge trips this. +/// +/// Applies to observers too: a frozen upstream feed is env-wide and observer nodes run the same +/// oracle client, so the alert should fire regardless of node role. +fn oracle_rate_frozen_alert( + name: &str, + title: &str, + rate_metric: &dyn MetricQueryName, + severity: impl Into, +) -> Alert { + Alert::new( + name, + title, + EvaluationRate::Default, + format!("sum(changes({}[1h]))", rate_metric.get_name_with_filter()), + vec![AlertCondition::new(AlertComparisonOp::LessThan, 1.0, AlertLogicalOp::And)], + PENDING_DURATION_DEFAULT, + severity, + ObserverApplicability::Applicable, + ) +}