From f4c4a022f5eb8cd59ee7afbc4835ee01adb28678 Mon Sep 17 00:00:00 2001 From: Matan Lior Date: Tue, 7 Jul 2026 11:20:36 +0300 Subject: [PATCH] apollo_dashboard: extract shared exchange-rate oracle alert builders (#14702) Co-authored-by: Claude Opus 4.8 (1M context) (cherry picked from commit 3251ad1bb3b49fc6ca6ffebf8754b2817b31637f) --- .../apollo_dashboard/src/alert_definitions.rs | 18 +---- .../src/alert_scenarios/l1_gas_prices.rs | 71 +++++++++++++++---- 2 files changed, 60 insertions(+), 29 deletions(-) diff --git a/crates/apollo_dashboard/src/alert_definitions.rs b/crates/apollo_dashboard/src/alert_definitions.rs index f04b2e1c7e8..f167b6bc24e 100644 --- a/crates/apollo_dashboard/src/alert_definitions.rs +++ b/crates/apollo_dashboard/src/alert_definitions.rs @@ -28,7 +28,6 @@ use apollo_l1_events::metrics::{ L1_MESSAGE_SCRAPER_REORG_DETECTED, }; use apollo_l1_gas_price::metrics::{ - ETH_TO_STRK_ERROR_COUNT, L1_GAS_PRICE_INFRA_METRICS, L1_GAS_PRICE_SCRAPER_BASELAYER_ERROR_COUNT, L1_GAS_PRICE_SCRAPER_REORG_DETECTED, @@ -71,6 +70,7 @@ use crate::alert_scenarios::infra_alerts::{ }; use crate::alert_scenarios::l1_endpoints::get_primary_l1_endpoint_down_too_long_alerts; use crate::alert_scenarios::l1_gas_prices::{ + get_eth_to_strk_error_count_alert, get_eth_to_strk_success_count_alert, get_l1_gas_price_provider_insufficient_history_alert, get_l1_gas_price_scraper_success_count_alert, @@ -310,22 +310,6 @@ fn get_consensus_retrospective_block_hash_mismatch() -> Alert { ) } -fn get_eth_to_strk_error_count_alert() -> Alert { - Alert::new( - "eth_to_strk_error_count", - "Eth to Strk error count", - EvaluationRate::Default, - format!( - "sum(increase({}[1h])) or vector(0)", - ETH_TO_STRK_ERROR_COUNT.get_name_with_filter() - ), - vec![AlertCondition::new(AlertComparisonOp::GreaterThan, 10.0, AlertLogicalOp::And)], - "1m", - AlertSeverity::Informational, - ObserverApplicability::NotApplicable, - ) -} - fn get_l1_gas_price_scraper_baselayer_error_count_alert() -> Alert { Alert::new( "l1_gas_price_scraper_baselayer_error_count", diff --git a/crates/apollo_dashboard/src/alert_scenarios/l1_gas_prices.rs b/crates/apollo_dashboard/src/alert_scenarios/l1_gas_prices.rs index 81f3d9a2ad4..176f0a3ef96 100644 --- a/crates/apollo_dashboard/src/alert_scenarios/l1_gas_prices.rs +++ b/crates/apollo_dashboard/src/alert_scenarios/l1_gas_prices.rs @@ -1,4 +1,5 @@ use apollo_l1_gas_price::metrics::{ + ETH_TO_STRK_ERROR_COUNT, ETH_TO_STRK_SUCCESS_COUNT, L1_GAS_PRICE_PROVIDER_INSUFFICIENT_HISTORY, L1_GAS_PRICE_SCRAPER_SUCCESS_COUNT, @@ -11,29 +12,29 @@ use crate::alerts::{ AlertComparisonOp, AlertCondition, AlertLogicalOp, + AlertSeverity, EvaluationRate, ObserverApplicability, PENDING_DURATION_DEFAULT, }; use crate::query_builder::sum_increase; -/// Alert if we have no successful eth to strk rates data from the last hour. -/// -/// Uses `sum_increase` instead of bare `increase` to avoid false positives on spot eviction: when -/// a pod is evicted and rescheduled, the new pod's counter resets to 0, so a bare `increase([1h])` -/// would return 0 until the first success. `sum` aggregates across all pod series, and the -/// evicted pod's data points remain in the TSDB for the full 1h window, keeping the sum ≥ 1. pub(crate) fn get_eth_to_strk_success_count_alert() -> Alert { const ALERT_NAME: &str = "eth_to_strk_success_count"; - Alert::new( + oracle_success_count_alert( ALERT_NAME, "Eth to Strk success count", - EvaluationRate::Default, - sum_increase(Ð_TO_STRK_SUCCESS_COUNT, "1h"), - vec![AlertCondition::new(AlertComparisonOp::LessThan, 1.0, AlertLogicalOp::And)], - PENDING_DURATION_DEFAULT, + Ð_TO_STRK_SUCCESS_COUNT, SeverityValueOrPlaceholder::Placeholder(ALERT_NAME.to_string()), - ObserverApplicability::NotApplicable, + ) +} + +pub(crate) fn get_eth_to_strk_error_count_alert() -> Alert { + oracle_error_count_alert( + "eth_to_strk_error_count", + "Eth to Strk error count", + Ð_TO_STRK_ERROR_COUNT, + AlertSeverity::Informational, ) } @@ -70,3 +71,49 @@ pub(crate) fn get_l1_gas_price_provider_insufficient_history_alert() -> Alert { ObserverApplicability::NotApplicable, ) } + +/// Alert if an exchange-rate oracle had no successful query in the last hour. +/// +/// Uses `sum_increase` instead of bare `increase` to avoid false positives on spot eviction: when +/// a pod is evicted and rescheduled, the new pod's counter resets to 0, so a bare `increase([1h])` +/// would return 0 until the first success. `sum` aggregates across all pod series, and the +/// evicted pod's data points remain in the TSDB for the full 1h window, keeping the sum ≥ 1. +fn oracle_success_count_alert( + name: &str, + title: &str, + success_count_metric: &dyn MetricQueryName, + severity: impl Into, +) -> Alert { + Alert::new( + name, + title, + EvaluationRate::Default, + sum_increase(success_count_metric, "1h"), + vec![AlertCondition::new(AlertComparisonOp::LessThan, 1.0, AlertLogicalOp::And)], + PENDING_DURATION_DEFAULT, + severity, + ObserverApplicability::NotApplicable, + ) +} + +/// Alert if an exchange-rate oracle exceeded the failure threshold in the last hour. +/// +/// `or vector(0)` keeps the query defined (evaluating to 0) when the metric has no samples yet, +/// so the alert stays silent instead of going to no-data before the first error is recorded. +fn oracle_error_count_alert( + name: &str, + title: &str, + error_count_metric: &dyn MetricQueryName, + severity: impl Into, +) -> Alert { + Alert::new( + name, + title, + EvaluationRate::Default, + format!("{} or vector(0)", sum_increase(error_count_metric, "1h")), + vec![AlertCondition::new(AlertComparisonOp::GreaterThan, 10.0, AlertLogicalOp::And)], + "1m", + severity, + ObserverApplicability::NotApplicable, + ) +}