From d99d2757bac68c418a2d1be6f42af2b4fb4d4fd9 Mon Sep 17 00:00:00 2001 From: obchain Date: Sat, 2 May 2026 23:42:18 +0530 Subject: [PATCH] chore(metrics): drop_reason::LIQUIDATOR_NOT_DEPLOYED + Grafana panel (closes #402) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A misconfigured fork or a runtime self-destruct on the deployed CharonLiquidator surfaced as a generic submit_failed (broadcast-stage estimateGas revert), leaving operators unable to distinguish "the liquidation logic reverted" from "no code at the configured address". Mechanism: - New constant drop_reason::LIQUIDATOR_NOT_DEPLOYED. Stable-name pin + typed-helper round-trip extended. - process_opportunity probes eth_getCode at the liquidator address upstream of encode_and_simulate. If empty, label drop liquidator_not_deployed and return early. Why upstream: geth/anvil reply to eth_call against a no-code address with Ok(0x), so a missing contract would otherwise pass the gate and queue. Reviewer caught the unreachable-branch regression in v1. - Cost: one eth_getCode per actionable opportunity (subset of liquidatable bucket). ~10/min on busy mainnet, well within budget. - New Grafana panel "Executor — drops by reason" using the reason metric (charon_opportunities_dropped_total, distinct from the stage-labelled charon_executor_opportunities_dropped_total). --- crates/charon-cli/src/main.rs | 38 +++++++++++++++++++++++++++++++ crates/charon-metrics/src/lib.rs | 13 +++++++++++ deploy/grafana/charon.json | 39 +++++++++++++++++++++++++++++++- 3 files changed, 89 insertions(+), 1 deletion(-) diff --git a/crates/charon-cli/src/main.rs b/crates/charon-cli/src/main.rs index 9870a7e..369a284 100644 --- a/crates/charon-cli/src/main.rs +++ b/crates/charon-cli/src/main.rs @@ -2349,6 +2349,44 @@ async fn process_opportunity( sim, provider: pipeline.provider.as_ref(), }; + + // Issue #402: probe `eth_getCode` at the liquidator address before + // the simulation gate. If empty, label the drop + // `liquidator_not_deployed` and return early; otherwise fall + // through to `encode_and_simulate` whose Err arm now genuinely + // means the liquidation logic reverted (`sim_revert`). + match pipeline + .provider + .as_ref() + .get_code_at(pipeline.liquidator) + .block_id(alloy::eips::BlockId::latest()) + .await + { + Ok(code) if code.is_empty() => { + charon_metrics::record_opportunity_dropped(chain, drop_stage::SIMULATION); + charon_metrics::record_opportunity_dropped_reason( + chain, + drop_reason::LIQUIDATOR_NOT_DEPLOYED, + ); + warn!( + chain, + liquidator = %pipeline.liquidator, + borrower = %pos.borrower, + "liquidator contract has empty bytecode at head — opportunity dropped" + ); + return Ok(false); + } + Ok(_) => {} + Err(err) => { + debug!( + chain, + liquidator = %pipeline.liquidator, + error = ?err, + "liquidator bytecode probe failed; deferring to simulator" + ); + } + } + let sim_block = pipeline .replay_block .map(BlockNumberOrTag::Number) diff --git a/crates/charon-metrics/src/lib.rs b/crates/charon-metrics/src/lib.rs index c1917be..2dc348d 100644 --- a/crates/charon-metrics/src/lib.rs +++ b/crates/charon-metrics/src/lib.rs @@ -270,6 +270,14 @@ pub mod drop_reason { /// opportunity passed every earlier gate but the broadcast /// stage rejected it. pub const SUBMIT_FAILED: &str = "submit_failed"; + /// Simulation reverted with the well-known "no code at address" + /// payload — the configured CharonLiquidator address has empty + /// bytecode on the connected chain, distinct from the liquidation + /// logic itself reverting (#402). The startup gate added in + /// #399 catches this at boot for chains with a router config; + /// this counter covers the runtime self-destruct edge case and + /// any scan-only / replay path that bypasses the startup gate. + pub const LIQUIDATOR_NOT_DEPLOYED: &str = "liquidator_not_deployed"; } /// Issue #397: outcome label on `EXECUTOR_BROADCASTS_TOTAL`. Mirrors @@ -1034,6 +1042,10 @@ mod tests { assert_eq!(drop_reason::GAS_CEILING, "gas_ceiling"); assert_eq!(drop_reason::TTL_EXPIRED, "ttl_expired"); assert_eq!(drop_reason::SUBMIT_FAILED, "submit_failed"); + assert_eq!( + drop_reason::LIQUIDATOR_NOT_DEPLOYED, + "liquidator_not_deployed" + ); } /// Issue #397: pin the public surface of the broadcast / inclusion @@ -1088,6 +1100,7 @@ mod tests { record_opportunity_dropped_reason("bnb", drop_reason::GAS_CEILING); record_opportunity_dropped_reason("bnb", drop_reason::TTL_EXPIRED); record_opportunity_dropped_reason("bnb", drop_reason::SUBMIT_FAILED); + record_opportunity_dropped_reason("bnb", drop_reason::LIQUIDATOR_NOT_DEPLOYED); // Issue #397: broadcast / inclusion / realised-profit record_broadcast("bnb", broadcast_result::SUBMITTED); record_broadcast("bnb", broadcast_result::TIMEOUT); diff --git a/deploy/grafana/charon.json b/deploy/grafana/charon.json index 1d9451b..95dd743 100644 --- a/deploy/grafana/charon.json +++ b/deploy/grafana/charon.json @@ -395,7 +395,7 @@ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (stage) (rate(charon_executor_opportunities_dropped_total{instance=~\"$instance\",chain=~\"$chain\",job=~\"$job\"}[$__rate_interval])) * 60", + "expr": "sum by (stage) (rate(charon_executor_opportunities_dropped_total{instance=~\"$instance\",chain=~\"$chain\",job=~\"$job\",stage!=\"\"}[$__rate_interval])) * 60", "legendFormat": "{{stage}}", "range": true, "refId": "B" @@ -404,6 +404,43 @@ "title": "Executor — opportunities queued vs dropped", "type": "timeseries" }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Drops broken out by root-cause reason label (issue #368 + #402). Distinguishes 'sim_revert' (the liquidation logic reverted) from 'liquidator_not_deployed' (no code at the configured CharonLiquidator address) so a misconfigured fork or a runtime self-destruct surfaces as its own series rather than collapsing into a generic SIM_REVERT.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisLabel": "opps / min", + "drawStyle": "bars", + "fillOpacity": 80, + "lineWidth": 1, + "showPoints": "never", + "stacking": { "group": "A", "mode": "normal" } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "id": 12, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (reason) (rate(charon_opportunities_dropped_total{instance=~\"$instance\",chain=~\"$chain\",job=~\"$job\"}[$__rate_interval])) * 60", + "legendFormat": "{{reason}}", + "range": true, + "refId": "A" + } + ], + "title": "Executor — drops by reason", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Distribution of per-opportunity net profit (USD cents). Heat-map uses the underlying histogram buckets.",