diff --git a/examples/CRISP/Readme.md b/examples/CRISP/Readme.md index 73d305800d..a943319c83 100644 --- a/examples/CRISP/Readme.md +++ b/examples/CRISP/Readme.md @@ -252,8 +252,8 @@ PROGRAM_SERVER_URL=http://127.0.0.1:13151 WS_RPC_URL=ws://127.0.0.1:8545 CHAIN_ID=31337 -# Bitquery API key -BITQUERY_API_KEY="" +# Etherscan API key +ETHERSCAN_API_KEY="" # Cron-job API key to trigger new rounds CRON_API_KEY=1234567890 @@ -275,8 +275,8 @@ E3_COMPUTE_PROVIDER_NAME="RISC0" E3_COMPUTE_PROVIDER_PARALLEL=false E3_COMPUTE_PROVIDER_BATCH_SIZE=4 # Must be a power of 2 -# Bitquery API Key (optional, leave empty if not using) -BITQUERY_API_KEY="" +# ETHERSCAN API Key (optional, leave empty if not using) +ETHERSCAN_API_KEY="" ``` ## Running Ciphernodes diff --git a/examples/CRISP/server/.env.example b/examples/CRISP/server/.env.example index 75e0932a93..8599e48c51 100644 --- a/examples/CRISP/server/.env.example +++ b/examples/CRISP/server/.env.example @@ -6,8 +6,8 @@ PROGRAM_SERVER_URL=http://127.0.0.1:13151 WS_RPC_URL=ws://127.0.0.1:8545 CHAIN_ID=31337 -# Bitquery API key -BITQUERY_API_KEY="" +# Etherscan API key +ETHERSCAN_API_KEY="" # Cron-job API key to trigger new rounds CRON_API_KEY=1234567890 diff --git a/examples/CRISP/server/src/config.rs b/examples/CRISP/server/src/config.rs index 904fb4abb6..446cba6d51 100644 --- a/examples/CRISP/server/src/config.rs +++ b/examples/CRISP/server/src/config.rs @@ -30,7 +30,7 @@ pub struct Config { pub e3_compute_provider_name: String, pub e3_compute_provider_parallel: bool, pub e3_compute_provider_batch_size: u32, - pub bitquery_api_key: String, + pub etherscan_api_key: String, } impl Config { diff --git a/examples/CRISP/server/src/lib.rs b/examples/CRISP/server/src/lib.rs index 4d9f3e47c4..77898ef967 100644 --- a/examples/CRISP/server/src/lib.rs +++ b/examples/CRISP/server/src/lib.rs @@ -4,6 +4,6 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +pub mod config; pub mod logger; pub mod server; -pub mod config; diff --git a/examples/CRISP/server/src/server/indexer.rs b/examples/CRISP/server/src/server/indexer.rs index 558d9c61a9..dd29635c6b 100644 --- a/examples/CRISP/server/src/server/indexer.rs +++ b/examples/CRISP/server/src/server/indexer.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::server::token_holders::{get_mock_token_holders, BitqueryClient}; +use crate::server::token_holders::{get_mock_token_holders, EtherscanClient}; use crate::server::{ models::{CurrentRound, CustomParams}, program_server_request::run_compute, @@ -14,7 +14,7 @@ use crate::server::{ }; use alloy::providers::{Provider, ProviderBuilder}; use alloy::sol_types::{sol_data, SolType}; -use alloy_primitives::Address; +use alloy_primitives::{Address, U256}; use e3_sdk::{ evm_helpers::{ contracts::{ @@ -74,8 +74,8 @@ pub async fn register_e3_requested( repo.initialize_round(custom_params.token_address, custom_params.balance_threshold) .await?; - // Get token holders from Bitquery API or mocked data. - let token_holders = if matches!(CONFIG.chain_id, 31337 | 1337 | 11155111) { + // Get token holders from Etherscan API or mocked data. + let token_holders = if matches!(CONFIG.chain_id, 31337 | 1337) { info!( "Using mocked token holders for local network (chain_id: {})", CONFIG.chain_id @@ -84,22 +84,28 @@ pub async fn register_e3_requested( get_mock_token_holders() } else { info!( - "Using Bitquery API for network (chain_id: {})", + "Using Etherscan API for network (chain_id: {})", CONFIG.chain_id ); - let bitquery_client = BitqueryClient::new(CONFIG.bitquery_api_key.clone()); - bitquery_client - .get_token_holders( + let etherscan_client = + EtherscanClient::new(CONFIG.etherscan_api_key.clone(), CONFIG.chain_id); + etherscan_client + .get_token_holders_with_voting_power( token_address, - balance_threshold, event.e3.requestBlock.to::(), - CONFIG.chain_id, - 10000, // TODO: this is fine for now, but we need pagination or chunking strategies - // to retrieve large datasets efficiently. + &CONFIG.http_rpc_url, + U256::from_str_radix(&balance_threshold.to_string(), 10).map_err( + |e| { + eyre::eyre!( + "Failed to convert balance threshold to U256: {}", + e + ) + }, + )?, ) .await - .with_context(|| "Bitquery error")? + .map_err(|e| eyre::eyre!("Etherscan error: {}", e))? }; if token_holders.is_empty() { diff --git a/examples/CRISP/server/src/server/program_server_request.rs b/examples/CRISP/server/src/server/program_server_request.rs index b3ccc2a4ad..10687f82e4 100644 --- a/examples/CRISP/server/src/server/program_server_request.rs +++ b/examples/CRISP/server/src/server/program_server_request.rs @@ -27,10 +27,7 @@ where serializer.serialize_str(&hex_string) } -fn serialize_hex_tuple( - tuples: &Vec<(Vec, u64)>, - serializer: S, -) -> Result +fn serialize_hex_tuple(tuples: &Vec<(Vec, u64)>, serializer: S) -> Result where S: Serializer, { diff --git a/examples/CRISP/server/src/server/repo.rs b/examples/CRISP/server/src/server/repo.rs index 3ead14e610..3336e44948 100644 --- a/examples/CRISP/server/src/server/repo.rs +++ b/examples/CRISP/server/src/server/repo.rs @@ -91,7 +91,11 @@ impl CrispE3Repository { self.set_crisp(e3_crisp).await } - pub async fn initialize_round(&mut self, token_address: String, balance_threshold: String) -> Result<()> { + pub async fn initialize_round( + &mut self, + token_address: String, + balance_threshold: String, + ) -> Result<()> { self.set_crisp(E3Crisp { has_voted: vec![], start_time: 0u64, @@ -254,7 +258,7 @@ impl CrispE3Repository { }) .await .map_err(|_| eyre::eyre!("Could not set token_holder_hashes for '{key}'"))?; - + Ok(()) } diff --git a/examples/CRISP/server/src/server/routes/voting.rs b/examples/CRISP/server/src/server/routes/voting.rs index 15692b61cf..0f571b0241 100644 --- a/examples/CRISP/server/src/server/routes/voting.rs +++ b/examples/CRISP/server/src/server/routes/voting.rs @@ -14,7 +14,7 @@ use crate::server::{ use actix_web::{web, HttpResponse, Responder}; use alloy::{ dyn_abi::DynSolValue, - primitives::{Bytes, U256, Address}, + primitives::{Address, Bytes, U256}, }; use e3_sdk::evm_helpers::contracts::{EnclaveContract, EnclaveWrite}; use eyre::Error; diff --git a/examples/CRISP/server/src/server/token_holders/bitquery.rs b/examples/CRISP/server/src/server/token_holders/bitquery.rs deleted file mode 100644 index 87c9e0ec4d..0000000000 --- a/examples/CRISP/server/src/server/token_holders/bitquery.rs +++ /dev/null @@ -1,454 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use alloy::primitives::Address; -use alloy::primitives::{ - utils::{parse_units, ParseUnits}, - U256, -}; - -use eyre::Result; -use num_bigint::BigUint; -use serde::{Deserialize, Serialize}; - -/// Represents a token holder with their address and balance. -/// Balance is stored as a string to preserve precision for large numbers. -#[derive(Debug, Serialize, Deserialize, PartialEq)] -pub struct TokenHolder { - pub address: String, - pub balance: String, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct BalanceUpdate { - #[serde(rename = "Address")] - pub address: String, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct Currency { - #[serde(rename = "Decimals")] - pub decimals: u8, -} - -/// Internal structure for deserializing Bitquery API response. -/// Contains both the balance and address information from the API. -#[derive(Debug, Serialize, Deserialize)] -pub struct BalanceUpdateResponse { - #[serde(rename = "Balance")] - pub balance: String, - #[serde(rename = "BalanceUpdate")] - pub balance_update: BalanceUpdate, - #[serde(rename = "Currency")] - pub currency: Currency, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct GraphQLResponse { - pub data: EvmData, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct EvmData { - #[serde(rename = "EVM")] - pub evm: EvmDataInner, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct EvmDataInner { - #[serde(rename = "BalanceUpdates")] - pub balance_updates: Vec, -} - -#[derive(Debug, Serialize)] -struct GraphQLRequest { - query: String, - variables: serde_json::Value, -} - -/// Client for querying token holder data from Bitquery GraphQL API. -pub struct BitqueryClient { - client: reqwest::Client, - api_key: String, -} - -impl BitqueryClient { - pub fn new(api_key: String) -> Self { - Self { - client: reqwest::Client::new(), - api_key, - } - } - - /// Maps chain IDs to Bitquery network names. - /// Returns an error for unsupported chains. - fn get_network_name(chain_id: u64) -> Result<&'static str> { - match chain_id { - 1 => Ok("eth"), - 11155111 => Ok("sepolia"), - 56 => Ok("bsc"), - 137 => Ok("matic"), - 250 => Ok("fantom"), - 43114 => Ok("avalanche"), - 42161 => Ok("arbitrum"), - 10 => Ok("optimism"), - _ => Err(eyre::eyre!("unsupported chain id: {}", chain_id)), - } - } - - /// Retrieves token holders for a specific token at a given block. - /// - /// # Arguments - /// * `token_address` - The token address - /// * `balance_threshold` - The balance threshold - /// * `block_number` - The block number to query - /// * `chain_id` - The blockchain network ID - /// * `limit` - Maximum number of holders to return - /// - /// # Returns - /// A vector of `TokenHolder` structs, or an error if the request fails. - pub async fn get_token_holders( - &self, - token_address: Address, - balance_threshold: BigUint, - block_number: u64, - chain_id: u64, - limit: u32, - ) -> Result> { - let network = Self::get_network_name(chain_id)?; - - // Build GraphQL query to fetch token holders. - let query = format!( - r#" - {{ - EVM(dataset: archive, network: {}) {{ - BalanceUpdates( - where: {{ - Block: {{ Number: {{ le: "{}" }} }} - Currency: {{ SmartContract: {{ is: "{}" }} }} - }} - orderBy: [ - {{ descendingByField: "Balance" }}, - {{ ascending: BalanceUpdate_Address }} - ] - limit: {{ count: {} }} - ) {{ - BalanceUpdate {{ - Address - }} - Balance: sum(of: BalanceUpdate_Amount) - Currency {{ - Decimals - }} - }} - }} - }} - "#, - network, block_number, token_address, limit - ); - - let request = GraphQLRequest { - query, - variables: serde_json::Value::Object(serde_json::Map::new()), - }; - - // Send authenticated request to Bitquery API. - let response = self - .client - .post("https://streaming.bitquery.io/graphql") - .header("Authorization", format!("Bearer {}", &self.api_key)) - .header("Content-Type", "application/json") - .json(&request) - .send() - .await - .map_err(|e| eyre::eyre!("Failed to send request to Bitquery: {}", e))?; - - // Check if the response is successful. - let status = response.status(); - let response_text = response - .text() - .await - .map_err(|e| eyre::eyre!("Failed to read response from Bitquery: {}", e))?; - if !status.is_success() { - return Err(eyre::eyre!("Bitquery HTTP {}: {}", status, response_text)); - } - - let graphql_response: GraphQLResponse = serde_json::from_str(&response_text) - .map_err(|e| eyre::eyre!("Failed to parse Bitquery response: {}", e))?; - - let balance_updates = graphql_response.data.evm.balance_updates; - - // Check if there are any balance updates. - if balance_updates.is_empty() { - return Err(eyre::eyre!("No balance updates found")); - } - - let decimals = balance_updates[0].currency.decimals; - let mut token_holders = Vec::new(); - - for token_holder in balance_updates { - // Parse Bitquery's string balance -> big int. The balance is a string with the decimals. - let balance_bigint: U256 = match parse_units(token_holder.balance.trim(), decimals) { - Ok(ParseUnits::U256(x)) => x, - Ok(ParseUnits::I256(x)) if x.is_negative() => { - return Err(eyre::eyre!( - "Negative balance found for address {}: {}", - token_holder.balance_update.address, - token_holder.balance - )); - } - Ok(ParseUnits::I256(x)) => x.unsigned_abs(), - Err(e) => { - return Err(eyre::eyre!( - "Failed to parse balance '{}' for address {}: {}", - token_holder.balance, - token_holder.balance_update.address, - e - )); - } - }; - - // Convert U256 to BigUint for comparison. - let balance_bigint = BigUint::from_bytes_be(&balance_bigint.to_be_bytes::<32>()); - - if balance_bigint >= balance_threshold { - token_holders.push(TokenHolder { - address: token_holder.balance_update.address.clone(), - balance: balance_bigint.to_string(), - }); - } - } - - println!("Token holders: {:#?}", token_holders); - - Ok(token_holders) - } -} - -/// Convenience function to get mocked token holder data for testing. -/// This is useful when you don't need a BitqueryClient instance. -/// -/// # Returns -/// A vector of 10 `TokenHolder` structs with realistic test data. -pub fn get_mock_token_holders() -> Vec { - vec![ - TokenHolder { - address: "0x1234567890123456789012345678901234567890".to_string(), - balance: "1000".to_string(), - }, - TokenHolder { - address: "0x2345678901234567890123456789012345678901".to_string(), - balance: "500".to_string(), - }, - TokenHolder { - address: "0x3456789012345678901234567890123456789012".to_string(), - balance: "250".to_string(), - }, - TokenHolder { - address: "0x4567890123456789012345678901234567890123".to_string(), - balance: "100".to_string(), - }, - TokenHolder { - address: "0x5678901234567890123456789012345678901234".to_string(), - balance: "75".to_string(), - }, - TokenHolder { - address: "0x6789012345678901234567890123456789012345".to_string(), - balance: "50".to_string(), - }, - TokenHolder { - address: "0x7890123456789012345678901234567890123456".to_string(), - balance: "25".to_string(), - }, - TokenHolder { - address: "0x8901234567890123456789012345678901234567".to_string(), - balance: "10".to_string(), - }, - TokenHolder { - address: "0x9012345678901234567890123456789012345678".to_string(), - balance: "5".to_string(), - }, - TokenHolder { - address: "0x0123456789012345678901234567890123456789".to_string(), - balance: "1".to_string(), - }, - ] -} - -#[cfg(test)] -mod tests { - //! Minimal tests for the Bitquery client. - //! - //! These include: - //! - A **live integration test** (ignored by default) that requires a valid `BITQUERY_API_KEY` - //! and exercises the real Bitquery API end‑to‑end. Run it manually with: - //! `cargo test --package crisp -- --ignored` - //! - A **negative test** that verifies proper erroring with an invalid API key, - //! without depending on any third‑party mocking framework. - //! - //! Rationale: - //! - Keep unit tests hermetic when possible; for external HTTP, run live tests only on demand. - //! - Avoid "always‑green" tests; failures should surface incorrect credentials or error handling. - - use super::*; - use std::env; - - /// Returns a known‑good tuple commonly used in examples: - /// - USDT contract on Ethereum mainnet. - /// - A historical block chosen to be well after deployment. - fn example_params() -> (Address, BigUint, u64, u64, u32) { - ( - // Token contract - "0xdAC17F958D2ee523a2206206994597C13D831ec7" - .parse() - .unwrap(), - // Balance threshold - BigUint::from(64707696530000u64), - // Historical block height (Ethereum) - 18_500_000, - // Chain id (Ethereum mainnet) - 1, - // Limit - 100, - ) - } - - /// Live end‑to‑end test hitting the real Bitquery GraphQL endpoint. - /// - /// Requirements: - /// - Set a valid environment variable `BITQUERY_API_KEY`. - /// - Network connectivity. - /// - /// Execution: - /// ```text - /// cargo test --package crisp -- --ignored - /// ``` - /// - /// Expectations: - /// - The request succeeds (no error). - /// - The response parses into a non‑empty vector OR an empty vector (both are valid states), - /// but the shape must be correct (i.e., no deserialization error). - #[tokio::test] - #[ignore] - async fn live_get_token_holders_succeeds_with_valid_key() { - let api_key = - env::var("BITQUERY_API_KEY").expect("Set BITQUERY_API_KEY to run this live test"); - - let client = BitqueryClient::new(api_key); - let (token, balance_threshold, block, chain_id, limit) = example_params(); - - let res = client - .get_token_holders(token, balance_threshold.clone(), block, chain_id, limit) - .await; - assert!(res.is_ok(), "Live call failed: {res:?}"); - - // Check shape: accessing the vector ensures deserialization happened. - let holders = res.unwrap(); - - // Verify the number of holders after filtering. - assert_eq!( - holders.len(), - 46, - "Expected exactly 46 holders, got {}", - holders.len() - ); - - // Verify that all holders have valid addresses and balances. - for holder in &holders { - assert!( - !holder.address.is_empty(), - "Holder address should not be empty" - ); - assert!( - !holder.balance.is_empty(), - "Holder balance should not be empty" - ); - // Verify address format (should start with 0x and be 42 characters) - assert!( - holder.address.starts_with("0x"), - "Address should start with 0x" - ); - assert_eq!( - holder.address.len(), - 42, - "Address should be 42 characters long" - ); - // Verify balance is a valid number string. - assert!( - holder.balance.parse::().is_ok(), - "Balance should be a valid number: {}", - holder.balance - ); - } - - // Verify sorting: holders should be sorted by balance in descending order - // (highest balance first), then by address in ascending order for ties. - for i in 1..holders.len() { - let prev_balance: BigUint = holders[i - 1].balance.parse().expect("Valid balance"); - let curr_balance: BigUint = holders[i].balance.parse().expect("Valid balance"); - - if prev_balance == curr_balance { - // For equal balances, addresses should be in ascending order. - assert!( - holders[i - 1].address < holders[i].address, - "For equal balances, addresses should be sorted in ascending order. \ - Found {} >= {} for balances of {}", - holders[i - 1].address, - holders[i].address, - prev_balance - ); - } else { - // Balances should be in descending order. - assert!( - prev_balance > curr_balance, - "Holders should be sorted by balance in descending order. \ - Found {} <= {} at positions {} and {}", - prev_balance, - curr_balance, - i - 1, - i - ); - } - } - - // Verify filtering: all holders should meet the balance threshold. - for holder in &holders { - let holder_balance: BigUint = holder.balance.parse().expect("Valid balance"); - assert!( - holder_balance >= balance_threshold, - "All holders should meet the balance threshold. \ - Found holder {} with balance {} below threshold {}", - holder.address, - holder_balance, - balance_threshold - ); - } - } - - /// Negative test to ensure invalid credentials are handled as an error. - /// - /// This does **not** call any private or unstable API. It simply uses an obviously invalid key - /// and expects the client to return an error (HTTP 401/403 or provider error mapped by the client). - /// - /// Why this test matters: - /// - Verifies that authentication failures are surfaced as errors instead of being silently swallowed. - /// - Does not depend on network flakiness; Bitquery consistently rejects invalid tokens. - #[tokio::test] - async fn get_token_holders_fails_with_invalid_key() { - // Use a clearly invalid key; do not rely on any env configuration. - let client = BitqueryClient::new("invalid_key_for_test_purposes".to_string()); - let (token, balance_threshold, block, chain_id, limit) = example_params(); - - let res = client - .get_token_holders(token, balance_threshold, block, chain_id, limit) - .await; - - assert!( - res.is_err(), - "Expected an authentication error with invalid key, but got success" - ); - } -} diff --git a/examples/CRISP/server/src/server/token_holders/etherscan.rs b/examples/CRISP/server/src/server/token_holders/etherscan.rs new file mode 100644 index 0000000000..9e1ca54245 --- /dev/null +++ b/examples/CRISP/server/src/server/token_holders/etherscan.rs @@ -0,0 +1,868 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::server::CONFIG; +use alloy::primitives::{Address, U256}; +use alloy::providers::ProviderBuilder; +use alloy::sol; +use eyre::{eyre, Context, Result}; // Add this import +use reqwest; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use tokio::time::{sleep, Duration}; + +// Define the Votes contract interface for getPastVotes +sol! { + #[derive(Debug)] + #[sol(rpc)] + contract ERC20Votes { + function getPastVotes(address account, uint256 timepoint) external view returns (uint256); + } +} + +// Config +pub const ETHERSCAN_API_URL: &str = "https://api.etherscan.io/v2/api"; +const ZERO_ADDRESS: Address = Address::ZERO; + +/// Represents a token holder with their address and balance. +/// Balance is stored as a string to preserve precision for large numbers. +#[derive(Debug, Serialize, Deserialize, PartialEq)] +pub struct TokenHolder { + pub address: String, + pub balance: String, +} + +// Response types +#[derive(Debug, Deserialize)] +struct EtherscanResponse { + status: String, + message: String, + result: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ContractCreation { + block_number: String, +} + +#[derive(Debug, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct TransferLog { + pub address: String, + pub topics: Vec, + pub data: String, + pub block_number: String, + pub transaction_hash: String, + pub transaction_index: String, + pub block_hash: String, + pub log_index: String, +} + +#[derive(Debug, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct DelegateVotesChangedLog { + pub address: String, + pub topics: Vec, + pub data: String, + pub block_number: String, + pub transaction_hash: String, + pub transaction_index: String, + pub block_hash: String, + pub log_index: String, +} + +/// Represents an address that may have voting power +#[derive(Debug, Clone)] +pub struct PotentialVoter { + pub address: Address, + pub token_balance: U256, + pub has_delegation: bool, +} + +/// Client for querying token holder data from Etherscan API. +pub struct EtherscanClient { + client: reqwest::Client, + api_key: String, + chain_id: u64, +} + +impl EtherscanClient { + /// Create a new EtherscanClient instance + pub fn new(api_key: String, chain_id: u64) -> Self { + Self { + client: reqwest::Client::new(), + api_key, + chain_id, + } + } + + /// Get the deployment block number for a contract + pub async fn get_deployment_block(&self, token: &str) -> Result { + let url = format!( + "{}?module=contract&action=getcontractcreation&contractaddresses={}&chainid={}&apikey={}", + ETHERSCAN_API_URL, token, self.chain_id, self.api_key + ); + + let response = self + .client + .get(&url) + .send() + .await + .context("Failed to send request to Etherscan")?; + let data: EtherscanResponse> = response + .json() + .await + .context("Failed to parse Etherscan response")?; + + if data.status != "1" { + return Err(eyre!("Deployment block not found: {}", data.message)); + } + + let result = data + .result + .and_then(|r| r.into_iter().next()) + .ok_or_else(|| eyre!("No deployment data found"))?; + + // Parse block number (could be hex or decimal) + let block_number = if result.block_number.starts_with("0x") { + u64::from_str_radix(&result.block_number[2..], 16) + .context("Failed to parse hex block number")? + } else { + result + .block_number + .parse::() + .context("Failed to parse decimal block number")? + }; + + Ok(block_number) + } + + /// Get transfer logs for a token + pub async fn get_transfer_logs( + &self, + token: &str, + from_block: u64, + to_block: u64, + ) -> Result> { + let mut all_logs = Vec::new(); + let mut page = 1; + + // ERC20 Transfer event signature + let transfer_topic = "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"; + + loop { + let url = format!( + "{}?module=logs&action=getLogs&address={}&fromBlock={}&toBlock={}&topic0={}&page={}&offset=10000&chainid={}&apikey={}", + ETHERSCAN_API_URL, token, from_block, to_block, transfer_topic, page, self.chain_id, self.api_key + ); + + let response = self + .client + .get(&url) + .send() + .await + .context("Failed to fetch transfer logs")?; + let data: EtherscanResponse> = response + .json() + .await + .context("Failed to parse transfer logs response")?; + + // Break if request failed + if data.status != "1" { + if data.message.eq_ignore_ascii_case("No records found") { + break; + } + + return Err(eyre!( + "Etherscan getLogs failed on page {}: {}", + page, + data.message + )); + } + + // Break if no results + let logs = match data.result { + Some(logs) if !logs.is_empty() => logs, + _ => break, + }; + + let log_count = logs.len(); + all_logs.extend(logs); + + // Break if we got less than the max page size + if log_count < 10000 { + break; + } + + page += 1; + + // Rate limiting - wait 100ms between requests + sleep(Duration::from_millis(100)).await; + } + + Ok(all_logs) + } + + /// Get DelegateVotesChanged logs for a token + pub async fn get_delegate_votes_changed_logs( + &self, + token: &str, + from_block: u64, + to_block: u64, + ) -> Result> { + let mut all_logs = Vec::new(); + let mut page = 1; + + // DelegateVotesChanged event signature + let delegate_votes_changed_topic = + "0xdec2bacdd2f05b59de34da9b523dff8be42e5e38e8af2ade71e1ddfc5c9f0e6f"; + + loop { + let url = format!( + "{}?module=logs&action=getLogs&address={}&fromBlock={}&toBlock={}&topic0={}&page={}&offset=10000&chainid={}&apikey={}", + ETHERSCAN_API_URL, token, from_block, to_block, delegate_votes_changed_topic, page, self.chain_id, self.api_key + ); + + let response = self + .client + .get(&url) + .send() + .await + .context("Failed to fetch delegation logs")?; + let data: EtherscanResponse> = response + .json() + .await + .context("Failed to parse delegation logs response")?; + + // Break if request failed + if data.status != "1" { + if data.message.eq_ignore_ascii_case("No records found") { + break; + } + + return Err(eyre!( + "Etherscan getLogs failed on page {}: {}", + page, + data.message + )); + } + + // Break if no results + let logs = match data.result { + Some(logs) if !logs.is_empty() => logs, + _ => break, + }; + + let log_count = logs.len(); + all_logs.extend(logs); + + // Break if we got less than the max page size + if log_count < 10000 { + break; + } + + page += 1; + + // Rate limiting - wait 100ms between requests + sleep(Duration::from_millis(100)).await; + } + + Ok(all_logs) + } + + /// Get all potential voters by combining token holders and delegates + pub fn get_potential_voters( + &self, + transfer_logs: &[TransferLog], + delegation_logs: &[DelegateVotesChangedLog], + ) -> Vec { + let balances = Self::compute_balances_from_logs(transfer_logs); + let delegates: HashSet
= Self::extract_delegates(delegation_logs) + .into_iter() + .collect(); + + let mut potential_voters = HashMap::new(); + + // Add all token holders + for (address, balance) in balances.iter() { + if *address != ZERO_ADDRESS { + potential_voters.insert( + *address, + PotentialVoter { + address: *address, + token_balance: *balance, + has_delegation: delegates.contains(address), + }, + ); + } + } + + // Add any delegates who might not have tokens themselves + for delegate in delegates.iter() { + if *delegate != ZERO_ADDRESS { + potential_voters.entry(*delegate).or_insert(PotentialVoter { + address: *delegate, + token_balance: U256::ZERO, + has_delegation: true, + }); + } + } + + potential_voters.into_values().collect() + } + + /// Verify voting power for multiple addresses + pub async fn verify_voting_power( + &self, + token_address: Address, + potential_voters: &[PotentialVoter], + block_number: u64, + rpc_url: &str, + threshold: U256, + ) -> Result> { + let mut token_holders: Vec = Vec::new(); + + for voter in potential_voters { + match Self::get_past_votes(token_address, voter.address, block_number, rpc_url).await { + Ok(votes) => { + if votes >= threshold { + token_holders.push(TokenHolder { + address: voter.address.to_string(), + balance: votes.to_string(), + }); + } + } + Err(e) => { + log::warn!("Failed to get votes for {}: {}", voter.address, e); + } + } + + // Rate limiting - small delay between RPC calls + sleep(Duration::from_millis(50)).await; + } + + Ok(token_holders) + } + + /// Extract unique addresses from transfer logs + fn extract_addresses(logs: &[TransferLog]) -> Vec
{ + let mut addresses = HashSet::new(); + + for log in logs { + if log.topics.len() >= 3 { + if let Ok(from) = Self::parse_address_from_topic(&log.topics[1]) { + if from != ZERO_ADDRESS { + addresses.insert(from); + } + } + + if let Ok(to) = Self::parse_address_from_topic(&log.topics[2]) { + if to != ZERO_ADDRESS { + addresses.insert(to); + } + } + } + } + + addresses.into_iter().collect() + } + + /// Extract delegate addresses from DelegateVotesChanged logs + fn extract_delegates(logs: &[DelegateVotesChangedLog]) -> Vec
{ + let mut delegates = HashSet::new(); + + for log in logs { + if log.topics.len() >= 2 { + if let Ok(delegate) = Self::parse_address_from_topic(&log.topics[1]) { + if delegate != ZERO_ADDRESS { + delegates.insert(delegate); + } + } + } + } + + delegates.into_iter().collect() + } + + /// Compute token balances from transfer logs + fn compute_balances_from_logs(logs: &[TransferLog]) -> HashMap { + let mut balances: HashMap = HashMap::new(); + + // Sort logs by block number + let mut sorted_logs = logs.to_vec(); + sorted_logs.sort_by(|a, b| { + let block_a = Self::parse_block_number(&a.block_number); + let block_b = Self::parse_block_number(&b.block_number); + block_a.cmp(&block_b) + }); + + for log in sorted_logs { + if log.topics.len() < 3 { + continue; + } + + let from = match Self::parse_address_from_topic(&log.topics[1]) { + Ok(addr) => addr, + Err(_) => continue, + }; + + let to = match Self::parse_address_from_topic(&log.topics[2]) { + Ok(addr) => addr, + Err(_) => continue, + }; + + let value = Self::parse_transfer_value(&log.data); + + // Update balances + if from != ZERO_ADDRESS { + let balance = balances.entry(from).or_insert(U256::ZERO); + *balance = balance.saturating_sub(value); + } + + if to != ZERO_ADDRESS { + let balance = balances.entry(to).or_insert(U256::ZERO); + *balance = balance.saturating_add(value); + } + } + + balances + } + + /// Verify actual voting power for an address at a specific block + async fn get_past_votes( + token_address: Address, + voter_address: Address, + block_number: u64, + rpc_url: &str, + ) -> Result { + let url = rpc_url.parse().context("Failed to parse RPC URL")?; + let provider = ProviderBuilder::new().connect_http(url); + let token = ERC20Votes::new(token_address, provider); + + let votes = token + .getPastVotes(voter_address, U256::from(block_number)) + .call() + .await + .context("Failed to call getPastVotes")?; + + Ok(votes) + } + + /// Parse address from 32-byte topic (last 20 bytes) + fn parse_address_from_topic(topic: &str) -> Result { + let hex = topic.strip_prefix("0x").unwrap_or(topic); + + if hex.len() >= 40 { + let addr_hex = &hex[hex.len() - 40..]; + addr_hex + .parse::
() + .map_err(|e| format!("Failed to parse address: {}", e)) + } else { + Err("Topic too short".to_string()) + } + } + + /// Parse block number from hex or decimal string + fn parse_block_number(block_number: &str) -> u64 { + if block_number.starts_with("0x") { + u64::from_str_radix(&block_number[2..], 16).unwrap_or(0) + } else { + block_number.parse::().unwrap_or(0) + } + } + + /// Parse transfer value from hex data string + fn parse_transfer_value(data: &str) -> U256 { + let hex_data = data.strip_prefix("0x").unwrap_or(data); + U256::from_str_radix(hex_data, 16).unwrap_or(U256::ZERO) + } + + /// Get all token holders with voting power at a specific block + pub async fn get_token_holders_with_voting_power( + &self, + token_address: Address, + snapshot_block: u64, + rpc_url: &str, + threshold: U256, + ) -> Result> { + log::info!("Starting token holder discovery for {}", token_address); + + // Step 1: Determine starting block + let start_block = self + .get_deployment_block(&token_address.to_string()) + .await + .context("Failed to get deployment block")?; + log::info!("Token deployed at block: {}", start_block); + + // Step 2: Fetch transfer logs + log::info!( + "Fetching transfer logs from block {} to {}...", + start_block, + snapshot_block + ); + let transfer_logs = self + .get_transfer_logs(&token_address.to_string(), start_block, snapshot_block) + .await + .context("Failed to fetch transfer logs")?; + log::info!("Found {} transfer events", transfer_logs.len()); + + // Step 3: Fetch delegation logs + log::info!("Fetching delegation logs..."); + let delegation_logs = self + .get_delegate_votes_changed_logs( + &token_address.to_string(), + start_block, + snapshot_block, + ) + .await + .context("Failed to fetch delegation logs")?; + log::info!("Found {} delegation events", delegation_logs.len()); + + // Step 4: Identify potential voters + log::info!("Identifying potential voters..."); + let potential_voters = self.get_potential_voters(&transfer_logs, &delegation_logs); + log::info!("Found {} potential voters", potential_voters.len()); + + // Step 5: Verify actual voting power + log::info!("Verifying voting power at block {}...", snapshot_block); + let token_holders = self + .verify_voting_power( + token_address, + &potential_voters, + snapshot_block, + rpc_url, + threshold, + ) + .await + .context("Failed to verify voting power")?; + + log::info!( + "Discovery complete: {} addresses with voting power above threshold", + token_holders.len() + ); + + Ok(token_holders) + } +} + +/// Convenience function to get mocked token holder data for testing. +pub fn get_mock_token_holders() -> Vec { + vec![ + TokenHolder { + address: "0x1234567890123456789012345678901234567890".to_string(), + balance: "1000".to_string(), + }, + TokenHolder { + address: "0x2345678901234567890123456789012345678901".to_string(), + balance: "500".to_string(), + }, + TokenHolder { + address: "0x3456789012345678901234567890123456789012".to_string(), + balance: "250".to_string(), + }, + TokenHolder { + address: "0x4567890123456789012345678901234567890123".to_string(), + balance: "100".to_string(), + }, + TokenHolder { + address: "0x5678901234567890123456789012345678901234".to_string(), + balance: "75".to_string(), + }, + TokenHolder { + address: "0x6789012345678901234567890123456789012345".to_string(), + balance: "50".to_string(), + }, + TokenHolder { + address: "0x7890123456789012345678901234567890123456".to_string(), + balance: "25".to_string(), + }, + TokenHolder { + address: "0x8901234567890123456789012345678901234567".to_string(), + balance: "10".to_string(), + }, + TokenHolder { + address: "0x9012345678901234567890123456789012345678".to_string(), + balance: "5".to_string(), + }, + TokenHolder { + address: "0x0123456789012345678901234567890123456789".to_string(), + balance: "1".to_string(), + }, + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_extract_addresses() { + let logs = vec![TransferLog { + address: "0xtoken".to_string(), + topics: vec![ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef".to_string(), + "0x000000000000000000000000a0b86991c6218b36c1d19d4a2e9eb0ce3606eb48".to_string(), + "0x000000000000000000000000dac17f958d2ee523a2206206994597c13d831ec7".to_string(), + ], + data: "0x0000000000000000000000000000000000000000000000000000000000000064".to_string(), + block_number: "0x1".to_string(), + transaction_hash: "0xhash".to_string(), + transaction_index: "0x0".to_string(), + block_hash: "0xblockhash".to_string(), + log_index: "0x0".to_string(), + }]; + + let addresses = EtherscanClient::extract_addresses(&logs); + assert_eq!(addresses.len(), 2); + + let addr1: Address = "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48" + .parse() + .unwrap(); + let addr2: Address = "0xdac17f958d2ee523a2206206994597c13d831ec7" + .parse() + .unwrap(); + + assert!(addresses.contains(&addr1)); + assert!(addresses.contains(&addr2)); + } + + #[test] + fn test_extract_delegates() { + let logs = vec![ + DelegateVotesChangedLog { + address: "0xtoken".to_string(), + topics: vec![ + "0xdec2bacdd2f05b59de34da9b523dff8be42e5e38e8af2ade71e1ddfc5c9f0e6f".to_string(), + "0x000000000000000000000000a0b86991c6218b36c1d19d4a2e9eb0ce3606eb48".to_string(), + ], + data: "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064".to_string(), + block_number: "0x1".to_string(), + transaction_hash: "0xhash".to_string(), + transaction_index: "0x0".to_string(), + block_hash: "0xblockhash".to_string(), + log_index: "0x0".to_string(), + }, + ]; + + let delegates = EtherscanClient::extract_delegates(&logs); + assert_eq!(delegates.len(), 1); + + let addr: Address = "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48" + .parse() + .unwrap(); + assert!(delegates.contains(&addr)); + } + + #[test] + fn test_compute_balances() { + let logs = vec![ + TransferLog { + address: "0xtoken".to_string(), + topics: vec![ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" + .to_string(), + "0x0000000000000000000000000000000000000000000000000000000000000000" + .to_string(), // from: zero address (mint) + "0x000000000000000000000000a0b86991c6218b36c1d19d4a2e9eb0ce3606eb48" + .to_string(), // to: address A + ], + data: "0x0000000000000000000000000000000000000000000000000000000000000064" + .to_string(), // 100 tokens + block_number: "0x1".to_string(), + transaction_hash: "0xhash1".to_string(), + transaction_index: "0x0".to_string(), + block_hash: "0xblock1".to_string(), + log_index: "0x0".to_string(), + }, + TransferLog { + address: "0xtoken".to_string(), + topics: vec![ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" + .to_string(), + "0x000000000000000000000000a0b86991c6218b36c1d19d4a2e9eb0ce3606eb48" + .to_string(), // from: address A + "0x000000000000000000000000dac17f958d2ee523a2206206994597c13d831ec7" + .to_string(), // to: address B + ], + data: "0x0000000000000000000000000000000000000000000000000000000000000032" + .to_string(), // 50 tokens + block_number: "0x2".to_string(), + transaction_hash: "0xhash2".to_string(), + transaction_index: "0x0".to_string(), + block_hash: "0xblock2".to_string(), + log_index: "0x0".to_string(), + }, + ]; + + let balances = EtherscanClient::compute_balances_from_logs(&logs); + + let addr_a: Address = "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48" + .parse() + .unwrap(); + let addr_b: Address = "0xdac17f958d2ee523a2206206994597c13d831ec7" + .parse() + .unwrap(); + + // Address A: received 100, sent 50 = 50 + assert_eq!(balances.get(&addr_a), Some(&U256::from(50))); + + // Address B: received 50 + assert_eq!(balances.get(&addr_b), Some(&U256::from(50))); + } + + #[test] + fn test_get_potential_voters() { + let client = EtherscanClient::new("test_key".to_string(), 1); + + let transfer_logs = vec![TransferLog { + address: "0xtoken".to_string(), + topics: vec![ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef".to_string(), + "0x0000000000000000000000000000000000000000000000000000000000000000".to_string(), + "0x000000000000000000000000a0b86991c6218b36c1d19d4a2e9eb0ce3606eb48".to_string(), + ], + data: "0x0000000000000000000000000000000000000000000000000000000000000064".to_string(), + block_number: "0x1".to_string(), + transaction_hash: "0xhash1".to_string(), + transaction_index: "0x0".to_string(), + block_hash: "0xblock1".to_string(), + log_index: "0x0".to_string(), + }]; + + let delegation_logs = vec![ + DelegateVotesChangedLog { + address: "0xtoken".to_string(), + topics: vec![ + "0xdec2bacdd2f05b59de34da9b523dff8be42e5e38e8af2ade71e1ddfc5c9f0e6f".to_string(), + "0x000000000000000000000000dac17f958d2ee523a2206206994597c13d831ec7".to_string(), // delegate B + ], + data: "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064".to_string(), + block_number: "0x2".to_string(), + transaction_hash: "0xhash2".to_string(), + transaction_index: "0x0".to_string(), + block_hash: "0xblock2".to_string(), + log_index: "0x0".to_string(), + }, + ]; + + let potential_voters = client.get_potential_voters(&transfer_logs, &delegation_logs); + + // Should have 2 voters: A (token holder) and B (delegate, may not have tokens) + assert_eq!(potential_voters.len(), 2); + + let addr_a: Address = "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48" + .parse() + .unwrap(); + let addr_b: Address = "0xdac17f958d2ee523a2206206994597c13d831ec7" + .parse() + .unwrap(); + + let voter_a = potential_voters + .iter() + .find(|v| v.address == addr_a) + .unwrap(); + assert_eq!(voter_a.token_balance, U256::from(100)); + assert!(!voter_a.has_delegation); // A is not a delegate + + let voter_b = potential_voters + .iter() + .find(|v| v.address == addr_b) + .unwrap(); + assert!(voter_b.has_delegation); // B is a delegate + } + + #[test] + fn test_parse_address_from_topic() { + let topic = "0x000000000000000000000000a0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"; + let addr = EtherscanClient::parse_address_from_topic(topic).unwrap(); + let expected: Address = "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48" + .parse() + .unwrap(); + assert_eq!(addr, expected); + } + + #[test] + fn test_parse_transfer_value() { + assert_eq!( + EtherscanClient::parse_transfer_value("0x64"), + U256::from(100) + ); + assert_eq!(EtherscanClient::parse_transfer_value("0x0"), U256::ZERO); + assert_eq!( + EtherscanClient::parse_transfer_value( + "0x0000000000000000000000000000000000000000000000000000000000000064" + ), + U256::from(100) + ); + } + + // Integration tests (requires valid API key) + #[tokio::test] + #[ignore] + async fn test_get_deployment_block() { + let token = "0xb0BE360719f84c5351621590B7FfBD8EB0B46B5d"; + let chain_id = 11155111; + let api_key = &CONFIG.etherscan_api_key; + + let client = EtherscanClient::new(api_key.to_string(), chain_id); + let result = client.get_deployment_block(token).await; + println!("Deployment block: {:?}", result); + assert!(result.is_ok()); + } + + #[tokio::test] + #[ignore] + async fn test_get_transfer_logs() { + let token = "0xC18360217D8F7Ab5e7c516566761Ea12Ce7F9D72"; + let from_block = 23680346; + let to_block = 23682281; + let chain_id = 1; + let api_key = &CONFIG.etherscan_api_key; + + let client = EtherscanClient::new(api_key.to_string(), chain_id); + let result = client.get_transfer_logs(token, from_block, to_block).await; + println!("Transfer logs: {:?}", result); + assert!(result.is_ok()); + } + + #[tokio::test] + #[ignore] + async fn test_get_delegate_votes_changed_logs() { + let token = "0xC18360217D8F7Ab5e7c516566761Ea12Ce7F9D72"; + let from_block = 23680346; + let to_block = 23682281; + let chain_id = 1; + let api_key = &CONFIG.etherscan_api_key; + + let client = EtherscanClient::new(api_key.to_string(), chain_id); + let result = client + .get_delegate_votes_changed_logs(token, from_block, to_block) + .await; + println!("Delegation logs: {:?}", result); + assert!(result.is_ok()); + } + + #[tokio::test] + #[ignore] + async fn test_get_token_holders_with_voting_power() { + let token = "0xb0BE360719f84c5351621590B7FfBD8EB0B46B5d"; + let token_address: Address = token.parse().unwrap(); + let chain_id = CONFIG.chain_id; + let api_key = &CONFIG.etherscan_api_key; + let rpc_url = &CONFIG.http_rpc_url; + let threshold = U256::ZERO; + let snapshot_block = 9564734; + + let client = EtherscanClient::new(api_key.to_string(), chain_id); + let res = client + .get_token_holders_with_voting_power(token_address, snapshot_block, rpc_url, threshold) + .await + .unwrap(); + + assert!(res.len() == 2); + } +} diff --git a/examples/CRISP/server/src/server/token_holders/hashes.rs b/examples/CRISP/server/src/server/token_holders/hashes.rs index 02f475c092..a40d8c55fd 100644 --- a/examples/CRISP/server/src/server/token_holders/hashes.rs +++ b/examples/CRISP/server/src/server/token_holders/hashes.rs @@ -11,7 +11,7 @@ use light_poseidon::{Poseidon, PoseidonHasher}; use num_bigint::BigUint; use std::str::FromStr; -use super::TokenHolder; +use super::etherscan::TokenHolder; /// Computes Poseidon hashes for token holder address + balance pairs. /// diff --git a/examples/CRISP/server/src/server/token_holders/mod.rs b/examples/CRISP/server/src/server/token_holders/mod.rs index db4701f7ac..3de8270d3d 100644 --- a/examples/CRISP/server/src/server/token_holders/mod.rs +++ b/examples/CRISP/server/src/server/token_holders/mod.rs @@ -4,10 +4,10 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -pub mod bitquery; +pub mod etherscan; pub mod hashes; pub mod merkle_tree; -pub use bitquery::*; +pub use etherscan::*; pub use hashes::*; pub use merkle_tree::*;