From 7a60bf780e5d499d8da72698ad2aa74456930f8e Mon Sep 17 00:00:00 2001 From: Amy Thomason Date: Mon, 9 Mar 2026 20:11:22 +0000 Subject: [PATCH 1/6] feat: add follower node support and testnet_follower_test target --- Makefile | 30 ++++-- crates/rbft-node/src/main.rs | 7 +- crates/rbft-node/src/rbft_consensus.rs | 32 +++++- crates/rbft-utils/src/main.rs | 9 ++ crates/rbft-utils/src/testnet.rs | 138 +++++++++++++++++++++++++ 5 files changed, 203 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index d19459e..c75395a 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ help: @echo " testnet_debug - Start a local testnet in debug mode" @echo " testnet_restart - Restart the testnet" @echo " testnet_load_test - Start testnet with transaction load testing and auto-exit" + @echo " testnet_follower_test - Start testnet and add follower nodes at blocks 5 and 15, exit at block 30" @echo " genesis - Generate a genesis file" @echo " node-gen - Generate node enodes and secret keys in CSV format" @echo " node-help - Show help for the rbft-node binary" @@ -67,6 +68,9 @@ help: @echo " RBFT_EXIT_AFTER_BLOCK= - Exit testnet after reaching this block height" @echo " RBFT_ADD_AT_BLOCKS= - Comma-separated list of block heights to add" @echo " validators at (e.g., '10,20,30')" + @echo " RBFT_ADD_FOLLOWER_AT= - Comma-separated list of block heights to add" + @echo " follower (non-validator) nodes at (e.g., '5,15')" + @echo " Uses key slots from nodes.csv starting at index num_nodes" @echo "" @echo " Examples:" @echo " RBFT_NUM_NODES=7 make genesis" @@ -151,13 +155,23 @@ testnet_debug: RUST_LOG=debug $(MAKE) CARGO_PROFILE=debug testnet_start -# Note this is controllable by environment variables. -# RBFT_NUM_NODES=10 make testnet_load_test -# RBFT_BLOCK_INTERVAL=0.1 make testnet_load_test -# RBFT_BASE_FEE=1000000000000 make testnet_load_test -# see target/release/rbft-utils genesis --help for details. -# Avoid setting log-level here as it is possible to set it before running make. -# RUST_LOG=debug make testnet_load_test +# Test follower node support: starts a 4-validator testnet, adds 2 non-validator follower +# nodes at blocks 5 and 15 via RBFT_ADD_FOLLOWER_AT, then exits at block 30. +# node-gen is given RBFT_NUM_NODES + 2 slots so nodes.csv has keys for the followers. +testnet_follower_test: + mkdir -p $(ASSETS_DIR) + $(CARGO) build --release --bin rbft-node + $(CARGO) build --release --bin rbft-utils + target/release/rbft-utils node-gen --assets-dir $(ASSETS_DIR) \ + --num-nodes $$(( $${RBFT_NUM_NODES:-4} + 2 )) + target/release/rbft-utils genesis --assets-dir $(ASSETS_DIR) \ + --initial-nodes $${RBFT_NUM_NODES:-4} + RBFT_ADD_FOLLOWER_AT=5,15 RBFT_EXIT_AFTER_BLOCK=30 \ + target/release/rbft-utils testnet --init \ + --assets-dir $(ASSETS_DIR) + target/release/rbft-utils logjam -q + +# Test transaction load handling: starts a 4-validator testnet with megatx enabled, which submits large transactions every block. testnet_load_test: mkdir -p $(ASSETS_DIR) $(CARGO) build --release --bin rbft-node @@ -317,4 +331,4 @@ docker-tag-registry: .PHONY: help docker-validate docker-build docker-build-dev docker-build-debug docker-run \ docker-run-dev docker-test docker-clean test fmt clippy clean docker-tag-registry \ dafny-translate validator_status status testnet_start testnet_restart genesis node-help \ - megatx validator-inspector testnet_load_test testnet_debug + megatx validator-inspector testnet_load_test testnet_follower_test testnet_debug diff --git a/crates/rbft-node/src/main.rs b/crates/rbft-node/src/main.rs index d5682d0..5f03807 100644 --- a/crates/rbft-node/src/main.rs +++ b/crates/rbft-node/src/main.rs @@ -36,7 +36,12 @@ use metrics::QbftMetrics; /// RBFT_TRUSTED_PEERS_REFRESH_SECS Peer refresh interval (default: 10) #[derive(Debug, Parser, Clone)] pub struct RbftNodeArgs { - /// Path to the validator private key file + /// Path to the validator private key file. + /// + /// If omitted, an ephemeral random key is generated at startup. The derived address will + /// not be present in the on-chain validator set, so the node operates as a follower: + /// it syncs blocks via `NewBlock` messages but never proposes, prepares, commits, or + /// triggers round changes. #[arg(long)] pub validator_key: Option, diff --git a/crates/rbft-node/src/rbft_consensus.rs b/crates/rbft-node/src/rbft_consensus.rs index 6f35aed..674306d 100644 --- a/crates/rbft-node/src/rbft_consensus.rs +++ b/crates/rbft-node/src/rbft_consensus.rs @@ -2618,10 +2618,34 @@ async fn attempt_peer_reconnection( fn get_private_key_and_id( args: &RbftNodeArgs, ) -> eyre::Result<(alloy_primitives::B256, alloy_primitives::Address)> { - let key = args - .validator_key - .as_ref() - .ok_or_else(|| eyre!("Validator key is required for QBFT nodes"))?; + // If no validator key file is provided, generate an ephemeral random key. + // The node's address will not appear in the on-chain validator set, so the QBFT + // state machine will treat it as a follower (non-validator): it will only run + // `upon_new_block` and never propose, prepare, commit, or trigger round changes. + let Some(key) = args.validator_key.as_ref() else { + let mut key_bytes = [0u8; 32]; + { + use std::io::Read; + let mut urandom = std::fs::File::open("/dev/urandom") + .map_err(|e| eyre!("Cannot open /dev/urandom for ephemeral follower key: {e}"))?; + urandom + .read_exact(&mut key_bytes) + .map_err(|e| eyre!("Cannot read from /dev/urandom: {e}"))?; + } + // Ensure the key is non-zero (truly random bytes are astronomically unlikely + // to be all-zero, but guard against it anyway). + key_bytes[0] |= 1; + let private_key = B256::from(key_bytes); + let signer = PrivateKeySigner::from_bytes(&private_key) + .map_err(|e| eyre!("Failed to create ephemeral follower signer: {e}"))?; + let id = signer.address(); + info!( + target: "rbft", + "No --validator-key provided; running as follower node with ephemeral address {id}" + ); + return Ok((private_key, id)); + }; + let key_data = std::fs::read_to_string(key) .map_err(|e| eyre!("Failed to read validator key file {key:?}: {e}"))?; let key_data = key_data.trim(); diff --git a/crates/rbft-utils/src/main.rs b/crates/rbft-utils/src/main.rs index 0c5d9ca..107c161 100644 --- a/crates/rbft-utils/src/main.rs +++ b/crates/rbft-utils/src/main.rs @@ -135,6 +135,13 @@ enum RbftCommands { #[arg(long, env = "RBFT_ADD_AT_BLOCKS")] add_at_blocks: Option, + /// Comma-separated list of block heights at which to add follower nodes (e.g., "10,20"). + /// Each trigger spawns one new node process whose key is NOT registered in the on-chain + /// validator contract, so the node observes consensus without participating in it. + /// Key files are taken from nodes.csv starting at index num_nodes. + #[arg(long, env = "RBFT_ADD_FOLLOWER_AT")] + add_followers_at: Option, + /// Number of initial validators (for tracking which validator to add next). /// Defaults to num_nodes if not specified. #[arg(long, env = "RBFT_INITIAL_NODES")] @@ -376,6 +383,7 @@ fn main() -> eyre::Result<()> { run_megatx, exit_after_block, add_at_blocks, + add_followers_at, initial_nodes, docker, kube, @@ -411,6 +419,7 @@ fn main() -> eyre::Result<()> { run_megatx, exit_after_block, add_at_blocks.as_deref(), + add_followers_at.as_deref(), initial_nodes, docker, kube, diff --git a/crates/rbft-utils/src/testnet.rs b/crates/rbft-utils/src/testnet.rs index eabd458..cd6fdad 100644 --- a/crates/rbft-utils/src/testnet.rs +++ b/crates/rbft-utils/src/testnet.rs @@ -570,6 +570,7 @@ pub async fn testnet( run_megatx: bool, exit_after_block: Option, add_at_blocks: Option<&str>, + add_followers_at: Option<&str>, initial_nodes: Option, docker: bool, kube: bool, @@ -922,6 +923,31 @@ pub async fn testnet( ); } + // Parse add_followers_at if provided + let add_followers_at_list: Vec = if let Some(blocks_str) = add_followers_at { + blocks_str + .split(',') + .filter_map(|s| s.trim().parse::().ok()) + .collect() + } else { + Vec::new() + }; + + // Track which blocks have already caused a follower to be spawned + let mut followers_added_at_blocks: std::collections::HashSet = + std::collections::HashSet::new(); + + // Running count of follower nodes launched so far (used to pick the node index). + // Follower node indices start at num_nodes (after all initial validator slots). + let mut current_follower_count: usize = 0; + + if !add_followers_at_list.is_empty() { + eprintln!( + "Will add follower nodes at blocks: {:?} (starting from node index {})", + add_followers_at_list, num_nodes + ); + } + loop { sleep(Duration::from_millis(1000)).await; monitor_tick += 1; @@ -1029,6 +1055,61 @@ pub async fn testnet( } } + // Check if we should add a follower node at this block height + if !add_followers_at_list.is_empty() { + let max_height = numeric_heights + .iter() + .filter_map(|h| *h) + .max() + .unwrap_or_default(); + + for &target_block in &add_followers_at_list { + if max_height >= target_block && !followers_added_at_blocks.contains(&target_block) + { + followers_added_at_blocks.insert(target_block); + let follower_index = num_nodes + current_follower_count as u32; + eprintln!( + "👀 Block {} reached, adding follower node at index {}...", + target_block, follower_index + ); + + // Short pause so the block is fully propagated before the new node connects + sleep(Duration::from_millis(500)).await; + + match add_next_follower( + follower_index, + base_http_port, + &assets, + &trusted_peers, + rbft_node_exe.as_path(), + &data_dir_path, + &logs_dir_path, + extra_args, + docker, + ) { + Ok((child, url)) => { + current_follower_count += 1; + let provider = ProviderBuilder::new() + .wallet(signer.clone()) + .connect_http(url.parse().expect("follower node URL is not valid")); + nodes.push(child); + providers.push(provider); + eprintln!( + "✅ Follower node {} started at block {} (total followers: {})", + follower_index, target_block, current_follower_count + ); + } + Err(e) => { + eprintln!( + "❌ Failed to start follower node at block {}: {:?}", + target_block, e + ); + } + } + } + } + } + let exit_condition = is_exit_condition_met(&numeric_heights, exit_after_block, &mut megatx_process); @@ -1170,6 +1251,63 @@ async fn add_next_validator( Ok((validator_address.to_string(), enode.to_string())) } +/// Spawn a follower node (non-validator) using key assets at `follower_node_index`. +/// +/// The node is started with the key files from `assets` at the given index (p2p + validator +/// key) and connects to the existing validators via `trusted_peers`. It is **not** registered +/// in the on-chain `QBFTValidatorSet` contract, so the QBFT state machine treats it as a +/// follower: it only runs `upon_new_block` and never participates in proposing or voting. +/// +/// Key files required: +/// `/validator-key.txt` +/// `/p2p-secret-key.txt` +/// +/// These are produced by `rbft-utils node-gen` / `make genesis`. Ensure nodes.csv was +/// generated with enough entries to cover this index. +/// +/// Returns `(Child process, RPC URL)` on success. +#[allow(clippy::too_many_arguments)] +fn add_next_follower( + follower_node_index: u32, + base_http_port: u16, + assets: &Path, + trusted_peers: &str, + current_exe: &Path, + data_dir_path: &Path, + logs_dir_path: &Path, + extra_args: Option<&[String]>, + docker: bool, +) -> eyre::Result<(Child, String)> { + let validator_key_path = assets.join(format!("validator-key{follower_node_index}.txt")); + let p2p_key_path = assets.join(format!("p2p-secret-key{follower_node_index}.txt")); + if !validator_key_path.exists() || !p2p_key_path.exists() { + return Err(eyre::eyre!( + "Missing key files for follower node index {follower_node_index}: expected {} and {} \ + in {}. Re-generate with a larger --num-nodes to create more key slots.", + validator_key_path.display(), + p2p_key_path.display(), + assets.display(), + )); + } + + // `start_new_validator` is purely about launching a node process; the `num_nodes` + // argument is only used for a port-overflow warning, so pass index+1 to suppress it. + let (child, url) = start_new_validator( + follower_node_index, + follower_node_index + 1, + base_http_port, + assets, + trusted_peers, + current_exe, + data_dir_path, + logs_dir_path, + extra_args, + docker, + ); + + Ok((child, url)) +} + fn is_exit_condition_met( numeric_heights: &[Option], exit_after_block: Option, From c9d292b602127130ac924fa8bf4fe7b09c609a0b Mon Sep 17 00:00:00 2001 From: Amy Thomason Date: Tue, 10 Mar 2026 10:26:56 +0000 Subject: [PATCH 2/6] feat: follower node support - private_key as Option in NodeState --- .gitignore | 1 + crates/rbft-node/src/main.rs | 38 +++++++++--------- crates/rbft-node/src/rbft_consensus.rs | 40 +++++-------------- crates/rbft-utils/src/testnet.rs | 29 ++++++++------ crates/rbft/src/node_auxilliary_functions.rs | 42 +++++++++++++++++--- crates/rbft/src/tests.rs | 4 +- crates/rbft/src/types/node_state.rs | 12 +++--- 7 files changed, 92 insertions(+), 74 deletions(-) diff --git a/.gitignore b/.gitignore index 1bd2f1c..b66f7f5 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,4 @@ generated_accounts.txt # Foundry/Soldeer dependencies lib/ +/~ diff --git a/crates/rbft-node/src/main.rs b/crates/rbft-node/src/main.rs index 5f03807..e69cb38 100644 --- a/crates/rbft-node/src/main.rs +++ b/crates/rbft-node/src/main.rs @@ -25,32 +25,28 @@ use metrics::QbftMetrics; // === Node args === /// RBFT node configuration /// -/// Environment Variables: -/// RBFT_BLOCK_INTERVAL Override block interval (seconds, float) -/// RBFT_GAS_LIMIT Override gas limit (unsigned integer) -/// RBFT_DEBUG_CATCHUP_BLOCK Only enable chain catchup after this -/// block height (unsigned integer) -/// RBFT_RESEND_AFTER Resend cached messages after this many -/// seconds without block commits (default: 0=disabled) -/// RBFT_FULL_LOGS Emit state logs on every advance (default: false) -/// RBFT_TRUSTED_PEERS_REFRESH_SECS Peer refresh interval (default: 10) +/// Environment Variables (all correspond to a `--flag` CLI arg of the same name): +/// RBFT_VALIDATOR_KEY Path to the validator private key file +/// RBFT_LOGS_DIR Directory for log files +/// RBFT_DB_DIR Directory for database files +/// RBFT_TRUSTED_PEERS_REFRESH_SECS Peer refresh interval in seconds (default: 10) +/// RBFT_FULL_LOGS Emit state logs on every advance (default: false) +/// RBFT_RESEND_AFTER Resend cached messages after N seconds without commits +/// RBFT_DISABLE_EXPRESS Disable express transaction delivery +/// RBFT_DEBUG_CATCHUP_BLOCK Only enable QBFT advance after this block height +/// (node 0 only; useful for debugging catch-up) #[derive(Debug, Parser, Clone)] pub struct RbftNodeArgs { - /// Path to the validator private key file. - /// - /// If omitted, an ephemeral random key is generated at startup. The derived address will - /// not be present in the on-chain validator set, so the node operates as a follower: - /// it syncs blocks via `NewBlock` messages but never proposes, prepares, commits, or - /// triggers round changes. - #[arg(long)] + /// Path to the validator private key file. If not provided, the node will be a follower node. + #[arg(long, env = "RBFT_VALIDATOR_KEY")] pub validator_key: Option, /// Directory for log files. Defaults to ~/.rbft/testnet/logs - #[arg(long)] + #[arg(long, env = "RBFT_LOGS_DIR")] pub logs_dir: Option, /// Directory for database files. Defaults to ~/.rbft/testnet/db - #[arg(long)] + #[arg(long, env = "RBFT_DB_DIR")] pub db_dir: Option, /// How often (in seconds) to refresh trusted peer DNS entries and reconnect on changes. @@ -80,6 +76,12 @@ pub struct RbftNodeArgs { /// transactions to the next block proposer). #[arg(long, env = "RBFT_DISABLE_EXPRESS")] pub disable_express: bool, + + /// Only enable QBFT advance on node 0 after the chain reaches this block height + /// (or another node has already seen a NewBlock at this height). Useful for + /// debugging catch-up behaviour without advancing immediately. + #[arg(long, value_name = "BLOCK", env = "RBFT_DEBUG_CATCHUP_BLOCK")] + pub debug_catchup_block: Option, } fn spawn_trusted_peer_refresh( diff --git a/crates/rbft-node/src/rbft_consensus.rs b/crates/rbft-node/src/rbft_consensus.rs index 674306d..15d6078 100644 --- a/crates/rbft-node/src/rbft_consensus.rs +++ b/crates/rbft-node/src/rbft_consensus.rs @@ -22,7 +22,6 @@ pub use rbft_utils::types::RbftConfig; use std::{ collections::{HashMap, VecDeque}, - env, time::{Duration, Instant, UNIX_EPOCH}, }; @@ -179,7 +178,7 @@ mod reload_tests { nodes: vec![qbft_beneficiary], ..Default::default() }; - let private_key = B256::from([1u8; 32]); + let private_key = Some(B256::from([1u8; 32])); let node_state = NodeState::new(blockchain, configuration, qbft_beneficiary, private_key, 0); @@ -1512,16 +1511,13 @@ where } } - let enabled = match env::var("RBFT_DEBUG_CATCHUP_BLOCK") { - Ok(value) => { - let threshold = value.parse::().map_err(|e| { - eyre!("RBFT_DEBUG_CATCHUP_BLOCK must be a valid unsigned integer: {e}") - })?; + let enabled = match self.args.debug_catchup_block { + Some(threshold) => { whoami != 0 || self.node_state.blockchain().height() >= threshold || highest_newblock_message >= threshold } - Err(_) => true, + None => true, }; if !enabled { @@ -2617,33 +2613,17 @@ async fn attempt_peer_reconnection( fn get_private_key_and_id( args: &RbftNodeArgs, -) -> eyre::Result<(alloy_primitives::B256, alloy_primitives::Address)> { - // If no validator key file is provided, generate an ephemeral random key. - // The node's address will not appear in the on-chain validator set, so the QBFT +) -> eyre::Result<(Option, alloy_primitives::Address)> { + // If no validator key file is provided, the node runs as a follower. + // Its address will not appear in the on-chain validator set, so the QBFT // state machine will treat it as a follower (non-validator): it will only run // `upon_new_block` and never propose, prepare, commit, or trigger round changes. let Some(key) = args.validator_key.as_ref() else { - let mut key_bytes = [0u8; 32]; - { - use std::io::Read; - let mut urandom = std::fs::File::open("/dev/urandom") - .map_err(|e| eyre!("Cannot open /dev/urandom for ephemeral follower key: {e}"))?; - urandom - .read_exact(&mut key_bytes) - .map_err(|e| eyre!("Cannot read from /dev/urandom: {e}"))?; - } - // Ensure the key is non-zero (truly random bytes are astronomically unlikely - // to be all-zero, but guard against it anyway). - key_bytes[0] |= 1; - let private_key = B256::from(key_bytes); - let signer = PrivateKeySigner::from_bytes(&private_key) - .map_err(|e| eyre!("Failed to create ephemeral follower signer: {e}"))?; - let id = signer.address(); info!( target: "rbft", - "No --validator-key provided; running as follower node with ephemeral address {id}" + "No --validator-key provided; running as follower node" ); - return Ok((private_key, id)); + return Ok((None, Address::ZERO)); }; let key_data = std::fs::read_to_string(key) @@ -2668,7 +2648,7 @@ fn get_private_key_and_id( let signer = PrivateKeySigner::from_bytes(&private_key) .map_err(|e| eyre!("Failed to create signer from private key: {e}"))?; let id = signer.address(); - Ok((private_key, id)) + Ok((Some(private_key), id)) } fn now() -> u64 { diff --git a/crates/rbft-utils/src/testnet.rs b/crates/rbft-utils/src/testnet.rs index cd6fdad..6c99229 100644 --- a/crates/rbft-utils/src/testnet.rs +++ b/crates/rbft-utils/src/testnet.rs @@ -280,6 +280,7 @@ fn start_new_validator( logs_dir_path: &Path, extra_args: Option<&[String]>, docker: bool, + is_follower: bool, ) -> (Child, String) { let db_path = data_dir_path.join(format!("d{}", index)); fs::create_dir_all(&db_path).expect("failed to create node datadir"); @@ -377,8 +378,10 @@ fn start_new_validator( cmd.push("/data/reth-config.toml".to_string()); cmd.push("--p2p-secret-key".to_string()); cmd.push(format!("/assets/p2p-secret-key{}.txt", index)); - cmd.push("--validator-key".to_string()); - cmd.push(format!("/assets/validator-key{}.txt", index)); + if !is_follower { + cmd.push("--validator-key".to_string()); + cmd.push(format!("/assets/validator-key{}.txt", index)); + } if num_nodes != 1 && !trusted_peers.is_empty() { cmd.push("--trusted-peers".to_string()); cmd.push(trusted_peers.to_string()); @@ -436,12 +439,14 @@ fn start_new_validator( .to_str() .expect("failed to convert p2p key path to string") )); - cmd.push(format!( - "--validator-key {}", - validator_key - .to_str() - .expect("failed to convert validator key path to string") - )); + if !is_follower { + cmd.push(format!( + "--validator-key {}", + validator_key + .to_str() + .expect("failed to convert validator key path to string") + )); + } if num_nodes != 1 && !trusted_peers.is_empty() { cmd.push(format!("--trusted-peers {trusted_peers}")); } @@ -723,6 +728,7 @@ pub async fn testnet( &logs_dir_path, extra_args, docker, + false, ); nodes.push(node); urls.push(url); @@ -1278,13 +1284,11 @@ fn add_next_follower( extra_args: Option<&[String]>, docker: bool, ) -> eyre::Result<(Child, String)> { - let validator_key_path = assets.join(format!("validator-key{follower_node_index}.txt")); let p2p_key_path = assets.join(format!("p2p-secret-key{follower_node_index}.txt")); - if !validator_key_path.exists() || !p2p_key_path.exists() { + if !p2p_key_path.exists() { return Err(eyre::eyre!( - "Missing key files for follower node index {follower_node_index}: expected {} and {} \ + "Missing p2p key file for follower node index {follower_node_index}: expected {} \ in {}. Re-generate with a larger --num-nodes to create more key slots.", - validator_key_path.display(), p2p_key_path.display(), assets.display(), )); @@ -1303,6 +1307,7 @@ fn add_next_follower( logs_dir_path, extra_args, docker, + true, ); Ok((child, url)) diff --git a/crates/rbft/src/node_auxilliary_functions.rs b/crates/rbft/src/node_auxilliary_functions.rs index b691cc5..892a1f2 100644 --- a/crates/rbft/src/node_auxilliary_functions.rs +++ b/crates/rbft/src/node_auxilliary_functions.rs @@ -44,7 +44,12 @@ pub fn sign_proposal(msg: &UnsignedProposal, node_state: &NodeState) -> SignedPr msg.encode(&mut encoded); // Create signature using the node's private key - let signature = Signature::sign_message(&encoded, &node_state.private_key()); + let signature = Signature::sign_message( + &encoded, + &node_state + .private_key() + .expect("validator node must have a private key to sign"), + ); SignedProposal { unsigned_payload: msg.clone(), @@ -66,7 +71,12 @@ pub fn sign_prepare(msg: &UnsignedPrepare, node_state: &NodeState) -> SignedPrep msg.encode(&mut encoded); // Create signature using the node's private key - let signature = Signature::sign_message(&encoded, &node_state.private_key()); + let signature = Signature::sign_message( + &encoded, + &node_state + .private_key() + .expect("validator node must have a private key to sign"), + ); SignedPrepare { unsigned_payload: msg.clone(), @@ -88,7 +98,12 @@ pub fn sign_commit(msg: &UnsignedCommit, node_state: &NodeState) -> SignedCommit msg.encode(&mut encoded); // Create signature using the node's private key - let signature = Signature::sign_message(&encoded, &node_state.private_key()); + let signature = Signature::sign_message( + &encoded, + &node_state + .private_key() + .expect("validator node must have a private key to sign"), + ); SignedCommit { unsigned_payload: msg.clone(), @@ -110,7 +125,12 @@ pub fn sign_round_change(msg: &UnsignedRoundChange, node_state: &NodeState) -> S msg.encode(&mut encoded); // Create signature using the node's private key - let signature = Signature::sign_message(&encoded, &node_state.private_key()); + let signature = Signature::sign_message( + &encoded, + &node_state + .private_key() + .expect("validator node must have a private key to sign"), + ); SignedRoundChange { unsigned_payload: msg.clone(), @@ -136,7 +156,12 @@ pub fn sign_new_block(block: &Block, node_state: &NodeState) -> SignedNewBlock { let mut encoded = Vec::new(); unsigned.encode(&mut encoded); - let signature = Signature::sign_message(&encoded, &node_state.private_key()); + let signature = Signature::sign_message( + &encoded, + &node_state + .private_key() + .expect("validator node must have a private key to sign"), + ); SignedNewBlock { unsigned_payload: unsigned, @@ -153,7 +178,12 @@ pub fn recover_signed_new_block_author(signature: &Signature) -> Address { /// Signs a hash and returns a signature. pub fn sign_hash(hash: &Hash, node_state: &NodeState) -> Signature { // Create signature using the node's private key - Signature::sign_message(hash.as_slice(), &node_state.private_key()) + Signature::sign_message( + hash.as_slice(), + &node_state + .private_key() + .expect("validator node must have a private key to sign"), + ) } /// Recovers the author of a signed hash. diff --git a/crates/rbft/src/tests.rs b/crates/rbft/src/tests.rs index c52ba42..4d4ae07 100644 --- a/crates/rbft/src/tests.rs +++ b/crates/rbft/src/tests.rs @@ -122,7 +122,7 @@ impl NodeSwarm { let blockchain = Blockchain::new(VecDeque::from([genesis_block])); - NodeState::new(blockchain, configuration, id, private_keys[i], 0) + NodeState::new(blockchain, configuration, id, Some(private_keys[i]), 0) }) .collect(); Self { @@ -1912,7 +1912,7 @@ fn test_message_author_validation() { updated_blockchain, updated_configuration, updated_validators[0], - B256::from([0x01; 32]), + Some(B256::from([0x01; 32])), 0, ); diff --git a/crates/rbft/src/types/node_state.rs b/crates/rbft/src/types/node_state.rs index 0645fb7..787afd8 100644 --- a/crates/rbft/src/types/node_state.rs +++ b/crates/rbft/src/types/node_state.rs @@ -77,8 +77,8 @@ pub struct NodeState { /// Added for practical implementation, not in the Dafny spec. /// This is used to sign messages sent by this node. - /// TODO: followers do not have a private key. This should be an Option. - private_key: PrivateKey, + /// `None` for follower nodes that do not participate in consensus. + private_key: Option, /// The first message at or above the current height. first_future_message: usize, @@ -92,7 +92,7 @@ impl NodeState { blockchain: Blockchain, configuration: Configuration, id: Address, - private_key: PrivateKey, + private_key: Option, local_time: u64, ) -> Self { let block_time = configuration.block_time; @@ -306,7 +306,7 @@ impl NodeState { /// Return the role of this node. fn role(&self) -> char { - let role = if self.private_key().is_zero() { + let role = if self.private_key().is_none() { // Follower node - no private key. 'f' } else if self.is_the_proposer_for_current_round() { @@ -467,7 +467,7 @@ impl NodeState { self.round_zero_last_progress_time } - pub fn private_key(&self) -> FixedBytes<32> { + pub fn private_key(&self) -> Option> { self.private_key } @@ -650,7 +650,7 @@ mod tests { blockchain, configuration, validators[0], - PrivateKey::default(), + Some(PrivateKey::default()), 0, ) } From 928d7515728d3857904f8039713d6ddce24e41df Mon Sep 17 00:00:00 2001 From: Amy Thomason Date: Tue, 10 Mar 2026 11:04:12 +0000 Subject: [PATCH 3/6] docs: add follower node docs and --chain to testnet_follower target --- Makefile | 15 ++++++++++++++- README.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c75395a..03ed907 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ help: @echo " testnet_restart - Restart the testnet" @echo " testnet_load_test - Start testnet with transaction load testing and auto-exit" @echo " testnet_follower_test - Start testnet and add follower nodes at blocks 5 and 15, exit at block 30" + @echo " testnet-follower - Run a follower node against a running testnet (uses enodes from nodes.csv)" @echo " genesis - Generate a genesis file" @echo " node-gen - Generate node enodes and secret keys in CSV format" @echo " node-help - Show help for the rbft-node binary" @@ -200,6 +201,18 @@ testnet_load_test: --extra-args "--rpc-cache.max-headers 100" target/release/rbft-utils logjam -q +# Run a minimal test follower node against an already-running testnet. +# Assumes the testnet was started with node-gen and genesis commands so nodes.csv and genesis.json exist in ASSETS_DIR. +# You may need to remove the reth database directory for the follower node to start successfully. +# +# --trusted-peers is set to all the enodes from nodes.csv for simplicity, but in a real scenario you would typically only trust a subset of validators. +# --chain is set to the same genesis file as the main testnet to ensure it can sync properly, but in a real scenario you would typically use a more minimal genesis for followers. +testnet_follower: + $(CARGO) build --release --bin rbft-node + target/release/rbft-node node --port 12345 \ + --chain $(ASSETS_DIR)/genesis.json \ + --trusted-peers "$$(awk -F',' 'NR>1{printf "%s%s",sep,$$5; sep=","}' $(ASSETS_DIR)/nodes.csv)" + genesis: mkdir -p $(ASSETS_DIR) $(CARGO) run --release --bin rbft-utils -- genesis --assets-dir $(ASSETS_DIR) @@ -331,4 +344,4 @@ docker-tag-registry: .PHONY: help docker-validate docker-build docker-build-dev docker-build-debug docker-run \ docker-run-dev docker-test docker-clean test fmt clippy clean docker-tag-registry \ dafny-translate validator_status status testnet_start testnet_restart genesis node-help \ - megatx validator-inspector testnet_load_test testnet_follower_test testnet_debug + megatx validator-inspector testnet_load_test testnet_follower_test testnet_debug testnet-follower diff --git a/README.md b/README.md index 98a0c2f..65a1e24 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,49 @@ This will: The nodes will log to `~/.rbft/testnet/logs/` and store data in `~/.rbft/testnet/db/`. +### Running a Follower Node + +A follower node connects to an existing validator network, receives blocks, and keeps a +full copy of the chain. It does not participate in consensus (no `--validator-key`), so it +can be added to or removed from the network at any time without affecting liveness. + +Before running a follower you need: +- A running RBFT testnet (e.g. started with `make testnet_start`) +- The `genesis.json` and `nodes.csv` from the testnet assets directory + (default: `~/.rbft/testnet/assets/`) + +Extract the validator enode URLs from `nodes.csv` (column 5): + +```bash +ENODES=$(awk -F',' 'NR>1{printf "%s%s",sep,$5; sep=","}' ~/.rbft/testnet/assets/nodes.csv) +``` + +Then start the follower: + +```bash +target/release/rbft-node node \ + --chain ~/.rbft/testnet/assets/genesis.json \ + --datadir /tmp/rbft-follower \ + --port 12345 \ + --authrpc.port 8651 \ + --http --http.port 8600 \ + --disable-discovery \ + --trusted-peers "$ENODES" +``` + +Key flags: +- `--chain` — path to the shared `genesis.json` (must match the running network) +- `--datadir` — a fresh directory for the follower's database; must not be shared with + a validator +- `--port` — P2P listen port; pick one that does not conflict with the validator nodes + (default validator ports start at 30303) +- `--authrpc.port` — engine API port; also must not conflict (validator default: 8551+) +- `--disable-discovery` — prevents unwanted discv4/discv5 discovery; peers are supplied + explicitly via `--trusted-peers` +- `--trusted-peers` — comma-separated enode URLs of the validators to connect to +- `--validator-key` is intentionally **omitted** — its absence is what makes this a + follower + ### Installing Cast (Foundry) Cast is a useful tool for interacting with the blockchain: From 82606f79083868b2a29ba093ee95cee2de5b8235 Mon Sep 17 00:00:00 2001 From: Amy Thomason Date: Tue, 10 Mar 2026 11:21:39 +0000 Subject: [PATCH 4/6] feat: add rbft-utils validator keygen subcommand --- README.md | 60 +++++++++++++++++++ crates/rbft-utils/src/main.rs | 17 ++++++ crates/rbft-utils/src/validator_management.rs | 35 +++++++++++ 3 files changed, 112 insertions(+) diff --git a/README.md b/README.md index 65a1e24..42d3f35 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,66 @@ Key flags: - `--validator-key` is intentionally **omitted** — its absence is what makes this a follower +### Adding a Validator Node + +To add a new validator to a running testnet you need a validator key (for signing QBFT +messages), a P2P secret key (for RLPx networking), and the enode URL derived from it. + +**1. Generate keys** + +```bash +target/release/rbft-utils validator keygen --ip --port 30305 +``` + +This prints JSON with all four values: + +```json +{ + "validator_address": "0xABCD...", + "validator_private_key": "0x1234...", + "p2p_secret_key": "abcd...", + "enode": "enode://@:30305" +} +``` + +Save the keys and capture the enode: + +```bash +echo "0x1234..." > validator-key-new.txt +echo "abcd..." > p2p-secret-key-new.txt +ENODE="enode://@:30305" +``` + +**2. Start the node** + +```bash +target/release/rbft-node node \ + --chain ~/.rbft/testnet/assets/genesis.json \ + --datadir /tmp/rbft-new-validator \ + --port 30305 \ + --authrpc.port 8652 \ + --http --http.port 8601 \ + --disable-discovery \ + --p2p-secret-key p2p-secret-key-new.txt \ + --validator-key validator-key-new.txt \ + --trusted-peers "$ENODES" # enodes of existing validators +``` + +**3. Register the validator in the contract** + +The default testnet admin key is `0x000...0001` +(address `0x7E5F4552091A69125d5DfCb7b8C2659029395Bdf`): + +```bash +target/release/rbft-utils validator add \ + --admin-key 0x0000000000000000000000000000000000000000000000000000000000000001 \ + --validator-address 0xABCD... \ + --enode "$ENODE" \ + --rpc-url http://localhost:8545 +``` + +The new validator becomes active at the next epoch boundary. + ### Installing Cast (Foundry) Cast is a useful tool for interacting with the blockchain: diff --git a/crates/rbft-utils/src/main.rs b/crates/rbft-utils/src/main.rs index 107c161..085b500 100644 --- a/crates/rbft-utils/src/main.rs +++ b/crates/rbft-utils/src/main.rs @@ -327,6 +327,20 @@ enum ValidatorCommand { #[arg(long, default_value = "http://localhost:8545")] rpc_url: String, }, + /// Generate a fresh validator key-pair and P2P secret key, print as JSON. + /// + /// The JSON output contains everything needed to start a new validator node + /// and register it with `validator add`: + /// validator_address, validator_private_key, p2p_secret_key, enode + Keygen { + /// IP address to embed in the enode URL + #[arg(long, default_value = "127.0.0.1")] + ip: String, + + /// P2P port to embed in the enode URL + #[arg(long, default_value_t = 30303)] + port: u16, + }, } fn main() -> eyre::Result<()> { @@ -521,6 +535,9 @@ fn main() -> eyre::Result<()> { value, rpc_url, } => validator_management::set_epoch_length(&admin_key, value, &rpc_url).await, + ValidatorCommand::Keygen { ip, port } => { + validator_management::keygen_validator(&ip, port) + } } }) } diff --git a/crates/rbft-utils/src/validator_management.rs b/crates/rbft-utils/src/validator_management.rs index 4e22503..ef6b7a2 100644 --- a/crates/rbft-utils/src/validator_management.rs +++ b/crates/rbft-utils/src/validator_management.rs @@ -6,6 +6,7 @@ use alloy_signer_local::PrivateKeySigner; use eyre::{eyre, Result}; use reth_network_peers::NodeRecord; use serde_json::json; +use std::net::SocketAddr; /// Default validator contract address (proxy contract) const DEFAULT_VALIDATOR_CONTRACT: Address = Address::new([ @@ -13,6 +14,40 @@ const DEFAULT_VALIDATOR_CONTRACT: Address = Address::new([ 0x00, 0x00, 0x10, 0x01, ]); +/// Generate a fresh validator key-pair + P2P secret key and print as JSON. +/// +/// Output fields: +/// validator_address – Ethereum address derived from the validator key +/// validator_private_key – hex-encoded validator private key (for --validator-key) +/// p2p_secret_key – hex-encoded P2P secret key (for --p2p-secret-key) +/// enode – enode URL derived from the P2P key +pub fn keygen_validator(ip: &str, port: u16) -> Result<()> { + // Generate validator key + let validator_signer = PrivateKeySigner::random(); + let validator_address = validator_signer.address(); + let validator_private_key = + format!("0x{}", alloy_primitives::hex::encode(validator_signer.to_bytes())); + + // Generate P2P secret key + let p2p_secret_key = reth_ethereum::network::config::rng_secret_key(); + let p2p_key_hex = alloy_primitives::hex::encode(p2p_secret_key.as_ref()); + + // Derive enode from P2P key + let socket_addr: SocketAddr = format!("{ip}:{port}") + .parse() + .map_err(|e| eyre!("Invalid IP/port '{}:{}': {}", ip, port, e))?; + let enode = NodeRecord::from_secret_key(socket_addr, &p2p_secret_key); + + let output = json!({ + "validator_address": format!("{:?}", validator_address), + "validator_private_key": validator_private_key, + "p2p_secret_key": p2p_key_hex, + "enode": enode.to_string(), + }); + println!("{}", serde_json::to_string_pretty(&output)?); + Ok(()) +} + /// Add a validator to the QBFTValidatorSet contract pub async fn add_validator( private_key: &str, From 69ad1ccd0eb14ff5794f152db0a58a26f337e24d Mon Sep 17 00:00:00 2001 From: Amy Thomason Date: Tue, 10 Mar 2026 11:44:59 +0000 Subject: [PATCH 5/6] docs: clarify RBFT_ADMIN_KEY usage in validator add docs --- README.md | 16 +++++++++++++--- crates/rbft-utils/src/constants.rs | 7 +++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 42d3f35..0627662 100644 --- a/README.md +++ b/README.md @@ -142,12 +142,22 @@ target/release/rbft-node node \ **3. Register the validator in the contract** -The default testnet admin key is `0x000...0001` -(address `0x7E5F4552091A69125d5DfCb7b8C2659029395Bdf`): +The admin key is taken from the `RBFT_ADMIN_KEY` environment variable, which must match +the key used when the genesis was generated. For a fresh local testnet started without +setting `RBFT_ADMIN_KEY`, the default is `0x000...0001`: ```bash target/release/rbft-utils validator add \ - --admin-key 0x0000000000000000000000000000000000000000000000000000000000000001 \ + --validator-address 0xABCD... \ + --enode "$ENODE" \ + --rpc-url http://localhost:8545 +``` + +If `RBFT_ADMIN_KEY` is not set, pass it explicitly: + +```bash +target/release/rbft-utils validator add \ + --admin-key \ --validator-address 0xABCD... \ --enode "$ENODE" \ --rpc-url http://localhost:8545 diff --git a/crates/rbft-utils/src/constants.rs b/crates/rbft-utils/src/constants.rs index ad96cfd..8cc7b04 100644 --- a/crates/rbft-utils/src/constants.rs +++ b/crates/rbft-utils/src/constants.rs @@ -1,9 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 //! Shared constants for RBFT tooling. -/// Well-known admin private key used for local testnets and load generation. +/// Fallback admin private key used when `RBFT_ADMIN_KEY` is not set. /// -/// This key is baked into genesis assets so tooling can assume a funded account. +/// Used by `genesis`, `validator add`, and other commands that require an admin key. +/// Override with the `RBFT_ADMIN_KEY` environment variable to use a different key. +/// **This key is only the actual genesis admin when the testnet was generated without +/// `RBFT_ADMIN_KEY` set.** pub const DEFAULT_ADMIN_KEY: &str = "0x0000000000000000000000000000000000000000000000000000000000000001"; From cad0b4f48966f2b93320d316cae0519001b2360e Mon Sep 17 00:00:00 2001 From: Amy Thomason Date: Thu, 12 Mar 2026 14:57:18 +0000 Subject: [PATCH 6/6] fix formatting --- crates/rbft-utils/src/testnet.rs | 4 ++-- crates/rbft-utils/src/validator_management.rs | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/rbft-utils/src/testnet.rs b/crates/rbft-utils/src/testnet.rs index 6c99229..3d2e4c7 100644 --- a/crates/rbft-utils/src/testnet.rs +++ b/crates/rbft-utils/src/testnet.rs @@ -1287,8 +1287,8 @@ fn add_next_follower( let p2p_key_path = assets.join(format!("p2p-secret-key{follower_node_index}.txt")); if !p2p_key_path.exists() { return Err(eyre::eyre!( - "Missing p2p key file for follower node index {follower_node_index}: expected {} \ - in {}. Re-generate with a larger --num-nodes to create more key slots.", + "Missing p2p key file for follower node index {follower_node_index}: expected {} in \ + {}. Re-generate with a larger --num-nodes to create more key slots.", p2p_key_path.display(), assets.display(), )); diff --git a/crates/rbft-utils/src/validator_management.rs b/crates/rbft-utils/src/validator_management.rs index ef6b7a2..3a119ec 100644 --- a/crates/rbft-utils/src/validator_management.rs +++ b/crates/rbft-utils/src/validator_management.rs @@ -25,8 +25,10 @@ pub fn keygen_validator(ip: &str, port: u16) -> Result<()> { // Generate validator key let validator_signer = PrivateKeySigner::random(); let validator_address = validator_signer.address(); - let validator_private_key = - format!("0x{}", alloy_primitives::hex::encode(validator_signer.to_bytes())); + let validator_private_key = format!( + "0x{}", + alloy_primitives::hex::encode(validator_signer.to_bytes()) + ); // Generate P2P secret key let p2p_secret_key = reth_ethereum::network::config::rng_secret_key();