diff --git a/Cargo.lock b/Cargo.lock index 69204ccaecd..660e9fbbd9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1232,8 +1232,8 @@ dependencies = [ "eth2_network_config", "ethereum_hashing", "ethereum_serde_utils", - "ethereum_ssz", - "ethereum_ssz_derive", + "ethereum_ssz 0.10.1", + "ethereum_ssz_derive 0.10.1", "execution_layer", "fixed_bytes", "fork_choice", @@ -1259,6 +1259,7 @@ dependencies = [ "proto_array", "rand 0.9.2", "rayon", + "reth-era", "safe_arith", "sensitive_url", "serde", @@ -1508,7 +1509,7 @@ dependencies = [ "blst", "ethereum_hashing", "ethereum_serde_utils", - "ethereum_ssz", + "ethereum_ssz 0.10.1", "fixed_bytes", "hex", "rand 0.9.2", @@ -1555,7 +1556,7 @@ dependencies = [ "clap", "clap_utils", "eth2_network_config", - "ethereum_ssz", + "ethereum_ssz 0.10.1", "hex", "lighthouse_network", "log", @@ -1613,7 +1614,7 @@ dependencies = [ "bls", "context_deserialize", "eth2", - "ethereum_ssz", + "ethereum_ssz 0.10.1", "lighthouse_version", "mockito", "reqwest", @@ -1881,7 +1882,7 @@ dependencies = [ "clap", "dirs", "eth2_network_config", - "ethereum_ssz", + "ethereum_ssz 0.10.1", "hex", "serde", "serde_json", @@ -1900,7 +1901,7 @@ dependencies = [ "environment", "eth2", "eth2_config", - "ethereum_ssz", + "ethereum_ssz 0.10.1", "execution_layer", "futures", "genesis", @@ -2334,6 +2335,16 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + [[package]] name = "darling" version = "0.21.3" @@ -2354,6 +2365,20 @@ dependencies = [ "darling_macro 0.23.0", ] +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.111", +] + [[package]] name = "darling_core" version = "0.21.3" @@ -2382,6 +2407,17 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.111", +] + [[package]] name = "darling_macro" version = "0.21.3" @@ -2506,7 +2542,7 @@ dependencies = [ "alloy-json-abi", "alloy-primitives", "bls", - "ethereum_ssz", + "ethereum_ssz 0.10.1", "hex", "reqwest", "serde_json", @@ -2842,8 +2878,8 @@ dependencies = [ "context_deserialize", "educe", "eth2_network_config", - "ethereum_ssz", - "ethereum_ssz_derive", + "ethereum_ssz 0.10.1", + "ethereum_ssz_derive 0.10.1", "execution_layer", "fork_choice", "fs2", @@ -3122,8 +3158,8 @@ dependencies = [ "eip_3076", "eth2_keystore", "ethereum_serde_utils", - "ethereum_ssz", - "ethereum_ssz_derive", + "ethereum_ssz 0.10.1", + "ethereum_ssz_derive 0.10.1", "futures", "futures-util", "mediatype", @@ -3207,7 +3243,7 @@ dependencies = [ "bytes", "discv5", "eth2_config", - "ethereum_ssz", + "ethereum_ssz 0.10.1", "fixed_bytes", "kzg", "pretty_reqwest_error", @@ -3272,6 +3308,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "ethereum_ssz" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dcddb2554d19cde19b099fadddde576929d7a4d0c1cd3512d1fd95cf174375c" +dependencies = [ + "alloy-primitives", + "ethereum_serde_utils", + "itertools 0.13.0", + "serde", + "serde_derive", + "smallvec", + "typenum", +] + [[package]] name = "ethereum_ssz" version = "0.10.1" @@ -3289,6 +3340,18 @@ dependencies = [ "typenum", ] +[[package]] +name = "ethereum_ssz_derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a657b6b3b7e153637dc6bdc6566ad9279d9ee11a15b12cfb24a2e04360637e9f" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "ethereum_ssz_derive" version = "0.10.1" @@ -3382,7 +3445,7 @@ dependencies = [ "bytes", "eth2", "ethereum_serde_utils", - "ethereum_ssz", + "ethereum_ssz 0.10.1", "fixed_bytes", "fork_choice", "hash-db", @@ -3580,8 +3643,8 @@ name = "fork_choice" version = "0.1.0" dependencies = [ "beacon_chain", - "ethereum_ssz", - "ethereum_ssz_derive", + "ethereum_ssz 0.10.1", + "ethereum_ssz_derive 0.10.1", "fixed_bytes", "logging", "metrics", @@ -3775,7 +3838,7 @@ version = "0.2.0" dependencies = [ "bls", "ethereum_hashing", - "ethereum_ssz", + "ethereum_ssz 0.10.1", "int_to_bytes", "merkle_proof", "rayon", @@ -4196,7 +4259,7 @@ dependencies = [ "either", "eth2", "ethereum_serde_utils", - "ethereum_ssz", + "ethereum_ssz 0.10.1", "execution_layer", "fixed_bytes", "futures", @@ -4836,8 +4899,8 @@ dependencies = [ "educe", "ethereum_hashing", "ethereum_serde_utils", - "ethereum_ssz", - "ethereum_ssz_derive", + "ethereum_ssz 0.10.1", + "ethereum_ssz_derive 0.10.1", "hex", "rayon", "rust_eth_kzg", @@ -4877,7 +4940,7 @@ dependencies = [ "eth2_network_config", "eth2_wallet", "ethereum_hashing", - "ethereum_ssz", + "ethereum_ssz 0.10.1", "execution_layer", "fixed_bytes", "hex", @@ -5414,8 +5477,8 @@ dependencies = [ "discv5", "either", "eth2", - "ethereum_ssz", - "ethereum_ssz_derive", + "ethereum_ssz 0.10.1", + "ethereum_ssz_derive 0.10.1", "fixed_bytes", "fnv", "futures", @@ -5758,8 +5821,8 @@ dependencies = [ "context_deserialize", "educe", "ethereum_hashing", - "ethereum_ssz", - "ethereum_ssz_derive", + "ethereum_ssz 0.10.1", + "ethereum_ssz_derive 0.10.1", "itertools 0.13.0", "parking_lot", "rayon", @@ -6059,7 +6122,7 @@ dependencies = [ "educe", "eth2", "eth2_network_config", - "ethereum_ssz", + "ethereum_ssz 0.10.1", "execution_layer", "fixed_bytes", "fnv", @@ -6452,8 +6515,8 @@ dependencies = [ "bitvec", "bls", "educe", - "ethereum_ssz", - "ethereum_ssz_derive", + "ethereum_ssz 0.10.1", + "ethereum_ssz_derive 0.10.1", "fixed_bytes", "itertools 0.14.0", "maplit", @@ -7018,8 +7081,8 @@ dependencies = [ name = "proto_array" version = "0.2.0" dependencies = [ - "ethereum_ssz", - "ethereum_ssz_derive", + "ethereum_ssz 0.10.1", + "ethereum_ssz_derive 0.10.1", "fixed_bytes", "safe_arith", "serde", @@ -7444,6 +7507,21 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" +[[package]] +name = "reth-era" +version = "1.9.3" +source = "git+https://github.com/paradigmxyz/reth.git?rev=62abfdaeb54e8a205a8ee085ddebd56047d93374#62abfdaeb54e8a205a8ee085ddebd56047d93374" +dependencies = [ + "alloy-consensus", + "alloy-eips", + "alloy-primitives", + "alloy-rlp", + "ethereum_ssz 0.9.1", + "ethereum_ssz_derive 0.9.1", + "snap", + "thiserror 2.0.17", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -8225,8 +8303,8 @@ dependencies = [ "bls", "byteorder", "educe", - "ethereum_ssz", - "ethereum_ssz_derive", + "ethereum_ssz 0.10.1", + "ethereum_ssz_derive 0.10.1", "filesystem", "fixed_bytes", "flate2", @@ -8378,7 +8456,7 @@ dependencies = [ "context_deserialize", "educe", "ethereum_serde_utils", - "ethereum_ssz", + "ethereum_ssz 0.10.1", "itertools 0.14.0", "serde", "serde_derive", @@ -8402,8 +8480,8 @@ dependencies = [ "bls", "educe", "ethereum_hashing", - "ethereum_ssz", - "ethereum_ssz_derive", + "ethereum_ssz 0.10.1", + "ethereum_ssz_derive 0.10.1", "fixed_bytes", "int_to_bytes", "integer-sqrt", @@ -8430,7 +8508,7 @@ version = "0.1.0" dependencies = [ "beacon_chain", "bls", - "ethereum_ssz", + "ethereum_ssz 0.10.1", "fixed_bytes", "state_processing", "tokio", @@ -8452,8 +8530,8 @@ dependencies = [ "criterion", "db-key", "directory", - "ethereum_ssz", - "ethereum_ssz_derive", + "ethereum_ssz 0.10.1", + "ethereum_ssz_derive 0.10.1", "fixed_bytes", "itertools 0.14.0", "leveldb", @@ -9256,7 +9334,7 @@ checksum = "f7fd51aa83d2eb83b04570808430808b5d24fdbf479a4d5ac5dee4a2e2dd2be4" dependencies = [ "alloy-primitives", "ethereum_hashing", - "ethereum_ssz", + "ethereum_ssz 0.10.1", "smallvec", "typenum", ] @@ -9321,8 +9399,8 @@ dependencies = [ "eth2_interop_keypairs", "ethereum_hashing", "ethereum_serde_utils", - "ethereum_ssz", - "ethereum_ssz_derive", + "ethereum_ssz 0.10.1", + "ethereum_ssz_derive 0.10.1", "fixed_bytes", "hex", "int_to_bytes", diff --git a/Cargo.toml b/Cargo.toml index 100a916c501..02dacf2c8ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -99,7 +99,7 @@ alloy-consensus = { version = "1", default-features = false } alloy-dyn-abi = { version = "1", default-features = false } alloy-json-abi = { version = "1", default-features = false } alloy-network = { version = "1", default-features = false } -alloy-primitives = { version = "1", default-features = false, features = ["rlp", "getrandom"] } +alloy-primitives = { version = "1.5", default-features = false, features = ["rlp", "getrandom"] } alloy-provider = { version = "1", default-features = false, features = ["reqwest"] } alloy-rlp = { version = "0.3", default-features = false } alloy-rpc-types-eth = { version = "1", default-features = false, features = ["serde"] } @@ -225,6 +225,7 @@ reqwest = { version = "0.12", default-features = false, features = [ "stream", "rustls-tls", ] } +reth-era = { git = "https://github.com/paradigmxyz/reth.git", rev = "62abfdaeb54e8a205a8ee085ddebd56047d93374" } ring = "0.17" rpds = "0.11" rusqlite = { version = "0.28", features = ["bundled"] } diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index 5e1c41b8302..b11463115db 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -47,6 +47,7 @@ parking_lot = { workspace = true } proto_array = { workspace = true } rand = { workspace = true } rayon = { workspace = true } +reth-era = { workspace = true } safe_arith = { workspace = true } sensitive_url = { workspace = true } serde = { workspace = true } diff --git a/beacon_node/beacon_chain/src/era_file_producer.rs b/beacon_node/beacon_chain/src/era_file_producer.rs new file mode 100644 index 00000000000..e3557fcdeb4 --- /dev/null +++ b/beacon_node/beacon_chain/src/era_file_producer.rs @@ -0,0 +1,318 @@ +use rand::random; +use reth_era::common::file_ops::{EraFileFormat, EraFileId, StreamWriter}; +use reth_era::era::file::{EraFile, EraWriter}; +use reth_era::era::types::consensus::{CompressedBeaconState, CompressedSignedBeaconBlock}; +use reth_era::era::types::group::{EraGroup, EraId, SlotIndex}; +use ssz::Encode; +use std::fs::{self, File, OpenOptions}; +use std::path::Path; +use store::{HotColdDB, ItemStore}; +use tracing::{error, info}; +use tree_hash::TreeHash; +use types::{BeaconState, EthSpec, Slot}; + +fn era_file_exists(dir: &Path, id: &EraId) -> bool { + dir.join(id.to_file_name()).exists() +} + +fn era_file_exists_for_number(dir: &Path, network_name: &str, era_number: u64) -> bool { + let prefix = format!("{}-{:05}-", network_name, era_number); + let Ok(entries) = fs::read_dir(dir) else { + return false; + }; + + for entry in entries.flatten() { + let file_name = entry.file_name(); + let Some(name) = file_name.to_str() else { + continue; + }; + if name.starts_with(&prefix) && name.ends_with(".era") { + return true; + } + } + false +} + +pub(crate) fn maybe_produce_reconstruction_eras< + E: EthSpec, + Hot: ItemStore, + Cold: ItemStore, +>( + db: &HotColdDB, + output_dir: &Path, +) { + let anchor = db.get_anchor_info(); + let max_era = anchor.state_lower_limit.as_u64() / E::slots_per_historical_root() as u64; + + for era_number in 0..=max_era { + if let Err(error) = create_era_file(db, era_number, output_dir) { + error!( + ?error, + era_number, "Era producer failed during reconstruction" + ); + break; + } + } +} + +pub(crate) fn maybe_produce_finalization_era, Cold: ItemStore>( + db: &HotColdDB, + output_dir: &Path, + finalized_slot: Slot, +) { + // This is the oldest slot for which we have a state and blocks available + let anchor_slot = db.get_anchor_info().anchor_slot; + // And finalized_slot is the most recent for which we have finalized state and blocks available + + // We can produce an era file for era_number if + // - anchor_slot <= start_slot(era_number) AND + // - finalized_slot >= end_slot(era_number) + let lowest_era_file = anchor_slot.as_u64() / E::slots_per_historical_root() as u64; + let max_era_file = (finalized_slot.as_u64() + 1) / E::slots_per_historical_root() as u64 - 1; + for era_number in lowest_era_file..=max_era_file { + if let Err(error) = create_era_file(db, era_number, output_dir) { + error!( + ?error, + era_number, "Era producer failed during finalization" + ); + break; + } + } +} + +fn create_era_file, Cold: ItemStore>( + db: &HotColdDB, + era_number: u64, + output_dir: &Path, +) -> Result<(), String> { + let network_name = db + .spec + .config_name + .clone() + .unwrap_or_else(|| "unknown".to_string()); + if era_file_exists_for_number(output_dir, &network_name, era_number) { + return Ok(()); + } + + let end_slot = Slot::new(era_number * E::slots_per_historical_root() as u64); + + let mut state = db + .load_cold_state_by_slot(end_slot) + .map_err(|error| format!("failed to load era state: {error:?}"))?; + + if state.slot() != end_slot { + return Err(format!( + "era state slot mismatch: expected {}, got {}", + end_slot, + state.slot() + )); + } + + let group = build_era_group(db, &mut state, era_number)?; + let file_id = era_file_id::(&network_name, era_number, &mut state)?; + let file = EraFile::new(group, file_id); + + fs::create_dir_all(output_dir) + .map_err(|error| format!("failed to create era files dir: {error}"))?; + + if era_file_exists(output_dir, file.id()) { + return Ok(()); + } + + write_era_file_atomic(output_dir, &file)?; + + info!( + era_number, + file = %file.id().to_file_name(), + "Wrote era file" + ); + + Ok(()) +} + +fn build_era_group, Cold: ItemStore>( + db: &HotColdDB, + state: &mut BeaconState, + era_number: u64, +) -> Result { + // Era file 0 goes from slot 0 to 0, genesis state only + let start_slot = + Slot::new(era_number.saturating_sub(1) * E::slots_per_historical_root() as u64); + let end_slot = Slot::new(era_number * E::slots_per_historical_root() as u64); + + let compressed_state = CompressedBeaconState::from_ssz(&state.as_ssz_bytes()) + .map_err(|error| format!("failed to compress state: {error:?}"))?; + + // Each entry has an 8-byte header; the version record is header-only. + let mut offset: i64 = 8; + let mut blocks: Vec = Vec::new(); + let mut block_data_starts: Vec<(Slot, i64)> = Vec::new(); + + // The era file number 0 contains the genesis state and nothing else + if era_number > 0 { + for slot_u64 in start_slot.as_u64()..end_slot.as_u64() { + let slot = Slot::new(slot_u64); + let block_root = state + .get_block_root(slot) + .map_err(|error| format!("failed to read block root {slot}: {error:?}"))?; + + if let Ok(prev_root) = state.get_block_root(Slot::new(slot_u64.saturating_sub(1))) + && prev_root == block_root + { + continue; + } + + let block = db + .get_full_block(block_root) + .map_err(|error| format!("failed to load block: {error:?}"))? + .ok_or_else(|| format!("missing block for root {block_root:?}"))?; + + // Skip blocks from previous era. At era boundaries, block_roots[slot] may + // contain a root from the previous era if there was no block at exactly + // `start_slot` (the root is the most recent block AT OR BEFORE that slot). + if block.slot() < start_slot { + continue; + } + + let compressed = CompressedSignedBeaconBlock::from_ssz(&block.as_ssz_bytes()) + .map_err(|error| format!("failed to compress block: {error:?}"))?; + + let data_len = compressed.data.len() as i64; + let data_start = offset + 8; + blocks.push(compressed); + block_data_starts.push((slot, data_start)); + offset += 8 + data_len; + } + } + + let state_data_len = compressed_state.data.len() as i64; + // Data starts after the 8-byte header. + let state_data_start = offset + 8; + offset += 8 + state_data_len; + + let block_index_start = offset; + let slot_count = E::slots_per_historical_root(); + // SlotIndex layout: starting_slot (8) + offsets (slot_count * 8) + count (8). + let block_index_len = 8 + slot_count as i64 * 8 + 8; + if era_number > 0 { + offset += 8 + block_index_len; + } + + let state_index_start = offset; + let state_offset = state_data_start - state_index_start; + let state_slot_index = SlotIndex::new(end_slot.as_u64(), vec![state_offset]); + + let group = if era_number > 0 { + let mut offsets = vec![0i64; slot_count]; + for (slot, data_start) in &block_data_starts { + let slot_index = slot + .as_u64() + .checked_sub(start_slot.as_u64()) + .ok_or_else(|| "slot underflow while building block index".to_string())? + as usize; + offsets[slot_index] = *data_start - block_index_start; + } + let block_index = SlotIndex::new(start_slot.as_u64(), offsets); + EraGroup::with_block_index(blocks, compressed_state, block_index, state_slot_index) + } else { + EraGroup::new(blocks, compressed_state, state_slot_index) + }; + Ok(group) +} + +fn short_historical_root( + state: &mut BeaconState, + era_number: u64, +) -> Result<[u8; 4], String> { + let root = if era_number == 0 { + state.genesis_validators_root() + } else { + let era_index = era_number + .checked_sub(1) + .ok_or_else(|| "era index underflow".to_string())?; + let roots_len = state.historical_roots_mut().len(); + if era_index < roots_len as u64 { + *state + .historical_roots_mut() + .get(era_index as usize) + .ok_or_else(|| "historical root missing".to_string())? + } else { + let summary_index = era_index + .checked_sub(roots_len as u64) + .ok_or_else(|| "historical summary index underflow".to_string())?; + let summaries = state + .historical_summaries_mut() + .map_err(|error| format!("failed to access historical summaries: {error:?}"))?; + let summary = summaries + .get(summary_index as usize) + .ok_or_else(|| "historical summary missing".to_string())?; + summary.tree_hash_root() + } + }; + + let mut short_hash = [0u8; 4]; + short_hash.copy_from_slice(&root.as_slice()[..4]); + Ok(short_hash) +} + +/// Write an era file atomically using a temp file + rename. +/// +/// If the process crashes mid-write, only the temp file is left behind; the final file is +/// created via rename, so it is either complete or absent. The era file existence check only +/// considers `.era` files, so partial temp files are ignored safely. +fn write_era_file_atomic(output_dir: &Path, file: &EraFile) -> Result<(), String> { + let filename = file.id().to_file_name(); + let final_path = output_dir.join(&filename); + if final_path.exists() { + return Ok(()); + } + + // Create a unique temp file and write the full era contents into it. + let tmp_name = format!("{filename}.tmp-{:016x}", random::()); + let tmp_path = output_dir.join(tmp_name); + let mut file_handle = OpenOptions::new() + .write(true) + .create_new(true) + .open(&tmp_path) + .map_err(|error| format!("failed to create temp file: {error}"))?; + { + let mut writer = EraWriter::new(&mut file_handle); + writer + .write_file(file) + .map_err(|error| format!("failed to write era file: {error:?}"))?; + } + file_handle + .sync_all() + .map_err(|error| format!("failed to fsync era temp file: {error}"))?; + + // Atomically publish; if another writer won, clean up and exit. + if let Err(error) = fs::rename(&tmp_path, &final_path) { + if error.kind() == std::io::ErrorKind::AlreadyExists && final_path.exists() { + let _ = fs::remove_file(&tmp_path); + return Ok(()); + } + return Err(format!("failed to rename era temp file: {error}")); + } + + // Best-effort directory sync to make the rename durable. + if let Ok(dir_handle) = File::open(output_dir) { + let _ = dir_handle.sync_all(); + } + + Ok(()) +} + +fn era_file_id( + network_name: &str, + era_number: u64, + state: &mut BeaconState, +) -> Result { + let end_slot = Slot::new(era_number * E::slots_per_historical_root() as u64); + let slot_count = if era_number == 0 { + 0 + } else { + E::slots_per_historical_root() as u32 + }; + let short_hash = short_historical_root(state, era_number)?; + Ok(EraId::new(network_name, end_slot.as_u64(), slot_count).with_hash(short_hash)) +} diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index e77739e2d53..b2307b739d7 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -20,6 +20,7 @@ pub mod custody_context; pub mod data_availability_checker; pub mod data_column_verification; mod early_attester_cache; +mod era_file_producer; mod errors; pub mod events; pub mod execution_payload; diff --git a/beacon_node/beacon_chain/src/migrate.rs b/beacon_node/beacon_chain/src/migrate.rs index 24258d2d31b..7fa12f68f77 100644 --- a/beacon_node/beacon_chain/src/migrate.rs +++ b/beacon_node/beacon_chain/src/migrate.rs @@ -1,3 +1,4 @@ +use crate::era_file_producer::{maybe_produce_finalization_era, maybe_produce_reconstruction_eras}; use crate::errors::BeaconChainError; use crate::summaries_dag::{DAGStateSummary, Error as SummariesDagError, StateSummariesDAG}; use parking_lot::Mutex; @@ -215,12 +216,15 @@ impl, Cold: ItemStore> BackgroundMigrator>, opt_tx: Option>, ) { match db.reconstruct_historic_states(Some(BLOCKS_PER_RECONSTRUCTION)) { Ok(()) => { + if let Some(era_files_dir) = &db.get_config().era_files_dir { + maybe_produce_reconstruction_eras(&db, era_files_dir); + } // Schedule another reconstruction batch if required and we have access to the // channel for requeueing. if let Some(tx) = opt_tx @@ -404,6 +408,10 @@ impl, Cold: ItemStore> BackgroundMigrator Command { .action(ArgAction::Set) .display_order(0) ) + .arg( + Arg::new("era-files-dir") + .long("era-files-dir") + .value_name("DIR") + .help("Directory containing .era files for backfill and where new .era files will be written.") + .action(ArgAction::Set) + .display_order(0) + ) /* * Network parameters. */ diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 752cf105505..3309abebe8e 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -350,6 +350,10 @@ pub fn get_config( client_config.freezer_db_path = Some(PathBuf::from(freezer_dir)); } + if let Some(era_files_dir) = cli_args.get_one::("era-files-dir") { + client_config.store.era_files_dir = Some(PathBuf::from(era_files_dir)); + } + if let Some(blobs_db_dir) = cli_args.get_one::("blobs-dir") { client_config.blobs_db_path = Some(PathBuf::from(blobs_db_dir)); } diff --git a/beacon_node/store/src/config.rs b/beacon_node/store/src/config.rs index 29705283fa9..34c895dd509 100644 --- a/beacon_node/store/src/config.rs +++ b/beacon_node/store/src/config.rs @@ -5,6 +5,7 @@ use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use std::io::{Read, Write}; use std::num::NonZeroUsize; +use std::path::PathBuf; use strum::{Display, EnumString, VariantNames}; use superstruct::superstruct; use types::EthSpec; @@ -64,6 +65,8 @@ pub struct StoreConfig { /// The margin for blob pruning in epochs. The oldest blobs are pruned up until /// data_availability_boundary - blob_prune_margin_epochs. Default: 0. pub blob_prune_margin_epochs: u64, + /// Directory to write era files to when enabled. + pub era_files_dir: Option, } /// Variant of `StoreConfig` that gets written to disk. Contains immutable configuration params. @@ -120,6 +123,7 @@ impl Default for StoreConfig { prune_blobs: true, epochs_per_blob_prune: DEFAULT_EPOCHS_PER_BLOB_PRUNE, blob_prune_margin_epochs: DEFAULT_BLOB_PUNE_MARGIN_EPOCHS, + era_files_dir: None, } } } diff --git a/book/src/help_bn.md b/book/src/help_bn.md index d3aa27c8a77..a3331781439 100644 --- a/book/src/help_bn.md +++ b/book/src/help_bn.md @@ -122,6 +122,9 @@ Options: The number of epochs to wait between running the migration of data from the hot DB to the cold DB. Less frequent runs can be useful for minimizing disk writes [default: 1] + --era-files-dir + Directory containing .era files for backfill and where new .era files + will be written. --execution-endpoint Server endpoint for an execution layer JWT-authenticated HTTP JSON-RPC connection. Uses the same endpoint to populate the deposit cache.