diff --git a/Cargo.lock b/Cargo.lock index ac888f026e..28a55245e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3106,6 +3106,7 @@ dependencies = [ "e3-trbfv", "e3-utils", "e3-zk-prover", + "libp2p", "once_cell", "rayon", "tempfile", @@ -3706,6 +3707,7 @@ dependencies = [ "fhe", "fhe-traits", "hex", + "libp2p", "rand 0.8.5", "rand_chacha 0.3.1", "tokio", diff --git a/crates/aggregator/src/publickey_aggregator.rs b/crates/aggregator/src/publickey_aggregator.rs index f4dad40569..2a3d6d33f2 100644 --- a/crates/aggregator/src/publickey_aggregator.rs +++ b/crates/aggregator/src/publickey_aggregator.rs @@ -42,13 +42,13 @@ pub enum PublicKeyAggregatorState { no_proof_parties: Vec, }, GeneratingC5Proof { - public_key: Vec, + public_key: ArcBytes, public_key_hash: [u8; 32], keyshare_bytes: Vec, nodes: OrderedSet, }, Complete { - public_key: Vec, + public_key: ArcBytes, keyshares: OrderedSet, nodes: OrderedSet, }, @@ -293,12 +293,13 @@ impl PublicKeyAggregator { let committee_h = honest_keyshares.len(); info!("Publishing PkAggregationProofPending for C5 proof generation..."); + let pubkey = ArcBytes::from_bytes(&pubkey); self.bus.publish( PkAggregationProofPending { e3_id: self.e3_id.clone(), proof_request: PkAggregationProofRequest { keyshare_bytes: honest_keyshares.clone(), - aggregated_pk_bytes: ArcBytes::from_bytes(&pubkey), + aggregated_pk_bytes: pubkey.clone(), params_preset: self.params_preset.clone(), // this field is not really used in the circuit, we only use H committee_n: committee_h, @@ -316,7 +317,7 @@ impl PublicKeyAggregator { // Transition to GeneratingC5Proof self.state.try_mutate(&ec, |_| { Ok(PublicKeyAggregatorState::GeneratingC5Proof { - public_key: pubkey, + public_key: pubkey.clone(), public_key_hash, keyshare_bytes: honest_keyshares, nodes: honest_nodes_set, diff --git a/crates/ciphernode-builder/Cargo.toml b/crates/ciphernode-builder/Cargo.toml index 0e86c91666..8e19b71d3d 100644 --- a/crates/ciphernode-builder/Cargo.toml +++ b/crates/ciphernode-builder/Cargo.toml @@ -29,6 +29,7 @@ e3-sortition.workspace = true e3-sync.workspace = true e3-trbfv.workspace = true e3-utils.workspace = true +libp2p.workspace = true rayon.workspace = true tempfile.workspace = true tokio.workspace = true diff --git a/crates/ciphernode-builder/src/ciphernode.rs b/crates/ciphernode-builder/src/ciphernode.rs index c4af8deaca..ddf1bfa7f9 100644 --- a/crates/ciphernode-builder/src/ciphernode.rs +++ b/crates/ciphernode-builder/src/ciphernode.rs @@ -8,7 +8,8 @@ use actix::Addr; use anyhow::Result; use e3_data::{DataStore, InMemStore, StoreAddr}; use e3_events::{BusHandle, EnclaveEvent, HistoryCollector}; -use tokio::task::JoinHandle; +use e3_net::NetChannelBridge; +use libp2p::PeerId; /// A Sharable handle to a Ciphernode. NOTE: clones are available for use in the CiphernodeSystem /// but they cannot await the task. @@ -19,10 +20,18 @@ pub struct CiphernodeHandle { pub bus: BusHandle, pub history: Option>>, pub errors: Option>>, - pub peer_id: String, - pub join_handle: JoinHandle>, + pub peer_id: PeerId, + pub channel_bridge: Option, } +impl PartialEq for CiphernodeHandle { + fn eq(&self, other: &Self) -> bool { + self.address == other.address && self.peer_id == other.peer_id + } +} + +impl Eq for CiphernodeHandle {} + impl CiphernodeHandle { pub fn new( address: String, @@ -30,8 +39,8 @@ impl CiphernodeHandle { bus: BusHandle, history: Option>>, errors: Option>>, - peer_id: String, - join_handle: JoinHandle>, + peer_id: PeerId, + channel_bridge: Option, ) -> Self { Self { address, @@ -40,7 +49,7 @@ impl CiphernodeHandle { history, errors, peer_id, - join_handle, + channel_bridge, } } @@ -64,8 +73,10 @@ impl CiphernodeHandle { &self.store } - pub fn split(self) -> (BusHandle, JoinHandle>) { - (self.bus, self.join_handle) + pub fn channel_bridge(&self) -> Result { + Ok(self.channel_bridge.clone().ok_or(anyhow::anyhow!( + "No channel bridge exists. We are likely not in test mode" + ))?) } pub fn in_mem_store(&self) -> Option<&Addr> { diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 1bd273cede..1cbb2ac0e9 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -23,7 +23,10 @@ use e3_fhe::ext::FheExtension; use e3_fhe_params::{BfvPreset, DEFAULT_BFV_PRESET}; use e3_keyshare::ext::ThresholdKeyshareExtension; use e3_multithread::{Multithread, MultithreadReport, TaskPool}; -use e3_net::{setup_net, NetRepositoryFactory}; +use e3_net::{ + create_channel_bridge, setup_libp2p_keypair, setup_net, setup_net_interface, + NetRepositoryFactory, +}; use e3_request::E3Router; use e3_sortition::{ CiphernodeSelector, CiphernodeSelectorFactory, FinalizedCommitteesRepositoryFactory, @@ -32,6 +35,7 @@ use e3_sortition::{ use e3_sync::sync; use e3_utils::SharedRng; use e3_zk_prover::{setup_zk_actors, ZkBackend}; +use libp2p::PeerId; use std::time::Duration; use std::{collections::HashMap, path::PathBuf, sync::Arc}; use tracing::{error, info}; @@ -496,28 +500,28 @@ impl CiphernodeBuilder { )) } - info!("building..."); - + info!("E3Router building..."); e3_builder.build().await?; - let (join_handle, peer_id) = if let Some(net_config) = self.net_config { + let topic = "enclave-gossip"; + let (peer_id, interface, channel_bridge) = if let Some(net_config) = self.net_config { + // Setup real net interface let repositories = store.repositories(); - setup_net( - bus.clone(), - net_config.peers, - &self.cipher, - net_config.quic_port, - repositories.libp2p_keypair(), - eventstore_ts, - ) - .await? + let keypair = setup_libp2p_keypair(repositories.libp2p_keypair(), &self.cipher).await?; + let peer_id = keypair.peer_id(); + let interface = + setup_net_interface(topic, keypair, net_config.peers, net_config.quic_port)?; + (peer_id, interface, None) } else { - ( - tokio::spawn(std::future::ready(Ok(()))), - "-not set-".to_string(), - ) + // Setup test net interface with random PeerId + let (interface, channel_bridge) = create_channel_bridge(); + let peer_id = PeerId::random(); + let channel_bridge = Some(channel_bridge); + (peer_id, interface, channel_bridge) }; + setup_net(topic, bus.clone(), eventstore_ts, interface)?; + // Run the sync routine sync( &bus, @@ -535,7 +539,7 @@ impl CiphernodeBuilder { history, errors, peer_id, - join_handle, + channel_bridge, )) } diff --git a/crates/entrypoint/src/helpers/shutdown.rs b/crates/entrypoint/src/helpers/shutdown.rs index 353ced3c75..b81f126092 100644 --- a/crates/entrypoint/src/helpers/shutdown.rs +++ b/crates/entrypoint/src/helpers/shutdown.rs @@ -7,53 +7,20 @@ use e3_ciphernode_builder::CiphernodeHandle; use e3_events::{prelude::*, Shutdown}; use std::time::Duration; -use tokio::{ - select, - signal::unix::{signal, SignalKind}, -}; +use tokio::signal::unix::{signal, SignalKind}; use tracing::{error, info}; pub async fn listen_for_shutdown(node: CiphernodeHandle) { - let (bus, mut handle) = node.split(); + let bus = node.bus; let mut sigterm = signal(SignalKind::terminate()).expect("Failed to create SIGTERM signal stream"); - select! { - _ = sigterm.recv() => { - info!("SIGTERM received, initiating graceful shutdown..."); + sigterm.recv().await; + info!("SIGTERM received, initiating graceful shutdown..."); - // Stop the actor system - match bus.publish_without_context(Shutdown){ - Ok(_) => (), - Err(e) => error!("Shutdown failed to publish! {e}") - } - - // Wait for all events to propagate - tokio::time::sleep(Duration::from_secs(2)).await; - - // Abort the spawned task - handle.abort(); - - // Wait for all actor processes to disconnect - tokio::time::sleep(Duration::from_secs(2)).await; - - // Wait for the task to finish - let _ = handle.await; - - info!("Graceful shutdown complete"); - - } - result = &mut handle => { - match result { - Ok(Ok(_)) => { - info!("Completed"); - } - Ok(Err(e)) => { - error!("Failed: {}", e); - } - Err(e) => { - error!("Panicked: {}", e); - } - } - } + if let Err(e) = bus.publish_without_context(Shutdown) { + error!("Shutdown failed to publish! {e}"); } + + tokio::time::sleep(Duration::from_secs(2)).await; + info!("Graceful shutdown complete"); } diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 94ccc6ba25..87ce6d08da 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -7,8 +7,8 @@ use actix::{Actor, Addr, Handler, Recipient}; use anyhow::Result; use derivative::Derivative; -use e3_utils::MAILBOX_LIMIT; -use std::marker::PhantomData; +use e3_utils::{actix::channel::oneshot, MAILBOX_LIMIT}; +use std::{future::Future, marker::PhantomData, pin::Pin}; use tracing::error; use crate::{ @@ -287,6 +287,20 @@ impl EventSubscriber> for BusHandle { self.event_bus .do_send(Unsubscribe::new(event_type, recipient)); } + + fn wait_for( + &self, + event_type: EventType, + ) -> Pin>> + Send>> { + let (addr, rx) = oneshot::>(); + self.subscribe(event_type, addr.clone()); + let bus = self.event_bus.clone(); + Box::pin(async move { + let r = rx.await?; + bus.do_send(Unsubscribe::new(event_type, addr)); + Ok(r) + }) + } } impl EventContextManager for BusHandle { diff --git a/crates/events/src/enclave_event/enable_effects.rs b/crates/events/src/enclave_event/enable_effects.rs index 4657fab0bd..e6830661b4 100644 --- a/crates/events/src/enclave_event/enable_effects.rs +++ b/crates/events/src/enclave_event/enable_effects.rs @@ -13,7 +13,7 @@ use std::fmt::{self, Display}; #[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub struct EffectsEnabled { - pub correlation_id: CorrelationId, + correlation_id: CorrelationId, } impl EffectsEnabled { diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index e9b4e0a242..3ae7093106 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -32,7 +32,7 @@ mod encryption_key_created; mod encryption_key_pending; mod encryption_key_received; mod keyshare_created; -mod net_sync_events_received; +mod net_ready; mod operator_activation_changed; mod outgoing_sync_requested; mod pk_aggregation_proof_pending; @@ -89,7 +89,7 @@ pub use encryption_key_created::*; pub use encryption_key_pending::*; pub use encryption_key_received::*; pub use keyshare_created::*; -pub use net_sync_events_received::*; +pub use net_ready::*; pub use operator_activation_changed::*; pub use outgoing_sync_requested::*; pub use pk_aggregation_proof_pending::*; @@ -258,12 +258,13 @@ pub enum EnclaveEventData { ShareVerificationDispatched(ShareVerificationDispatched), ShareVerificationComplete(ShareVerificationComplete), OutgoingSyncRequested(OutgoingSyncRequested), - NetSyncEventsReceived(NetSyncEventsReceived), HistoricalEvmSyncStart(HistoricalEvmSyncStart), HistoricalNetSyncStart(HistoricalNetSyncStart), + HistoricalNetSyncEventsReceived(HistoricalNetSyncEventsReceived), SyncEffect(SyncEffect), SyncEnded(SyncEnded), EffectsEnabled(EffectsEnabled), + NetReady(NetReady), DecryptionShareProofSigned(DecryptionShareProofSigned), ShareDecryptionProofPending(ShareDecryptionProofPending), PkAggregationProofPending(PkAggregationProofPending), @@ -401,6 +402,14 @@ impl EnclaveEvent { } } +impl TryFrom> for EnclaveEvent { + type Error = bincode::Error; + + fn try_from(value: Vec) -> Result { + EnclaveEvent::from_bytes(&value) + } +} + #[cfg(feature = "test-helpers")] impl EnclaveEvent { /// test-helpers only utility function to create a new sequenced event @@ -598,12 +607,13 @@ impl_event_types!( ShareVerificationDispatched, ShareVerificationComplete, OutgoingSyncRequested, - NetSyncEventsReceived, HistoricalEvmSyncStart, HistoricalNetSyncStart, + HistoricalNetSyncEventsReceived, SyncEffect, SyncEnded, EffectsEnabled, + NetReady, DecryptionShareProofSigned, ShareDecryptionProofPending, PkAggregationProofPending, @@ -680,3 +690,128 @@ impl EventConstructorWithTimestamp for EnclaveEvent { } } } + +#[cfg(feature = "test-helpers")] +impl EnclaveEvent { + /// Create a test event using the TestEventBuilder struct + pub fn test_event(label: &str) -> TestEventBuilder { + TestEventBuilder::::new(label) + } +} + +/// Build out a test event +pub struct TestEventBuilder { + label: String, + seq: S::Seq, + id: Option, + data: Option, + aggregate_id: Option, + e3_id: Option, + ts: Option, +} + +impl TestEventBuilder { + /// Create a new test event + pub fn new(label: &str) -> Self { + Self { + label: label.to_owned(), + seq: (), + id: None, + aggregate_id: None, + data: None, + e3_id: None, + ts: None, + } + } + + /// make it a sequenced event + pub fn seq(self, seq: u64) -> TestEventBuilder { + TestEventBuilder:: { + seq, + label: self.label, + id: self.id, + data: self.data, + aggregate_id: self.aggregate_id, + e3_id: self.e3_id, + ts: self.ts, + } + } +} + +impl TestEventBuilder { + /// Add an e3_id based on a u64 this takes preference over e3_id() + pub fn id(mut self, id: u64) -> Self { + self.id = Some(id); + self + } + + /// Ensure the event holds the given aggregate_id this takes preference over e3_id() + pub fn aggregate_id(mut self, id: u64) -> Self { + self.aggregate_id = Some(id); + self + } + + /// Ensure the event holds the given e3_id. + pub fn e3_id(mut self, e3_id: E3id) -> Self { + self.e3_id = Some(e3_id); + self + } + + /// Ensure the event holds a ts + pub fn ts(mut self, ts: u128) -> Self { + self.ts = Some(ts); + self + } + + /// Ensure the event holds the given EnclaveEventData object. This overrides all other params + /// aiside from seq(n) + pub fn data(mut self, data: impl Into) -> Self { + self.data = Some(data.into()); + self + } + + fn get_built_event(self) -> EnclaveEvent { + let event = self.data.unwrap_or( + TestEvent { + msg: self.label, + entropy: self.id.unwrap_or(0), + e3_id: resolve_e3_id(self.e3_id, self.id, self.aggregate_id), + } + .into(), + ); + + EnclaveEvent::::new_with_timestamp( + event, + None, + self.ts.unwrap_or(0), + None, + EventSource::Evm, + ) + } +} + +impl TestEventBuilder { + /// Build the event + pub fn build(self) -> EnclaveEvent { + self.get_built_event() + } +} + +impl TestEventBuilder { + /// Build the event + pub fn build(self) -> EnclaveEvent { + let seq = self.seq; + let unseq = self.get_built_event(); + unseq.into_sequenced(seq) + } +} + +fn resolve_e3_id(e3_id: Option, id: Option, aggregate_id: Option) -> Option { + match (e3_id, id, aggregate_id) { + (Some(_), Some(id), Some(agg)) if agg != 0 => Some(E3id::new(id.to_string(), agg)), + (Some(e3), Some(id), _) => Some(E3id::new(id.to_string(), e3.chain_id())), + (Some(e3), _, Some(agg)) if agg != 0 => Some(E3id::new(e3.e3_id(), agg)), + (None, Some(id), Some(agg)) if agg != 0 => Some(E3id::new(id.to_string(), agg)), + (e3, _, _) => e3, + } +} diff --git a/crates/events/src/enclave_event/net_sync_events_received.rs b/crates/events/src/enclave_event/net_ready.rs similarity index 61% rename from crates/events/src/enclave_event/net_sync_events_received.rs rename to crates/events/src/enclave_event/net_ready.rs index eefe1828fb..c8205df15c 100644 --- a/crates/events/src/enclave_event/net_sync_events_received.rs +++ b/crates/events/src/enclave_event/net_ready.rs @@ -4,25 +4,27 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use crate::CorrelationId; use actix::Message; use serde::{Deserialize, Serialize}; use std::fmt::{self, Display}; -use super::{EnclaveEvent, Unsequenced}; - +/// Dispatched once effects (side-effects) should be activated after a sync pass #[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] -pub struct NetSyncEventsReceived { - pub events: Vec>, +pub struct NetReady { + correlation_id: CorrelationId, } -impl NetSyncEventsReceived { - pub fn new(events: Vec>) -> Self { - Self { events } +impl NetReady { + pub fn new() -> Self { + Self { + correlation_id: CorrelationId::new(), + } } } -impl Display for NetSyncEventsReceived { +impl Display for NetReady { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } diff --git a/crates/events/src/enclave_event/pk_aggregation_proof_pending.rs b/crates/events/src/enclave_event/pk_aggregation_proof_pending.rs index 2ee5b1afe8..52dd677868 100644 --- a/crates/events/src/enclave_event/pk_aggregation_proof_pending.rs +++ b/crates/events/src/enclave_event/pk_aggregation_proof_pending.rs @@ -12,6 +12,7 @@ //! `PkAggregationProofSigned`. use crate::{E3id, OrderedSet, PkAggregationProofRequest}; +use e3_utils::ArcBytes; use serde::{Deserialize, Serialize}; /// PublicKeyAggregator -> ProofRequestActor: generate and sign C5 proof. @@ -19,7 +20,7 @@ use serde::{Deserialize, Serialize}; pub struct PkAggregationProofPending { pub e3_id: E3id, pub proof_request: PkAggregationProofRequest, - pub public_key: Vec, + pub public_key: ArcBytes, pub public_key_hash: [u8; 32], pub nodes: OrderedSet, } diff --git a/crates/events/src/enclave_event/publickey_aggregated.rs b/crates/events/src/enclave_event/publickey_aggregated.rs index 128e8f368f..bcf4563890 100644 --- a/crates/events/src/enclave_event/publickey_aggregated.rs +++ b/crates/events/src/enclave_event/publickey_aggregated.rs @@ -7,6 +7,7 @@ use crate::{E3id, OrderedSet, Proof}; use actix::Message; use derivative::Derivative; +use e3_utils::ArcBytes; use serde::{Deserialize, Serialize}; use std::fmt::{self, Display}; @@ -15,8 +16,9 @@ use std::fmt::{self, Display}; #[rtype(result = "()")] pub struct PublicKeyAggregated { #[derivative(Debug(format_with = "e3_utils::formatters::hexf"))] - pub pubkey: Vec, - pub public_key_hash: [u8; 32], + pub pubkey: ArcBytes, // TODO: ArcBytes ? + #[derivative(Debug(format_with = "e3_utils::formatters::hexf"))] + pub public_key_hash: [u8; 32], // TODO: ArcBytes32 ? pub e3_id: E3id, pub nodes: OrderedSet, /// C5 proof: proof of correct pk aggregation. diff --git a/crates/events/src/enclave_event/sync_end.rs b/crates/events/src/enclave_event/sync_end.rs index 881cf0dd71..5507705815 100644 --- a/crates/events/src/enclave_event/sync_end.rs +++ b/crates/events/src/enclave_event/sync_end.rs @@ -13,7 +13,7 @@ use std::fmt::{self, Display}; #[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub struct SyncEnded { - pub correlation_id: CorrelationId, + correlation_id: CorrelationId, } impl SyncEnded { diff --git a/crates/events/src/enclave_event/sync_start.rs b/crates/events/src/enclave_event/sync_start.rs index 89993094ed..1db06bd508 100644 --- a/crates/events/src/enclave_event/sync_start.rs +++ b/crates/events/src/enclave_event/sync_start.rs @@ -63,23 +63,11 @@ impl Display for HistoricalEvmSyncStart { #[rtype(result = "()")] pub struct HistoricalNetSyncStart { pub since: BTreeMap, - #[serde(skip)] - /// We include the sender here so that the evm can communicate directly with the sync actor - pub sender: Option>, // Must be Option to allow serde deserialize on - // EnclaveEvent as Default is required to be - // implemented this is fine as this event is never - // shared } impl HistoricalNetSyncStart { - pub fn new( - sender: impl Into>, - since: BTreeMap, - ) -> Self { - Self { - since, - sender: Some(sender.into()), - } + pub fn new(since: BTreeMap) -> Self { + Self { since } } } @@ -110,17 +98,17 @@ impl Display for HistoricalEvmEventsReceived { #[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] -pub struct HistoricalNetEventsReceived { +pub struct HistoricalNetSyncEventsReceived { pub events: Vec>, } -impl HistoricalNetEventsReceived { +impl HistoricalNetSyncEventsReceived { pub fn new(events: Vec>) -> Self { Self { events } } } -impl Display for HistoricalNetEventsReceived { +impl Display for HistoricalNetSyncEventsReceived { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } diff --git a/crates/events/src/eventbus.rs b/crates/events/src/eventbus.rs index 6760976f8e..14eb21939e 100644 --- a/crates/events/src/eventbus.rs +++ b/crates/events/src/eventbus.rs @@ -9,9 +9,11 @@ use crate::EventType; use actix::prelude::*; use bloom::{BloomFilter, ASMS}; use e3_utils::{colorize, Color, MAILBOX_LIMIT, MAILBOX_LIMIT_LARGE}; -use std::collections::{HashMap, VecDeque}; +use std::collections::HashMap; +use std::fmt; use std::marker::PhantomData; -use tracing::info; +use std::time::Duration; +use tokio::sync::mpsc; ////////////////////////////////////////////////////////////////////////////// // Configuration @@ -251,40 +253,41 @@ impl GetEvents { } #[derive(Message)] -#[rtype(result = "Vec")] +#[rtype(result = "TakeEventsResult")] pub struct TakeEvents { amount: usize, + timeout: Duration, _d: PhantomData, } +#[derive(Debug)] +pub struct TakeEventsResult { + pub events: Vec, + pub timed_out: bool, +} + impl TakeEvents { pub fn new(amount: usize) -> Self { Self { amount, + timeout: Duration::from_secs(1), _d: PhantomData, } } -} -struct PendingTake { - count: usize, - collected: Vec, - responder: tokio::sync::oneshot::Sender>, + pub fn with_per_evt_timeout(amount: usize, timeout: Duration) -> Self { + Self { + amount, + timeout, + _d: PhantomData, + } + } } #[derive(Message)] #[rtype(result = "()")] pub struct ResetHistory; -impl Handler for HistoryCollector { - type Result = (); - - fn handle(&mut self, _: ResetHistory, _: &mut Context) { - self.history.clear(); - self.pending_takes.clear(); - } -} - #[derive(Message)] #[rtype(result = "Vec")] pub struct GetErrors(PhantomData); @@ -299,122 +302,106 @@ impl GetErrors { // History Collector ////////////////////////////////////////////////////////////////////////////// -/// Actor to subscribe to EventBus to capture all history -pub struct HistoryCollector { - history: VecDeque, - pending_takes: Vec>, +struct HistoryCollectorWaiter { + rx: Option>, } -impl HistoryCollector { - pub fn new() -> Self { - Self { - history: VecDeque::new(), - pending_takes: Vec::new(), - } - } - - fn try_fulfill_pending_takes(&mut self) { - let mut completed = Vec::new(); - - // For each pending take, try to fulfill it - for (idx, pending) in self.pending_takes.iter_mut().enumerate() { - // Fill from history first - while pending.collected.len() < pending.count && !self.history.is_empty() { - pending.collected.push(self.history.pop_front().unwrap()); - } +impl Actor for HistoryCollectorWaiter { + type Context = Context; +} - // If we have enough, mark as complete - if pending.collected.len() >= pending.count { - completed.push(idx); +impl Handler> for HistoryCollectorWaiter { + type Result = ResponseActFuture>; + fn handle(&mut self, msg: TakeEvents, _: &mut Context) -> Self::Result { + let count = msg.amount; + let timeout = msg.timeout; + let mut rx = self.rx.take().unwrap(); + Box::pin( + async move { + let mut events = Vec::with_capacity(count); + let mut timed_out = false; + for _ in 0..count { + match tokio::time::timeout(timeout, rx.recv()).await { + Ok(Some(e)) => events.push(e), + Ok(None) => break, + Err(_) => { + timed_out = true; + break; + } + } + } + (TakeEventsResult { events, timed_out }, rx) } - } - - // Send responses for completed takes (in reverse order to maintain indices) - for idx in completed.into_iter().rev() { - let pending = self.pending_takes.swap_remove(idx); - let events = pending.collected.into_iter().take(pending.count).collect(); - let _ = pending.responder.send(events); - } + .into_actor(self) + .map(|(result, rx), actor, _| { + actor.rx = Some(rx); + result + }), + ) } +} - fn add_event(&mut self, event: E) { - // First try to give to pending takes - for pending in &mut self.pending_takes { - if pending.collected.len() < pending.count { - info!( - "Received event {}. Pushing to pending take {}/{}...", - event.event_type(), - pending.collected.len() + 1, - pending.count - ); - pending.collected.push(event); - self.try_fulfill_pending_takes(); - return; - } +impl Handler for HistoryCollectorWaiter { + type Result = (); + fn handle(&mut self, _: ResetHistory, _: &mut Context) { + if let Some(ref mut rx) = self.rx { + while rx.try_recv().is_ok() {} } - - // No pending take needed it, add to history - self.history.push_back(event); } } -impl Handler> for HistoryCollector { - type Result = Vec; - - fn handle(&mut self, _: GetEvents, _: &mut Context) -> Vec { - self.history.iter().cloned().collect() - } +pub struct HistoryCollector { + history: Vec, + tx: mpsc::UnboundedSender, + waiter: Addr>, } -impl Handler> for HistoryCollector { - type Result = ResponseActFuture>; - - fn handle(&mut self, msg: TakeEvents, _: &mut Context) -> Self::Result { - let count = msg.amount; - - // If we have enough events in history, return immediately - if self.history.len() >= count { - let events: Vec = self.history.drain(..count).collect(); - return Box::pin(async move { events }.into_actor(self)); - } - - info!( - "Requesting {} events but only {} in the buffer. waiting for more...", - msg.amount, - self.history.len() - ); - - // Create a tokio oneshot channel for the response - let (tx, rx) = tokio::sync::oneshot::channel(); - - // Collect what we can from history - let mut collected = Vec::new(); - while !self.history.is_empty() && collected.len() < count { - collected.push(self.history.pop_front().unwrap()); +impl HistoryCollector { + pub fn new() -> Self { + let (tx, rx) = mpsc::unbounded_channel(); + let waiter = HistoryCollectorWaiter { rx: Some(rx) }.start(); + Self { + history: Vec::new(), + tx, + waiter, } - - // Store the pending request - self.pending_takes.push(PendingTake { - count, - collected, - responder: tx, - }); - - // Return future that waits for the response - Box::pin(async move { rx.await.unwrap_or_else(|_| Vec::new()) }.into_actor(self)) } } impl Actor for HistoryCollector { type Context = Context; fn started(&mut self, ctx: &mut Self::Context) { - ctx.set_mailbox_capacity(MAILBOX_LIMIT) + ctx.set_mailbox_capacity(MAILBOX_LIMIT); } } impl Handler for HistoryCollector { type Result = E::Result; fn handle(&mut self, msg: E, _ctx: &mut Self::Context) -> Self::Result { - self.add_event(msg); + self.history.push(msg.clone()); + let _ = self.tx.send(msg); + } +} + +impl Handler for HistoryCollector { + type Result = (); + fn handle(&mut self, _: ResetHistory, _: &mut Context) { + self.history.clear(); + self.waiter.do_send(ResetHistory); + } +} + +impl Handler> for HistoryCollector { + type Result = ResponseActFuture>; + fn handle(&mut self, msg: TakeEvents, _: &mut Context) -> Self::Result { + let fut = self.waiter.send(msg); + Box::pin(async move { fut.await.unwrap() }.into_actor(self)) + } +} + +impl Handler> for HistoryCollector { + type Result = Vec; + fn handle(&mut self, _: GetEvents, _: &mut Context) -> Vec { + self.history.clone() } } diff --git a/crates/events/src/events.rs b/crates/events/src/events.rs index 3ff819c7ea..24d6b33142 100644 --- a/crates/events/src/events.rs +++ b/crates/events/src/events.rs @@ -8,7 +8,13 @@ use std::collections::HashMap; use actix::{Message, Recipient}; -use crate::{AggregateId, CorrelationId, EnclaveEvent, Sequenced, Unsequenced}; +use crate::traits::EventContextAccessors; +use crate::{AggregateId, CorrelationId, EnclaveEvent, EventSource, Sequenced, Unsequenced}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EventStoreFilter { + Source(EventSource), +} /// Direct event received by the EventStore to store an event #[derive(Message, Debug)] @@ -98,6 +104,8 @@ pub struct EventStoreQueryBy { correlation_id: CorrelationId, query: Q::Shape, sender: Recipient, + limit: Option, + filter: Option, } impl EventStoreQueryBy { @@ -110,12 +118,32 @@ impl EventStoreQueryBy { correlation_id, query, sender: sender.into(), + limit: None, + filter: None, } } pub fn query(&self) -> &HashMap { &self.query } + + pub fn limit(&self) -> Option { + self.limit + } + + pub fn filter(&self) -> Option<&EventStoreFilter> { + self.filter.as_ref() + } + + pub fn with_limit(mut self, limit: u64) -> Self { + self.limit = Some(limit); + self + } + + pub fn with_filter(mut self, filter: EventStoreFilter) -> Self { + self.filter = Some(filter); + self + } } impl EventStoreQueryBy { @@ -128,12 +156,32 @@ impl EventStoreQueryBy { correlation_id, query, sender: sender.into(), + limit: None, + filter: None, } } pub fn query(&self) -> &HashMap { &self.query } + + pub fn limit(&self) -> Option { + self.limit + } + + pub fn filter(&self) -> Option<&EventStoreFilter> { + self.filter.as_ref() + } + + pub fn with_limit(mut self, limit: u64) -> Self { + self.limit = Some(limit); + self + } + + pub fn with_filter(mut self, filter: EventStoreFilter) -> Self { + self.filter = Some(filter); + self + } } impl EventStoreQueryBy { @@ -146,12 +194,32 @@ impl EventStoreQueryBy { correlation_id, query, sender: sender.into(), + limit: None, + filter: None, } } pub fn query(&self) -> u128 { self.query } + + pub fn limit(&self) -> Option { + self.limit + } + + pub fn filter(&self) -> Option<&EventStoreFilter> { + self.filter.as_ref() + } + + pub fn with_limit(mut self, limit: u64) -> Self { + self.limit = Some(limit); + self + } + + pub fn with_filter(mut self, filter: EventStoreFilter) -> Self { + self.filter = Some(filter); + self + } } impl EventStoreQueryBy { @@ -164,12 +232,32 @@ impl EventStoreQueryBy { correlation_id, query, sender: sender.into(), + limit: None, + filter: None, } } pub fn query(&self) -> u64 { self.query } + + pub fn limit(&self) -> Option { + self.limit + } + + pub fn filter(&self) -> Option<&EventStoreFilter> { + self.filter.as_ref() + } + + pub fn with_limit(mut self, limit: u64) -> Self { + self.limit = Some(limit); + self + } + + pub fn with_filter(mut self, filter: EventStoreFilter) -> Self { + self.filter = Some(filter); + self + } } impl EventStoreQueryBy { @@ -180,4 +268,14 @@ impl EventStoreQueryBy { pub fn sender(self) -> Recipient { self.sender } + + pub fn with_options(mut self, limit: Option, filter: Option) -> Self { + if let Some(l) = limit { + self.limit = Some(l); + } + if let Some(f) = filter { + self.filter = Some(f); + } + self + } } diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index fb52fe74c4..93751e3aa5 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -6,8 +6,8 @@ use crate::{ events::{StoreEventRequested, StoreEventResponse}, - EventContextAccessors, EventLog, EventStoreQueryBy, EventStoreQueryResponse, Seq, - SequenceIndex, Ts, + EnclaveEvent, EventContextAccessors, EventLog, EventStoreFilter, EventStoreQueryBy, + EventStoreQueryResponse, Seq, SequenceIndex, Sequenced, Ts, Unsequenced, }; use actix::{Actor, Handler}; use anyhow::{bail, Result}; @@ -22,9 +22,12 @@ pub struct EventStore { } impl EventStore { - pub fn handle_store_event_requested(&mut self, msg: StoreEventRequested) -> Result<()> { - let event = msg.event; - let sender = msg.sender; + /// Attempt to store an event. Returns the sequenced event on success, + /// `None` if the event was a duplicate, or an error on failure. + pub fn store_event( + &mut self, + event: EnclaveEvent, + ) -> Result>> { let ts = event.ts(); if let Some(_) = self.index.get(ts)? { warn!("Event already stored at timestamp {ts}! This might happen when recovering from a snapshot. Skipping storage"); @@ -35,48 +38,57 @@ impl EventStore { self.storage_errors ); } - return Ok(()); + return Ok(None); } let seq = self.log.append(&event)?; self.index.insert(ts, seq)?; - sender.do_send(StoreEventResponse(event.into_sequenced(seq))); - Ok(()) + Ok(Some(event.into_sequenced(seq))) } - pub fn handle_event_store_query_ts(&mut self, msg: EventStoreQueryBy) -> Result<()> { - // if there are no events after the timestamp return an empty vector - let id = msg.id(); - let Some(seq) = self.index.seek(msg.query())? else { - msg.sender() - .try_send(EventStoreQueryResponse::new(id, vec![]))?; - return Ok(()); - }; - // read and return the events - let evts = self - .log - .read_from(seq) - .map(|(s, e)| e.into_sequenced(s)) - .collect::>(); + fn collect_events( + &self, + iter: Box)>>, + filter: Option, + limit: Option, + ) -> Vec> { + let iter = iter.map(|(s, e)| e.into_sequenced(s)); - msg.sender() - .try_send(EventStoreQueryResponse::new(id, evts))?; - Ok(()) + match filter { + Some(EventStoreFilter::Source(source)) => { + let iter = iter.filter(move |e| e.get_ctx().source() == source); + match limit { + Some(lim) => iter.take(lim as usize).collect(), + None => iter.collect(), + } + } + None => match limit { + Some(lim) => iter.take(lim as usize).collect(), + None => iter.collect(), + }, + } } - pub fn handle_event_store_query_seq(&mut self, msg: EventStoreQueryBy) -> Result<()> { - // if there are no events after the timestamp return an empty vector - let id = msg.id(); - - // read and return the events - let evts = self - .log - .read_from(msg.query()) - .map(|(s, e)| e.into_sequenced(s)) - .collect::>(); + /// Query events by timestamp. Returns events at or after the given timestamp. + pub fn query_by_ts( + &self, + query: u128, + filter: Option, + limit: Option, + ) -> Result>> { + let Some(seq) = self.index.seek(query)? else { + return Ok(vec![]); + }; + Ok(self.collect_events(self.log.read_from(seq), filter, limit)) + } - msg.sender() - .try_send(EventStoreQueryResponse::new(id, evts))?; - Ok(()) + /// Query events by sequence number. Returns events at or after the given sequence. + pub fn query_by_seq( + &self, + query: u64, + filter: Option, + limit: Option, + ) -> Vec> { + self.collect_events(self.log.read_from(query), filter, limit) } } @@ -97,10 +109,15 @@ impl Actor for EventStore { impl Handler for EventStore { type Result = (); fn handle(&mut self, msg: StoreEventRequested, _: &mut Self::Context) -> Self::Result { - if let Err(e) = self.handle_store_event_requested(msg) { - // Log append or index insert failed — storage is broken. - error!("Event storage failed: {e}"); - panic!("Unrecoverable event storage failure: {e}"); + match self.store_event(msg.event) { + Ok(Some(sequenced)) => { + msg.sender.do_send(StoreEventResponse(sequenced)); + } + Ok(None) => {} // duplicate — already warned inside store_event + Err(e) => { + error!("Event storage failed: {e}"); + panic!("Unrecoverable event storage failure: {e}"); + } } } } @@ -108,8 +125,18 @@ impl Handler for EventStore< impl Handler> for EventStore { type Result = (); fn handle(&mut self, msg: EventStoreQueryBy, _: &mut Self::Context) -> Self::Result { - if let Err(e) = self.handle_event_store_query_ts(msg) { - error!("{e}"); + let query = msg.query(); + let id = msg.id(); + let limit = msg.limit(); + let filter = msg.filter().cloned(); + let sender = msg.sender(); + match self.query_by_ts(query, filter, limit) { + Ok(evts) => { + if let Err(e) = sender.try_send(EventStoreQueryResponse::new(id, evts)) { + error!("{e}"); + } + } + Err(e) => error!("{e}"), } } } @@ -117,8 +144,360 @@ impl Handler> for EventStor impl Handler> for EventStore { type Result = (); fn handle(&mut self, msg: EventStoreQueryBy, _: &mut Self::Context) -> Self::Result { - if let Err(e) = self.handle_event_store_query_seq(msg) { + let id = msg.id(); + let query = msg.query(); + let limit = msg.limit(); + let filter = msg.filter().cloned(); + let sender = msg.sender(); + let evts = self.query_by_seq(query, filter, limit); + if let Err(e) = sender.try_send(EventStoreQueryResponse::new(id, evts)) { error!("{e}"); } } } + +#[cfg(test)] +mod tests { + use crate::{EventConstructorWithTimestamp, EventSource, TestEvent}; + + use super::*; + use anyhow::Result; + use std::collections::BTreeMap; + + // --------------------------------------------------------------------------- + // Mock SequenceIndex backed by BTreeMap + // --------------------------------------------------------------------------- + struct MockIndex(BTreeMap); + + impl MockIndex { + fn new() -> Self { + Self(BTreeMap::new()) + } + } + + impl SequenceIndex for MockIndex { + fn insert(&mut self, key: u128, value: u64) -> Result<()> { + self.0.insert(key, value); + Ok(()) + } + + fn get(&self, key: u128) -> Result> { + Ok(self.0.get(&key).copied()) + } + + fn seek(&self, key: u128) -> Result> { + Ok(self.0.range(key..).next().map(|(_, &v)| v)) + } + } + + // --------------------------------------------------------------------------- + // Mock EventLog backed by Vec + // --------------------------------------------------------------------------- + struct MockLog(Vec>); + + impl MockLog { + fn new() -> Self { + Self(Vec::new()) + } + } + + impl EventLog for MockLog { + fn append(&mut self, event: &EnclaveEvent) -> Result { + let seq = self.0.len() as u64; + self.0.push(event.clone()); + Ok(seq) + } + + fn read_from( + &self, + from: u64, + ) -> Box)>> { + let items: Vec<_> = self + .0 + .iter() + .enumerate() + .filter(move |(i, _)| *i as u64 >= from) + .map(|(i, e)| (i as u64, e.clone())) + .collect(); + Box::new(items.into_iter()) + } + } + + // --------------------------------------------------------------------------- + // Test helpers + // --------------------------------------------------------------------------- + fn make_event(ts: u128, source: EventSource) -> EnclaveEvent { + EnclaveEvent::::new_with_timestamp( + TestEvent::new("test", 1).into(), + None, + ts, + None, + source, + ) + } + + fn make_local_event(ts: u128) -> EnclaveEvent { + make_event(ts, EventSource::Local) + } + + fn make_network_event(ts: u128) -> EnclaveEvent { + make_event(ts, EventSource::Net) + } + + fn new_store() -> EventStore { + EventStore::new(MockIndex::new(), MockLog::new()) + } + + fn populated_store(events: &[EnclaveEvent]) -> EventStore { + let mut store = new_store(); + for event in events { + store.store_event(event.clone()).unwrap(); + } + store + } + + // =========================================================================== + // store_event + // =========================================================================== + + #[test] + fn store_event_returns_sequenced_event() { + let mut store = new_store(); + let event = make_local_event(100); + + let result = store.store_event(event).unwrap().unwrap(); + + assert_eq!(result.get_ctx().ts(), 100); + } + + #[test] + fn store_event_assigns_incrementing_sequence_numbers() { + let mut store = new_store(); + + let _a = store.store_event(make_local_event(100)).unwrap().unwrap(); + let _b = store.store_event(make_local_event(200)).unwrap().unwrap(); + let _c = store.store_event(make_local_event(300)).unwrap().unwrap(); + + assert_eq!(store.index.get(100).unwrap(), Some(0)); + assert_eq!(store.index.get(200).unwrap(), Some(1)); + assert_eq!(store.index.get(300).unwrap(), Some(2)); + } + + #[test] + fn store_event_appends_to_log() { + let mut store = new_store(); + store.store_event(make_local_event(100)).unwrap(); + store.store_event(make_local_event(200)).unwrap(); + + let logged: Vec<_> = store.log.read_from(0).collect(); + assert_eq!(logged.len(), 2); + } + + #[test] + fn store_event_returns_none_for_duplicate_timestamp() { + let mut store = new_store(); + store.store_event(make_local_event(100)).unwrap(); + + let result = store.store_event(make_local_event(100)).unwrap(); + + assert!(result.is_none()); + assert_eq!(store.storage_errors, 1); + // Log should still have only one event + assert_eq!(store.log.read_from(0).count(), 1); + } + + #[test] + fn store_event_bails_after_max_storage_errors() { + let mut store = new_store(); + store.store_event(make_local_event(100)).unwrap(); + + for _ in 0..MAX_STORAGE_ERRORS { + let result = store.store_event(make_local_event(100)).unwrap(); + assert!(result.is_none()); + } + + assert_eq!(store.storage_errors, MAX_STORAGE_ERRORS); + + let result = store.store_event(make_local_event(100)); + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("too many storage errors")); + } + + // =========================================================================== + // query_by_seq + // =========================================================================== + + #[test] + fn seq_query_returns_all_events() { + let store = populated_store(&[ + make_local_event(100), + make_local_event(200), + make_local_event(300), + ]); + + let events = store.query_by_seq(0, None, None); + + assert_eq!(events.len(), 3); + } + + #[test] + fn seq_query_reads_from_given_offset() { + let store = populated_store(&[ + make_local_event(100), + make_local_event(200), + make_local_event(300), + make_local_event(400), + ]); + + let events = store.query_by_seq(2, None, None); + + assert_eq!(events.len(), 2); + } + + #[test] + fn seq_query_with_source_filter() { + let store = populated_store(&[ + make_local_event(100), + make_network_event(200), + make_local_event(300), + make_network_event(400), + ]); + + let events = + store.query_by_seq(0, Some(EventStoreFilter::Source(EventSource::Local)), None); + + assert_eq!(events.len(), 2); + for e in &events { + assert_eq!(e.get_ctx().source(), EventSource::Local); + } + } + + #[test] + fn seq_query_with_limit() { + let store = populated_store(&[ + make_local_event(100), + make_local_event(200), + make_local_event(300), + make_local_event(400), + make_local_event(500), + ]); + + let events = store.query_by_seq(0, None, Some(2)); + + assert_eq!(events.len(), 2); + } + + #[test] + fn seq_query_with_filter_and_limit() { + let store = populated_store(&[ + make_local_event(100), + make_network_event(200), + make_local_event(300), + make_local_event(400), + make_network_event(500), + ]); + + let events = store.query_by_seq( + 0, + Some(EventStoreFilter::Source(EventSource::Local)), + Some(2), + ); + + assert_eq!(events.len(), 2); + for e in &events { + assert_eq!(e.get_ctx().source(), EventSource::Local); + } + } + + #[test] + fn seq_query_on_empty_log_returns_empty() { + let store = new_store(); + + let events = store.query_by_seq(0, None, None); + + assert!(events.is_empty()); + } + + // =========================================================================== + // query_by_ts + // =========================================================================== + + #[test] + fn ts_query_returns_events_from_exact_timestamp() { + let store = populated_store(&[ + make_local_event(100), + make_local_event(200), + make_local_event(300), + make_local_event(400), + ]); + + let events = store.query_by_ts(200, None, None).unwrap(); + + assert_eq!(events.len(), 3); + } + + #[test] + fn ts_query_seeks_to_nearest_future_timestamp() { + let store = populated_store(&[ + make_local_event(100), + make_local_event(300), + make_local_event(500), + ]); + + // ts=200 has no match; seek finds ts=300 onwards + let events = store.query_by_ts(200, None, None).unwrap(); + + assert_eq!(events.len(), 2); + } + + #[test] + fn ts_query_returns_empty_when_no_matching_timestamp() { + let store = new_store(); + + let events = store.query_by_ts(999, None, None).unwrap(); + + assert!(events.is_empty()); + } + + #[test] + fn ts_query_returns_empty_when_past_all_events() { + let store = populated_store(&[make_local_event(100), make_local_event(200)]); + + let events = store.query_by_ts(999, None, None).unwrap(); + + assert!(events.is_empty()); + } + + #[test] + fn ts_query_with_filter() { + let store = populated_store(&[ + make_local_event(100), + make_network_event(200), + make_local_event(300), + ]); + + let events = store + .query_by_ts(100, Some(EventStoreFilter::Source(EventSource::Net)), None) + .unwrap(); + + assert_eq!(events.len(), 1); + assert_eq!(events[0].get_ctx().source(), EventSource::Net); + } + + #[test] + fn ts_query_with_limit() { + let store = populated_store(&[ + make_local_event(100), + make_local_event(200), + make_local_event(300), + make_local_event(400), + ]); + + let events = store.query_by_ts(100, None, Some(2)).unwrap(); + + assert_eq!(events.len(), 2); + } +} diff --git a/crates/events/src/eventstore_router.rs b/crates/events/src/eventstore_router.rs index 3679957b50..83253746f3 100644 --- a/crates/events/src/eventstore_router.rs +++ b/crates/events/src/eventstore_router.rs @@ -119,6 +119,8 @@ impl EventStoreRouter { debug!("Received request for timestamp query."); let parent_id = msg.id(); let query = msg.query().clone(); + let limit = msg.limit(); + let filter = msg.filter().cloned(); let sender = msg.sender(); let sub_queries: Vec<_> = query @@ -145,7 +147,8 @@ impl EventStoreRouter { for (aggregate_id, ts, sub_query_id, store_addr) in sub_queries { let get_events_msg = - EventStoreQueryBy::::new(sub_query_id, ts, aggregator_addr.clone().recipient()); + EventStoreQueryBy::::new(sub_query_id, ts, aggregator_addr.clone().recipient()) + .with_options(limit, filter.clone()); debug!("Sending query for aggregate {:?}", aggregate_id); store_addr.do_send(get_events_msg); } @@ -161,6 +164,8 @@ impl EventStoreRouter { debug!("Received request for sequence query."); let parent_id = msg.id(); let query = msg.query().clone(); + let limit = msg.limit(); + let filter = msg.filter().cloned(); let sender = msg.sender(); let sub_queries: Vec<_> = query @@ -190,7 +195,8 @@ impl EventStoreRouter { sub_query_id, seq, aggregator_addr.clone().recipient(), - ); + ) + .with_options(limit, filter.clone()); debug!("Sending query for aggregate {:?}", aggregate_id); store_addr.do_send(get_events_msg); } diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index 73e33314da..6e8b6cb40b 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -81,6 +81,7 @@ mod tests { assert_eq!( events + .events .iter() .map(EnclaveEvent::strip_ts) .collect::>(), diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 5ac78dd3ed..599562d12f 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -6,8 +6,9 @@ use actix::{Message, Recipient}; use anyhow::Result; -use std::fmt::Display; use std::hash::Hash; +use std::pin::Pin; +use std::{fmt::Display, future::Future}; use crate::{ event_context::{AggregateId, EventContext}, @@ -140,6 +141,8 @@ pub trait EventSubscriber { fn subscribe_all(&self, event_types: &[EventType], recipient: Recipient); /// Subscribe the recipient to events matching the given event type fn unsubscribe(&self, event_type: &str, recipient: Recipient); + /// Return a future that waits for a specific event + fn wait_for(&self, event_type: EventType) -> Pin> + Send>>; } /// Trait to create an event with a timestamp from its associated type data diff --git a/crates/evm/src/ciphernode_registry_sol.rs b/crates/evm/src/ciphernode_registry_sol.rs index d56a9e6c4f..624c7aed39 100644 --- a/crates/evm/src/ciphernode_registry_sol.rs +++ b/crates/evm/src/ciphernode_registry_sol.rs @@ -23,7 +23,7 @@ use e3_events::{ EffectsEnabled, EnclaveEvent, EnclaveEventData, EventSubscriber, EventType, OrderedSet, PublicKeyAggregated, Seed, Shutdown, TicketGenerated, TicketId, }; -use e3_utils::{NotifySync, MAILBOX_LIMIT}; +use e3_utils::{ArcBytes, NotifySync, MAILBOX_LIMIT}; use tracing::{error, info, trace}; sol!( @@ -501,11 +501,11 @@ pub async fn publish_committee_to_registry, - public_key: Vec, + public_key: ArcBytes, public_key_hash: [u8; 32], ) -> Result { let e3_id_u256: U256 = e3_id.try_into()?; - let public_key_bytes = Bytes::from(public_key); + let public_key_bytes = Bytes::from(public_key.extract_bytes()); let public_key_hash_fixed = FixedBytes::from(public_key_hash); let nodes_vec: Vec
= nodes .into_iter() diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index 65c7677fc4..e192883a29 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -246,6 +246,8 @@ impl Handler for EvmChainGateway { #[cfg(test)] mod tests { + use std::time::Duration; + use crate::EvmEvent; use super::*; @@ -349,6 +351,7 @@ mod tests { let full = history_collector.send(TakeEvents::new(5)).await?; let test_events: Vec = full + .events .iter() .filter_map(|e| { if let EnclaveEventData::TestEvent(TestEvent { msg, .. }) = e.get_data() { @@ -364,7 +367,7 @@ mod tests { vec!["Before Complete", "Before SyncEnded", "After SyncEnded"] ); - let event_types: Vec = full.iter().map(|e| e.event_type()).collect(); + let event_types: Vec = full.events.iter().map(|e| e.event_type()).collect(); assert_eq!( event_types, diff --git a/crates/net/src/bin/p2p_test.rs b/crates/net/src/bin/p2p_test.rs index 6621795184..3b8d0b6247 100644 --- a/crates/net/src/bin/p2p_test.rs +++ b/crates/net/src/bin/p2p_test.rs @@ -7,7 +7,7 @@ use anyhow::{Context, Result}; use e3_events::CorrelationId; use e3_net::events::{GossipData, NetCommand, NetEvent}; -use e3_net::{ContentHash, NetInterface}; +use e3_net::{ContentHash, Libp2pKeypair, Libp2pNetInterface, NetInterface}; use e3_utils::ArcBytes; use libp2p::gossipsub::IdentTopic; use std::sync::atomic::{AtomicU8, Ordering}; @@ -221,11 +221,11 @@ impl TestPeer { let topic = IdentTopic::new("test"); let peers: Vec = dial_to.iter().cloned().collect(); - let id = libp2p::identity::Keypair::generate_ed25519(); - - let mut peer = NetInterface::new(&id, peers, udp_port, &topic.to_string())?; - let tx = peer.tx(); - let mut rx = peer.rx(); + let keypair = Libp2pKeypair::generate(); + let mut peer = Libp2pNetInterface::new(keypair, peers, udp_port, &topic.to_string())?; + let handle = peer.handle(); + let tx = handle.tx(); + let mut rx = handle.rx(); tokio::spawn({ let name = name.clone(); diff --git a/crates/net/src/dialer.rs b/crates/net/src/dialer.rs index bae685397a..d28ae911ec 100644 --- a/crates/net/src/dialer.rs +++ b/crates/net/src/dialer.rs @@ -19,7 +19,10 @@ use tracing::trace; use tracing::warn; use crate::events::{NetCommand, NetEvent}; -use e3_utils::{retry_with_backoff, to_retry, RetryError, BACKOFF_DELAY, BACKOFF_MAX_RETRIES}; +use e3_utils::{retry_with_backoff, to_retry, OnceTake, RetryError}; + +const DIAL_DELAY: u64 = 3000; +const DIAL_RETRIES: u32 = 10; /// Dial a single Multiaddr with retries and return an error should those retries not work async fn dial_multiaddr( @@ -31,8 +34,8 @@ async fn dial_multiaddr( info!("Now dialing in to {}", multiaddr); retry_with_backoff( || attempt_connection(cmd_tx, event_tx, multiaddr), - BACKOFF_MAX_RETRIES, - BACKOFF_DELAY, + DIAL_RETRIES, + DIAL_DELAY, ) .await?; Ok(()) @@ -79,7 +82,7 @@ async fn attempt_connection( dial_connection ); cmd_tx - .send(NetCommand::Dial(opts)) + .send(NetCommand::Dial(OnceTake::new(opts))) .await .map_err(to_retry)?; wait_for_connection(&mut event_rx, dial_connection).await diff --git a/crates/net/src/direct_requester.rs b/crates/net/src/direct_requester.rs new file mode 100644 index 0000000000..2c099659f1 --- /dev/null +++ b/crates/net/src/direct_requester.rs @@ -0,0 +1,569 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use std::{fmt, marker::PhantomData, sync::Arc, time::Duration}; + +use anyhow::{anyhow, Result}; +use e3_events::CorrelationId; +use e3_utils::{retry_with_backoff, to_retry}; +use tokio::sync::{broadcast, mpsc}; + +use crate::events::{ + call_and_await_response, NetCommand, NetEvent, OutgoingRequest, OutgoingRequestFailed, + OutgoingRequestSucceeded, PeerTarget, ProtocolResponse, +}; + +pub trait DirectRequesterOutput: TryFrom> + Send + Sync + 'static {} + +pub trait DirectRequesterInput: + TryInto> + Clone + Send + Sync + fmt::Debug + 'static +{ +} + +impl DirectRequesterOutput for T where T: TryFrom> + Send + Sync + 'static {} + +impl DirectRequesterInput for T where + T: TryInto> + Clone + Send + Sync + fmt::Debug + 'static +{ +} + +pub struct WithoutPeer; +pub struct WithPeer(PeerTarget); + +/// DirectRequester is used to send direct requests to a specific peer. +/// +/// # Example +/// +/// ```ignore +/// use crate::net::direct_requester::DirectRequester; +/// use crate::events::PeerTarget; +/// use libp2p::PeerId; +/// +/// // Create a requester with default settings +/// let requester = DirectRequester::builder(net_cmds, net_events) +/// .request_timeout(Duration::from_secs(30)) +/// .max_retries(4) +/// .build(); +/// +/// // Target a specific peer (use Random to pick a random peer) +/// let peer_id: PeerId = "12D3KooWEZiPVmEZkwCFEWYxPL6xts6LnPHRFqsSEDGmt1vQ17By".parse().unwrap(); +/// let peer_requester = requester.to(PeerTarget::Specific(peer_id)); +/// +/// // Make a request (any type implementing TryInto> and TryFrom> works) +/// let response: MyResponse = peer_requester.request(my_request).await?; +/// ``` +pub struct DirectRequester { + net_cmds: mpsc::Sender, + net_events: Arc>, + request_timeout: Duration, + max_retries: u32, + retry_timeout: Duration, + peer: PeerTarget, + _state: PhantomData, +} + +impl DirectRequester { + /// Creates a new DirectRequester builder. + /// + /// Default settings: + /// - request_timeout: 30 seconds + /// - max_retries: 4 + /// - retry_timeout: 5 seconds + pub fn builder( + net_cmds: mpsc::Sender, + net_events: Arc>, + ) -> DirectRequesterBuilder { + DirectRequesterBuilder { + net_cmds: Some(net_cmds), + net_events: Some(net_events), + request_timeout: Some(Duration::from_secs(30)), + max_retries: Some(4), + retry_timeout: Some(Duration::from_millis(5000)), + } + } + + /// Sets the target peer for requests. + /// + /// Use `PeerTarget::Random` to send to a random peer, or + /// `PeerTarget::Specific(peer_id)` to target a specific peer. + pub fn to(&self, peer: PeerTarget) -> DirectRequester { + DirectRequester { + net_cmds: self.net_cmds.clone(), + net_events: self.net_events.clone(), + request_timeout: self.request_timeout, + max_retries: self.max_retries, + retry_timeout: self.retry_timeout, + peer, + _state: PhantomData, + } + } +} + +impl DirectRequester { + /// Sends a direct request to the peer and waits for a response. + /// + /// The request type must implement `TryInto>` with `Clone + Send + Sync + Debug`. + /// The response type must implement `TryFrom>`. + /// + /// # Errors + /// + /// Returns an error if request serialization fails, the peer responds with an error, + /// or if the request times out after retries. + pub async fn request(&self, request: R) -> Result + where + T: DirectRequesterOutput, + R: DirectRequesterInput, + { + let payload: Vec = request + .clone() + .try_into() + .map_err(|_| anyhow!("Request serialization failed for request: {:?}", request))?; + + let response = self.request_with_retry(payload).await?; + + match response { + ProtocolResponse::Ok(data) => Ok(data + .try_into() + .map_err(|_| anyhow!("Could not deserialize ProtocolResponse"))?), + ProtocolResponse::BadRequest(msg) => Err(anyhow!("BadRequest: {}", msg)), + ProtocolResponse::Error(msg) => Err(anyhow!("ProtocolError: {}", msg)), + } + } + + async fn request_with_retry(&self, payload: Vec) -> Result { + let request_timeout = self.request_timeout; + let peer = self.peer; + retry_with_backoff( + || { + let net_cmds = self.net_cmds.clone(); + let net_events = self.net_events.clone(); + let payload = payload.clone(); + let request_timeout = request_timeout; + async move { + do_request(net_cmds, net_events, peer, payload, request_timeout) + .await + .map_err(to_retry) + } + }, + self.max_retries, + self.retry_timeout.as_millis() as u64, + ) + .await + } +} + +pub struct DirectRequesterBuilder { + net_cmds: Option>, + net_events: Option>>, + request_timeout: Option, + max_retries: Option, + retry_timeout: Option, +} + +impl DirectRequesterBuilder { + /// Sets the timeout for each request attempt. + pub fn request_timeout(mut self, request_timeout: Duration) -> Self { + self.request_timeout = Some(request_timeout); + self + } + + /// Sets the maximum number of retry attempts. + pub fn max_retries(mut self, max_retries: u32) -> Self { + self.max_retries = Some(max_retries); + self + } + + /// Sets the timeout between retry attempts. + pub fn retry_timeout(mut self, retry_timeout: Duration) -> Self { + self.retry_timeout = Some(retry_timeout); + self + } + + pub fn build(self) -> DirectRequester { + DirectRequester { + net_cmds: self.net_cmds.expect("net_cmds is required"), + net_events: self.net_events.expect("net_events is required"), + request_timeout: self.request_timeout.unwrap_or(Duration::from_secs(30)), + max_retries: self.max_retries.unwrap_or(4), + retry_timeout: self.retry_timeout.unwrap_or(Duration::from_millis(5000)), + peer: PeerTarget::Random, + _state: PhantomData, + } + } +} + +async fn do_request( + net_cmds: mpsc::Sender, + net_events: Arc>, + target: PeerTarget, + payload: Vec, + timeout: Duration, +) -> Result { + let correlation_id = CorrelationId::new(); + + let response = call_and_await_response( + net_cmds, + net_events, + NetCommand::OutgoingRequest(OutgoingRequest { + correlation_id, + payload, + target, + }), + |e| match e { + NetEvent::OutgoingRequestSucceeded(value) => { + if value.correlation_id == correlation_id { + Some(Ok(value.payload.clone())) + } else { + None + } + } + NetEvent::OutgoingRequestFailed(value) => { + if value.correlation_id == correlation_id { + Some(Err(anyhow!("Request failed: {}", value.error))) + } else { + None + } + } + _ => None, + }, + timeout, + ) + .await + .map_err(|e| anyhow!("Request failed: {}", e))?; + + Ok(response) +} + +struct Expectation { + expected_request: Vec, + response: Result, String>, +} + +pub(crate) struct DirectRequesterTester { + net_cmds_rx: mpsc::Receiver, + net_events_tx: broadcast::Sender, + respond_with: Option>, + responses: Vec>, + expectations: Vec, + error_on: Option, + num_requests: Option, +} + +pub(crate) struct ExpectationBuilder { + tester: DirectRequesterTester, + expected_request: Vec, +} + +impl ExpectationBuilder { + pub fn respond_with>>(mut self, payload: T) -> DirectRequesterTester + where + >>::Error: std::fmt::Debug, + { + self.tester.expectations.push(Expectation { + expected_request: self.expected_request, + response: Ok(payload.try_into().unwrap()), + }); + self.tester + } + + pub fn error_with(mut self, error: impl Into) -> DirectRequesterTester { + self.tester.expectations.push(Expectation { + expected_request: self.expected_request, + response: Err(error.into()), + }); + self.tester + } +} + +impl DirectRequesterTester { + pub fn new( + net_cmds_rx: mpsc::Receiver, + net_events_tx: broadcast::Sender, + ) -> Self { + Self { + net_cmds_rx, + net_events_tx, + respond_with: None, + responses: Vec::new(), + expectations: Vec::new(), + error_on: None, + num_requests: None, + } + } + + pub fn expect_request>>(self, payload: T) -> ExpectationBuilder + where + >>::Error: std::fmt::Debug, + { + ExpectationBuilder { + tester: self, + expected_request: payload.try_into().unwrap(), + } + } + + pub fn respond_with>>(mut self, payload: T) -> Self + where + >>::Error: std::fmt::Debug, + { + self.respond_with = Some(payload.try_into().unwrap()); + self + } + + pub fn respond_with_each>>( + mut self, + payloads: impl IntoIterator, + ) -> Self + where + >>::Error: std::fmt::Debug, + { + self.responses = payloads + .into_iter() + .map(|p| p.try_into().unwrap()) + .collect(); + self + } + + pub fn error_with(mut self, error: impl Into) -> Self { + self.error_on = Some(error.into()); + self + } + + pub fn num_requests(mut self, n: usize) -> Self { + self.num_requests = Some(n); + self + } + + pub fn spawn(mut self) -> tokio::task::JoinHandle<()> { + let num_requests = self.num_requests.unwrap_or_else(|| { + if !self.expectations.is_empty() { + self.expectations.len() + } else { + usize::MAX + } + }); + // Reverse expectations so we can pop from the back in order. + self.expectations.reverse(); + self.responses.reverse(); + + tokio::spawn(async move { + let mut remaining = num_requests; + while remaining > 0 { + if let Some(cmd) = self.net_cmds_rx.recv().await { + if let NetCommand::OutgoingRequest(req) = cmd { + remaining -= 1; + let response = if let Some(expectation) = self.expectations.pop() { + assert_eq!( + req.payload, expectation.expected_request, + "DirectRequesterTester: expected request {:?} but got {:?}", + expectation.expected_request, req.payload, + ); + match expectation.response { + Ok(payload) => { + NetEvent::OutgoingRequestSucceeded(OutgoingRequestSucceeded { + payload: ProtocolResponse::Ok(payload), + correlation_id: req.correlation_id, + }) + } + Err(error) => { + NetEvent::OutgoingRequestFailed(OutgoingRequestFailed { + error, + correlation_id: req.correlation_id, + }) + } + } + } else if let Some(payload) = self.respond_with.clone() { + NetEvent::OutgoingRequestSucceeded(OutgoingRequestSucceeded { + payload: ProtocolResponse::Ok(payload), + correlation_id: req.correlation_id, + }) + } else if let Some(payload) = self.responses.pop() { + NetEvent::OutgoingRequestSucceeded(OutgoingRequestSucceeded { + payload: ProtocolResponse::Ok(payload), + correlation_id: req.correlation_id, + }) + } else if let Some(error) = self.error_on.clone() { + NetEvent::OutgoingRequestFailed(OutgoingRequestFailed { + error, + correlation_id: req.correlation_id, + }) + } else { + panic!("DirectRequesterTester: no response configured"); + }; + let _ = self.net_events_tx.send(response); + } + } else { + break; + } + } + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::events::PeerTarget; + use tokio::sync::broadcast; + + #[tokio::test] + async fn test_successful_request() { + let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); + let (net_events_tx, net_events_rx) = broadcast::channel::(16); + let net_events = Arc::new(net_events_rx); + + let requester = DirectRequester::builder(net_cmds_tx, net_events).build(); + + let handle = DirectRequesterTester::new(net_cmds_rx, net_events_tx) + .respond_with(b"world".to_vec()) + .num_requests(1) + .spawn(); + + let response: Vec = requester + .to(PeerTarget::Random) + .request(b"hello".to_vec()) + .await + .unwrap(); + + handle.await.unwrap(); + + assert_eq!(response, b"world"); + } + + #[tokio::test] + async fn test_request_with_peer_target() { + let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); + let (net_events_tx, net_events_rx) = broadcast::channel::(16); + let net_events = Arc::new(net_events_rx); + + let requester = DirectRequester::builder(net_cmds_tx, net_events).build(); + + let handle = DirectRequesterTester::new(net_cmds_rx, net_events_tx) + .respond_with(b"pong".to_vec()) + .num_requests(1) + .spawn(); + + let _: Vec = requester + .to(PeerTarget::Random) + .request(b"ping".to_vec()) + .await + .unwrap(); + + handle.await.unwrap(); + } + + #[tokio::test] + async fn test_peer_requester_reuse_across_requests() { + let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); + let (net_events_tx, net_events_rx) = broadcast::channel::(16); + let net_events = Arc::new(net_events_rx); + + let requester = DirectRequester::builder(net_cmds_tx, net_events) + .request_timeout(Duration::from_secs(10)) + .max_retries(3) + .retry_timeout(Duration::from_secs(5)) + .build(); + + let peer_requester = requester.to(PeerTarget::Random); + + let handle = DirectRequesterTester::new(net_cmds_rx, net_events_tx) + .respond_with(b"ok".to_vec()) + .num_requests(2) + .spawn(); + + let response1: Vec = peer_requester.request(b"first".to_vec()).await.unwrap(); + let response2: Vec = peer_requester.request(b"second".to_vec()).await.unwrap(); + + handle.await.unwrap(); + + assert_eq!(response1, b"ok"); + assert_eq!(response2, b"ok"); + } + + #[tokio::test] + async fn test_expect_request() { + let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); + let (net_events_tx, net_events_rx) = broadcast::channel::(16); + let net_events = Arc::new(net_events_rx); + + let requester = DirectRequester::builder(net_cmds_tx, net_events).build(); + + let handle = DirectRequesterTester::new(net_cmds_rx, net_events_tx) + .expect_request(b"hello".to_vec()) + .respond_with(b"world".to_vec()) + .expect_request(b"ping".to_vec()) + .respond_with(b"pong".to_vec()) + .spawn(); + + let peer = requester.to(PeerTarget::Random); + + let r1: Vec = peer.request(b"hello".to_vec()).await.unwrap(); + let r2: Vec = peer.request(b"ping".to_vec()).await.unwrap(); + + handle.await.unwrap(); + + assert_eq!(r1, b"world"); + assert_eq!(r2, b"pong"); + } + + #[tokio::test] + async fn test_request_failure() { + let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); + let (net_events_tx, net_events_rx) = broadcast::channel::(16); + let net_events = Arc::new(net_events_rx); + + let requester = DirectRequester::builder(net_cmds_tx, net_events) + .max_retries(0) + .build(); + + let handle = DirectRequesterTester::new(net_cmds_rx, net_events_tx) + .error_with("connection refused") + .num_requests(1) + .spawn(); + + let result: std::result::Result, _> = requester + .to(PeerTarget::Random) + .request(b"hello".to_vec()) + .await; + + handle.await.unwrap(); + + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("connection refused")); + } + + #[tokio::test] + async fn test_respond_with_each() { + let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); + let (net_events_tx, net_events_rx) = broadcast::channel::(16); + let net_events = Arc::new(net_events_rx); + + let requester = DirectRequester::builder(net_cmds_tx, net_events).build(); + + let handle = DirectRequesterTester::new(net_cmds_rx, net_events_tx) + .respond_with_each(vec![ + b"first_response".to_vec(), + b"second_response".to_vec(), + b"third_response".to_vec(), + ]) + .num_requests(3) + .spawn(); + + let peer = requester.to(PeerTarget::Random); + + let r1: Vec = peer.request(b"req1".to_vec()).await.unwrap(); + let r2: Vec = peer.request(b"req2".to_vec()).await.unwrap(); + let r3: Vec = peer.request(b"req3".to_vec()).await.unwrap(); + + handle.await.unwrap(); + + assert_eq!(r1, b"first_response"); + assert_eq!(r2, b"second_response"); + assert_eq!(r3, b"third_response"); + } +} diff --git a/crates/net/src/direct_responder.rs b/crates/net/src/direct_responder.rs new file mode 100644 index 0000000000..f45fe2095a --- /dev/null +++ b/crates/net/src/direct_responder.rs @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::events::{IncomingResponse, NetCommand, ProtocolResponse}; +use anyhow::{anyhow, Context, Result}; +use e3_utils::OnceTake; +use libp2p::request_response::{InboundRequestId, ResponseChannel}; +use tokio::sync::mpsc; + +/// Helper trait to extract id from libp2p things like InboundRequestId +pub trait IntoId { + fn into_id(self) -> u64; +} + +impl IntoId for u64 { + fn into_id(self) -> u64 { + self + } +} + +impl IntoId for InboundRequestId { + fn into_id(self) -> u64 { + format!("{:?}", self) + .chars() + .filter(|c| c.is_ascii_digit()) + .collect::() + .parse::() + .expect("Failed to extract u64 from InboundRequestId") + } +} + +#[derive(Debug)] +pub enum ChannelType { + Test(String), // For testing + Channel(ResponseChannel), // actual libp2p response channel +} + +#[derive(Debug)] +/// DirectResponder is used to respond to incoming libp2p requests. +/// +/// # Example +/// +/// ``` +/// # use tokio::sync::mpsc; +/// use e3_net::direct_responder::DirectResponder; +/// # use e3_net::direct_responder::ChannelType; +/// # fn main() -> anyhow::Result<()> { +/// # let request_id = 6; +/// # let channel_orig = ChannelType::Test("channel".to_string()); +/// # let channel = ChannelType::Test("channel".to_string()); +/// # let (cmd_tx, _rx) = mpsc::channel(400); +/// +/// // We create a responder and send it over our event channel +/// let responder = DirectResponder::new( +/// // request_id comes from libp2p anything that looks like a u64 will work +/// request_id, +/// // Likely ResponseChannel from libp2p event but does not matter will just get passed on +/// channel, +/// // Our NetCommand channel Sender +/// &cmd_tx +/// ); +/// +/// // Now in our handlers we can respond with ok() or bad_request() this will consume the responder +/// responder.ok(String::from("Something that implements TryInto>"))?; +/// # let responder = DirectResponder::new(request_id, channel_orig, &cmd_tx); +/// // or +/// responder.bad_request("It was pretty bad.")?; +/// # Ok(()) +/// # } +/// ``` +pub struct DirectResponder { + id: u64, + request: Vec, + response: Option, + channel: OnceTake, + net_cmds: mpsc::Sender, +} +impl Clone for DirectResponder { + fn clone(&self) -> Self { + Self { + id: self.id.clone(), + request: self.request.clone(), + response: self.response.clone(), + channel: self.channel.clone(), + net_cmds: self.net_cmds.clone(), + } + } +} + +impl DirectResponder { + /// Creates a new responder for an incoming request. + /// + /// * `id` - is the request identifier used for debugging (e.g., `InboundRequestId` or `u64`). + /// * `channel` - is usually the response channel provided by libp2p but can be anything that is passed along with the response + /// * `net_cmds` - sender is used to send the response back to the net interface. + pub fn new(id: impl IntoId, channel: ChannelType, net_cmds: &mpsc::Sender) -> Self { + Self { + id: id.into_id(), + request: Vec::new(), + response: None, + channel: OnceTake::new(channel), + net_cmds: net_cmds.clone(), + } + } + + /// Sets the request data on the responder. + /// + /// This should be called when creating a responder for an incoming request, + /// passing the raw request bytes. + pub fn with_request(mut self, request: Vec) -> Self { + self.request = request; + self + } + + /// Get the request data + pub fn request(&self) -> Vec { + self.request.clone() + } + + /// Get the request data + pub fn try_request_into(&self) -> Result + where + T: TryFrom>, + { + self.request + .clone() + .try_into() + .map_err(|_| anyhow!("Could not serialize request bytes")) + } + + /// Extract the payload information to send to swarm + pub fn to_response(mut self) -> Result<(ChannelType, ProtocolResponse)> { + let channel = self.channel.try_take()?; + let response = self + .response + .take() + .context("No response found on responder")?; + Ok((channel, response)) + } + + /// Consumes self and responds + pub fn respond(mut self, value: ProtocolResponse) -> Result<()> { + let response = value; + self.response = Some(response); + let cmds = self.net_cmds.clone(); + let incoming = IncomingResponse::new(self); + Ok(cmds + .clone() + .try_send(NetCommand::IncomingResponse(incoming)) + .map_err(|e| anyhow!("Failed to send response command {:?}", e))?) + } + + /// Request is ok returning response + pub fn ok>>(self, data: T) -> Result<()> { + let bytes: Vec = data + .try_into() + .map_err(|_| anyhow!("Could not serialize response."))?; + self.respond(ProtocolResponse::Ok(bytes)) + } + + /// Return a bad request + pub fn bad_request(self, reason: impl Into) -> Result<()> { + self.respond(ProtocolResponse::BadRequest(reason.into())) + } + + /// Get the id (for logging purposes) + pub fn id(&self) -> u64 { + self.id + } +} + +#[cfg(test)] +mod tests { + use super::*; + use anyhow::bail; + use tokio::sync::mpsc; + + fn make_responder() -> (DirectResponder, mpsc::Receiver) { + let (tx, rx) = mpsc::channel::(16); + let responder = + DirectResponder::new(42u64, ChannelType::Test("test_channel".to_string()), &tx); + (responder, rx) + } + + fn extract_response(rx: &mut mpsc::Receiver) -> Result<(String, ProtocolResponse)> { + let cmd = rx.try_recv().unwrap(); + match cmd { + NetCommand::IncomingResponse(incoming) => { + let (channel, response) = incoming.responder.to_response().unwrap(); + let ChannelType::Test(channel) = channel else { + bail!("bad channel"); + }; + Ok((channel, response)) + } + + other => panic!("Expected IncomingResponse, got {:?}", other), + } + } + + #[test] + fn to_response_fails_without_response_set() { + let (responder, _rx) = make_responder(); + assert!(responder.to_response().is_err()); + } + + #[test] + fn channel_can_only_be_taken_once() { + let (mut responder, _rx) = make_responder(); + responder.response = Some(ProtocolResponse::Ok(Vec::new())); + let cloned = responder.clone(); + let _ = responder.to_response().unwrap(); + assert!(cloned.to_response().is_err()); + } + + #[test] + fn ok_sends_serialized_payload() { + let (responder, mut rx) = make_responder(); + responder.ok(b"foo".to_vec()).unwrap(); + let (channel, response) = extract_response(&mut rx).unwrap(); + assert_eq!(channel, "test_channel"); + assert!(matches!(response, ProtocolResponse::Ok(v) if v == b"foo")); + } + + #[test] + fn respond_sends_bad_request() { + let (responder, mut rx) = make_responder(); + responder.bad_request("bad").unwrap(); + let (channel, response) = extract_response(&mut rx).unwrap(); + assert_eq!(channel, "test_channel"); + assert!(matches!(response, ProtocolResponse::BadRequest(r) if r == "bad")); + } + + #[test] + fn respond_fails_when_receiver_dropped() { + let (responder, rx) = make_responder(); + drop(rx); + assert!(responder.respond(ProtocolResponse::Ok(vec![])).is_err()); + } +} diff --git a/crates/net/src/document_publisher.rs b/crates/net/src/document_publisher.rs index b2ce0d5451..7f848f7d98 100644 --- a/crates/net/src/document_publisher.rs +++ b/crates/net/src/document_publisher.rs @@ -40,7 +40,7 @@ use tracing::{debug, info}; const KADEMLIA_PUT_TIMEOUT: Duration = Duration::from_secs(30); const KADEMLIA_GET_TIMEOUT: Duration = Duration::from_secs(30); const KADEMLIA_BROADCAST_TIMEOUT: Duration = Duration::from_secs(30); -/// DocumentPublisher is an actor that monitors events from both the NetInterface and the Enclave +/// DocumentPublisher is an actor that monitors events from both the Libp2pNetInterface and the Enclave /// EventBus in order to manage document publishing interactions. In particular this involves the /// interactions of publishing a document and listening for notifications, determining if the node /// is interested in a specific document and fetching the document to broadcast on the local event @@ -48,9 +48,9 @@ const KADEMLIA_BROADCAST_TIMEOUT: Duration = Duration::from_secs(30); pub struct DocumentPublisher { /// Enclave EventBus bus: BusHandle, - /// NetCommand sender to forward commands to the NetInterface + /// NetCommand sender to forward commands to the Libp2pNetInterface tx: mpsc::Sender, - /// NetEvent receiver to resubscribe for events from the NetInterface. This is in an Arc so + /// NetEvent receiver to resubscribe for events from the Libp2pNetInterface. This is in an Arc so /// that we do not do excessive resubscribes without actually listening for events. rx: Arc>, /// The gossipsub broadcast topic @@ -333,7 +333,7 @@ pub async fn handle_document_published_notification( Ok(()) } -/// Call DhtPutRecord Command on the NetInterface and handle the results +/// Call DhtPutRecord Command on the Libp2pNetInterface and handle the results async fn put_record( net_cmds: mpsc::Sender, net_events: Arc>, @@ -363,7 +363,7 @@ async fn put_record( .await } -/// Call DhtPutRecord Command on the NetInterface and handle the results +/// Call DhtGetRecord Command on the Libp2pNetInterface and handle the results async fn get_record( net_cmds: mpsc::Sender, net_events: Arc>, @@ -389,7 +389,7 @@ async fn get_record( .await } -/// Broadcasts document published notification on NetInterface +/// Broadcasts document published notification on Libp2pNetInterface async fn broadcast_document_published_notification( net_cmds: mpsc::Sender, net_events: Arc>, @@ -507,6 +507,7 @@ impl EventConverter { Ok(()) } + fn handle_encryption_key_created(&self, msg: TypedEvent) -> Result<()> { let (msg, ctx) = msg.into_components(); if msg.external { @@ -535,6 +536,7 @@ impl EventConverter { Ok(()) } + // TODO: Split this off to a separate module/actor to make each component unidirectional /// Convert received document to internal events. /// Note: Filtering already happened in DocumentPublisher before DHT fetch. fn handle_document_received(&self, msg: TypedEvent) -> Result<()> { @@ -745,7 +747,7 @@ mod tests { value: value.clone(), })?; - // 2. Document publisher should have asked the NetInterface to put the doc on Kademlia + // 2. Document publisher should have asked the Libp2pNetInterface to put the doc on Kademlia let Some(NetCommand::DhtPutRecord { correlation_id, expires, @@ -850,7 +852,7 @@ mod tests { // wait for events to settle let errors = errors.send(TakeEvents::new(1)).await?; - let error: EnclaveError = errors.first().unwrap().try_into()?; + let error: EnclaveError = errors.events.first().unwrap().try_into()?; assert_eq!( error.message, "Operation failed after 4 attempts. Last error: DHT get record failed: Timeout { key: Key(b\"\\xda-\\xe1\\xc0T\\x11$X\\x05\\xd1\\xd4\\xa6C\\x86\\x96\\xb7e\\xd9j\\x96\\x1bD\\xc8P#\\x0f\\\"\\xea A@b\") }" @@ -907,7 +909,7 @@ mod tests { // Expect error to exist let errors = errors.send(TakeEvents::new(1)).await?; - let error: EnclaveError = errors.first().unwrap().try_into()?; + let error: EnclaveError = errors.events.first().unwrap().try_into()?; assert_eq!( error.message, "Operation failed after 4 attempts. Last error: DHT put record failed: PutRecordError(QuorumFailed { key: Key(b\"I got the secret\"), success: [], quorum: 1 })" @@ -934,7 +936,7 @@ mod tests { ..CiphernodeSelected::default() })?; - // 2. Dispatch a NetEvent from the NetInterface signaling that a document was published + // 2. Dispatch a NetEvent from the Libp2pNetInterface signaling that a document was published net_evt_tx.send(NetEvent::GossipData( GossipData::DocumentPublishedNotification(DocumentPublishedNotification { key: ContentHash::from_content(&b"wrong document".to_vec()), @@ -952,7 +954,7 @@ mod tests { let result = timeout(Duration::from_secs(1), net_cmd_rx.recv()).await; assert!(result.is_err(), "Expected timeout but received a message"); - // 4. Dispatch a NetEvent from the NetInterface signaling that a document we ARE interested + // 4. Dispatch a NetEvent from the Libp2pNetInterface signaling that a document we ARE interested // in was published net_evt_tx.send(NetEvent::GossipData( GossipData::DocumentPublishedNotification(DocumentPublishedNotification { diff --git a/crates/net/src/events.rs b/crates/net/src/events.rs index c172f1f595..4083c2b0d4 100644 --- a/crates/net/src/events.rs +++ b/crates/net/src/events.rs @@ -4,18 +4,18 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::ContentHash; +use crate::{direct_responder::DirectResponder, ContentHash}; use actix::Message; -use anyhow::{bail, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use e3_events::{ - AggregateId, CorrelationId, DocumentMeta, EnclaveEvent, EventContextAccessors, EventSource, - Sequenced, Unsequenced, + CorrelationId, DocumentMeta, EnclaveEvent, EventContextAccessors, EventSource, Sequenced, + Unsequenced, }; use e3_utils::{ArcBytes, OnceTake}; use libp2p::{ gossipsub::{MessageId, PublishError, TopicHash}, kad::{store, GetRecordError, PutRecordError}, - request_response::{InboundRequestId, ResponseChannel}, + request_response::ResponseChannel, swarm::{dial_opts::DialOpts, ConnectionId, DialError}, }; use serde::{Deserialize, Serialize}; @@ -28,6 +28,14 @@ use std::{ use tokio::sync::{broadcast, mpsc}; use tracing::{error, trace, warn}; +use libp2p::PeerId; + +#[derive(Clone, Copy, Debug)] +pub enum PeerTarget { + Random, + Specific(PeerId), +} + /// Incoming/Outgoing GossipData. We disambiguate on concerns relative to the net package. #[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] pub enum GossipData { @@ -68,39 +76,84 @@ impl TryFrom for EnclaveEvent { } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct SyncRequestValue { - pub since: HashMap, +pub enum ProtocolResponse { + Ok(Vec), + BadRequest(String), + Error(String), } -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct SyncResponseValue { - pub events: Vec, - pub ts: u128, -} +pub type ProtocolResponseChannel = ResponseChannel; #[derive(Message, Clone, Debug)] #[rtype("()")] -pub struct SyncRequestReceived { - pub request_id: InboundRequestId, - pub value: SyncRequestValue, - pub channel: OnceTake>, +/// Remote has sent us a request +pub struct IncomingRequest { + pub responder: DirectResponder, +} + +#[derive(Clone, Debug)] +/// We are responding to a remote request +pub struct IncomingResponse { + pub responder: DirectResponder, +} + +impl IncomingResponse { + pub fn new(responder: DirectResponder) -> Self { + Self { responder } + } +} + +#[derive(Debug, Clone)] +pub struct OutgoingRequest { + pub correlation_id: CorrelationId, + pub payload: Vec, + pub target: PeerTarget, +} + +impl OutgoingRequest { + pub fn new_with_correlation( + id: CorrelationId, + target: PeerTarget, + payload: impl TryInto>, + ) -> Result { + Ok(Self { + correlation_id: id, + payload: payload.try_into().map_err(|_| { + anyhow!( + "could not serialize payload for outgoing request with correlation_id={id} and target={target:?}." + ) + })?, + target, + }) + } + pub fn to_random_peer(payload: impl TryInto>) -> Result { + Self::new_with_correlation(CorrelationId::new(), PeerTarget::Random, payload) + } + + pub fn new(target: PeerId, payload: impl TryInto>) -> Result { + Self::new_with_correlation(CorrelationId::new(), PeerTarget::Specific(target), payload) + } } #[derive(Message, Clone, Debug)] #[rtype("()")] -pub struct OutgoingSyncRequestSucceeded { - pub value: SyncResponseValue, +pub struct OutgoingRequestSucceeded { + pub payload: ProtocolResponse, pub correlation_id: CorrelationId, } #[derive(Debug, Clone)] -pub struct OutgoingSyncRequestFailed { +pub struct OutgoingRequestFailed { pub correlation_id: CorrelationId, pub error: String, } -/// NetInterface Commands are sent to the network peer over a mspc channel -#[derive(Debug)] +#[derive(Message, Debug, Clone)] +#[rtype("()")] +pub struct AllPeersDialed; + +/// Libp2pNetInterface Commands are sent to the network peer over a mspc channel +#[derive(Debug, Clone)] pub enum NetCommand { /// Publish message to gossipsub GossipPublish { @@ -109,7 +162,7 @@ pub enum NetCommand { correlation_id: CorrelationId, }, /// Dial peer - Dial(DialOpts), + Dial(OnceTake), /// Command to PublishDocument to Kademlia DhtPutRecord { correlation_id: CorrelationId, @@ -123,20 +176,14 @@ pub enum NetCommand { key: ContentHash, }, /// Remove DHT records associated with a completed E3 - DhtRemoveRecords { keys: Vec }, + DhtRemoveRecords { + keys: Vec, + }, /// Shutdown signal Shutdown, - /// Called from the syning node to request libp2p events from a random peer node starting - /// from the given timestamp. - OutgoingSyncRequest { - correlation_id: CorrelationId, - value: SyncRequestValue, - }, - /// Send libp2p events back to a peer that requested a sync. - SyncResponse { - value: SyncResponseValue, - channel: OnceTake>, - }, + /// Send a request to a peer and await response + OutgoingRequest(OutgoingRequest), + IncomingResponse(IncomingResponse), } impl NetCommand { @@ -146,7 +193,7 @@ impl NetCommand { N::DhtPutRecord { correlation_id, .. } => Some(*correlation_id), N::DhtGetRecord { correlation_id, .. } => Some(*correlation_id), N::GossipPublish { correlation_id, .. } => Some(*correlation_id), - N::OutgoingSyncRequest { correlation_id, .. } => Some(*correlation_id), + N::OutgoingRequest(OutgoingRequest { correlation_id, .. }) => Some(*correlation_id), _ => None, } } @@ -207,12 +254,11 @@ pub enum NetEvent { count: usize, topic: TopicHash, }, - /// A peer node is requesting gossipsub events since the given timestamp. - /// Use the provided channel to send a `SyncResponse - SyncRequestReceived(SyncRequestReceived), - /// Received gossipsub events from a peer in response to a `SyncRequest`. - OutgoingSyncRequestSucceeded(OutgoingSyncRequestSucceeded), - OutgoingSyncRequestFailed(OutgoingSyncRequestFailed), + /// A peer made a request to this node + IncomingRequest(IncomingRequest), + /// Received response from a peer in response to an outgoing request + OutgoingRequestSucceeded(OutgoingRequestSucceeded), + OutgoingRequestFailed(OutgoingRequestFailed), AllPeersDialed, } @@ -232,8 +278,8 @@ impl NetEvent { N::DhtGetRecordSucceeded { correlation_id, .. } => Some(*correlation_id), N::DhtPutRecordError { correlation_id, .. } => Some(*correlation_id), N::DhtPutRecordSucceeded { correlation_id, .. } => Some(*correlation_id), - N::OutgoingSyncRequestSucceeded(msg) => Some(msg.correlation_id), - N::OutgoingSyncRequestFailed(msg) => Some(msg.correlation_id), + N::OutgoingRequestSucceeded(msg) => Some(msg.correlation_id), + N::OutgoingRequestFailed(msg) => Some(msg.correlation_id), _ => None, } } @@ -288,7 +334,7 @@ where // We don't have access to this later and we cannot clone command let debug_cmd = format!("{:?}", command); - // Send the command to NetInterface + // Send the command to Libp2pNetInterface trace!( "call_and_await_response: sending command {:?} with timeout {:?}", command, diff --git a/crates/net/src/lib.rs b/crates/net/src/lib.rs index 45f44453c2..ed496bc56a 100644 --- a/crates/net/src/lib.rs +++ b/crates/net/src/lib.rs @@ -7,11 +7,15 @@ mod cid; mod correlator; mod dialer; +pub mod direct_requester; +pub mod direct_responder; mod document_publisher; pub mod events; +mod net_event_batch; mod net_event_buffer; mod net_event_translator; mod net_interface; +mod net_interface_handle; mod net_sync_manager; mod repo; @@ -19,31 +23,25 @@ use std::sync::Arc; use actix::Recipient; use anyhow::bail; +use anyhow::Result; pub use cid::ContentHash; pub use document_publisher::*; use e3_crypto::Cipher; use e3_data::Repository; use e3_events::{run_once, BusHandle, EffectsEnabled, EventStoreQueryBy, EventSubscriber, TsAgg}; -use libp2p::identity::ed25519; use net_event_buffer::NetEventBuffer; pub use net_event_translator::*; pub use net_interface::*; +pub use net_interface_handle::*; use net_sync_manager::NetSyncManager; pub use repo::*; +use tracing::error; use tracing::{info, instrument}; -/// Spawn a Libp2p interface and hook it up to this actor -#[instrument(name = "libp2p", skip_all)] -pub async fn setup_net( - bus: BusHandle, - peers: Vec, - cipher: &Arc, - quic_port: u16, +pub async fn setup_libp2p_keypair( repository: Repository>, - eventstore: impl Into>>, -) -> anyhow::Result<(tokio::task::JoinHandle>, String)> { - let topic = "enclave-gossip"; - + cipher: &Arc, +) -> Result { // Get existing keypair or generate a new one let mut bytes = match repository.read().await? { Some(bytes) => { @@ -52,14 +50,36 @@ pub async fn setup_net( } None => bail!("No network keypair found in repository, please generate a new one using `enclave net generate-key`"), }; + Libp2pKeypair::try_from_bytes(&mut bytes) +} + +pub fn setup_net_interface( + topic: &str, + keypair: Libp2pKeypair, + peers: Vec, + quic_port: u16, +) -> Result { + let mut interface = Libp2pNetInterface::new(keypair, peers, Some(quic_port), topic)?; + + let handle = interface.handle(); - // Create peer from keypair - let keypair: libp2p::identity::Keypair = - ed25519::Keypair::try_from_bytes(&mut bytes)?.try_into()?; + actix::spawn(async move { + if let Err(e) = interface.start().await { + error!("{e}"); + } + }); - // Generate a new interface to read and write peer events to - let mut interface = NetInterface::new(&keypair, peers, Some(quic_port), topic)?; + Ok(handle) +} +/// Spawn a Libp2p interface and hook it up to this actor +#[instrument(name = "libp2p", skip_all)] +pub fn setup_net( + topic: &str, + bus: BusHandle, + eventstore: impl Into>>, + interface: impl NetInterface, +) -> Result<()> { // NOTE: Pass the unbuffered rx to SyncManager as it must operate before live events are // processed let _net_sync = NetSyncManager::setup( @@ -87,8 +107,5 @@ pub async fn setup_net( bus.subscribe(e3_events::EventType::EffectsEnabled, runner.recipient()); - // TODO: actix::spawn might avoid all the cleanup code - let handle = tokio::spawn(async move { Ok(interface.start().await?) }); - - Ok((handle, keypair.public().to_peer_id().to_string())) + Ok(()) } diff --git a/crates/net/src/net_event_batch.rs b/crates/net/src/net_event_batch.rs new file mode 100644 index 0000000000..b1b934c02c --- /dev/null +++ b/crates/net/src/net_event_batch.rs @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use std::fmt::Debug; + +use anyhow::{Context, Result}; +use e3_events::AggregateId; +use tracing::info; + +use crate::{ + direct_requester::{DirectRequester, WithPeer, WithoutPeer}, + events::PeerTarget, +}; + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +pub enum BatchCursor { + Done, + Next(u128), +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +pub struct EventBatch { + pub events: Vec, + pub next: BatchCursor, + pub aggregate_id: AggregateId, +} + +impl TryFrom> for EventBatch +where + E: serde::de::DeserializeOwned, +{ + type Error = anyhow::Error; + + fn try_from(value: Vec) -> Result { + bincode::deserialize(&value).context("failed to deserialize EventBatch") + } +} + +impl TryFrom> for Vec +where + E: serde::Serialize, +{ + type Error = anyhow::Error; + + fn try_from(value: EventBatch) -> Result { + bincode::serialize(&value).context("failed to serialize EventBatch") + } +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +pub struct FetchEventsSince { + aggregate_id: AggregateId, + since: u128, + limit: usize, +} + +impl FetchEventsSince { + pub fn new(aggregate_id: AggregateId, since: u128, limit: usize) -> Self { + Self { + aggregate_id, + since, + limit, + } + } + + pub fn aggregate_id(&self) -> AggregateId { + self.aggregate_id + } + + pub fn since(&self) -> u128 { + self.since + } + + pub fn limit(&self) -> usize { + self.limit + } +} + +impl TryFrom for Vec { + type Error = anyhow::Error; + + fn try_from(value: FetchEventsSince) -> Result { + bincode::serialize(&value).context("failed to serialize FetchEventsSince") + } +} + +impl TryFrom> for FetchEventsSince { + type Error = anyhow::Error; + + fn try_from(value: Vec) -> Result { + bincode::deserialize(&value).context("failed to deserialize FetchEventsSince") + } +} + +pub async fn fetch_events_since( + requester: &DirectRequester, + request: FetchEventsSince, +) -> Result> +where + E: TryFrom> + Send + Sync + 'static, + EventBatch: TryFrom>, +{ + requester.request(request).await +} + +pub async fn fetch_all_batched_events( + requester: DirectRequester, + peer: PeerTarget, + aggregate_id: AggregateId, + since: u128, + batch_size: usize, +) -> Result> +where + E: TryFrom> + Send + Sync + 'static, + EventBatch: TryFrom>, +{ + let requester = requester.to(peer); + let mut all_events = Vec::new(); + let mut cursor = since; + + loop { + let request = FetchEventsSince::new(aggregate_id, cursor, batch_size); + info!( + "Fetching batch aggregate={} cursor={} batch_size={}", + aggregate_id, cursor, batch_size + ); + let batch = fetch_events_since(&requester, request).await?; + info!( + "Batch received with {} events for aggregate={} cursor={}", + batch.events.len(), + aggregate_id, + cursor + ); + + all_events.extend(batch.events); + + match batch.next { + BatchCursor::Done => break, + BatchCursor::Next(next_cursor) => cursor = next_cursor, + } + } + + info!("Batch is done returning {} events", all_events.len()); + + Ok(all_events) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::direct_requester::DirectRequesterTester; + use crate::events::{NetCommand, NetEvent, PeerTarget}; + use std::sync::Arc; + use tokio::sync::{broadcast, mpsc}; + + #[tokio::test] + async fn test_fetch_all_batched_events() { + let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); + let (net_events_tx, net_events_rx) = broadcast::channel::(16); + let net_events = Arc::new(net_events_rx); + + let requester = DirectRequester::builder(net_cmds_tx, net_events).build(); + + let batch1 = EventBatch { + events: vec![b"event1".to_vec(), b"event2".to_vec()], + next: BatchCursor::Next(100), + aggregate_id: AggregateId::new(1), + }; + let batch2 = EventBatch { + events: vec![b"event3".to_vec()], + next: BatchCursor::Done, + aggregate_id: AggregateId::new(1), + }; + + let handle = DirectRequesterTester::new(net_cmds_rx, net_events_tx) + .expect_request(FetchEventsSince::new(AggregateId::new(1), 0, 100)) + .respond_with(batch1) + .expect_request(FetchEventsSince::new(AggregateId::new(1), 100, 100)) + .respond_with(batch2) + .spawn(); + + let events: Vec> = + fetch_all_batched_events(requester, PeerTarget::Random, AggregateId::new(1), 0, 100) + .await + .unwrap(); + + handle.await.unwrap(); + + assert_eq!( + events, + vec![b"event1".to_vec(), b"event2".to_vec(), b"event3".to_vec(),] + ); + } +} diff --git a/crates/net/src/net_event_translator.rs b/crates/net/src/net_event_translator.rs index 709d68e410..cb08ad5662 100644 --- a/crates/net/src/net_event_translator.rs +++ b/crates/net/src/net_event_translator.rs @@ -11,19 +11,19 @@ use anyhow::Result; use bloom::{BloomFilter, ASMS}; use e3_events::{ prelude::*, trap, BusHandle, CorrelationId, EType, EnclaveEvent, EnclaveEventData, Event, - EventContextAccessors, EventSource, EventType, Unsequenced, + EventContextAccessors, EventSource, EventType, NetReady, Unsequenced, }; use e3_utils::MAILBOX_LIMIT; use std::sync::Arc; use tokio::sync::broadcast; use tokio::sync::mpsc; -use tracing::{trace, warn}; +use tracing::{info, trace, warn}; // TODO: store event filtering here on this actor instead of is_local_only() on the event. We // should do this as this functionality is not global and ramifications should stay local to here /// NetEventTranslator Actor converts between EventBus events and Libp2p events forwarding them to a -/// NetInterface for propagation over the p2p network +/// Libp2pNetInterface for propagation over the p2p network pub struct NetEventTranslator { bus: BusHandle, tx: mpsc::Sender, @@ -38,7 +38,7 @@ impl Actor for NetEventTranslator { } } -/// Libp2pEvent is used to send data to the NetInterface from the NetEventTranslator +/// Libp2pEvent is used to send data to the Libp2pNetInterface from the NetEventTranslator #[derive(Message, Clone, Debug, PartialEq, Eq)] #[rtype(result = "()")] struct LibP2pEvent(pub GossipData); @@ -65,7 +65,7 @@ impl NetEventTranslator { // Listen on all events bus.subscribe(EventType::All, addr.clone().recipient()); - + info!("NetEventTranslator is running"); tokio::spawn({ let addr = addr.clone(); async move { diff --git a/crates/net/src/net_interface.rs b/crates/net/src/net_interface.rs index 156b8a867d..c0f39d516c 100644 --- a/crates/net/src/net_interface.rs +++ b/crates/net/src/net_interface.rs @@ -4,16 +4,21 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::correlator::Correlator; -use anyhow::{bail, Result}; +use crate::{ + correlator::Correlator, + direct_responder::{ChannelType, DirectResponder}, + events::{IncomingResponse, OutgoingRequest, ProtocolResponse}, + net_interface_handle::NetInterfaceHandle, +}; +use anyhow::{bail, Context, Result}; use e3_events::CorrelationId; -use e3_utils::{ArcBytes, OnceTake}; +use e3_utils::ArcBytes; use libp2p::{ connection_limits::{self, ConnectionLimits}, futures::StreamExt, gossipsub, identify::{Behaviour as IdentifyBehaviour, Config as IdentifyConfig}, - identity::Keypair, + identity::{ed25519, Keypair}, kad::{ self, store::{MemoryStore, MemoryStoreConfig, RecordStore}, @@ -21,8 +26,8 @@ use libp2p::{ Record, RecordKey, }, request_response::{ - self, cbor::Behaviour as CborRequestResponse, Event as RequestResponseEvent, - Message as RequestResponseMessage, ProtocolSupport, ResponseChannel, + self, cbor, Event as RequestResponseEvent, Message as RequestResponseMessage, + ProtocolSupport, }, swarm::{dial_opts::DialOpts, DialError, NetworkBehaviour, SwarmEvent}, PeerId, StreamProtocol, Swarm, @@ -34,7 +39,10 @@ use std::{ sync::Arc, time::{Duration, Instant}, }; -use tokio::{select, sync::broadcast, sync::mpsc}; +use tokio::{ + select, + sync::{broadcast, mpsc}, +}; use tracing::{debug, error, info, trace, warn}; const PROTOCOL_NAME: StreamProtocol = StreamProtocol::new("/enclave/kad/1.0.0"); @@ -46,9 +54,8 @@ const MAX_CONSECUTIVE_DIAL_FAILURES: u32 = 3; use crate::{ dialer::dial_peers, events::{ - estimate_hashmap_size, GossipData, NetCommand, NetEvent, OutgoingSyncRequestFailed, - OutgoingSyncRequestSucceeded, PutOrStoreError, SyncRequestReceived, SyncRequestValue, - SyncResponseValue, + GossipData, IncomingRequest, NetCommand, NetEvent, OutgoingRequestFailed, + OutgoingRequestSucceeded, PeerTarget, PutOrStoreError, }, ContentHash, }; @@ -59,12 +66,13 @@ pub struct NodeBehaviour { kademlia: KademliaBehaviour, connection_limits: connection_limits::Behaviour, identify: IdentifyBehaviour, - sync: CborRequestResponse, + /// Send bytes reply with enumeration for errors + request_response: cbor::Behaviour, ProtocolResponse>, } /// Manage the peer to peer connection. This struct wraps a libp2p Swarm and enables communication /// with it using channels. -pub struct NetInterface { +pub struct Libp2pNetInterface { /// The Libp2p Swarm instance swarm: Swarm, /// A list of peers to automatically dial @@ -75,15 +83,15 @@ pub struct NetInterface { topic: gossipsub::IdentTopic, /// Broadcast channel to report NetEvents to listeners event_tx: broadcast::Sender, - /// Transmission channel to send NetCommands to the NetInterface + /// Transmission channel to send NetCommands to the Libp2pNetInterface cmd_tx: mpsc::Sender, /// Local receiver to process NetCommands from cmd_rx: mpsc::Receiver, } -impl NetInterface { +impl Libp2pNetInterface { pub fn new( - id: &Keypair, + id: Libp2pKeypair, peers: Vec, udp_port: Option, topic: &str, @@ -91,7 +99,7 @@ impl NetInterface { let (event_tx, _) = broadcast::channel(1000); // TODO : tune this param let (cmd_tx, cmd_rx) = mpsc::channel(1000); // TODO : tune this param - let swarm = libp2p::SwarmBuilder::with_existing_identity(id.clone()) + let swarm = libp2p::SwarmBuilder::with_existing_identity(id.into_keypair()) .with_tokio() .with_quic() .with_dns() @@ -113,12 +121,8 @@ impl NetInterface { }) } - pub fn rx(&mut self) -> broadcast::Receiver { - self.event_tx.subscribe() - } - - pub fn tx(&self) -> mpsc::Sender { - self.cmd_tx.clone() + pub fn handle(&self) -> NetInterfaceHandle { + NetInterfaceHandle::new(self.cmd_tx.clone(), self.event_tx.subscribe()) } pub async fn start(&mut self) -> Result<()> { @@ -146,6 +150,7 @@ impl NetInterface { trace!("Peers to dial: {:?}", self.peers); tokio::spawn({ let event_tx = event_tx.clone(); + let cmd_tx = cmd_tx.clone(); let peers = self.peers.clone(); async move { dial_peers(&cmd_tx, &event_tx, &peers).await?; @@ -171,7 +176,7 @@ impl NetInterface { } // Process events event = self.swarm.select_next_some() => { - match process_swarm_event(&mut self.swarm, &event_tx, &mut correlator, &mut peer_failures, event).await { + match process_swarm_event(&mut self.swarm, &event_tx, &cmd_tx, &mut correlator, &mut peer_failures, event).await { Ok(_) => (), Err(e) => error!("Error processing NetEvent: {e}") } @@ -185,6 +190,33 @@ impl NetInterface { } } +pub struct Libp2pKeypair { + keypair: libp2p::identity::Keypair, +} + +impl Libp2pKeypair { + pub fn new(keypair: libp2p::identity::Keypair) -> Self { + Self { keypair } + } + + pub fn generate() -> Self { + let id = libp2p::identity::Keypair::generate_ed25519(); + Self::new(id) + } + + pub fn try_from_bytes(bytes: &mut [u8]) -> Result { + let keypair: libp2p::identity::Keypair = + ed25519::Keypair::try_from_bytes(bytes)?.try_into()?; + Ok(Self { keypair }) + } + + pub fn into_keypair(self) -> libp2p::identity::Keypair { + self.keypair + } + pub fn peer_id(&self) -> PeerId { + self.keypair.public().to_peer_id() + } +} /// Create the libp2p behaviour fn create_behaviour( key: &Keypair, @@ -210,7 +242,7 @@ fn create_behaviour( let request_response_config = request_response::Config::default().with_request_timeout(Duration::from_secs(30)); - let sync = CborRequestResponse::::new( + let request_response = cbor::Behaviour::, ProtocolResponse>::new( [( StreamProtocol::new("/enclave/sync/0.0.1"), ProtocolSupport::Full, @@ -228,9 +260,6 @@ fn create_behaviour( max_provided_keys: DHT_MAX_RECORDS, }; let store = MemoryStore::with_config(peer_id, store_config); - // Force Server mode: in a private network all nodes should fully participate - // in DHT routing. Auto-detect (None) would classify containerized/NAT'd nodes - // as Clients, preventing peer discovery and record replication. let mut kademlia = KademliaBehaviour::with_config(peer_id, store, config); kademlia.set_mode(Some(kad::Mode::Server)); @@ -239,7 +268,7 @@ fn create_behaviour( kademlia, connection_limits, identify, - sync, + request_response, }) } @@ -247,6 +276,7 @@ fn create_behaviour( async fn process_swarm_event( swarm: &mut Swarm, event_tx: &broadcast::Sender, + cmd_tx: &mpsc::Sender, correlator: &mut Correlator, peer_failures: &mut PeerFailureTracker, event: SwarmEvent, @@ -418,46 +448,49 @@ async fn process_swarm_event( event_tx.send(NetEvent::GossipSubscribed { count, topic })?; } - SwarmEvent::Behaviour(NodeBehaviourEvent::Sync(RequestResponseEvent::Message { - message: - RequestResponseMessage::Request { - request, - channel, - request_id, - }, - .. - })) => { - debug!("Incoming sync request received (id={})", request_id); + SwarmEvent::Behaviour(NodeBehaviourEvent::RequestResponse( + RequestResponseEvent::Message { + message: + RequestResponseMessage::Request { + request, + channel, + request_id, + }, + .. + }, + )) => { + debug!("Incoming request received (id={})", request_id); + let responder = + DirectResponder::new(request_id, ChannelType::Channel(channel), &cmd_tx) + .with_request(request); // received a request for events - event_tx.send(NetEvent::SyncRequestReceived(SyncRequestReceived { - request_id, - channel: OnceTake::new(channel), - value: request, - }))?; + event_tx.send(NetEvent::IncomingRequest(IncomingRequest { responder }))?; } - SwarmEvent::Behaviour(NodeBehaviourEvent::Sync(RequestResponseEvent::Message { - message: - RequestResponseMessage::Response { - request_id, - response, - .. - }, - .. - })) => { - debug!("Outgoing sync response received (id={request_id})"); + SwarmEvent::Behaviour(NodeBehaviourEvent::RequestResponse( + RequestResponseEvent::Message { + message: + RequestResponseMessage::Response { + request_id, + response, + .. + }, + .. + }, + )) => { + debug!("Response received (id={request_id})"); let correlation_id = correlator.expire(request_id)?; - debug!("Correlated sync response: {correlation_id}"); - event_tx.send(NetEvent::OutgoingSyncRequestSucceeded( - OutgoingSyncRequestSucceeded { - value: response, + debug!("Correlated response: {correlation_id}"); + event_tx.send(NetEvent::OutgoingRequestSucceeded( + OutgoingRequestSucceeded { + payload: response, correlation_id, }, ))?; } - SwarmEvent::Behaviour(NodeBehaviourEvent::Sync( + SwarmEvent::Behaviour(NodeBehaviourEvent::RequestResponse( RequestResponseEvent::OutboundFailure { peer, request_id, @@ -465,34 +498,33 @@ async fn process_swarm_event( }, )) => { warn!( - "Outbound sync request failed: peer={}, id={}, error={:?}", + "Outbound request failed: peer={}, id={}, error={:?}", peer, request_id, error ); let correlation_id = correlator.expire(request_id)?; - event_tx.send(NetEvent::OutgoingSyncRequestFailed( - OutgoingSyncRequestFailed { - correlation_id, - error: format!("Outbound sync request failed: {:?}", error), - }, - ))?; + event_tx.send(NetEvent::OutgoingRequestFailed(OutgoingRequestFailed { + correlation_id, + error: format!("Outbound request failed: {:?}", error), + }))?; } - SwarmEvent::Behaviour(NodeBehaviourEvent::Sync(RequestResponseEvent::InboundFailure { - peer, - request_id, - error, - })) => { + SwarmEvent::Behaviour(NodeBehaviourEvent::RequestResponse( + RequestResponseEvent::InboundFailure { + peer, + request_id, + error, + }, + )) => { warn!( - "Inbound sync request failed: peer={}, id={}, error={:?}", + "Inbound request failed: peer={}, id={}, error={:?}", peer, request_id, error ); } - SwarmEvent::Behaviour(NodeBehaviourEvent::Sync(RequestResponseEvent::ResponseSent { - peer, - request_id, - })) => { - debug!("Sync response sent to peer={}, id={}", peer, request_id); + SwarmEvent::Behaviour(NodeBehaviourEvent::RequestResponse( + RequestResponseEvent::ResponseSent { peer, request_id }, + )) => { + debug!("Response sent to peer={}, id={}", peer, request_id); } unknown => { @@ -518,7 +550,8 @@ async fn process_swarm_command( handle_gossip_publish(swarm, event_tx, data, topic, correlation_id)?; Ok(()) } - NetCommand::Dial(multi) => { + NetCommand::Dial(env) => { + let multi = env.take().context("Dial received without payload")?; handle_dial(swarm, event_tx, multi)?; Ok(()) } @@ -550,19 +583,27 @@ async fn process_swarm_command( handle_remove_records(swarm, keys); Ok(()) } - NetCommand::OutgoingSyncRequest { + NetCommand::OutgoingRequest(OutgoingRequest { correlation_id, - value, - } => { - handle_outgoing_sync_request(swarm, correlator, correlation_id, value)?; + payload, + target, + }) => { + if let Err(e) = + handle_outgoing_request(swarm, correlator, correlation_id, payload, target) + { + event_tx.send(NetEvent::OutgoingRequestFailed(OutgoingRequestFailed { + correlation_id, + error: e.to_string(), + }))?; + }; Ok(()) } - NetCommand::SyncResponse { value, channel } => { - handle_sync_response(swarm, channel, value)?; + NetCommand::IncomingResponse(IncomingResponse { responder }) => { + handle_response(swarm, responder)?; Ok(()) } NetCommand::Shutdown => { - unreachable!("shutdown command must be handled in NetInterface::start") + unreachable!("shutdown command must be handled in Libp2pNetInterface::start") } } } @@ -753,55 +794,48 @@ fn handle_shutdown(swarm: &mut Swarm) -> Result<()> { Ok(()) } -fn handle_outgoing_sync_request( +fn handle_outgoing_request( swarm: &mut Swarm, correlator: &mut Correlator, correlation_id: CorrelationId, - value: SyncRequestValue, + payload: Vec, + target: PeerTarget, ) -> Result<()> { - debug!("Outgoing sync request (cid={})", correlation_id); - // TODO: - // This is a first pass. - // Lots of stuff to work through here: - // How can I know events are correct? - // How can I trust this peer? - // Can I validate events with another peer? - // Should I use an OrderedSet with a hash and request the hash from a second peer? - - // Pick a random peer - let Some(peer) = swarm - .connected_peers() - .choose(&mut rand::thread_rng()) - .copied() - else { - bail!("No peer found on swarm!") + let peer = match target { + PeerTarget::Random => swarm + .connected_peers() + .choose(&mut rand::thread_rng()) + .copied() + .context("No connected peers available")?, + PeerTarget::Specific(peer_id) => peer_id, }; - debug!( - "Sync request payload size: {:?}", - estimate_hashmap_size(&value.since) - ); + debug!("Outgoing request payload size: {:?}", payload.len()); // Request events - let query_id = swarm.behaviour_mut().sync.send_request(&peer, value); + let query_id = swarm + .behaviour_mut() + .request_response + .send_request(&peer, payload); debug!( - "Sync request sent: query_id={}, correlation_id={}", + "Outgoing request sent: query_id={}, correlation_id={}", query_id, correlation_id ); correlator.track(query_id, correlation_id); Ok(()) } -fn handle_sync_response( - swarm: &mut Swarm, - channel: OnceTake>, - value: SyncResponseValue, -) -> Result<()> { - debug!("Sending sync response"); - let channel = channel.try_take()?; - if let Err(value) = swarm.behaviour_mut().sync.send_response(channel, value) { - error!("Failed to send sync response: {:?}", value); - } +fn handle_response(swarm: &mut Swarm, responder: DirectResponder) -> Result<()> { + debug!("Sending response to {}", responder.id()); + let (channel, response) = responder.to_response()?; + let ChannelType::Channel(channel) = channel else { + bail!("responder did not return the correct type of channel"); + }; + swarm + .behaviour_mut() + .request_response + .send_response(channel, response) + .map_err(|payload| anyhow::anyhow!("Failed to send response: {:?}", payload))?; Ok(()) } diff --git a/crates/net/src/net_interface_handle.rs b/crates/net/src/net_interface_handle.rs new file mode 100644 index 0000000000..f90b44ea4c --- /dev/null +++ b/crates/net/src/net_interface_handle.rs @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use std::time::Duration; + +use tokio::{ + sync::{broadcast, mpsc}, + time::sleep, +}; + +use crate::events::{NetCommand, NetEvent}; + +#[derive(Debug)] +pub struct NetInterfaceHandle { + tx: mpsc::Sender, + rx: broadcast::Receiver, +} +impl NetInterfaceHandle { + pub fn new(tx: mpsc::Sender, rx: broadcast::Receiver) -> Self { + Self { tx, rx } + } +} + +pub trait NetInterface: Sized { + fn tx(&self) -> mpsc::Sender; + fn rx(&self) -> broadcast::Receiver; + fn handle(&self) -> NetInterfaceHandle { + NetInterfaceHandle::from(self) + } +} + +#[derive(Debug, Clone)] +/// Allow Net events and commands to be bridged between nodes. This is used for testing purposes to +/// simulate libp2p without running libp2p. +pub struct NetChannelBridge { + cmd_tx: broadcast::Sender, + tx: mpsc::Sender, + event_tx: broadcast::Sender, +} + +impl NetInterfaceHandle { + pub fn from(interface: &impl NetInterface) -> Self { + Self { + tx: interface.tx(), + rx: interface.rx(), + } + } +} +impl NetInterface for NetInterfaceHandle { + fn rx(&self) -> broadcast::Receiver { + self.rx.resubscribe() + } + + fn tx(&self) -> mpsc::Sender { + self.tx.clone() + } +} + +/// This creates a channel bridge which allows for network events to be connected between test nodes +pub fn create_channel_bridge() -> (NetInterfaceHandle, NetChannelBridge) { + let (m_cmd_tx, mut m_cmd_rx) = mpsc::channel::(1000); + let (b_evt_tx, _) = broadcast::channel(1000); + let (b_cmd_tx, _) = broadcast::channel(1000); + + let tx = b_cmd_tx.clone(); + let startup_event_tx = b_evt_tx.clone(); + let keep_alive = b_cmd_tx.subscribe(); + + // Bridge from mpsc channel to broadcast channel simulating AllPeersDialed for each node + tokio::spawn(async move { + let _rx_guard = keep_alive; + sleep(Duration::from_millis(100)).await; + let _ = startup_event_tx.send(NetEvent::AllPeersDialed); + while let Some(cmd) = m_cmd_rx.recv().await { + let _ = tx.send(cmd); + } + }); + + let handle = NetInterfaceHandle { + tx: m_cmd_tx.clone(), + rx: b_evt_tx.subscribe(), + }; + + let inverted = NetChannelBridge { + tx: m_cmd_tx, + cmd_tx: b_cmd_tx, + event_tx: b_evt_tx, + }; + + (handle, inverted) +} + +pub trait NetInterfaceInverted: Sized { + fn tx(&self) -> mpsc::Sender; + fn event_tx(&self) -> broadcast::Sender; //U + fn event_rx(&self) -> broadcast::Receiver; + fn cmd_tx(&self) -> broadcast::Sender; + fn cmd_rx(&self) -> broadcast::Receiver; //U + + fn into_handle_inverted(self) -> NetChannelBridge { + NetChannelBridge { + tx: self.tx(), + event_tx: self.event_tx(), + cmd_tx: self.cmd_tx(), + } + } +} + +impl NetInterfaceInverted for NetChannelBridge { + fn tx(&self) -> mpsc::Sender { + self.tx.clone() + } + + fn cmd_rx(&self) -> broadcast::Receiver { + self.cmd_tx.subscribe() + } + fn event_tx(&self) -> broadcast::Sender { + self.event_tx.clone() + } + fn cmd_tx(&self) -> broadcast::Sender { + self.cmd_tx.clone() + } + fn event_rx(&self) -> broadcast::Receiver { + self.event_tx.subscribe() + } +} diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs index eb2291bdc6..483b13507e 100644 --- a/crates/net/src/net_sync_manager.rs +++ b/crates/net/src/net_sync_manager.rs @@ -5,33 +5,62 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use actix::{Actor, Addr, AsyncContext, Handler, Message, Recipient, ResponseFuture}; -use anyhow::{anyhow, bail, Result}; +use anyhow::{bail, Context, Result}; use e3_events::{ prelude::*, trap, trap_fut, AggregateId, BusHandle, CorrelationId, EType, EnclaveEvent, EnclaveEventData, EventSource, EventStoreQueryBy, EventStoreQueryResponse, EventType, - HistoricalNetSyncStart, NetSyncEventsReceived, TsAgg, TypedEvent, Unsequenced, + HistoricalNetSyncEventsReceived, HistoricalNetSyncStart, NetReady, TsAgg, TypedEvent, + Unsequenced, }; -use e3_utils::{retry_with_backoff, to_retry, OnceTake, MAILBOX_LIMIT}; -use futures::TryFutureExt; -use libp2p::request_response::ResponseChannel; -use std::{collections::HashMap, sync::Arc, time::Duration}; +use e3_utils::MAILBOX_LIMIT; +use serde::{Deserialize, Serialize}; +use std::{collections::HashMap, convert::TryInto, sync::Arc, time::Duration}; use tokio::sync::{broadcast, mpsc}; -use tracing::{debug, info, warn}; +use tracing::{debug, info}; -use crate::events::{ - await_event, call_and_await_response, NetCommand, NetEvent, OutgoingSyncRequestSucceeded, - SyncRequestReceived, SyncRequestValue, SyncResponseValue, +use crate::{ + direct_requester::DirectRequester, + direct_responder::DirectResponder, + events::{await_event, IncomingRequest, NetCommand, NetEvent, PeerTarget}, + net_event_batch::{fetch_all_batched_events, BatchCursor, EventBatch, FetchEventsSince}, }; +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SyncResponseValue { + pub events: Vec>, + pub ts: u128, +} + +impl TryInto> for SyncResponseValue { + type Error = anyhow::Error; + + fn try_into(self) -> Result, Self::Error> { + bincode::serialize(&self).context("failed to serialize sync response") + } +} + +impl TryFrom> for SyncResponseValue { + type Error = anyhow::Error; + + fn try_from(value: Vec) -> Result { + bincode::deserialize(&value).context("failed to deserialize sync response") + } +} + +#[derive(Debug, Clone)] +pub struct SyncRequestSucceeded { + pub response: SyncResponseValue, +} + pub struct NetSyncManager { /// Enclave EventBus bus: BusHandle, - /// NetCommand sender to forward commands to the NetInterface + /// NetCommand sender to forward commands to the Libp2pNetInterface tx: mpsc::Sender, /// NetEvents receiver to receive events rx: Arc>, eventstore: Recipient>, - requests: HashMap>>, + requests: HashMap, peers_ready: bool, } @@ -72,7 +101,7 @@ impl NetSyncManager { debug!("Received event {:?}", event); match event { // Someone is asking for our sync - NetEvent::SyncRequestReceived(value) => addr.do_send(value), + NetEvent::IncomingRequest(value) => addr.do_send(value), NetEvent::AllPeersDialed => addr.do_send(AllPeersDialed), _ => (), } @@ -128,26 +157,22 @@ impl Handler> for NetSyncManager { } /// We have received the sync response from the remote peer -impl Handler> for NetSyncManager { +impl Handler> for NetSyncManager { type Result = (); fn handle( &mut self, - msg: TypedEvent, + msg: TypedEvent, _: &mut Self::Context, ) -> Self::Result { trap(EType::Net, &self.bus.with_ec(msg.get_ctx()), || { + info!("SYNC REQUEST SUCCEEDED"); let (msg, ctx) = msg.into_components(); + let response = msg.response; self.bus.publish_from_remote_as_response( - NetSyncEventsReceived { - events: msg - .value - .events - .iter() - .cloned() - .map(|data| data.try_into()) - .collect::>>>()?, + HistoricalNetSyncEventsReceived { + events: response.events.iter().cloned().collect(), }, - msg.value.ts, + response.ts, ctx, None, EventSource::Net, @@ -159,18 +184,19 @@ impl Handler> for NetSyncManager { } /// We have received a sync request from a remote peer -impl Handler for NetSyncManager { +impl Handler for NetSyncManager { type Result = (); - fn handle(&mut self, msg: SyncRequestReceived, ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: IncomingRequest, ctx: &mut Self::Context) -> Self::Result { trap(EType::Net, &self.bus, || { - info!("GOT SyncRequestReceived"); let id = CorrelationId::new(); - info!("STORING channel in requests map..."); - self.requests.insert(id, msg.channel); - info!("QUERYING eventstore..."); + info!("Processing incoming request with correlation={}", id); + let fetch_request: FetchEventsSince = msg.responder.try_request_into()?; + self.requests.insert(id, msg.responder); + let query: HashMap = + HashMap::from([(fetch_request.aggregate_id(), fetch_request.since())]); self.eventstore.try_send(EventStoreQueryBy::::new( id, - msg.value.since, + query, ctx.address().recipient(), ))?; Ok(()) @@ -184,24 +210,37 @@ impl Handler for NetSyncManager { fn handle(&mut self, msg: EventStoreQueryResponse, _: &mut Self::Context) -> Self::Result { trap(EType::Net, &self.bus.clone(), || { info!("Received response from eventstore."); - let Some(channel) = self.requests.get(&msg.id()) else { - bail!("request not found with {}", msg.id()); + let Some(responder) = self.requests.remove(&msg.id()) else { + bail!("responder not found for {}", msg.id()); }; - debug!("Sending SyncResponse with channel={:?}", channel); - if let Err(e) = self.tx.try_send(NetCommand::SyncResponse { - value: SyncResponseValue { - events: msg - .into_events() - .into_iter() - .filter(|e| e.source() == EventSource::Net) - .map(|ev| ev.try_into()) - .collect::>()?, - ts: self.bus.ts()?, // NOTE: We are storing a local timestamp on this response - }, - channel: channel.to_owned(), - }) { - warn!("Failed to send SyncResponse (channel full or closed): {e}"); + + let fetch_request: FetchEventsSince = responder.try_request_into()?; + let limit = fetch_request.limit(); + if limit == 0 { + responder.bad_request("limit must be greater than 0")?; + return Ok(()); } + let aggregate_id = fetch_request.aggregate_id(); + let events: Vec> = msg + .into_events() + .into_iter() + .filter(|e| e.source() == EventSource::Net) + .take(limit) + .map(|ev| ev.clone_unsequenced()) + .collect(); + + let next = if events.len() == limit { + let last_event_ts = events.last().map(|e| e.ts()).unwrap_or(0); + BatchCursor::Next(last_event_ts) + } else { + BatchCursor::Done + }; + + responder.ok(EventBatch { + events, + next, + aggregate_id, + })?; Ok(()) }) @@ -211,8 +250,12 @@ impl Handler for NetSyncManager { impl Handler for NetSyncManager { type Result = (); fn handle(&mut self, _: AllPeersDialed, _: &mut Self::Context) -> Self::Result { - info!("Received handler: All peers dialed"); - self.peers_ready = true; + trap(EType::Sync, &self.bus.clone(), || { + info!("NetSyncManager: AllPeersDialed"); + self.peers_ready = true; + self.bus.publish_without_context(NetReady::new())?; + Ok(()) + }) } } @@ -220,44 +263,16 @@ impl Handler for NetSyncManager { #[rtype(result = "()")] struct AllPeersDialed; -const SYNC_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); - -async fn sync_request( - net_cmds: mpsc::Sender, - net_events: Arc>, - since: HashMap, -) -> Result { - info!("RUNNING sync request..."); - let id = CorrelationId::new(); - call_and_await_response( - net_cmds, - net_events, - NetCommand::OutgoingSyncRequest { - correlation_id: id, - value: SyncRequestValue { since }, - }, - |e| match e.clone() { - NetEvent::OutgoingSyncRequestSucceeded(value) => Some(Ok(value)), - NetEvent::OutgoingSyncRequestFailed(error) => { - Some(Err(anyhow!("Outgoing sync request failed: {:?}", error))) - } - _ => None, - }, - SYNC_REQUEST_TIMEOUT, - ) - .await -} - async fn handle_sync_request_event( net_cmds: mpsc::Sender, net_events: Arc>, event: TypedEvent, - address: impl Into>>, + address: impl Into>>, wait_for_event: bool, ) -> Result<()> { info!("Sync request event received"); let (event, ctx) = event.into_components(); - info!("Waiting for peers to have been contacted..."); + info!("Checking for AllPeersDialed..."); if wait_for_event { await_event( &net_events, @@ -273,26 +288,50 @@ async fn handle_sync_request_event( ) .await?; } - info!("handle_sync_request_event: All peers have been dialed."); - - // Make the sync request - // value returned includes the timestamp from the remote peer - let value = retry_with_backoff( - || { - info!("Running SYNC REQUEST!!"); - sync_request( - net_cmds.clone(), - net_events.clone(), - event.since.clone().into_iter().collect(), - ) - .map_err(to_retry) + info!("handle_sync_request_event: AllPeersDialed"); + + let mut all_events: Vec> = Vec::new(); + let mut latest_timestamp: u128 = 0; + + for (aggregate_id, since) in event.since.iter() { + info!( + "Requesting batched events for aggregate_id={} since={}", + aggregate_id, since + ); + let requester = DirectRequester::builder(net_cmds.clone(), net_events.clone()).build(); + let events: Vec> = + fetch_all_batched_events(requester, PeerTarget::Random, *aggregate_id, *since, 100) + .await?; + + info!( + "Received {} events for aggregate_id={}", + events.len(), + aggregate_id + ); + + for enclave_event in events { + let ts = enclave_event.ts(); + if ts > latest_timestamp { + latest_timestamp = ts; + } + all_events.push(enclave_event); + } + } + + info!( + "Sync complete: collected {} events across {} aggregates, latest_timestamp={}", + all_events.len(), + event.since.len(), + latest_timestamp + ); + + let value = SyncRequestSucceeded { + response: SyncResponseValue { + events: all_events, + ts: latest_timestamp, }, - 4, - 5000, - ) - .await?; + }; - // send the sync request succeeded to ourselves address.into().try_send(TypedEvent::new(value, ctx))?; Ok(()) } diff --git a/crates/sync/Cargo.toml b/crates/sync/Cargo.toml index 268d7b1aa3..c06369c819 100644 --- a/crates/sync/Cargo.toml +++ b/crates/sync/Cargo.toml @@ -17,5 +17,6 @@ tokio.workspace = true tracing.workspace = true [dev-dependencies] +e3-events = { workspace = true, features = ["test-helpers"] } e3-ciphernode-builder.workspace = true e3-test-helpers.workspace = true diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index adadeda54c..b5f20e0b56 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -6,13 +6,14 @@ use crate::SyncRepositoryFactory; use actix::{Message, Recipient}; -use anyhow::Result; +use anyhow::{bail, Result}; use e3_data::Repositories; use e3_events::{ - AggregateConfig, AggregateId, BusHandle, CorrelationId, EffectsEnabled, EnclaveEvent, + AggregateConfig, AggregateId, BusHandle, CorrelationId, E3id, EffectsEnabled, EnclaveEvent, EnclaveEventData, Event, EventContextAccessors, EventPublisher, EventStoreQueryBy, - EventStoreQueryResponse, EvmEventConfig, EvmEventConfigChain, HistoricalEvmEventsReceived, - HistoricalEvmSyncStart, SeqAgg, SyncEnded, Unsequenced, + EventStoreQueryResponse, EventSubscriber, EventType, EvmEventConfig, EvmEventConfigChain, + HistoricalEvmEventsReceived, HistoricalEvmSyncStart, HistoricalNetSyncStart, SeqAgg, SyncEnded, + Unsequenced, }; use e3_utils::actix::channel as actix_toolbox; use std::{ @@ -38,6 +39,9 @@ pub async fn sync( aggregate_config: &AggregateConfig, eventstore: &Recipient>, ) -> Result<()> { + // 0. start listening early for net ready + let net_ready = bus.wait_for(EventType::NetReady); + // 1. Load snapsshot metadata info!("Loading snapshot metadata..."); let snapshot = @@ -91,22 +95,30 @@ pub async fn sync( "{} historical blockchain events loaded.", historical_evm_events.len() ); - - // XXX: Skipping as we have bugs in libp2p netevent requests + let net_config = find_net_hlc(&historical_evm_events); // 6. Load the historical libp2p events to memory - // info!("Loading historical libp2p events..."); - // let (addr, rx) = actix_toolbox::oneshot::(); - // bus.publish_without_context(HistoricalNetSyncStart::new(addr, net_config.clone()))?; - // let historical_net_events = rx.await?.events; - // info!( - // "{} historical libp2p events loaded.", - // historical_net_events.len() - // ); + info!("Waiting until NetReady..."); + net_ready.await?; + info!("NetReady!"); + info!("Loading historical libp2p events..."); + // let (addr, rx) = actix_toolbox::oneshot::(); + let events_received = bus.wait_for(EventType::HistoricalNetSyncEventsReceived); + bus.publish_without_context(HistoricalNetSyncStart::new(net_config.clone()))?; + let EnclaveEventData::HistoricalNetSyncEventsReceived(event) = + events_received.await?.into_data() + else { + bail!("failed to get HistoricalNetSyncEventsReceived"); + }; + let historical_net_events = event.events; + info!( + "{} historical libp2p events loaded.", + historical_net_events.len() + ); // 7. Sort both the evm and libp2p events together by HLC timestamp let mut historical = historical_evm_events .into_iter() - // .chain(historical_net_events) // Commenting out to skip + .chain(historical_net_events) .collect::>(); historical.sort_by_key(|event| event.ts()); @@ -177,6 +189,27 @@ pub async fn collect_historical_evm_events( results } +fn find_net_hlc(events: &[EnclaveEvent]) -> BTreeMap { + // find all E3s that are closed + let e3s: Vec = events + .iter() + .filter_map(|e| match e.get_data() { + EnclaveEventData::E3Failed(d) => Some(d.e3_id.clone()), + EnclaveEventData::E3RequestComplete(d) => Some(d.e3_id.clone()), + _ => None, + }) + .collect(); + events + .iter() + .filter(|e| e.get_e3_id().map_or(true, |id| !e3s.contains(&id))) + .fold(BTreeMap::new(), |mut acc, e| { + acc.entry(e.aggregate_id()) + .and_modify(|ts| *ts = (*ts).max(e.ts())) + .or_insert(e.ts()); + acc + }) +} + /// Latest event information in store #[derive(Clone)] pub struct AggregateState { @@ -278,29 +311,16 @@ impl SnapshotLoaded { Self { snapshot } } } - #[cfg(test)] mod tests { - use super::is_infrastructure_event; + use super::*; use e3_ciphernode_builder::EventSystem; use e3_events::{ - EffectsEnabled, EnclaveEvent, EnclaveEventData, Event, EventConstructorWithTimestamp, - EventSource, EvmEventConfig, HistoricalEvmSyncStart, SyncEnded, TakeEvents, TestEvent, + E3Failed, E3RequestComplete, E3Stage, E3id, EffectsEnabled, EnclaveEvent, EnclaveEventData, + Event, EvmEventConfig, FailureReason, HistoricalEvmSyncStart, SyncEnded, TakeEvents, Unsequenced, }; - fn make_sequenced(data: impl Into, seq: u64) -> EnclaveEvent { - EnclaveEvent::::new_with_timestamp( - data.into(), - None, - 1000, - None, - EventSource::Local, - ) - .into_sequenced(seq) - } - - /// `sender` is `Option>` — `None` is safe here since we're not dispatching. fn make_historical_evm_sync_start() -> HistoricalEvmSyncStart { HistoricalEvmSyncStart { evm_config: EvmEventConfig::new(), @@ -310,10 +330,22 @@ mod tests { #[test] fn infrastructure_events_are_detected() { - let sync_ended = make_sequenced(SyncEnded::new(), 1); - let effects_enabled = make_sequenced(EffectsEnabled::new(), 2); - let evm_sync_start = make_sequenced(make_historical_evm_sync_start(), 3); - let test_event = make_sequenced(TestEvent::new("hello", 42), 4); + let sync_ended = EnclaveEvent::::test_event("sync") + .data(SyncEnded::new()) + .seq(1) + .build(); + let effects_enabled = EnclaveEvent::::test_event("fx") + .data(EffectsEnabled::new()) + .seq(2) + .build(); + let evm_sync_start = EnclaveEvent::::test_event("evm") + .data(make_historical_evm_sync_start()) + .seq(3) + .build(); + let test_event = EnclaveEvent::::test_event("hello") + .id(42) + .seq(4) + .build(); assert!(is_infrastructure_event(&sync_ended)); assert!(is_infrastructure_event(&effects_enabled)); @@ -321,9 +353,6 @@ mod tests { assert!(!is_infrastructure_event(&test_event)); } - /// Regression: infrastructure events replayed from the EventStore must be filtered before - /// they reach the bus. If they aren't, the bloom-filter deduplicates the copy that `sync()` - /// re-publishes later, causing it to be silently dropped. #[actix::test] async fn infrastructure_events_are_filtered_during_replay() -> anyhow::Result<()> { let system = EventSystem::new().with_fresh_bus(); @@ -331,11 +360,26 @@ mod tests { let history = bus.history(); let events: Vec = vec![ - make_sequenced(TestEvent::new("before", 1), 1), - make_sequenced(SyncEnded::new(), 2), - make_sequenced(EffectsEnabled::new(), 3), - make_sequenced(make_historical_evm_sync_start(), 4), - make_sequenced(TestEvent::new("after", 2), 5), + EnclaveEvent::::test_event("before") + .id(1) + .seq(1) + .build(), + EnclaveEvent::::test_event("sync") + .data(SyncEnded::new()) + .seq(2) + .build(), + EnclaveEvent::::test_event("fx") + .data(EffectsEnabled::new()) + .seq(3) + .build(), + EnclaveEvent::::test_event("evm") + .data(make_historical_evm_sync_start()) + .seq(4) + .build(), + EnclaveEvent::::test_event("after") + .id(2) + .seq(5) + .build(), ]; for event in events { @@ -348,6 +392,7 @@ mod tests { let received = history.send(TakeEvents::new(2)).await?; let event_types: Vec<&'static str> = received + .events .iter() .map(|e| match e.get_data() { EnclaveEventData::TestEvent(_) => "TestEvent", @@ -361,6 +406,7 @@ mod tests { assert_eq!(event_types, vec!["TestEvent", "TestEvent"]); let msgs: Vec = received + .events .iter() .filter_map(|e| { if let EnclaveEventData::TestEvent(t) = e.get_data() { @@ -370,8 +416,81 @@ mod tests { } }) .collect(); - assert_eq!(msgs, vec!["before", "after"]); + assert_eq!(msgs, vec!["before", "after"]); Ok(()) } + + #[test] + fn test_find_net_hlc() { + let closed_1 = E3id::new("1", 1); + let closed_2 = E3id::new("2", 2); + let open_1 = E3id::new("3", 3); + let open_2 = E3id::new("4", 4); + + let events = vec![ + // closed e3s -> should be filtered out + EnclaveEvent::::test_event("a") + .e3_id(closed_1.clone()) + .ts(1000) + .build(), + EnclaveEvent::::test_event("a") + .e3_id(closed_1.clone()) + .ts(2000) + .build(), + EnclaveEvent::::test_event("complete") + .data(E3RequestComplete { + e3_id: closed_1.clone(), + }) + .ts(3000) + .build(), + EnclaveEvent::::test_event("b") + .e3_id(closed_2.clone()) + .ts(1500) + .build(), + EnclaveEvent::::test_event("failed") + .data(E3Failed { + e3_id: closed_2.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::InsufficientCommitteeMembers, + }) + .ts(2500) + .build(), + // open e3s -> should be kept + EnclaveEvent::::test_event("c") + .e3_id(open_1.clone()) + .ts(4000) + .build(), + EnclaveEvent::::test_event("c") + .e3_id(open_1.clone()) + .ts(5000) + .build(), + EnclaveEvent::::test_event("d") + .e3_id(open_2.clone()) + .ts(6000) + .build(), + // no e3_id -> aggregate 0, always kept + EnclaveEvent::::test_event("e") + .ts(7000) + .build(), + EnclaveEvent::::test_event("e") + .ts(8000) + .build(), + ]; + + let result = find_net_hlc(&events); + + // closed e3s excluded + assert!(!result.contains_key(&AggregateId::new(1))); + assert!(!result.contains_key(&AggregateId::new(2))); + + // open e3s kept with max ts + assert_eq!(result[&AggregateId::new(3)], 5000); + assert_eq!(result[&AggregateId::new(4)], 6000); + + // no-e3 events kept with max ts + assert_eq!(result[&AggregateId::new(0)], 8000); + + assert_eq!(result.len(), 3); + } } diff --git a/crates/test-helpers/Cargo.toml b/crates/test-helpers/Cargo.toml index 6cf492e796..8d8c85b31f 100644 --- a/crates/test-helpers/Cargo.toml +++ b/crates/test-helpers/Cargo.toml @@ -31,6 +31,7 @@ e3-sortition = { workspace = true } fhe = { workspace = true } fhe-traits = { workspace = true } hex = { workspace = true } +libp2p = { workspace = true } rand = { workspace = true } rand_chacha = { workspace = true } tokio = { workspace = true } diff --git a/crates/test-helpers/src/ciphernode_system.rs b/crates/test-helpers/src/ciphernode_system.rs index 042a34da09..3a5a079323 100644 --- a/crates/test-helpers/src/ciphernode_system.rs +++ b/crates/test-helpers/src/ciphernode_system.rs @@ -5,10 +5,13 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::simulate_libp2p_net; -use anyhow::*; +use anyhow::bail; +use anyhow::Context; +use anyhow::Result; use e3_ciphernode_builder::CiphernodeHandle; use e3_events::Event; use e3_events::{EnclaveEvent, GetEvents, ResetHistory, TakeEvents}; +use std::u64; use std::{future::Future, ops::Deref, pin::Pin, time::Duration}; use tokio::time::timeout; @@ -81,7 +84,7 @@ impl<'a> CiphernodeSystemBuilder<'a> { } if self.simulate { - simulate_libp2p_net(&nodes); + simulate_libp2p_net(&nodes).await; } for then_fn in self.thens { @@ -116,11 +119,74 @@ impl CiphernodeSystem { .await } + /// expect events to fire with the default timeout 1000sec per event + pub async fn expect_events(&self, expected: &[&str]) -> Result { + let h = self + .take_history_with_timeouts( + 0, + expected.len(), + Some(Duration::from_secs(1000)), + Some(Duration::from_secs(1000)), + ) + .await + .map_err(|e| anyhow::anyhow!("FAILURE: {expected:?} : {e}"))?; + + println!(">> {:?} == {:?}", h.event_types(), expected.to_vec()); + h.expect(expected.to_vec()); + Ok(h) + } + + pub async fn expect_events_without_timeout( + &self, + expected: &[&str], + ) -> Result { + let h = self + .take_history_with_timeouts(0, expected.len(), None, None) + .await + .map_err(|e| anyhow::anyhow!("FAILURE: {expected:?} : {e}"))?; + + println!(">> {:?} == {:?}", h.event_types(), expected.to_vec()); + h.expect(expected.to_vec()); + Ok(h) + } + + pub async fn expect_events_with_timeouts( + &self, + expected: &[&str], + total_to: Duration, // total + per_evt_to: Duration, // per event + ) -> Result { + let h = self + .take_history_with_timeouts(0, expected.len(), Some(total_to), Some(per_evt_to)) + .await + .map_err(|e| anyhow::anyhow!("FAILURE: {expected:?} : {e}"))?; + println!(">> {:?} == {:?}", h.event_types(), expected.to_vec()); + + h.expect(expected.to_vec()); + Ok(h) + } + pub async fn take_history_with_timeout( &self, index: usize, count: usize, - tout: Duration, + total_to: Duration, + ) -> Result { + self.take_history_with_timeouts( + index, + count, + Some(total_to), + Some(Duration::from_millis(1000)), + ) + .await + } + + pub async fn take_history_with_timeouts( + &self, + index: usize, + count: usize, + total_to: Option, + event_to: Option, ) -> Result { let Some(node) = self.0.get(index) else { bail!("No node found"); @@ -130,14 +196,28 @@ impl CiphernodeSystem { return Ok(CiphernodeHistory(vec![])); }; - let history = timeout(tout, history.send(TakeEvents::new(count))) - .await - .context(format!( - "Could not take {} events from node {}", - count, index - ))??; + let history = timeout( + total_to.unwrap_or(Duration::from_secs(u64::MAX)), // No timeout + history.send(TakeEvents::with_per_evt_timeout( + count, + event_to.unwrap_or(Duration::from_secs(u64::MAX)), + )), + ) + .await + .context(format!( + "Could not take {} events from node {}", + count, index + ))??; + + if history.timed_out { + bail!( + "Take History timed out was trying to take {} events. Returned {:?}", + count, + history + ); + }; - Ok(CiphernodeHistory(history)) + Ok(CiphernodeHistory(history.events)) } pub async fn flush_all_history(&self, millis: u64) -> Result<()> { let nodes = &self.0; @@ -146,15 +226,25 @@ impl CiphernodeSystem { break; }; loop { + println!("IN FLUSH LOOP..."); let nhs = history.send(TakeEvents::new(1)); let tr = timeout(Duration::from_millis(millis), nhs).await; - if !tr.is_ok() { - break; + match tr { + Ok(Ok(result)) if result.timed_out => { + println!("PER-EVENT TIMEOUT, BREAKING LOOP..."); + break; + } + Err(_) => { + println!("OUTER TIMEOUT, BREAKING LOOP..."); + break; + } + _ => { + // Got events, keep draining + } } } history.send(ResetHistory).await?; } - Ok(()) } } @@ -182,6 +272,10 @@ impl CiphernodeHistory { pub fn event_types(&self) -> Vec { self.0.iter().map(|e| e.event_type()).collect() } + + pub fn expect(&self, event_types: Vec<&str>) { + assert_eq!(self.event_types(), event_types); + } } impl Deref for CiphernodeHistory { @@ -199,7 +293,7 @@ mod tests { use e3_ciphernode_builder::EventSystem; use e3_data::InMemStore; use e3_events::{EventBus, EventBusConfig}; - use tokio::task::JoinHandle; + use libp2p::PeerId; async fn mock_setup_node(address: String) -> Result { // Create mock actors for the test @@ -212,7 +306,6 @@ mod tests { .with_event_bus(bus) .handle()? .enable("test"); - let handle: JoinHandle> = tokio::spawn(async { Ok(()) }); Ok(CiphernodeHandle { address, @@ -220,8 +313,8 @@ mod tests { bus, history: Some(history), errors: Some(errors), - join_handle: handle, - peer_id: "-unknown peer id-".to_string(), + peer_id: PeerId::random(), + channel_bridge: None, }) } diff --git a/crates/test-helpers/src/lib.rs b/crates/test-helpers/src/lib.rs index a90588bada..0b932e79fc 100644 --- a/crates/test-helpers/src/lib.rs +++ b/crates/test-helpers/src/lib.rs @@ -6,18 +6,18 @@ pub mod application; pub mod ciphernode_system; +pub mod libp2p_mock; mod plaintext_writer; mod public_key_writer; pub mod usecase_helpers; mod utils; use actix::prelude::*; use alloy::primitives::Address; -use anyhow::*; +use anyhow::Result; use e3_ciphernode_builder::{CiphernodeHandle, EventSystem}; use e3_events::{ BusHandle, CiphernodeAdded, Enabled, EnclaveEvent, EnclaveEventData, EventBus, EventBusConfig, - EventContextAccessors, EventPublisher, EventSubscriber, EventType, HistoryCollector, Seed, - Sequenced, Subscribe, + EventContextAccessors, EventPublisher, EventType, HistoryCollector, Seed, Sequenced, Subscribe, }; use e3_fhe_params::BfvParamSet; use e3_fhe_params::DEFAULT_BFV_PRESET; @@ -28,13 +28,13 @@ use fhe::bfv::{BfvParameters, Ciphertext, Encoding, Plaintext, PublicKey}; use fhe::mbfv::CommonRandomPoly; use fhe_traits::Serialize; use fhe_traits::{FheEncoder, FheEncrypter}; +use libp2p_mock::Libp2pMock; pub use plaintext_writer::*; pub use public_key_writer::*; use rand::Rng; use rand_chacha::rand_core::SeedableRng; use rand_chacha::ChaCha20Rng; use std::sync::Arc; -use tracing::trace; pub use utils::*; pub fn create_shared_rng_from_u64(value: u64) -> Arc> { @@ -127,44 +127,13 @@ impl Handler> for SimulatedNetPipe { } } -/// Simulate libp2p by taking output events on each local bus and filter for !is_local_only() and forward remaining events back to the event bus -/// deduplication will remove previously seen events. -/// This sets up a set of cyphernodes without libp2p. -/// The way it works is that it feeds back all events from -/// all nodes filteres by whether they are broadcastible or not -/// ```txt -/// -/// ┌─────┐ -/// │ BUS │ -/// └─────┘ -/// │ -/// ┌────────────┼────────────┐ -/// │ │ │ -/// ▼ ▼ ▼ -/// ┌────┐ ┌────┐ ┌────┐ -/// │ B1 │ │ B2 │ │ B3 │◀──┐ -/// └────┘ └────┘ └────┘ │ -/// │ │ │ │ -/// │ │ │ │ -/// └────────────┼────────────┘ │ -/// │ │ -/// ▼ │ -/// ┌─────┐ │ -/// │ FIL │───────────────┘ -/// └─────┘ -/// ``` -pub fn simulate_libp2p_net(nodes: &[CiphernodeHandle]) { +/// Simulate libp2p by taking output net commands and converting them to net events sending them to +/// the other nodes +pub async fn simulate_libp2p_net(nodes: &[CiphernodeHandle]) { + let mock = Libp2pMock::new(); for node in nodes.iter() { - let source = node.bus(); - for (_, node) in nodes.iter().enumerate() { - let dest = node.bus(); - if source != dest { - let pipe = SimulatedNetPipe { dest: dest.clone() }.start(); - source.subscribe(EventType::All, pipe.into()); - } else { - trace!("Source = Dest! Not piping bus to itself"); - } - } + let interface = node.channel_bridge().unwrap(); + mock.add_node(node.peer_id, interface).await; } } @@ -224,7 +193,7 @@ pub fn encrypt_ciphertext( .map(|pt| { pubkey .try_encrypt(&pt, &mut rng) - .map_err(|e| anyhow!("{e}")) + .map_err(|e| anyhow::anyhow!("{e}")) }) .collect::>>()?; Ok((ciphertext, plaintext)) diff --git a/crates/test-helpers/src/libp2p_mock.rs b/crates/test-helpers/src/libp2p_mock.rs new file mode 100644 index 0000000000..92442b60b9 --- /dev/null +++ b/crates/test-helpers/src/libp2p_mock.rs @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use std::{collections::HashMap, sync::Arc}; + +use e3_net::{ + events::{NetCommand, NetEvent}, + ContentHash, NetChannelBridge, NetInterfaceInverted, +}; +use e3_utils::ArcBytes; +use libp2p::{gossipsub::MessageId, kad::GetRecordError, PeerId}; +use tokio::sync::{broadcast, RwLock}; +use tracing::{error, warn}; + +#[derive(Debug, Clone)] +pub struct Libp2pMock { + store: Arc>>, + nodes: Arc>>, +} + +impl Libp2pMock { + pub fn new() -> Self { + Self { + store: Arc::new(RwLock::new(HashMap::new())), + nodes: Arc::new(RwLock::new(HashMap::new())), + } + } + + pub async fn add_node(&self, peer_id: PeerId, handle: NetChannelBridge) { + self.nodes.write().await.insert(peer_id, handle.clone()); + + let src_event_tx = handle.event_tx(); + let mut src_cmd_rx = handle.cmd_rx(); + let store = self.store.clone(); + let nodes = self.nodes.clone(); + let self_peer_id = peer_id; + + tokio::spawn(async move { + loop { + match src_cmd_rx.recv().await { + Ok(NetCommand::GossipPublish { + data, + correlation_id, + .. + }) => { + // Broadcast to all other nodes + let peers = nodes.read().await; + for (id, peer) in peers.iter() { + if *id == self_peer_id { + continue; + } + if let Err(e) = peer.event_tx().send(NetEvent::GossipData(data.clone())) + { + error!("Libp2pMock: failed to forward GossipData to {id}: {e}"); + } + } + + let message_id = + MessageId::new(&format!("{correlation_id:?}").into_bytes()); + if let Err(e) = src_event_tx.send(NetEvent::GossipPublished { + correlation_id, + message_id, + }) { + error!("Libp2pMock: failed to send GossipPublished: {e}"); + } + } + Ok(NetCommand::DhtPutRecord { + correlation_id, + key, + value, + .. + }) => { + store.write().await.insert(key.clone(), value); + + if let Err(e) = src_event_tx.send(NetEvent::DhtPutRecordSucceeded { + key, + correlation_id, + }) { + error!("Libp2pMock: failed to send DhtPutRecordSucceeded: {e}"); + } + } + Ok(NetCommand::DhtGetRecord { + correlation_id, + key, + }) => { + let maybe_value = store.read().await.get(&key).cloned(); + + if let Some(value) = maybe_value { + if let Err(e) = src_event_tx.send(NetEvent::DhtGetRecordSucceeded { + key, + correlation_id, + value, + }) { + error!("Libp2pMock: failed to send DhtGetRecordSucceeded: {e}"); + } + } else { + if let Err(e) = src_event_tx.send(NetEvent::DhtGetRecordError { + correlation_id, + error: GetRecordError::NotFound { + key: libp2p::kad::RecordKey::new(&key.into_inner()), + closest_peers: vec![], + }, + }) { + error!("Libp2pMock: failed to send DhtGetRecordError: {e}"); + } + } + } + Ok(NetCommand::DhtRemoveRecords { keys }) => { + let mut s = store.write().await; + for key in keys { + s.remove(&key); + } + } + Err(broadcast::error::RecvError::Lagged(n)) => { + warn!("Libp2pMock: cmd receiver lagged by {n} messages"); + continue; + } + Err(_) => break, + _ => continue, + } + } + }); + } +} diff --git a/crates/tests/tests/integration.rs b/crates/tests/tests/integration.rs index 5fa0fa62a1..e123893ec9 100644 --- a/crates/tests/tests/integration.rs +++ b/crates/tests/tests/integration.rs @@ -28,8 +28,8 @@ use e3_test_helpers::{ create_seed_from_u64, create_shared_rng_from_u64, with_tracing, AddToCommittee, }; use e3_trbfv::helpers::calculate_error_size; -use e3_utils::rand_eth_addr; use e3_utils::utility_types::ArcBytes; +use e3_utils::{colorize, rand_eth_addr, Color}; use e3_zk_prover::test_utils::get_tempdir; use e3_zk_prover::ZkBackend; use fhe::bfv::PublicKey; @@ -389,31 +389,62 @@ async fn setup_score_sortition_environment( Ok(()) } -fn serialize_report(report: &[(&str, Duration)]) -> String { - let max_key_len = report.iter().map(|(k, _)| k.len()).max().unwrap_or(0); +#[derive(Default)] +struct Report { + inner: Vec<(String, Duration)>, +} - report - .iter() - .map(|(key, duration)| { - format!( - "{:width$}: {:.3}s", - key, - duration.as_secs_f64(), - width = max_key_len - ) - }) - .collect::>() - .join("\n") +fn repeat(ch: char, num: usize) -> String { + let mut s = String::new(); + while s.len() < num { + s.push(ch); + } + s +} + +impl Report { + pub fn push(&mut self, repo: (&str, Duration)) { + let (label, dur) = repo; + self.show(label); + self.inner.push((label.to_owned(), dur)); + } + + pub fn show(&self, label: &str) { + println!( + "\n\n {}\n {}{}{}\n {}\n", + colorize(repeat('#', label.len() + 6), Color::Yellow), + colorize("## ", Color::Yellow), + colorize(label.to_uppercase(), Color::White), + colorize(" ##", Color::Yellow), + colorize(repeat('#', label.len() + 6), Color::Yellow), + ); + } + + pub fn serialize(&self) -> String { + let max_key_len = self.inner.iter().map(|(k, _)| k.len()).max().unwrap_or(0); + + self.inner + .iter() + .map(|(key, duration)| { + format!( + "{:width$}: {:.3}s", + key, + duration.as_secs_f64(), + width = max_key_len + ) + }) + .collect::>() + .join("\n") + } } /// Test trbfv #[actix::test] #[serial_test::serial] async fn test_trbfv_actor() -> Result<()> { - println!("Running test_trbfv_actor..."); - let mut report: Vec<(&str, Duration)> = vec![]; + let mut report = Report::default(); + report.push(("Starting trbfv actor test", Duration::from_secs(0))); let whole_test = Instant::now(); - let _guard = with_tracing("info"); // NOTE: Here we are trying to make it as clear as possible as to what is going on so attempting to @@ -520,7 +551,7 @@ async fn test_trbfv_actor() -> Result<()> { .build() .await?; - report.push(("Setup", setup.elapsed())); + report.push(("Setup completed", setup.elapsed())); let committee_setup = Instant::now(); let chain_id = 1u64; @@ -545,8 +576,9 @@ async fn test_trbfv_actor() -> Result<()> { setup_score_sortition_environment(&bus, ð_addrs, chain_id).await?; // Flush all events - nodes.flush_all_history(100).await?; - report.push(("Committee Setup", committee_setup.elapsed())); + nodes.flush_all_history(10000).await?; + + report.push(("Committee Setup Completed", committee_setup.elapsed())); /////////////////////////////////////////////////////////////////////////////////// // 2. Trigger E3Requested @@ -586,16 +618,13 @@ async fn test_trbfv_actor() -> Result<()> { &collector_addr, )?; - println!( + report.show(&format!( "Committee selected: {} nodes, {} buffer nodes", committee.len(), buffer_nodes.len() - ); + )); - let expected = vec!["E3Requested"]; - let _ = nodes - .take_history_with_timeout(0, expected.len(), Duration::from_secs(1000)) - .await?; + nodes.expect_events(&["E3Requested"]).await?; bus.publish_without_context(CommitteeFinalized { e3_id: e3_id.clone(), @@ -605,62 +634,13 @@ async fn test_trbfv_actor() -> Result<()> { let committee_finalized_timer = Instant::now(); - let expected = vec!["CommitteeFinalized"]; - let _ = nodes - .take_history_with_timeout(0, expected.len(), Duration::from_secs(1000)) - .await?; + nodes.expect_events(&["CommitteeFinalized"]).await?; report.push(( - "Committee Finalization", + "Committee Finalization Complete", committee_finalized_timer.elapsed(), )); - // First, wait for all EncryptionKeyCreated events (BFV key exchange) - // The collector (node 0) only sees events forwarded by simulate_libp2p: - // - EncryptionKeyCreated × 5 (one per party, passes is_document_publisher_event filter) - // Internal events (EncryptionKeyPending, ComputeRequest/Response) stay on committee nodes' local buses. - let encryption_keys_timer = Instant::now(); - let expected = vec![ - "EncryptionKeyCreated", - "EncryptionKeyCreated", - "EncryptionKeyCreated", - "EncryptionKeyCreated", - "EncryptionKeyCreated", - ]; - let _ = nodes - .take_history_with_timeout(0, expected.len(), Duration::from_secs(1000)) - .await?; - report.push(( - "All EncryptionKeyCreated events", - encryption_keys_timer.elapsed(), - )); - - // Then wait for all ThresholdShareCreated events - // Each of the 5 parties publishes 5 events (one per target party) = 25 total - // Only ThresholdShareCreated passes the simulate_libp2p filter (is_document_publisher_event). - // Internal events (ComputeRequest/Response for GenPk, GenEsi, ZK proofs, ThresholdSharePending, - // PkGenerationProofSigned, DkgProofSigned) stay on committee nodes' local buses. - let shares_timer = Instant::now(); - let expected: Vec<&str> = (0..25).map(|_| "ThresholdShareCreated").collect(); - let _ = nodes - .take_history_with_timeout(0, expected.len(), Duration::from_secs(3000)) - .await?; - report.push(("All ThresholdShareCreated events", shares_timer.elapsed())); - - // Wait for DecryptionKeyShared (Exchange #3) events - // - DecryptionKeyShared × 5 (passes is_document_publisher_event filter) - // Each committee node publishes DecryptionKeyShared after computing its decryption key - // and generating C4 (share decryption) proofs. - let decryption_key_shared_timer = Instant::now(); - let expected: Vec<&str> = (0..5).map(|_| "DecryptionKeyShared").collect(); - let _ = nodes - .take_history_with_timeout(0, expected.len(), Duration::from_secs(1000)) - .await?; - report.push(( - "All DecryptionKeyShared events", - decryption_key_shared_timer.elapsed(), - )); - // Wait for KeyshareCreated + C1 verification + C5 proof + PublicKeyAggregated // - KeyshareCreated × 5 (forwarded from committee nodes) // - ShareVerificationDispatched (C1 proof verification dispatched by PublicKeyAggregator) @@ -673,24 +653,27 @@ async fn test_trbfv_actor() -> Result<()> { // - PkAggregationProofSigned (C5 proof signed by ProofRequestActor) // - PublicKeyAggregated × 1 let shares_to_pubkey_agg_timer = Instant::now(); - let expected = vec![ - "KeyshareCreated", - "KeyshareCreated", - "KeyshareCreated", - "KeyshareCreated", - "KeyshareCreated", - "ShareVerificationDispatched", - "ComputeRequest", - "ComputeResponse", - "ShareVerificationComplete", - "PkAggregationProofPending", - "ComputeRequest", - "ComputeResponse", - "PkAggregationProofSigned", - "PublicKeyAggregated", - ]; let h = nodes - .take_history_with_timeout(0, expected.len(), Duration::from_secs(1000)) + .expect_events_with_timeouts( + &[ + "KeyshareCreated", + "KeyshareCreated", + "KeyshareCreated", + "KeyshareCreated", + "KeyshareCreated", + "ShareVerificationDispatched", + "ComputeRequest", + "ComputeResponse", + "ShareVerificationComplete", + "PkAggregationProofPending", + "ComputeRequest", + "ComputeResponse", + "PkAggregationProofSigned", + "PublicKeyAggregated", + ], + Duration::from_secs(5000), + Duration::from_secs(600), + ) .await?; report.push(( @@ -780,7 +763,12 @@ async fn test_trbfv_actor() -> Result<()> { let expected_count = 1 + 5 + 1 + 2 + 1 + 2 + 1 + 2 + 1 + 1; let h = nodes - .take_history_with_timeout(0, expected_count, Duration::from_secs(1000)) + .take_history_with_timeouts( + 0, + expected_count, + Some(Duration::from_secs(1000)), + Some(Duration::from_secs(1000)), + ) .await?; report.push(( @@ -833,7 +821,7 @@ async fn test_trbfv_actor() -> Result<()> { println!("{}", mt_report); report.push(("Entire Test", whole_test.elapsed())); - println!("{}", serialize_report(&report)); + println!("{}", report.serialize()); Ok(()) } @@ -926,6 +914,7 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { assert_eq!( history + .events .into_iter() .map(|e| e.into_data()) .collect::>(), @@ -972,6 +961,7 @@ async fn test_p2p_actor_forwards_events_to_bus() -> Result<()> { assert_eq!( history + .events .into_iter() .map(|e| e.into_data()) .collect::>(), @@ -987,6 +977,7 @@ async fn test_p2p_actor_forwards_events_to_bus() -> Result<()> { /// Test that stopped keyshares retain their state after restart. /// This test needs to be ported to the new trBFV system once Sync is completed. +// XXX: ENABLE THIS!! #[actix::test] #[ignore = "Needs to be ported to trBFV system after Sync is completed"] async fn test_stopped_keyshares_retain_state() -> Result<()> { @@ -1050,7 +1041,7 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { .await?; result.push(tuple); } - simulate_libp2p_net(&result); + simulate_libp2p_net(&result).await; Ok(result) } @@ -1145,12 +1136,13 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { ) .await?; let history_collector = cn1.history().unwrap(); - simulate_libp2p_net(&[cn1, cn2]); + simulate_libp2p_net(&[cn1, cn2]).await; println!("getting collector from cn1.6"); // get the public key from history. let pubkey: PublicKey = history + .events .iter() .filter_map(|evt| match evt.get_data() { EnclaveEventData::KeyshareCreated(data) => { @@ -1177,6 +1169,7 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { .await?; let actual = history + .events .into_iter() .filter_map(|e| match e.into_data() { EnclaveEventData::PlaintextAggregated(data) => Some(data), @@ -1268,7 +1261,7 @@ async fn test_duplicate_e3_id_with_different_chain_id() -> Result<()> { .await?; result.push(tuple); } - simulate_libp2p_net(&result); + simulate_libp2p_net(&result).await; Ok(result) } @@ -1350,9 +1343,9 @@ async fn test_duplicate_e3_id_with_different_chain_id() -> Result<()> { .await?; assert_eq!( - history.last().cloned().unwrap().into_data(), + history.events.last().cloned().unwrap().into_data(), PublicKeyAggregated { - pubkey: test_pubkey.to_bytes(), + pubkey: ArcBytes::from_bytes(&test_pubkey.to_bytes()), public_key_hash, e3_id: E3id::new("1234", 1), nodes: OrderedSet::from(eth_addrs.clone()), @@ -1393,9 +1386,9 @@ async fn test_duplicate_e3_id_with_different_chain_id() -> Result<()> { .await?; assert_eq!( - history.last().cloned().unwrap().into_data(), + history.events.last().cloned().unwrap().into_data(), PublicKeyAggregated { - pubkey: test_pubkey.to_bytes(), + pubkey: ArcBytes::from_bytes(&test_pubkey.to_bytes()), public_key_hash, e3_id: E3id::new("1234", 2), nodes: OrderedSet::from(eth_addrs.clone()), diff --git a/crates/utils/src/retry.rs b/crates/utils/src/retry.rs index 6eb82724e8..f8bcd1c046 100644 --- a/crates/utils/src/retry.rs +++ b/crates/utils/src/retry.rs @@ -19,7 +19,7 @@ pub fn to_retry(e: impl Into) -> RetryError { RetryError::Retry(e.into()) } -pub const BACKOFF_DELAY: u64 = 500; +pub const BACKOFF_DELAY: u64 = 3000; pub const BACKOFF_MAX_RETRIES: u32 = 10; /// Retries an async operation with exponential backoff diff --git a/crates/zk-prover/tests/backend_tests.rs b/crates/zk-prover/tests/backend_tests.rs index b3cfc3b2df..e0f7dea600 100644 --- a/crates/zk-prover/tests/backend_tests.rs +++ b/crates/zk-prover/tests/backend_tests.rs @@ -6,6 +6,8 @@ mod common; +use std::env; + use common::test_backend; use e3_zk_prover::{test_utils::get_tempdir, ZkConfig, ZkProver}; use tokio::fs; @@ -84,6 +86,11 @@ async fn test_work_dir_path_traversal_protection() { #[test] fn test_prover_requires_bb() { + if env::var("E3_CUSTOM_BB").is_ok() { + // Cannot run this test when E3_CUSTOM_BB is set + return; + } + let temp = get_tempdir().unwrap(); let backend = test_backend(temp.path(), ZkConfig::default()); let prover = ZkProver::new(&backend); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2ae7325694..b0fc4211f6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -728,7 +728,7 @@ importers: version: 5.3.0 '@risc0/ethereum': specifier: file:lib/risc0-ethereum - version: file:templates/default/lib/risc0-ethereum + version: risc0-ethereum@file:templates/default/lib/risc0-ethereum '@types/chai': specifier: ^4.2.0 version: 4.3.20 @@ -3100,9 +3100,6 @@ packages: '@reown/appkit@1.7.8': resolution: {integrity: sha512-51kTleozhA618T1UvMghkhKfaPcc9JlKwLJ5uV+riHyvSoWPKPRIa5A6M1Wano5puNyW0s3fwywhyqTHSilkaA==} - '@risc0/ethereum@file:templates/default/lib/risc0-ethereum': - resolution: {directory: templates/default/lib/risc0-ethereum, type: directory} - '@rolldown/pluginutils@1.0.0-beta.27': resolution: {integrity: sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==} @@ -8786,6 +8783,9 @@ packages: resolution: {integrity: sha512-5Di9UC0+8h1L6ZD2d7awM7E/T4uA1fJRlx6zk/NvdCCVEoAnFqvHmCuNeIKoCeIixBX/q8uM+6ycDvF8woqosA==} engines: {node: '>= 0.8'} + risc0-ethereum@file:templates/default/lib/risc0-ethereum: + resolution: {directory: templates/default/lib/risc0-ethereum, type: directory} + robust-predicates@3.0.2: resolution: {integrity: sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg==} @@ -13241,8 +13241,6 @@ snapshots: - utf-8-validate - zod - '@risc0/ethereum@file:templates/default/lib/risc0-ethereum': {} - '@rolldown/pluginutils@1.0.0-beta.27': {} '@rollup/plugin-inject@5.0.5(rollup@4.52.5)': @@ -20114,7 +20112,7 @@ snapshots: '@noble/hashes': 1.8.0 '@scure/bip32': 1.7.0 '@scure/bip39': 1.6.0 - abitype: 1.1.1(typescript@5.8.3)(zod@3.25.76) + abitype: 1.1.1(typescript@5.8.3)(zod@3.22.4) eventemitter3: 5.0.1 optionalDependencies: typescript: 5.8.3 @@ -20968,6 +20966,8 @@ snapshots: hash-base: 3.1.2 inherits: 2.0.4 + risc0-ethereum@file:templates/default/lib/risc0-ethereum: {} + robust-predicates@3.0.2: {} rollup@4.52.5: @@ -22168,7 +22168,7 @@ snapshots: '@noble/hashes': 1.8.0 '@scure/bip32': 1.7.0 '@scure/bip39': 1.6.0 - abitype: 1.1.0(typescript@5.8.3)(zod@3.25.76) + abitype: 1.1.0(typescript@5.8.3)(zod@3.22.4) isows: 1.0.7(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@5.0.10)) ox: 0.9.6(typescript@5.8.3) ws: 8.18.3(bufferutil@4.0.9)(utf-8-validate@5.0.10)